From e7492c624dcca3e27ef6ecc132ed8ed4bbeb2e9f Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Mon, 7 Jun 2021 10:47:57 -0600 Subject: [PATCH 01/98] separate mpi gv test; passing pointers; const (#1262) Co-authored-by: Stephen Herbener <32968781+srherbener@users.noreply.github.com> Co-authored-by: Ryan Honeyager --- src/ufo/GeoVaLs.cc | 7 ++++--- src/ufo/GeoVaLs.h | 2 +- src/ufo/Locations.cc | 5 +++-- src/ufo/Locations.h | 2 +- test/CMakeLists.txt | 9 ++++++++- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/ufo/GeoVaLs.cc b/src/ufo/GeoVaLs.cc index b636b697b..4a8bb4c99 100644 --- a/src/ufo/GeoVaLs.cc +++ b/src/ufo/GeoVaLs.cc @@ -9,6 +9,7 @@ #include #include +#include #include #include "eckit/config/Configuration.h" @@ -29,9 +30,9 @@ namespace ufo { // ----------------------------------------------------------------------------- /*! \brief Default constructor - does not allocate fields */ -GeoVaLs::GeoVaLs(const std::shared_ptr dist, +GeoVaLs::GeoVaLs(std::shared_ptr dist, const oops::Variables & vars) - : keyGVL_(-1), vars_(vars), dist_(dist) + : keyGVL_(-1), vars_(vars), dist_(std::move(dist)) { oops::Log::trace() << "GeoVaLs default constructor starting" << std::endl; ufo_geovals_default_constr_f90(keyGVL_); @@ -218,7 +219,7 @@ double GeoVaLs::dot_product_with(const GeoVaLs & other) const { assert(vars_ == other.vars_); auto accumulator = dist_->createAccumulator(); std::vector this_values(nlocs), other_values(nlocs); - double missing = util::missingValue(missing); + const double missing = util::missingValue(missing); // loop over all variables in geovals for (size_t jvar = 0; jvar < vars_.size(); ++jvar) { const size_t nlevs = this->nlevs(vars_[jvar]); diff --git a/src/ufo/GeoVaLs.h b/src/ufo/GeoVaLs.h index 57e59278f..cbce8e926 100644 --- a/src/ufo/GeoVaLs.h +++ b/src/ufo/GeoVaLs.h @@ -40,7 +40,7 @@ class GeoVaLs : public util::Printable, public: static const std::string classname() {return "ufo::GeoVaLs";} - GeoVaLs(const std::shared_ptr, const oops::Variables &); + GeoVaLs(std::shared_ptr, const oops::Variables &); GeoVaLs(const Locations &, const oops::Variables &); GeoVaLs(const eckit::Configuration &, const ioda::ObsSpace &, const oops::Variables &); diff --git a/src/ufo/Locations.cc b/src/ufo/Locations.cc index 8c98ec740..0cd88f931 100644 --- a/src/ufo/Locations.cc +++ b/src/ufo/Locations.cc @@ -7,6 +7,7 @@ #include "ufo/Locations.h" +#include #include #include "eckit/config/Configuration.h" @@ -24,8 +25,8 @@ namespace ufo { Locations::Locations(const std::vector & lons, const std::vector & lats, const std::vector & times, - const std::shared_ptr dist) - : dist_(dist), lons_(lons), lats_(lats), times_(times) { + std::shared_ptr dist) + : dist_(std::move(dist)), lons_(lons), lats_(lats), times_(times) { ASSERT(lons_.size() == lats_.size()); ASSERT(lats_.size() == times_.size()); oops::Log::trace() << "ufo::Locations::Locations done" << std::endl; diff --git a/src/ufo/Locations.h b/src/ufo/Locations.h index f0c9bf118..c33cce88b 100644 --- a/src/ufo/Locations.h +++ b/src/ufo/Locations.h @@ -34,7 +34,7 @@ class Locations : public util::Printable, /// \brief constructor from passed \p lons, \p lats, \p times Locations(const std::vector & lons, const std::vector & lats, const std::vector & times, - const std::shared_ptr); + std::shared_ptr); /// \brief constructor used in oops tests Locations(const eckit::Configuration &, const eckit::mpi::Comm &); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c9d4f9a0f..266e8e2b3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -483,7 +483,14 @@ ecbuild_add_test( TARGET test_ufo_geovals ARGS "testinput/geovals.yaml" ENVIRONMENT OOPS_TRAPFPE=1 LIBS ufo - MPI 2 + TEST_DEPENDS ufo_get_ufo_test_data ) + +ecbuild_add_test( TARGET test_ufo_geovals_mpi + SOURCES mains/TestGeoVaLs.cc + ARGS "testinput/geovals.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + MPI 4 + LIBS ufo TEST_DEPENDS ufo_get_ufo_test_data ) ecbuild_add_test( TARGET test_ufo_locations From f21366bffed7d88f91035d881d8da6bf31fa4bd8 Mon Sep 17 00:00:00 2001 From: Greg Thompson <63315545+gthompsnJCSDA@users.noreply.github.com> Date: Mon, 7 Jun 2021 13:20:31 -0600 Subject: [PATCH 02/98] New **composite** obs operators of SfcPCorrected and GsiSfcModel (#1259) * unit test yaml for composite of VertInterp and SfcPCorrected * follow Wojciech changes (PR736) to make sfcpcorrected a composite component * adjust the RMS value to match these data * perform reorderz if needed and make it possible to use atmsfcinterp as a composite operator * add missing var declaration * test yaml for composite operator * add in code changes (follow Wojciech) for GsiSfcModel to permit composite obs oper; also update yamls with test files and tolerances * two new ctests added for new composite obs oper * spelling typos corrected Co-authored-by: gthompsnJCSDA Co-authored-by: Ryan Honeyager --- src/ufo/atmsfcinterp/ObsAtmSfcInterp.cc | 10 ++++- src/ufo/atmsfcinterp/ObsAtmSfcInterp.h | 3 ++ .../ObsAtmSfcInterp.interface.F90 | 15 ++++--- .../atmsfcinterp/ObsAtmSfcInterp.interface.h | 23 +++++++++- src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 | 45 ++++++++++++++----- src/ufo/sfcpcorrected/ObsSfcPCorrected.cc | 10 ++++- src/ufo/sfcpcorrected/ObsSfcPCorrected.h | 3 ++ .../ObsSfcPCorrected.interface.F90 | 15 ++++--- .../ObsSfcPCorrected.interface.h | 23 +++++++++- .../sfcpcorrected/ufo_sfcpcorrected_mod.F90 | 10 +++-- test/CMakeLists.txt | 16 +++++++ .../composite_GsiSfcModel_SfcPCorrected.yaml | 35 +++++++++++++++ .../composite_vertInterp_SfcPCorrected.yaml | 35 +++++++++++++++ 13 files changed, 215 insertions(+), 28 deletions(-) create mode 100644 test/testinput/unit_tests/composite_GsiSfcModel_SfcPCorrected.yaml create mode 100644 test/testinput/unit_tests/composite_vertInterp_SfcPCorrected.yaml diff --git a/src/ufo/atmsfcinterp/ObsAtmSfcInterp.cc b/src/ufo/atmsfcinterp/ObsAtmSfcInterp.cc index 847348f17..48fb9b6cd 100644 --- a/src/ufo/atmsfcinterp/ObsAtmSfcInterp.cc +++ b/src/ufo/atmsfcinterp/ObsAtmSfcInterp.cc @@ -8,13 +8,16 @@ #include "ufo/atmsfcinterp/ObsAtmSfcInterp.h" #include +#include #include "oops/util/Logger.h" #include "ioda/ObsVector.h" +#include "ufo/filters/Variables.h" #include "ufo/GeoVaLs.h" #include "ufo/ObsDiagnostics.h" +#include "ufo/utils/OperatorUtils.h" // for getOperatorVariables namespace ufo { @@ -26,7 +29,12 @@ ObsAtmSfcInterp::ObsAtmSfcInterp(const ioda::ObsSpace & odb, const eckit::Config : ObsOperatorBase(odb, config), keyOperAtmSfcInterp_(0), odb_(odb), varin_() { - ufo_atmsfcinterp_setup_f90(keyOperAtmSfcInterp_, config, odb.obsvariables(), varin_); + std::vector operatorVarIndices; + getOperatorVariables(config, odb.obsvariables(), operatorVars_, operatorVarIndices); + + ufo_atmsfcinterp_setup_f90(keyOperAtmSfcInterp_, config, + operatorVars_, operatorVarIndices.data(), operatorVarIndices.size(), + varin_); oops::Log::trace() << "ObsAtmSfcInterp created." << std::endl; } diff --git a/src/ufo/atmsfcinterp/ObsAtmSfcInterp.h b/src/ufo/atmsfcinterp/ObsAtmSfcInterp.h index 3b7aa1243..d53febe8b 100644 --- a/src/ufo/atmsfcinterp/ObsAtmSfcInterp.h +++ b/src/ufo/atmsfcinterp/ObsAtmSfcInterp.h @@ -47,6 +47,8 @@ class ObsAtmSfcInterp : public ObsOperatorBase, // Other const oops::Variables & requiredVars() const override {return varin_;} + oops::Variables simulatedVars() const override {return operatorVars_;} + int & toFortran() {return keyOperAtmSfcInterp_;} const int & toFortran() const {return keyOperAtmSfcInterp_;} @@ -55,6 +57,7 @@ class ObsAtmSfcInterp : public ObsOperatorBase, F90hop keyOperAtmSfcInterp_; const ioda::ObsSpace& odb_; oops::Variables varin_; + oops::Variables operatorVars_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/atmsfcinterp/ObsAtmSfcInterp.interface.F90 b/src/ufo/atmsfcinterp/ObsAtmSfcInterp.interface.F90 index 1330207fb..052c5ae5e 100644 --- a/src/ufo/atmsfcinterp/ObsAtmSfcInterp.interface.F90 +++ b/src/ufo/atmsfcinterp/ObsAtmSfcInterp.interface.F90 @@ -37,13 +37,16 @@ module ufo_atmsfcinterp_mod_c ! ------------------------------------------------------------------------------ -subroutine ufo_atmsfcinterp_setup_c(c_key_self, c_conf, c_obsvars, c_geovars) bind(c,name='ufo_atmsfcinterp_setup_f90') +subroutine ufo_atmsfcinterp_setup_c(c_key_self, c_conf, c_obsvars, c_obsvarindices, c_nobsvars, & + c_geovars) bind(c,name='ufo_atmsfcinterp_setup_f90') use oops_variables_mod implicit none -integer(c_int), intent(inout) :: c_key_self -type(c_ptr), value, intent(in) :: c_conf -type(c_ptr), value, intent(in) :: c_obsvars ! variables to be simulated -type(c_ptr), value, intent(in) :: c_geovars ! variables requested from the model +integer(c_int), intent(inout) :: c_key_self +type(c_ptr), value, intent(in) :: c_conf +type(c_ptr), value, intent(in) :: c_obsvars ! variables to be simulated +integer(c_int), intent(in), value :: c_nobsvars +integer(c_int), intent(in) :: c_obsvarindices(c_nobsvars) ! ... and their global indices +type(c_ptr), value, intent(in) :: c_geovars ! variables requested from the model type(ufo_atmsfcinterp), pointer :: self type(fckit_configuration) :: f_conf @@ -52,6 +55,8 @@ subroutine ufo_atmsfcinterp_setup_c(c_key_self, c_conf, c_obsvars, c_geovars) bi f_conf = fckit_configuration(c_conf) self%obsvars = oops_variables(c_obsvars) +allocate(self%obsvarindices(self%obsvars%nvars())) +self%obsvarindices(:) = c_obsvarindices(:) + 1 ! Convert from C to Fortran indexing self%geovars = oops_variables(c_geovars) call self%setup(f_conf) diff --git a/src/ufo/atmsfcinterp/ObsAtmSfcInterp.interface.h b/src/ufo/atmsfcinterp/ObsAtmSfcInterp.interface.h index d23903f73..16ae2b064 100644 --- a/src/ufo/atmsfcinterp/ObsAtmSfcInterp.interface.h +++ b/src/ufo/atmsfcinterp/ObsAtmSfcInterp.interface.h @@ -20,8 +20,29 @@ extern "C" { // ----------------------------------------------------------------------------- +// ----------------------------------------------------------------------------- +// AtmSfcInterp observation operator +// ----------------------------------------------------------------------------- + + /// \param operatorVars + /// Variables to be simulated by this operator. + /// \param operatorVarIndices + /// Indices of the variables from \p operatorVar in the list of all simulated + /// variables in the ObsSpace. + /// \param numOperatorVarIndices + /// Size of the \p operatorVarIndices array (must be the same as the number of variables in + /// \p operatorVars). + /// \param[out] requiredVars + /// GeoVaLs required for the simulation of the variables \p operatorVars. + /// + /// Example: if the list of simulated variables in the ObsSpace is + /// [air_temperature, northward_wind, eastward_wind] and \p operatorVars is + /// [northward_wind, eastward_wind], then \p operatorVarIndices should be set to [1, 2]. + void ufo_atmsfcinterp_setup_f90(F90hop &, const eckit::Configuration &, - const oops::Variables &, oops::Variables &); + const oops::Variables &operatorVars, + const int *operatorVarIndices, const int numOperatorVarIndices, + oops::Variables &requiredVars); void ufo_atmsfcinterp_delete_f90(F90hop &); void ufo_atmsfcinterp_simobs_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, const int &, const int &, double &); diff --git a/src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 b/src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 index 6c852b153..afd83e458 100644 --- a/src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 +++ b/src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 @@ -17,7 +17,9 @@ module ufo_atmsfcinterp_mod !> Fortran derived type for the observation type type, public :: ufo_atmsfcinterp private - type(oops_variables), public :: obsvars + type(oops_variables), public :: obsvars ! Variables to be simulated + integer, allocatable, public :: obsvarindices(:) ! Indices of obsvars in the list of all + ! simulated variables in the ObsSpace type(oops_variables), public :: geovars logical :: use_fact10 real(kind_real) :: magl @@ -44,6 +46,11 @@ subroutine atmsfcinterp_setup_(self, f_conf) self%use_fact10 = .true. end if + nvars = self%obsvars%nvars() + do ivar = 1, nvars + call self%geovars%push_back(self%obsvars%variable(ivar)) + enddo + !> add geopotential height call self%geovars%push_back(var_z) !> need skin temperature for near-surface interpolations @@ -63,37 +70,36 @@ subroutine atmsfcinterp_setup_(self, f_conf) call self%geovars%push_back(var_v) call self%geovars%push_back(var_sfc_lfrac) if (self%use_fact10) call self%geovars%push_back(var_sfc_fact10) - nvars = self%obsvars%nvars() - do ivar = 1, nvars - call self%geovars%push_back(self%obsvars%variable(ivar)) - enddo end subroutine atmsfcinterp_setup_ ! ------------------------------------------------------------------------------ -subroutine atmsfcinterp_simobs_(self, geovals, obss, nvars, nlocs, hofx) +subroutine atmsfcinterp_simobs_(self, geovals_in, obss, nvars, nlocs, hofx) use atmsfc_mod, only : calc_conv_vel_gsi, sfc_wind_fact_gsi, & calc_psi_vars_gsi use thermo_utils_mod, only: calc_theta, gsi_tp_to_qs use ufo_constants_mod, only: grav, rv, rd, rd_over_cp, von_karman - use ufo_geovals_mod, only: ufo_geovals, ufo_geoval, ufo_geovals_get_var + use ufo_geovals_mod, only: ufo_geovals, ufo_geoval, ufo_geovals_get_var, ufo_geovals_copy, ufo_geovals_reorderzdir use ufo_utils_mod, only: cmp_strings use obsspace_mod use iso_c_binding + use fckit_log_module, only: fckit_log implicit none class(ufo_atmsfcinterp), intent(in) :: self integer, intent(in) :: nvars, nlocs - type(ufo_geovals), intent(in) :: geovals + type(ufo_geovals), intent(in) :: geovals_in real(c_double), intent(inout) :: hofx(nvars, nlocs) type(c_ptr), value, intent(in) :: obss type(ufo_geoval), pointer :: phi, hgt, tsfc, roughlen, psfc, prs, prsi, & tsen, tv, q, u, v, landmask, & profile, rad10 - integer :: ivar, iobs + type(ufo_geovals):: geovals + integer :: ivar, iobs, iobsvar real(kind_real), allocatable :: obselev(:), obshgt(:) real(kind_real), parameter :: minroughlen = 1.0e-4_kind_real character(len=MAXVARLEN) :: geovar + character(len=MAXVARLEN) :: var_zdir real(kind_real) :: thv1, thv2, th1, thg, thvg, rib, V2, agl, zbot real(kind_real) :: gzsoz0, gzzoz0 real(kind_real) :: redfac, psim, psimz, psih, psihz @@ -103,6 +109,19 @@ subroutine atmsfcinterp_simobs_(self, geovals, obss, nvars, nlocs, hofx) real(kind_real) :: psit, psitz, ust, psiq, psiqz real(kind_real), parameter :: zint0 = 0.01_kind_real ! default roughness over land real(kind_real), parameter :: ka = 2.4e-5_kind_real + character(1024) :: debug_msg + + ! Notes: + ! (1) Set desired vertical coordinate direction (top2bottom or bottom2top) based + ! on vertical coodinate variable and reload geovals according to the set + ! direction + ! (2) This is done because this observation operator assumes pressure levels + ! are from bottom to top (bottom2top) with model pressure(1) for surface and + ! model pressure(nsig+1) for model top + + call ufo_geovals_copy(geovals_in, geovals) ! dont want to change geovals_in + var_zdir = var_prsi ! vertical coordinate variable + call ufo_geovals_reorderzdir(geovals, var_zdir, "bottom2top") ! to compute the value near the surface we are going to use ! similarity theory which requires a number of near surface parameters @@ -171,9 +190,13 @@ subroutine atmsfcinterp_simobs_(self, geovals, obss, nvars, nlocs, hofx) ! calculate parameters regardless of variable call calc_psi_vars_gsi(rib, gzsoz0, gzzoz0, thv1, thv2, V2, th1, & thg, zbot, agl, psim, psih, psimz, psihz) - do ivar = 1, nvars + + do iobsvar = 1, size(self%obsvarindices) + ! Get the index of the row of hofx to fill + ivar = self%obsvarindices(iobsvar) + ! Get the name of input variable in geovals - geovar = self%obsvars%variable(ivar) + geovar = self%obsvars%variable(iobsvar) ! Get profile for this variable from geovals call ufo_geovals_get_var(geovals, geovar, profile) diff --git a/src/ufo/sfcpcorrected/ObsSfcPCorrected.cc b/src/ufo/sfcpcorrected/ObsSfcPCorrected.cc index 1b99bd0f7..81db432b0 100644 --- a/src/ufo/sfcpcorrected/ObsSfcPCorrected.cc +++ b/src/ufo/sfcpcorrected/ObsSfcPCorrected.cc @@ -8,13 +8,16 @@ #include "ufo/sfcpcorrected/ObsSfcPCorrected.h" #include +#include #include "ioda/ObsVector.h" #include "oops/base/Variables.h" +#include "ufo/filters/Variables.h" #include "ufo/GeoVaLs.h" #include "ufo/ObsDiagnostics.h" +#include "ufo/utils/OperatorUtils.h" // for getOperatorVariables namespace ufo { @@ -26,7 +29,12 @@ ObsSfcPCorrected::ObsSfcPCorrected(const ioda::ObsSpace & odb, const eckit::Configuration & config) : ObsOperatorBase(odb, config), keyOper_(0), odb_(odb), varin_() { - ufo_sfcpcorrected_setup_f90(keyOper_, config, odb.obsvariables(), varin_); + std::vector operatorVarIndices; + getOperatorVariables(config, odb.obsvariables(), operatorVars_, operatorVarIndices); + + ufo_sfcpcorrected_setup_f90(keyOper_, config, + operatorVars_, operatorVarIndices.data(), operatorVarIndices.size(), + varin_); oops::Log::trace() << "ObsSfcPCorrected created." << std::endl; } diff --git a/src/ufo/sfcpcorrected/ObsSfcPCorrected.h b/src/ufo/sfcpcorrected/ObsSfcPCorrected.h index 1eddb044d..96e0e1a9e 100644 --- a/src/ufo/sfcpcorrected/ObsSfcPCorrected.h +++ b/src/ufo/sfcpcorrected/ObsSfcPCorrected.h @@ -47,6 +47,8 @@ class ObsSfcPCorrected : public ObsOperatorBase, // Other const oops::Variables & requiredVars() const override {return varin_;} + oops::Variables simulatedVars() const override {return operatorVars_;} + int & toFortran() {return keyOper_;} const int & toFortran() const {return keyOper_;} @@ -55,6 +57,7 @@ class ObsSfcPCorrected : public ObsOperatorBase, F90hop keyOper_; const ioda::ObsSpace& odb_; oops::Variables varin_; + oops::Variables operatorVars_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/sfcpcorrected/ObsSfcPCorrected.interface.F90 b/src/ufo/sfcpcorrected/ObsSfcPCorrected.interface.F90 index 5016dd62f..4684c923e 100644 --- a/src/ufo/sfcpcorrected/ObsSfcPCorrected.interface.F90 +++ b/src/ufo/sfcpcorrected/ObsSfcPCorrected.interface.F90 @@ -32,14 +32,17 @@ module ufo_sfcpcorrected_mod_c ! ------------------------------------------------------------------------------ -subroutine ufo_sfcpcorrected_setup_c(c_key_self, c_conf, c_obsvars, c_geovars) bind(c,name='ufo_sfcpcorrected_setup_f90') +subroutine ufo_sfcpcorrected_setup_c(c_key_self, c_conf, c_obsvars, c_obsvarindices, c_nobsvars, & + c_geovars) bind(c,name='ufo_sfcpcorrected_setup_f90') use fckit_configuration_module, only: fckit_configuration use oops_variables_mod implicit none -integer(c_int), intent(inout) :: c_key_self -type(c_ptr), value, intent(in) :: c_conf -type(c_ptr), value, intent(in) :: c_obsvars ! variables to be simulated -type(c_ptr), value, intent(in) :: c_geovars ! variables requested from the model +integer(c_int), intent(inout) :: c_key_self +type(c_ptr), value, intent(in) :: c_conf +type(c_ptr), value, intent(in) :: c_obsvars ! variables to be simulated +integer(c_int), value, intent(in) :: c_nobsvars +integer(c_int), intent(in) :: c_obsvarindices(c_nobsvars) ! ... and their global indices +type(c_ptr), value, intent(in) :: c_geovars ! variables requested from the model type(ufo_sfcpcorrected), pointer :: self type(fckit_configuration) :: f_conf @@ -48,6 +51,8 @@ subroutine ufo_sfcpcorrected_setup_c(c_key_self, c_conf, c_obsvars, c_geovars) b f_conf = fckit_configuration(c_conf) self%obsvars = oops_variables(c_obsvars) +allocate(self%obsvarindices(self%obsvars%nvars())) +self%obsvarindices(:) = c_obsvarindices(:) + 1 ! Convert from C to Fortran indexing self%geovars = oops_variables(c_geovars) call self%setup(f_conf) diff --git a/src/ufo/sfcpcorrected/ObsSfcPCorrected.interface.h b/src/ufo/sfcpcorrected/ObsSfcPCorrected.interface.h index 8472ceec1..e22c95168 100644 --- a/src/ufo/sfcpcorrected/ObsSfcPCorrected.interface.h +++ b/src/ufo/sfcpcorrected/ObsSfcPCorrected.interface.h @@ -18,10 +18,31 @@ namespace ufo { extern "C" { +// ----------------------------------------------------------------------------- +// SfcPCorrected observation operator +// ----------------------------------------------------------------------------- + + /// \param operatorVars + /// Variables to be simulated by this operator. + /// \param operatorVarIndices + /// Indices of the variables from \p operatorVar in the list of all simulated + /// variables in the ObsSpace. + /// \param numOperatorVarIndices + /// Size of the \p operatorVarIndices array (must be the same as the number of variables in + /// \p operatorVars). + /// \param[out] requiredVars + /// GeoVaLs required for the simulation of the variables \p operatorVars. + /// + /// Example: if the list of simulated variables in the ObsSpace is + /// [air_temperature, northward_wind, eastward_wind] and \p operatorVars is + /// [northward_wind, eastward_wind], then \p operatorVarIndices should be set to [1, 2]. + // ----------------------------------------------------------------------------- void ufo_sfcpcorrected_setup_f90(F90hop &, const eckit::Configuration &, - const oops::Variables &, oops::Variables &); + const oops::Variables&operatorVars, + const int *operatorVarIndices, const int numOperatorVarIndices, + oops::Variables &requiredVars); void ufo_sfcpcorrected_delete_f90(F90hop &); void ufo_sfcpcorrected_simobs_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, const int &, const int &, double &); diff --git a/src/ufo/sfcpcorrected/ufo_sfcpcorrected_mod.F90 b/src/ufo/sfcpcorrected/ufo_sfcpcorrected_mod.F90 index af1c75ffb..8629a3822 100644 --- a/src/ufo/sfcpcorrected/ufo_sfcpcorrected_mod.F90 +++ b/src/ufo/sfcpcorrected/ufo_sfcpcorrected_mod.F90 @@ -22,7 +22,9 @@ module ufo_sfcpcorrected_mod !> Fortran derived type for the observation type type, public :: ufo_sfcpcorrected private - type(oops_variables), public :: obsvars + type(oops_variables), public :: obsvars ! Variables to be simulated + integer, allocatable, public :: obsvarindices(:) ! Indices of obsvars in the list of all + ! simulated variables in the ObsSpace type(oops_variables), public :: geovars character(len=MAXVARLEN) :: da_psfc_scheme contains @@ -87,7 +89,7 @@ subroutine ufo_sfcpcorrected_simobs(self, geovals, obss, nvars, nlocs, hofx) ! Local variables real(c_double) :: missing real(kind_real) :: H2000 = 2000.0 -integer :: nobs, iobs, ivar, k, kbot, idx_geop +integer :: nobs, iobs, ivar, iobsvar, k, kbot, idx_geop real(kind_real), allocatable :: cor_psfc(:) type(ufo_geoval), pointer :: model_ps, model_p, model_sfc_geomz, model_tv, model_geomz character(len=*), parameter :: myname_="ufo_sfcpcorrected_simobs" @@ -265,7 +267,9 @@ subroutine ufo_sfcpcorrected_simobs(self, geovals, obss, nvars, nlocs, hofx) end select ! update the obs surface pressure -do ivar = 1, nvars +do iobsvar = 1, size(self%obsvarindices) + ! Get the index of the row of hofx to fill + ivar = self%obsvarindices(iobsvar) do iobs = 1, nlocs if ( cor_psfc(iobs) /= missing) then hofx(ivar,iobs) = obs_psfc(iobs) - cor_psfc(iobs) + model_psfc(iobs) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 266e8e2b3..d13244373 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -266,6 +266,8 @@ list( APPEND ufo_test_input testinput/tropomi_no2.yaml testinput/variables.yaml testinput/windprof.yaml + testinput/unit_tests/composite_GsiSfcModel_SfcPCorrected.yaml + testinput/unit_tests/composite_vertInterp_SfcPCorrected.yaml testinput/unit_tests/function_clwret_ssmis.yaml testinput/unit_tests/function_tropo_estimate.yaml testinput/unit_tests/function_lam_domaincheck.yaml @@ -1504,6 +1506,20 @@ ecbuild_add_test( TARGET test_ufo_linopr_composite DEPENDS test_ObsOperatorTLAD.x TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_opr_compositeSfc + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x + ARGS "testinput/unit_tests/composite_GsiSfcModel_SfcPCorrected.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsOperator.x + TEST_DEPENDS ufo_get_ufo_test_data ) + +ecbuild_add_test( TARGET test_ufo_opr_compositeSonde + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x + ARGS "testinput/unit_tests/composite_vertInterp_SfcPCorrected.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsOperator.x + TEST_DEPENDS ufo_get_ufo_test_data ) + ecbuild_add_test( TARGET test_ufo_opr_categorical COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x ARGS "testinput/categorical.yaml" diff --git a/test/testinput/unit_tests/composite_GsiSfcModel_SfcPCorrected.yaml b/test/testinput/unit_tests/composite_GsiSfcModel_SfcPCorrected.yaml new file mode 100644 index 000000000..56d41608b --- /dev/null +++ b/test/testinput/unit_tests/composite_GsiSfcModel_SfcPCorrected.yaml @@ -0,0 +1,35 @@ +window begin: 2020-12-14T20:30:00Z +window end: 2020-12-15T03:30:00Z + +observations: + +# Composite operator (mix SfcPCorrected with GsiSfcModel for all other variables.) +- obs space: + name: SurfaceComposite + obsdatain: + obsfile: Data/ufo/testinput_tier_1/sfc_obs_2020121500_s.nc + obsdataout: + obsfile: Data/sfc_obs_2020121500_out.nc + simulated variables: [eastward_wind, northward_wind, air_temperature, specific_humidity, surface_pressure] + geovals: + filename: Data/ufo/testinput_tier_1/sfc_geoval_2020121500_s.nc + obs operator: + name: Composite + components: + - name: GSISfcModel + variables: + - name: air_temperature + - name: specific_humidity + - name: eastward_wind + - name: northward_wind + - name: SfcPCorrected + variables: + - name: surface_pressure + da_psfc_scheme: UKMO + geovar_geomz: geopotential_height + geovar_sfc_geomz: surface_geopotential_height + # The reference value is calculated as + # sqrt( (rms(air_temperature)^2 + rms(specific_humidity)^2 + rms(eastward_wind)^2 + # rms(northward_wind)^2 + rms(surface_pressure)^2) / 5) + rms ref: 43876.78924 + tolerance: 1.0e-05 diff --git a/test/testinput/unit_tests/composite_vertInterp_SfcPCorrected.yaml b/test/testinput/unit_tests/composite_vertInterp_SfcPCorrected.yaml new file mode 100644 index 000000000..38aa09298 --- /dev/null +++ b/test/testinput/unit_tests/composite_vertInterp_SfcPCorrected.yaml @@ -0,0 +1,35 @@ +window begin: 2020-12-14T21:00:00Z +window end: 2020-12-15T03:00:00Z + +observations: + +# Composite operator (mix SfcPCorrected with VertInterp for all other variables.) +- obs space: + name: SondeComposite + obsdatain: + obsfile: Data/ufo/testinput_tier_1/sondes_obs_2020121500_s.nc + obsdataout: + obsfile: Data/sondes_obs_2020121500_out.nc + simulated variables: [eastward_wind, northward_wind, air_temperature, specific_humidity, surface_pressure] + geovals: + filename: Data/ufo/testinput_tier_1/sondes_geoval_2020121500_s.nc + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: air_temperature + - name: specific_humidity + - name: eastward_wind + - name: northward_wind + - name: SfcPCorrected + variables: + - name: surface_pressure + da_psfc_scheme: UKMO + geovar_geomz: geopotential_height + geovar_sfc_geomz: surface_geopotential_height + # The reference value is calculated as + # sqrt( (rms(air_temperature)^2 + rms(specific_humidity)^2 + rms(eastward_wind)^2 + # rms(northward_wind)^2 + rms(surface_pressure)^2) / 5) + rms ref: 43735.7376 + tolerance: 1.0e-05 From 04966c55d7b174340a4df1719ce0020554e59d30 Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Mon, 7 Jun 2021 22:16:22 +0100 Subject: [PATCH 03/98] GeoVaLs getAtLevel and putAtLevel (#1236) * putAtLocation and associated tests * comment * replace std::cout * coding norms * Modified test values * add getAtLevel, putAtLevel; add test * location -> level * move index + 1 to interface * modify a few more indices * modifications of data.get(geovals) * remove index modification * fix offset * comments * remove newline at start of file * improve debug output * modifications to ensure compilation Co-authored-by: Ryan Honeyager --- src/ufo/GeoVaLs.cc | 79 +++++++++++-------- src/ufo/GeoVaLs.h | 19 +++-- src/ufo/ObsDiagnostics.cc | 4 +- src/ufo/filters/ObsFilterData.cc | 2 +- src/ufo/gnssro/QC/ROobserror.cc | 2 +- .../marine/chleuzintegr/ObsChlEuzIntegr.cc | 4 +- .../seaicefraction/ObsSeaIceFraction.cc | 2 +- .../seaicefraction/ObsSeaIceFractionTLAD.cc | 4 +- src/ufo/predictors/LapseRate.cc | 2 +- src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc | 12 +-- src/ufo/rttovcpp/rttovcpp_interface.cc | 6 +- test/ufo/GeoVaLs.h | 32 ++++---- test/ufo/ObsFilterData.h | 2 +- test/ufo/ProfileConsistencyChecks.h | 4 +- 14 files changed, 98 insertions(+), 76 deletions(-) diff --git a/src/ufo/GeoVaLs.cc b/src/ufo/GeoVaLs.cc index 4a8bb4c99..33b73eb69 100644 --- a/src/ufo/GeoVaLs.cc +++ b/src/ufo/GeoVaLs.cc @@ -226,8 +226,8 @@ double GeoVaLs::dot_product_with(const GeoVaLs & other) const { assert(nlevs == other.nlevs(vars_[jvar])); // loop over all levels for this variable for (size_t jlev = 0; jlev < nlevs; ++jlev) { - this->get(this_values, vars_[jvar], jlev+1); - other.get(other_values, vars_[jvar], jlev+1); + this->getAtLevel(this_values, vars_[jvar], jlev+1); + other.getAtLevel(other_values, vars_[jvar], jlev+1); // loop over all locations for (size_t jloc = 0; jloc < nlocs; ++jloc) { if ((this_values[jloc] != missing) && (other_values[jloc] != missing)) { @@ -313,23 +313,32 @@ void GeoVaLs::get(std::vector & vals, const std::string & var) const { } // ----------------------------------------------------------------------------- /*! \brief Return all values for a specific variable and level */ -void GeoVaLs::get(std::vector & vals, const std::string & var, const int lev) const { - oops::Log::trace() << "GeoVaLs::get starting" << std::endl; +void GeoVaLs::getAtLevel(std::vector & vals, const std::string & var, const int lev) const { + oops::Log::trace() << "GeoVaLs::getAtLevel(double) starting" << std::endl; size_t nlocs; ufo_geovals_nlocs_f90(keyGVL_, nlocs); ASSERT(vals.size() == nlocs); - ufo_geovals_get_f90(keyGVL_, var.size(), var.c_str(), lev, nlocs, vals[0]); - oops::Log::trace() << "GeoVaLs::get done" << std::endl; + ufo_geovals_getdouble_f90(keyGVL_, var.size(), var.c_str(), lev, nlocs, vals[0]); + oops::Log::trace() << "GeoVaLs::getAtLevel(double) done" << std::endl; } // ----------------------------------------------------------------------------- -/*! \brief Return all values for a specific variable and level */ -void GeoVaLs::get(std::vector & vals, const std::string & var, const int lev) const { - oops::Log::trace() << "GeoVaLs::get starting" << std::endl; +/*! \brief Return all values for a specific variable and level and convert to float */ +void GeoVaLs::getAtLevel(std::vector & vals, const std::string & var, const int lev) const { + oops::Log::trace() << "GeoVaLs::getAtLevel(float) starting" << std::endl; size_t nlocs; ufo_geovals_nlocs_f90(keyGVL_, nlocs); ASSERT(vals.size() == nlocs); - ufo_geovals_getdouble_f90(keyGVL_, var.size(), var.c_str(), lev, nlocs, vals[0]); - oops::Log::trace() << "GeoVaLs::get done" << std::endl; + ufo_geovals_get_f90(keyGVL_, var.size(), var.c_str(), lev, nlocs, vals[0]); + oops::Log::trace() << "GeoVaLs::getAtLevel(float) done" << std::endl; +} +// ----------------------------------------------------------------------------- +/*! \brief Return all values for a specific variable and level and convert to int */ +void GeoVaLs::getAtLevel(std::vector & vals, const std::string & var, const int lev) const { + oops::Log::trace() << "GeoVaLs::getAtLevel(int) starting" << std::endl; + std::vector doublevals(vals.size()); + this->getAtLevel(doublevals, var, lev); + vals.assign(doublevals.begin(), doublevals.end()); + oops::Log::trace() << "GeoVaLs::getAtLevel(int) done" << std::endl; } // ----------------------------------------------------------------------------- /*! \brief Return all values for a specific 2D variable */ @@ -357,101 +366,107 @@ void GeoVaLs::get(std::vector & vals, const std::string & var) const { void GeoVaLs::getAtLocation(std::vector & vals, const std::string & var, const int loc) const { - oops::Log::trace() << "GeoVaLs::getAtLocation starting" << std::endl; + oops::Log::trace() << "GeoVaLs::getAtLocation(double) starting" << std::endl; const size_t nlevs = this->nlevs(var); ASSERT(vals.size() == nlevs); ASSERT(loc >= 0 && loc < this->nlocs()); ufo_geovals_get_loc_f90(keyGVL_, var.size(), var.c_str(), loc, nlevs, vals[0]); - oops::Log::trace() << "GeoVaLs::getAtLocation done" << std::endl; + oops::Log::trace() << "GeoVaLs::getAtLocation(double) done" << std::endl; } // ----------------------------------------------------------------------------- /*! \brief Return all values for a specific variable and location and convert to float */ void GeoVaLs::getAtLocation(std::vector & vals, const std::string & var, const int loc) const { - oops::Log::trace() << "GeoVaLs::getAtLocation starting" << std::endl; + oops::Log::trace() << "GeoVaLs::getAtLocation(float) starting" << std::endl; std::vector doublevals(vals.size()); this->getAtLocation(doublevals, var, loc); vals.assign(doublevals.begin(), doublevals.end()); - oops::Log::trace() << "GeoVaLs::getAtLocation done" << std::endl; + oops::Log::trace() << "GeoVaLs::getAtLocation(float) done" << std::endl; } // ----------------------------------------------------------------------------- /*! \brief Return all values for a specific variable and location and convert to int */ void GeoVaLs::getAtLocation(std::vector & vals, const std::string & var, const int loc) const { - oops::Log::trace() << "GeoVaLs::getAtLocation starting" << std::endl; + oops::Log::trace() << "GeoVaLs::getAtLocation(int) starting" << std::endl; std::vector doublevals(vals.size()); this->getAtLocation(doublevals, var, loc); vals.assign(doublevals.begin(), doublevals.end()); - oops::Log::trace() << "GeoVaLs::getAtLocation done" << std::endl; + oops::Log::trace() << "GeoVaLs::getAtLocation(int) done" << std::endl; } // ----------------------------------------------------------------------------- /*! \brief Put double values for a specific variable and level */ -void GeoVaLs::put(const std::vector & vals, const std::string & var, const int lev) const { - oops::Log::trace() << "GeoVaLs::put starting" << std::endl; +void GeoVaLs::putAtLevel(const std::vector & vals, + const std::string & var, + const int lev) const { + oops::Log::trace() << "GeoVaLs::putAtLevel(double) starting" << std::endl; size_t nlocs; ufo_geovals_nlocs_f90(keyGVL_, nlocs); ASSERT(vals.size() == nlocs); ufo_geovals_putdouble_f90(keyGVL_, var.size(), var.c_str(), lev, nlocs, vals[0]); - oops::Log::trace() << "GeoVaLs::put done" << std::endl; + oops::Log::trace() << "GeoVaLs::putAtLevel(double) done" << std::endl; } // ----------------------------------------------------------------------------- /*! \brief Put float values for a specific variable and level */ -void GeoVaLs::put(const std::vector & vals, const std::string & var, const int lev) const { - oops::Log::trace() << "GeoVaLs::put starting" << std::endl; +void GeoVaLs::putAtLevel(const std::vector & vals, + const std::string & var, + const int lev) const { + oops::Log::trace() << "GeoVaLs::putAtLevel(float) starting" << std::endl; size_t nlocs; ufo_geovals_nlocs_f90(keyGVL_, nlocs); ASSERT(vals.size() == nlocs); std::vector doublevals(vals.begin(), vals.end()); ufo_geovals_putdouble_f90(keyGVL_, var.size(), var.c_str(), lev, nlocs, doublevals[0]); - oops::Log::trace() << "GeoVaLs::put done" << std::endl; + oops::Log::trace() << "GeoVaLs::putAtLevel(float) done" << std::endl; } // ----------------------------------------------------------------------------- /*! \brief Put int values for a specific variable and level */ -void GeoVaLs::put(const std::vector & vals, const std::string & var, const int lev) const { - oops::Log::trace() << "GeoVaLs::put starting" << std::endl; +void GeoVaLs::putAtLevel(const std::vector & vals, + const std::string & var, + const int lev) const { + oops::Log::trace() << "GeoVaLs::putAtLevel(int) starting" << std::endl; size_t nlocs; ufo_geovals_nlocs_f90(keyGVL_, nlocs); ASSERT(vals.size() == nlocs); std::vector doublevals(vals.begin(), vals.end()); ufo_geovals_putdouble_f90(keyGVL_, var.size(), var.c_str(), lev, nlocs, doublevals[0]); - oops::Log::trace() << "GeoVaLs::put done" << std::endl; + oops::Log::trace() << "GeoVaLs::putAtLevel(int) done" << std::endl; } /*! \brief Put double values for a specific variable and location */ void GeoVaLs::putAtLocation(const std::vector & vals, const std::string & var, const int loc) const { - oops::Log::trace() << "GeoVaLs::putAtLocation starting" << std::endl; + oops::Log::trace() << "GeoVaLs::putAtLocation(double) starting" << std::endl; const size_t nlevs = this->nlevs(var); ASSERT(vals.size() == nlevs); ASSERT(loc >= 0 && loc < this->nlocs()); ufo_geovals_put_loc_f90(keyGVL_, var.size(), var.c_str(), loc, nlevs, vals[0]); - oops::Log::trace() << "GeoVaLs::putAtLocation done" << std::endl; + oops::Log::trace() << "GeoVaLs::putAtLocation(double) done" << std::endl; } /*! \brief Put float values for a specific variable and location */ void GeoVaLs::putAtLocation(const std::vector & vals, const std::string & var, const int loc) const { - oops::Log::trace() << "GeoVaLs::putAtLocation starting" << std::endl; + oops::Log::trace() << "GeoVaLs::putAtLocation(float) starting" << std::endl; const size_t nlevs = this->nlevs(var); ASSERT(vals.size() == nlevs); ASSERT(loc >= 0 && loc < this->nlocs()); std::vector doublevals(vals.begin(), vals.end()); ufo_geovals_put_loc_f90(keyGVL_, var.size(), var.c_str(), loc, nlevs, doublevals[0]); - oops::Log::trace() << "GeoVaLs::putAtLocation done" << std::endl; + oops::Log::trace() << "GeoVaLs::putAtLocation(float) done" << std::endl; } /*! \brief Put int values for a specific variable and location */ void GeoVaLs::putAtLocation(const std::vector & vals, const std::string & var, const int loc) const { - oops::Log::trace() << "GeoVaLs::putAtLocation starting" << std::endl; + oops::Log::trace() << "GeoVaLs::putAtLocation(int) starting" << std::endl; const size_t nlevs = this->nlevs(var); ASSERT(vals.size() == nlevs); ASSERT(loc >= 0 && loc < this->nlocs()); std::vector doublevals(vals.begin(), vals.end()); ufo_geovals_put_loc_f90(keyGVL_, var.size(), var.c_str(), loc, nlevs, doublevals[0]); - oops::Log::trace() << "GeoVaLs::putAtLocation done" << std::endl; + oops::Log::trace() << "GeoVaLs::putAtLocation(int) done" << std::endl; } // ----------------------------------------------------------------------------- /*! \brief Read GeoVaLs from the file */ diff --git a/src/ufo/GeoVaLs.h b/src/ufo/GeoVaLs.h index cbce8e926..233b7675d 100644 --- a/src/ufo/GeoVaLs.h +++ b/src/ufo/GeoVaLs.h @@ -77,26 +77,34 @@ class GeoVaLs : public util::Printable, const oops::Variables & getVars() const {return vars_;} size_t nlevs(const std::string & var) const; - void get(std::vector &, const std::string &, const int) const; - void get(std::vector &, const std::string &, const int) const; /// Get 2D GeoVaLs for variable \p var (fails for 3D GeoVaLs) void get(std::vector &, const std::string & var) const; /// Get 2D GeoVaLs for variable \p var (fails for 3D GeoVaLs), and convert to float void get(std::vector &, const std::string & var) const; /// Get 2D GeoVaLs for variable \p var (fails for 3D GeoVaLs), and convert to int void get(std::vector &, const std::string & var) const; + + /// Get GeoVaLs at a specified level + void getAtLevel(std::vector &, const std::string &, const int) const; + /// Get GeoVaLs at a specified level and convert to float + void getAtLevel(std::vector &, const std::string &, const int) const; + /// Get GeoVaLs at a specified level and convert to int + void getAtLevel(std::vector &, const std::string &, const int) const; + /// Get GeoVaLs at a specified location void getAtLocation(std::vector &, const std::string &, const int) const; /// Get GeoVaLs at a specified location and convert to float void getAtLocation(std::vector &, const std::string &, const int) const; /// Get GeoVaLs at a specified location and convert to int void getAtLocation(std::vector &, const std::string &, const int) const; + /// Put GeoVaLs for double variable \p var at level \p lev. - void put(const std::vector & vals, const std::string & var, const int lev) const; + void putAtLevel(const std::vector & vals, const std::string & var, const int lev) const; /// Put GeoVaLs for float variable \p var at level \p lev. - void put(const std::vector & vals, const std::string & var, const int lev) const; + void putAtLevel(const std::vector & vals, const std::string & var, const int lev) const; /// Put GeoVaLs for int variable \p var at level \p lev. - void put(const std::vector & vals, const std::string & var, const int lev) const; + void putAtLevel(const std::vector & vals, const std::string & var, const int lev) const; + /// Put GeoVaLs for double variable \p var at location \p loc. void putAtLocation(const std::vector & vals, const std::string & var, const int loc) const; @@ -104,6 +112,7 @@ class GeoVaLs : public util::Printable, void putAtLocation(const std::vector & vals, const std::string & var, const int loc) const; /// Put GeoVaLs for int variable \p var at location \p loc. void putAtLocation(const std::vector & vals, const std::string & var, const int loc) const; + void read(const eckit::Configuration &, const ioda::ObsSpace &); void write(const eckit::Configuration &) const; size_t nlocs() const; diff --git a/src/ufo/ObsDiagnostics.cc b/src/ufo/ObsDiagnostics.cc index 0da5bcb8a..3a61c89f9 100644 --- a/src/ufo/ObsDiagnostics.cc +++ b/src/ufo/ObsDiagnostics.cc @@ -43,7 +43,7 @@ void ObsDiagnostics::allocate(const int nlev, const oops::Variables & vars) { void ObsDiagnostics::save(const std::vector & vals, const std::string & var, const int lev) { - gdiags_.put(vals, var, lev); + gdiags_.putAtLevel(vals, var, lev); } // ----------------------------------------------------------------------------- @@ -62,7 +62,7 @@ void ObsDiagnostics::get(std::vector & vals, const std::string & var) con void ObsDiagnostics::get(std::vector & vals, const std::string & var, const int lev) const { - gdiags_.get(vals, var, lev); + gdiags_.getAtLevel(vals, var, lev); } // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/ObsFilterData.cc b/src/ufo/filters/ObsFilterData.cc index 4f52dc39f..3c041aec3 100644 --- a/src/ufo/filters/ObsFilterData.cc +++ b/src/ufo/filters/ObsFilterData.cc @@ -244,7 +244,7 @@ void ObsFilterData::get(const Variable & varname, const int level, /// For GeoVaLs read from GeoVaLs (should be available) if (grp == "GeoVaLs") { ASSERT(gvals_); - gvals_->get(values, var, level); + gvals_->getAtLevel(values, var, level); /// For ObsDiag get from ObsDiagnostics } else if (grp == "ObsDiag" || grp == "ObsBiasTerm") { ASSERT(diags_); diff --git a/src/ufo/gnssro/QC/ROobserror.cc b/src/ufo/gnssro/QC/ROobserror.cc index 6272ab540..e3b742d1e 100644 --- a/src/ufo/gnssro/QC/ROobserror.cc +++ b/src/ufo/gnssro/QC/ROobserror.cc @@ -89,7 +89,7 @@ Eigen::ArrayXXf ROobserror::get_geovals(const std::string& var_name) const { std::vector single_geoval(nlocs); for (int ilev=0; ilev < static_cast(nlevs); ilev++) { oops::Log::debug() << "Getting data for level " << ilev+1 << std::endl; - data_.getGeoVaLs()->get(single_geoval, Variable(var_name).variable(), ilev+1); + data_.getGeoVaLs()->getAtLevel(single_geoval, Variable(var_name).variable(), ilev+1); all_geovals.col(ilev) = Eigen::VectorXf::Map(single_geoval.data(), single_geoval.size()); } return all_geovals; diff --git a/src/ufo/marine/chleuzintegr/ObsChlEuzIntegr.cc b/src/ufo/marine/chleuzintegr/ObsChlEuzIntegr.cc index 710547963..273ddbe56 100644 --- a/src/ufo/marine/chleuzintegr/ObsChlEuzIntegr.cc +++ b/src/ufo/marine/chleuzintegr/ObsChlEuzIntegr.cc @@ -54,9 +54,9 @@ void ObsChlEuzIntegr::simulateObs(const GeoVaLs & gv, ioda::ObsVector & ovec, std::vector> chl; std::vector> h; for ( std::size_t k = 0; k < nlevs; ++k ) { - gv.get(tmp, "sea_water_cell_thickness", k+1); + gv.getAtLevel(tmp, "sea_water_cell_thickness", k+1); h.push_back(tmp); - gv.get(tmp, "mass_concentration_of_chlorophyll_in_sea_water", k+1); + gv.getAtLevel(tmp, "mass_concentration_of_chlorophyll_in_sea_water", k+1); chl.push_back(tmp); } diff --git a/src/ufo/marine/seaicefraction/ObsSeaIceFraction.cc b/src/ufo/marine/seaicefraction/ObsSeaIceFraction.cc index 697f14ae1..801c9cf1b 100644 --- a/src/ufo/marine/seaicefraction/ObsSeaIceFraction.cc +++ b/src/ufo/marine/seaicefraction/ObsSeaIceFraction.cc @@ -48,7 +48,7 @@ void ObsSeaIceFraction::simulateObs(const GeoVaLs & gv, ioda::ObsVector & ovec, std::vector aicen(nlocs); for ( std::size_t k = 1; k < nlevs+1; ++k ) { - gv.get(aicen, "sea_ice_category_area_fraction", k); + gv.getAtLevel(aicen, "sea_ice_category_area_fraction", k); for ( std::size_t i = 0; i < nlocs; ++i ) { ovec[i] += aicen[i]; } diff --git a/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc b/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc index 466e75dc7..02e6cf1bd 100644 --- a/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc +++ b/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc @@ -60,7 +60,7 @@ void ObsSeaIceFractionTLAD::simulateObsTL(const GeoVaLs & gv, ioda::ObsVector & std::vector aicen(nlocs); for ( std::size_t k = 1; k < nlevs+1; ++k ) { - gv.get(aicen, "sea_ice_category_area_fraction", k); + gv.getAtLevel(aicen, "sea_ice_category_area_fraction", k); for ( std::size_t i = 0; i < nlocs; ++i ) { ovec[i] += aicen[i]; } @@ -84,7 +84,7 @@ void ObsSeaIceFractionTLAD::simulateObsAD(GeoVaLs & gv, const ioda::ObsVector & else { aicen[i] = 0.0; } } - gv.put(aicen, "sea_ice_category_area_fraction", k); + gv.putAtLevel(aicen, "sea_ice_category_area_fraction", k); } oops::Log::trace() << "ObsSeaIceFractionTLAD: adjoint observation operator run" << std::endl; } diff --git a/src/ufo/predictors/LapseRate.cc b/src/ufo/predictors/LapseRate.cc index 464a30ebf..3d688f70e 100644 --- a/src/ufo/predictors/LapseRate.cc +++ b/src/ufo/predictors/LapseRate.cc @@ -111,7 +111,7 @@ void LapseRate::compute(const ioda::ObsSpace & odb, std::vector> tvp; std::size_t nlevs = geovals.nlevs("air_temperature"); for (std::size_t js = 0; js < nlevs; ++js) { - geovals.get(pred, "air_temperature", js+1); + geovals.getAtLevel(pred, "air_temperature", js+1); tvp.push_back(pred); } nlevs = geovals.nlevs("air_pressure"); diff --git a/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc b/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc index 775798cf4..45f36f504 100644 --- a/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc +++ b/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc @@ -85,13 +85,13 @@ void ObsRadianceRTTOVCPPTLAD::simulateObsTL(const GeoVaLs & dx, ioda::ObsVector // Retrieve temperature increment in K for (std::size_t i = 0; i < nlevels; ++i) { - dx.get(tmpvar2d, "air_temperature", i+1); // get one level T + dx.getAtLevel(tmpvar2d, "air_temperature", i+1); // get one level T dT.push_back(tmpvar2d); // push one level T into 3D T } // Retrieve specific humidity increment in kg/kg for (std::size_t i = 0; i < nlevels; ++i) { - dx.get(tmpvar2d, "specific_humidity", i+1); // get one level Q + dx.getAtLevel(tmpvar2d, "specific_humidity", i+1); // get one level Q dQ.push_back(tmpvar2d); // push one level Q into 3D Q } @@ -136,13 +136,13 @@ void ObsRadianceRTTOVCPPTLAD::simulateObsAD(GeoVaLs & dx, const ioda::ObsVector // Retrieve temperature increment in K for (std::size_t i = 0; i < nlevels; ++i) { - dx.get(tmpvar2d, "air_temperature", i+1); // get one level T + dx.getAtLevel(tmpvar2d, "air_temperature", i+1); // get one level T dT.push_back(tmpvar2d); // push one level T into 3D T } // Retrieve specific humidity increment in kg/kg for (std::size_t i = 0; i < nlevels; ++i) { - dx.get(tmpvar2d, "specific_humidity", i+1); // get one level Q + dx.getAtLevel(tmpvar2d, "specific_humidity", i+1); // get one level Q dQ.push_back(tmpvar2d); // push one level Q into 3D Q } @@ -177,7 +177,7 @@ void ObsRadianceRTTOVCPPTLAD::simulateObsAD(GeoVaLs & dx, const ioda::ObsVector for (size_t p = 0; p < nprofiles; p++) { tmpvar2d[p] = dT[l][p]; } - dx.put(tmpvar2d, "air_temperature", l+1); // put one level T + dx.putAtLevel(tmpvar2d, "air_temperature", l+1); // put one level T } // Put specific humidity increment in kg/kg @@ -185,7 +185,7 @@ void ObsRadianceRTTOVCPPTLAD::simulateObsAD(GeoVaLs & dx, const ioda::ObsVector for (size_t p = 0; p < nprofiles; p++) { tmpvar2d[p] = dQ[l][p]; } - dx.put(tmpvar2d, "specific_humidity", l+1); // put one level Q + dx.putAtLevel(tmpvar2d, "specific_humidity", l+1); // put one level Q } oops::Log::trace() << "ObsRadianceRTTOVCPPTLAD::simulateObsAD done" << std::endl; diff --git a/src/ufo/rttovcpp/rttovcpp_interface.cc b/src/ufo/rttovcpp/rttovcpp_interface.cc index ab18bb453..817d6bf08 100644 --- a/src/ufo/rttovcpp/rttovcpp_interface.cc +++ b/src/ufo/rttovcpp/rttovcpp_interface.cc @@ -116,7 +116,7 @@ void rttovcpp_interface(const GeoVaLs & geovals, const ioda::ObsSpace & odb_, // 3.1.1 Retrieve pressure in hPa std::vector> tmpvar3d; // [nlevels][nprofiles] for (std::size_t i = 0; i < nlevels; ++i) { - geovals.get(tmpvar2d, "air_pressure", i+1); // get one level P + geovals.getAtLevel(tmpvar2d, "air_pressure", i+1); // get one level P tmpvar3d.push_back(tmpvar2d); // push one level P into 3D P } for (std::size_t i = 0; i < nprofiles; ++i) { @@ -132,7 +132,7 @@ void rttovcpp_interface(const GeoVaLs & geovals, const ioda::ObsSpace & odb_, // 3.1.2 Retrieve temperature in K std::vector> tmpvar3d_T; // [nlevels][nprofiles] for (std::size_t i = 0; i < nlevels; ++i) { - geovals.get(tmpvar2d, "air_temperature", i+1); // get one level T + geovals.getAtLevel(tmpvar2d, "air_temperature", i+1); // get one level T tmpvar3d_T.push_back(tmpvar2d); // push one level T into 3D T } for (std::size_t i = 0; i < nprofiles; ++i) { @@ -146,7 +146,7 @@ void rttovcpp_interface(const GeoVaLs & geovals, const ioda::ObsSpace & odb_, // 3.1.3 Retrieve specific humidity in kg/kg std::vector> tmpvar3d_Q; // [nlevels][nprofiles] for (std::size_t i = 0; i < nlevels; ++i) { - geovals.get(tmpvar2d, "specific_humidity", i+1); + geovals.getAtLevel(tmpvar2d, "specific_humidity", i+1); tmpvar3d_Q.push_back(tmpvar2d); } for (std::size_t i = 0; i < nprofiles; ++i) { diff --git a/test/ufo/GeoVaLs.h b/test/ufo/GeoVaLs.h index 6a739c166..ce47f725e 100644 --- a/test/ufo/GeoVaLs.h +++ b/test/ufo/GeoVaLs.h @@ -72,7 +72,7 @@ void testGeoVaLs() { for (std::size_t i = 0; i < ind.size(); ++i) { GeoVaLs gv_one(gval, ind[i]); std::vector gv_val(1); - gv_one.get(gv_val, var, 1); + gv_one.getAtLevel(gv_val, var, 1); EXPECT(oops::is_close_absolute(gv_val[0], values[i], oneloctol, 0, verbosity)); } } else { @@ -96,8 +96,8 @@ void testGeoVaLs() { sum = 0; for (size_t k = 0; k < nlevs; ++k) { size_t kk = nlevs - k; - gv.get(gvar, ingeovars[i], k+1); - gval.get(gvarref, ingeovars[i], kk); + gv.getAtLevel(gvar, ingeovars[i], k+1); + gval.getAtLevel(gvarref, ingeovars[i], kk); for (size_t j = 0; j < nobs; ++j) { gvar[j] = gvar[j] - gvarref[j]; sum += sum + gvar[j]; @@ -165,7 +165,8 @@ void testGeoVaLs() { } } -/// \brief Tests GeoVaLs::allocate, GeoVaLs::put, GeoVaLs::get, +/// \brief Tests GeoVaLs::allocate, GeoVals::put, GeoVaLs::get, +/// GeoVaLs::putAtLevel, GeoVaLs::getAtLevel, /// GeoVaLs::putAtLocation and GeoVaLs::getAtLocation. void testGeoVaLsAllocatePutGet() { const eckit::LocalConfiguration conf(::test::TestEnvironment::config()); @@ -209,7 +210,7 @@ void testGeoVaLsAllocatePutGet() { const double fillvalue_double = 3.01234567890123; oops::Log::test() << "Put(double) fill value: " << fillvalue_double << std::endl; const std::vector refvalues_double(gval.nlocs(), fillvalue_double); - gval.put(refvalues_double, var2, 1); + gval.putAtLevel(refvalues_double, var2, 1); std::vector testvalues_double(gval.nlocs(), 0); gval.get(testvalues_double, var2); oops::Log::test() << "Get(double) result: " << testvalues_double << std::endl; @@ -218,7 +219,7 @@ void testGeoVaLsAllocatePutGet() { const float fillvalue_float = 4.1f; oops::Log::test() << "Put(float) fill value: " << fillvalue_float << std::endl; const std::vector refvalues_float(gval.nlocs(), fillvalue_float); - gval.put(refvalues_float, var2, 1); + gval.putAtLevel(refvalues_float, var2, 1); std::vector testvalues_float(gval.nlocs(), 0); gval.get(testvalues_float, var2); oops::Log::test() << "Get(float) result: " << testvalues_float << std::endl; @@ -227,7 +228,7 @@ void testGeoVaLsAllocatePutGet() { const int fillvalue_int = 5; oops::Log::test() << "Put(int) fill value: " << fillvalue_int << std::endl; const std::vector refvalues_int(gval.nlocs(), fillvalue_int); - gval.put(refvalues_int, var2, 1); + gval.putAtLevel(refvalues_int, var2, 1); std::vector testvalues_int(gval.nlocs(), 0); gval.get(testvalues_int, var2); oops::Log::test() << "Get(int) result: " << testvalues_int << std::endl; @@ -238,7 +239,7 @@ void testGeoVaLsAllocatePutGet() { for (size_t jlev = 0; jlev < nlevs1; ++jlev) { std::vector refvalues_loc_double(gval.nlocs()); std::iota(refvalues_loc_double.begin(), refvalues_loc_double.end(), jlev); - gval.put(refvalues_loc_double, var1, jlev+1); + gval.putAtLevel(refvalues_loc_double, var1, jlev+1); } for (size_t jloc = 0; jloc < gval.nlocs(); ++jloc) { // Get the test vector at this location. @@ -279,7 +280,7 @@ void testGeoVaLsAllocatePutGet() { for (size_t jlev = 0; jlev < gval.nlevs(var1); ++jlev) { // Get the test vector on this level. std::vector testvalues_double(gval.nlocs()); - gval.get(testvalues_double, var1, jlev + 1); + gval.getAtLevel(testvalues_double, var1, jlev + 1); // Recreate reference vector for this level. std::vector refvalues_double(gval.nlocs()); std::iota(refvalues_double.begin(), refvalues_double.end(), jlev); @@ -298,7 +299,7 @@ void testGeoVaLsAllocatePutGet() { for (size_t jlev = 0; jlev < gval.nlevs(var1); ++jlev) { // Get the test vector on this level. std::vector testvalues_float(gval.nlocs()); - gval.get(testvalues_float, var1, jlev + 1); + gval.getAtLevel(testvalues_float, var1, jlev + 1); // Recreate reference vector for this level. std::vector refvalues_float(gval.nlocs()); std::iota(refvalues_float.begin(), refvalues_float.end(), jlev + 1); @@ -316,11 +317,8 @@ void testGeoVaLsAllocatePutGet() { oops::Log::test() << "testing putAtLoction with ints" << std::endl; for (size_t jlev = 0; jlev < gval.nlevs(var1); ++jlev) { // Get the test vector on this level. - // The get method has not been implemented for integers, so get a vector of floats - // and convert it. - std::vector testvalues_float(gval.nlocs()); - gval.get(testvalues_float, var1, jlev + 1); - std::vector testvalues_int(testvalues_float.begin(), testvalues_float.end()); + std::vector testvalues_int(gval.nlocs()); + gval.getAtLevel(testvalues_int, var1, jlev + 1); // Recreate reference vector for this level. std::vector refvalues_int(gval.nlocs()); std::iota(refvalues_int.begin(), refvalues_int.end(), jlev + 2); @@ -344,10 +342,10 @@ void testGeoVaLsAllocatePutGet() { for (size_t jlev = 0; jlev < nlevs1; ++jlev) { const float fillvalue = 3.0*(jlev+1); const std::vector refvalues(gval.nlocs(), fillvalue); - gval.put(refvalues, var1, jlev+1); + gval.putAtLevel(refvalues, var1, jlev + 1); oops::Log::test() << jlev << " level: put fill value: " << fillvalue << std::endl; std::vector testvalues(gval.nlocs(), 0); - gval.get(testvalues, var1, jlev+1); + gval.getAtLevel(testvalues, var1, jlev + 1); oops::Log::test() << jlev << " level: get result: " << testvalues << std::endl; EXPECT_EQUAL(testvalues, refvalues); } diff --git a/test/ufo/ObsFilterData.h b/test/ufo/ObsFilterData.h index b4b3b2508..4f5afea32 100644 --- a/test/ufo/ObsFilterData.h +++ b/test/ufo/ObsFilterData.h @@ -172,7 +172,7 @@ void testObsFilterData() { std::vector vec; data.get(geovars.variable(jvar), nlevs, vec); std::vector ref(ospace.nlocs()); - gval.get(ref, geovars.variable(jvar).variable(), nlevs); + gval.getAtLevel(ref, geovars.variable(jvar).variable(), nlevs); EXPECT(vec == ref); } } diff --git a/test/ufo/ProfileConsistencyChecks.h b/test/ufo/ProfileConsistencyChecks.h index 4b3c67c7b..cd4378799 100644 --- a/test/ufo/ProfileConsistencyChecks.h +++ b/test/ufo/ProfileConsistencyChecks.h @@ -259,7 +259,7 @@ void testProfileConsistencyChecks(const eckit::LocalConfiguration &conf) { // Calculate level heights for GeoVaLs. std::vector orogGeoVaLs(obsspace.nlocs(), 0.0); - geovals->get(orogGeoVaLs, ufo::VariableNames::geovals_orog, 1); + geovals->getAtLevel(orogGeoVaLs, ufo::VariableNames::geovals_orog, 1); std::vector zRhoGeoVaLs; std::vector zThetaGeoVaLs; ufo::CalculateModelHeight(options.DHParameters.ModParameters, @@ -276,7 +276,7 @@ void testProfileConsistencyChecks(const eckit::LocalConfiguration &conf) { const size_t gvnlevs = geovals->nlevs(ufo::VariableNames::geovals_pressure); std::vector pressureGeoVaLs_column; for (int jlev = 1; jlev < gvnlevs + 1; ++jlev) { - geovals->get(pressureGeoVaLs, ufo::VariableNames::geovals_pressure, jlev); + geovals->getAtLevel(pressureGeoVaLs, ufo::VariableNames::geovals_pressure, jlev); pressureGeoVaLs_column.push_back(pressureGeoVaLs[0]); } From d327f50b9b466cad4c94cba31ff31458dc5e1c64 Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Mon, 7 Jun 2021 16:26:56 -0600 Subject: [PATCH 04/98] bump up tolerance on gnssroBendMetOffice test (#1269) * increase tol * change tol in gnssrobendmetoffice.yaml back to 1e-14 * increase tol in gnssrobendmetoffice.yaml --- test/testinput/gnssrobendmetoffice_nopseudo.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/testinput/gnssrobendmetoffice_nopseudo.yaml b/test/testinput/gnssrobendmetoffice_nopseudo.yaml index c217731a6..35e51bbde 100644 --- a/test/testinput/gnssrobendmetoffice_nopseudo.yaml +++ b/test/testinput/gnssrobendmetoffice_nopseudo.yaml @@ -20,5 +20,5 @@ observations: linear obs operator test: coef TL: 1.0e-4 iterations TL: 10 - tolerance TL: 1.0e-14 - tolerance AD: 1.0e-14 + tolerance TL: 1.5e-14 + tolerance AD: 1.5e-14 From afc55b4f635bcf6e13d3fe969c30f9d847936c18 Mon Sep 17 00:00:00 2001 From: adammartins Date: Tue, 15 Jun 2021 17:55:49 +0100 Subject: [PATCH 05/98] Feature/PGEtotalweight (#1158) * working filter not yaml * Add test * Fix vector declarations and improve error handling * Catch ReportPGE <= 0 * Bug fixes * Tidying * add obstype * add metoffice obstypes include * correct passedmark * remove output netcdf * change filter name * Include documentation * Update yaml * Response to ctgh review comments * trigger ci * Float Declarations and intialization on single line * Coding norms fixes * Response to review commentsXSXC * indexing fix * bug fix and resposne to review comments * test output * Add comparison test and bug fix Remove diagnostic output file * Empty commit to retrigger test with updated data Co-authored-by: ldhawknesssmith Co-authored-by: mmiesch Co-authored-by: Ryan Honeyager --- src/ufo/filters/CMakeLists.txt | 2 + .../ProbabilityGrossErrorWholeReport.cc | 229 ++++++++++++++++++ .../ProbabilityGrossErrorWholeReport.h | 125 ++++++++++ src/ufo/instantiateObsFilterFactory.h | 3 + test/CMakeLists.txt | 8 + test/testinput/qc_grosserrorwholereport.yaml | 65 +++++ 6 files changed, 432 insertions(+) create mode 100644 src/ufo/filters/ProbabilityGrossErrorWholeReport.cc create mode 100644 src/ufo/filters/ProbabilityGrossErrorWholeReport.h create mode 100644 test/testinput/qc_grosserrorwholereport.yaml diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index becd199f4..11e342138 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -57,6 +57,8 @@ set ( filters_files ObsFilterData.h PreQC.cc PreQC.h + ProbabilityGrossErrorWholeReport.cc + ProbabilityGrossErrorWholeReport.h QCflags.h QCmanager.cc QCmanager.h diff --git a/src/ufo/filters/ProbabilityGrossErrorWholeReport.cc b/src/ufo/filters/ProbabilityGrossErrorWholeReport.cc new file mode 100644 index 000000000..ee1872664 --- /dev/null +++ b/src/ufo/filters/ProbabilityGrossErrorWholeReport.cc @@ -0,0 +1,229 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/filters/ProbabilityGrossErrorWholeReport.h" + +#include +#include + +#include "eckit/config/Configuration.h" + +#include "ioda/ObsDataVector.h" +#include "ioda/ObsSpace.h" + +#include "oops/util/FloatCompare.h" +#include "oops/util/Logger.h" +#include "ufo/filters/QCflags.h" +#include "ufo/utils/metoffice/MetOfficeObservationIDs.h" +#include "ufo/utils/metoffice/MetOfficeQCFlags.h" + +namespace ufo { + +// ----------------------------------------------------------------------------- +ProbabilityGrossErrorWholeReport::ProbabilityGrossErrorWholeReport(ioda::ObsSpace & obsdb, + const Parameters_ & parameters, + std::shared_ptr > flags, + std::shared_ptr > obserr) + : FilterBase(obsdb, parameters, flags, obserr), + parameters_(parameters) +{ + oops::Log::trace() << "ProbabilityGrossErrorWholeReport contructor starting" << std::endl; +} + +// ----------------------------------------------------------------------------- + +ProbabilityGrossErrorWholeReport::~ProbabilityGrossErrorWholeReport() { + oops::Log::trace() << "ProbabilityGrossErrorWholeReport destructed" << std::endl; +} + +// ----------------------------------------------------------------------------- + +void ProbabilityGrossErrorWholeReport::applyFilter(const std::vector & apply, + const Variables & filtervars, + std::vector> & flagged) const { + oops::Log::trace() << "ProbabilityGrossErrorWholeReport preProcess" << std::endl; + // Missing value indicator + const float missingValueFloat = util::missingValue(missingValueFloat); + // Dimensions + const size_t nlocs = obsdb_.nlocs(); + const size_t nvars = filtervars.nvars(); + // Missing data indicator for stored PGEs. + const float PGEMDI = 1.111f; + // PGE multiplication factor used to store PGE values for later use. + const float PGEMult = 1000.0; + // PGE rejection limit. + const float PGECrit = parameters_.PGEParameters.PGE_PGECrit.value(); + + // Initial Probability of Gross error in whole report + std::vector PBadRep(nlocs); + // Overall Probability density for whole report + std::vector PdReport(nlocs, 1.0f); + // Probability density of bad observations for each variable + std::vector PdBad(nvars); + // Probability density of bad synop observations for each variable + std::vector PdBad_synop(nvars); + // Probability density of bad bogus observations for each variable + std::vector PdBad_bogus(nvars); + // Probabilty density of bad observations used for each observation + std::vector> PdBad_used(nvars, std::vector(nlocs)); + // Is this variable used in the whole report error calculation? + std::vector notUsed(nvars); + // Vector holding filter variable names + std::vector varname(nvars); + // Vector holding probability of gross error for input to Buddy check + std::vector> varPGEBd(nvars, std::vector(nlocs)); + // Vector holding combined probability densities + std::vector> varPGE(nvars, std::vector(nlocs)); + // Vector holding packed PGEs + std::vector> varPGEFinal(nvars, std::vector(nlocs)); + // Vector holding OPS style QCflags for each variable + std::vector> varQCflags(nvars, std::vector(nlocs)); + + for (size_t ivar = 0; ivar < filtervars.nvars(); ++ivar) { + varname[ivar] = filtervars.variable(ivar).variable(); + // Get Gross Error Probability values and Met Office QCFlags from ObsSpace + if (obsdb_.has("GrossErrorProbability", varname[ivar]) && + obsdb_.has("GrossErrorProbabilityTotal", varname[ivar]) && + obsdb_.has("GrossErrorProbabilityFinal", varname[ivar]) && + obsdb_.has("QCFlags", varname[ivar])) { + obsdb_.get_db("GrossErrorProbability", varname[ivar], varPGEBd[ivar]); + obsdb_.get_db("GrossErrorProbabilityTotal", varname[ivar], varPGE[ivar]); + obsdb_.get_db("GrossErrorProbabilityFinal", varname[ivar], varPGEFinal[ivar]); + obsdb_.get_db("QCFlags", varname[ivar], varQCflags[ivar]); + } else { + std::stringstream errormessage; + errormessage << "GrossErrorProbability/" + varname[ivar] + ", " + << "GrossErrorProbabilityTotal/" + varname[ivar] + ", " + << "GrossErrorProbabilityFinal/" + varname[ivar] + ", and " + << "QCFlags/" + varname[ivar] + " must all be present" + << std::endl; + throw eckit::BadValue(errormessage.str(), Here()); + } + // Get variable options + PdBad[ivar] = filtervars[ivar].options().getFloat("probability_density_bad", 0.1f); + if (filtervars[ivar].options().has("synop_probability_density_bad")) { + PdBad_synop[ivar] = filtervars[ivar].options().getFloat("synop_probability_density_bad"); + } else { + PdBad_synop[ivar] = PdBad[ivar]; + } + if (filtervars[ivar].options().has("bogus_probability_density_bad")) { + PdBad_bogus[ivar] = filtervars[ivar].options().getFloat("bogus_probability_density_bad"); + } else { + PdBad_bogus[ivar] = PdBad[ivar]; + } + // + notUsed[ivar] = filtervars[ivar].options().getBool("not_used_in_whole_report", false); + } + // Get input probability of gross error affecting whole report + std::vector ReportPGE(nlocs); + if (obsdb_.has("MetaData", "GrossErrorProbabilityReport")) { + obsdb_.get_db("MetaData", "GrossErrorProbabilityReport", ReportPGE); + } else { + throw eckit::BadValue("MetaData/GrossErrorProbabilityReport not present", Here()); + } + // Get ObsType + std::vector ObsType(nlocs); + if (obsdb_.has("MetaData", "ObsType")) { + obsdb_.get_db("MetaData", "ObsType", ObsType); + } else { + throw eckit::BadValue("MetaData/ObsType not present", Here()); + } + + // Calculate probability that whole report is affected by gross error + for (size_t jobs = 0; jobs < nlocs; ++jobs) { + if (apply[jobs] && (ReportPGE[jobs] > 0)) { + // Probability density given Gross error in whole report + float PdBadRep = 1.0f; + for (size_t ivar = 0; ivar < filtervars.nvars(); ++ivar) { + if (!notUsed[ivar]) { + PdReport[jobs] *= varPGE[ivar][jobs]; + if (ObsType[jobs] == MetOfficeObsIDs::Bogus::Bogus) { + PdBad_used[ivar][jobs] = PdBad_bogus[ivar]; + } else if (ObsType[jobs] == MetOfficeObsIDs::Surface::SynopManual || + ObsType[jobs] == MetOfficeObsIDs::Surface::SynopAuto || + ObsType[jobs] == MetOfficeObsIDs::Surface::MetarManual || + ObsType[jobs] == MetOfficeObsIDs::Surface::MetarAuto || + ObsType[jobs] == MetOfficeObsIDs::Surface::SynopMob || + ObsType[jobs] == MetOfficeObsIDs::Surface::SynopBufr || + ObsType[jobs] == MetOfficeObsIDs::Surface::WOW) { + PdBad_used[ivar][jobs] = PdBad_synop[ivar]; + } else { + PdBad_used[ivar][jobs] = PdBad[ivar]; + } + PdBadRep *= PdBad_used[ivar][jobs]; + } + } + PBadRep[jobs] = ReportPGE[jobs]; + PdBadRep *= PBadRep[jobs]; + PdReport[jobs] = PdBadRep + ((1.0f - PBadRep[jobs]) * PdReport[jobs]); + if (PdReport[jobs] > 0.0f) { + ReportPGE[jobs] = PdBadRep / PdReport[jobs]; + } + } + } + + // Calculate updated probability that individual observation OR + // whole report is affected by gross error + bool secondComponentOfTwo = false; + for (size_t ivar = 0; ivar < filtervars.nvars(); ++ivar) { + secondComponentOfTwo = filtervars[ivar].options().getBool("second_component_of_two", false); + if (secondComponentOfTwo) { + varPGEBd[ivar] = varPGEBd[ivar - 1]; + varPGEFinal[ivar] = varPGEFinal[ivar - 1]; + varQCflags[ivar] = varQCflags[ivar - 1]; + flagged[ivar] = flagged[ivar - 1]; + } else { + for (size_t jobs = 0; jobs < nlocs; ++jobs) { + if (apply[jobs] && (ReportPGE[jobs] > 0) && (PdReport[jobs] > 0)) { + if (varPGEFinal[ivar][jobs] < PGEMult) { + float intpart; + // Initial Probability of Gross error in element + float const PGE0 = std::modf(varPGEFinal[ivar][jobs], &intpart); + float PdProduct = 1.0f; + for (size_t jvar = 0; jvar < filtervars.nvars(); ++jvar) { + if ((jvar != ivar) && !notUsed[jvar]) { + PdProduct *= varPGE[jvar][jobs]; + } + } + // PGE in element or whole report + float PGE1; + if (notUsed[ivar]) { + PGE1 = ReportPGE[jobs] + varPGEBd[ivar][jobs] - + ReportPGE[jobs] * varPGEBd[ivar][jobs]; + } else { + PGE1 = ReportPGE[jobs] + (PdBad_used[ivar][jobs] * PdProduct) * PGE0 * + (1.0f - PBadRep[jobs])/PdReport[jobs]; + } + varPGEFinal[ivar][jobs] = floor(PGE1 * PGEMult) + PGE0; // PGE packing + if (PGE1 >= PGECrit) { + varQCflags[ivar][jobs] |= ufo::MetOfficeQCFlags::Elem::BackRejectFlag; + flagged[ivar][jobs] = true; + } + if (std::abs(varPGEBd[ivar][jobs] - PGEMDI) > 0.001f) { + varPGEBd[ivar][jobs] = PGE1; + } + } + } + } + } + // Save updated gross error probabilities and QCFlags to ObsSpace + obsdb_.put_db("GrossErrorProbability", varname[ivar], varPGEBd[ivar]); + obsdb_.put_db("GrossErrorProbabilityFinal", varname[ivar], varPGEFinal[ivar]); + obsdb_.put_db("QCFlags", varname[ivar], varQCflags[ivar]); + } + obsdb_.put_db("MetaData", "GrossErrorProbabilityReport", ReportPGE); +} + +// ----------------------------------------------------------------------------- + +void ProbabilityGrossErrorWholeReport::print(std::ostream & os) const { + os << "ProbabilityGrossErrorWholeReport: config = " << parameters_ << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/filters/ProbabilityGrossErrorWholeReport.h b/src/ufo/filters/ProbabilityGrossErrorWholeReport.h new file mode 100644 index 000000000..1bd53db8d --- /dev/null +++ b/src/ufo/filters/ProbabilityGrossErrorWholeReport.h @@ -0,0 +1,125 @@ +/* + * (C) Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_FILTERS_PROBABILITYGROSSERRORWHOLEREPORT_H_ +#define UFO_FILTERS_PROBABILITYGROSSERRORWHOLEREPORT_H_ + +#include +#include +#include +#include + +#include "oops/util/ObjectCounter.h" +#include "ufo/filters/FilterBase.h" +#include "ufo/filters/QCflags.h" +#include "ufo/filters/Variable.h" +#include "ufo/utils/ProbabilityOfGrossErrorParameters.h" + +namespace eckit { + class Configuration; +} + +namespace ioda { + template class ObsDataVector; + class ObsSpace; +} + +namespace ufo { + +/// Parameters controlling the operation of the ProbabilityGrossErrorWholeReport filter. +class ProbabilityGrossErrorWholeReportParameters : public FilterParametersBase { + OOPS_CONCRETE_PARAMETERS(ProbabilityGrossErrorWholeReportParameters, FilterParametersBase) + + public: + /// Parameters related to PGE calculations. The value of \p PGECrit + /// is obtained from here. + ProbabilityOfGrossErrorParameters PGEParameters{this}; +}; + +/// \brief This filter calculates the probability that an entire report is affected by gross error. +/// \details Synoptic stations typically provide reports at regular intervals. A report is a +/// combination of variables observed by different sensors at a single location. Reports may +/// include some, but not necessarily all, of pressure, temperature, dew point and wind speed +/// and direction. +/// +/// The probability that the whole report is affected by gross error is calculated +/// through the Bayesian combination of the probability of gross error of individual +/// observations. This is based on the logic that if multiple observations within a report +/// appear dubious based on a Bayesian Background check, it is likely that the whole report +/// is affected by, for example, location error. This filter should be called after the +/// Bayesian Background Check. +/// +/// Variables which are to have their probability of gross error updated should be specified +/// using the "filter variables" YAML option. All variables included in "filter variables" will +/// be used to calculate the probability that the whole report is affected by gross error unless +/// the option \c not_used_in_whole_report is set to true for that variable. +/// +/// Variables can be either scalar or vector (with two Cartesian components, such as the eastward +/// and northward wind components). In the latter case the two components need to specified one +/// after the other in the "filter variables" list, with the second component having the +/// \c second_component_of_two option set to true. For each variable, the option +/// \c Probability_Density_Bad is used to set the prior probability density of that variable being +/// "bad". The filter can also apply a specific prior probability density of bad observations for +/// the following Met Office SubTypes: +/// * Bogus +/// * Synop (SynopManual, SynopAuto, MetarManual, MetarAuto, SynopMob, SynopBufr, WOW) +/// +/// Example: +/// +/// \code{.yaml} +///- filter: Bayesian Whole Report +/// filter variables: +/// - name: pressure_at_model_surface +/// options: +/// Probability_Density_Bad: 0.1 +/// Bogus_Probability_Density_Bad: 0.1 +/// - name: air_temperature_at_2m +/// options: +/// Probability_Density_Bad: 0.1 +/// - name: eastward_wind +/// options: +/// Probability_Density_Bad: 0.1 +/// Synop_Probability_Density_Bad: 0.1 +/// Bogus_Probability_Density_Bad: 0.1 +/// - name: northward_wind +/// options: +/// not_used_in_whole_report: true +/// second_component_of_two: true +/// - name: relative_humidity_at_2m +/// options: +/// not_used_in_whole_report: true +/// Probability_Density_Bad: 0.1 +/// PGE threshold: 0.15 +/// \endcode +/// + +class ProbabilityGrossErrorWholeReport : public FilterBase, + private util::ObjectCounter { + public: + /// The type of parameters accepted by the constructor of this filter. + /// This typedef is used by the FilterFactory. + typedef ProbabilityGrossErrorWholeReportParameters Parameters_; + + static const std::string classname() {return "ufo::ProbabilityGrossErrorWholeReport";} + + ProbabilityGrossErrorWholeReport(ioda::ObsSpace &, const Parameters_ &, + std::shared_ptr>, + std::shared_ptr>); + ~ProbabilityGrossErrorWholeReport(); + + private: + void print(std::ostream &) const override; + void applyFilter(const std::vector &, const Variables &, + std::vector> &) const override; + int qcFlag() const override {return QCflags::bayesianQC;} + + Parameters_ parameters_; +}; + +} // namespace ufo + +#endif // UFO_FILTERS_PROBABILITYGROSSERRORWHOLEREPORT_H_ diff --git a/src/ufo/instantiateObsFilterFactory.h b/src/ufo/instantiateObsFilterFactory.h index caf31a774..9562fb5b5 100644 --- a/src/ufo/instantiateObsFilterFactory.h +++ b/src/ufo/instantiateObsFilterFactory.h @@ -31,6 +31,7 @@ #include "ufo/filters/PerformAction.h" #include "ufo/filters/PoissonDiskThinning.h" #include "ufo/filters/PreQC.h" +#include "ufo/filters/ProbabilityGrossErrorWholeReport.h" #include "ufo/filters/ProfileBackgroundCheck.h" #include "ufo/filters/ProfileConsistencyChecks.h" #include "ufo/filters/ProfileFewObsCheck.h" @@ -123,6 +124,8 @@ template void instantiateObsFilterFactory() { performActionMaker("Perform Action"); static oops::FilterMaker > BayesianBackgroundQCFlagsMaker("Bayesian Background QC Flags"); + static oops::FilterMaker > + ProbabilityGrossErrorWholeReportMaker("Bayesian Whole Report"); static oops::FilterMaker > ImpactHeightCheckMaker("GNSSRO Impact Height Check"); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3e4cfb9b4..29fb1e199 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -176,6 +176,7 @@ list( APPEND ufo_test_input testinput/qc_differencecheck.yaml testinput/qc_gauss_thinning.yaml testinput/qc_gauss_thinning_unittests.yaml + testinput/qc_grosserrorwholereport.yaml testinput/qc_historycheck_unittests.yaml testinput/qc_met_office_buddy_check.yaml testinput/qc_met_office_buddy_check_unittests.yaml @@ -1604,6 +1605,13 @@ ecbuild_add_test( TARGET test_ufo_qc_gen_boundscheck DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_qc_grosserror_wholereport + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x + ARGS "testinput/qc_grosserrorwholereport.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFilters.x + TEST_DEPENDS ufo_get_ufo_test_data ) + ecbuild_add_test( TARGET test_ufo_qc_velocitybounds COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x ARGS "testinput/qc_velocitycheck.yaml" diff --git a/test/testinput/qc_grosserrorwholereport.yaml b/test/testinput/qc_grosserrorwholereport.yaml new file mode 100644 index 000000000..156fef092 --- /dev/null +++ b/test/testinput/qc_grosserrorwholereport.yaml @@ -0,0 +1,65 @@ +window begin: 2020-12-31T23:59:00Z +window end: 2021-01-01T00:01:00Z + +observations: +- obs space: + name: Surface + obsdatain: + obsfile: Data/ufo/testinput_tier_1/grosserrorwholereport_filter_testdata.nc4 + simulated variables: [pressure_at_model_surface, + air_temperature_at_2m, + eastward_wind, + northward_wind, + relative_humidity_at_2m] + obs filters: + - filter: Bayesian Whole Report + filter variables: + - name: pressure_at_model_surface + options: + probability_density_bad: 0.1 + bogus_probability_density_bad: 0.1 + - name: air_temperature_at_2m + options: + probability_density_bad: 0.1 + - name: eastward_wind + options: + probability_density_bad: 0.1 + synop_probability_density_bad: 0.1 + bogus_probability_density_bad: 0.1 + - name: northward_wind + options: + not_used_in_whole_report: true + second_component_of_two: true + - name: relative_humidity_at_2m + options: + not_used_in_whole_report: true + probability_density_bad: 0.1 + PGE threshold: 0.15 + passedBenchmark: 11 + compareVariables: # test output matches ops + - reference: + name: pressure_at_model_surface@GrossErrorProbabilityTestReference + test: + name: pressure_at_model_surface@GrossErrorProbability + absTol: 5.0e-4 + - reference: + name: air_temperature_at_2m@GrossErrorProbabilityTestReference + test: + name: air_temperature_at_2m@GrossErrorProbability + absTol: 5.0e-5 + - reference: + name: eastward_wind@GrossErrorProbabilityTestReference + test: + name: eastward_wind@GrossErrorProbability + absTol: 5.0e-5 + - reference: + name: northward_wind@GrossErrorProbabilityTestReference + test: + name: northward_wind@GrossErrorProbability + absTol: 5.0e-5 + - reference: + name: relative_humidity_at_2m@GrossErrorProbabilityTestReference + test: + name: relative_humidity_at_2m@GrossErrorProbability + absTol: 5.0e-5 + From 2021100251cf2c34a35bde5ce8ccd77f5f0a8455 Mon Sep 17 00:00:00 2001 From: James Cotton <64855970+james-cotton@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:56:11 +0100 Subject: [PATCH 06/98] Feature/satwind inversion (#1181) * First version - test passes * Tidy the code and add documentation * Now suitable for pull request * Update yaml data locations * Fix typos in comments * Improve initialisation of original pressure * Put cloud motion methods in enum and move test to start of loop * Declare GeoVal columns before for loop and assign in one go * use std::numeric_limits::max() * add missing include statement * Add further comments * Update test/testinput/satwind_inversion_correction.yaml * Refactor to use ioda::Accumulator Co-authored-by: Ryan Honeyager --- src/ufo/filters/CMakeLists.txt | 2 + src/ufo/filters/SatwindInversionCorrection.cc | 242 ++++++++++++++++++ src/ufo/filters/SatwindInversionCorrection.h | 95 +++++++ src/ufo/instantiateObsFilterFactory.h | 3 + test/CMakeLists.txt | 8 + .../satwind_inversion_correction.yaml | 32 +++ 6 files changed, 382 insertions(+) create mode 100644 src/ufo/filters/SatwindInversionCorrection.cc create mode 100644 src/ufo/filters/SatwindInversionCorrection.h create mode 100644 test/testinput/satwind_inversion_correction.yaml diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index 11e342138..ad720e231 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -73,6 +73,8 @@ set ( filters_files ProfileFewObsCheck.h SatName.h SatName.cc + SatwindInversionCorrection.cc + SatwindInversionCorrection.h StuckCheck.cc StuckCheck.h StuckCheckParameters.h diff --git a/src/ufo/filters/SatwindInversionCorrection.cc b/src/ufo/filters/SatwindInversionCorrection.cc new file mode 100644 index 000000000..9ccd9d20e --- /dev/null +++ b/src/ufo/filters/SatwindInversionCorrection.cc @@ -0,0 +1,242 @@ +/* ----------------------------------------------------------------------------- + * (C) British Crown Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/filters/SatwindInversionCorrection.h" + +#include + +#include "eckit/config/Configuration.h" + +#include "ioda/distribution/Accumulator.h" +#include "ioda/ObsDataVector.h" +#include "ioda/ObsSpace.h" + +#include "oops/base/ObsFilterBase.h" +#include "oops/interface/ObsFilter.h" +#include "oops/util/Logger.h" + +#include "ufo/GeoVaLs.h" +#include "ufo/utils/metoffice/MetOfficeQCFlags.h" + +namespace ufo { + +// ----------------------------------------------------------------------------- + +SatwindInversionCorrection::SatwindInversionCorrection(ioda::ObsSpace & obsdb, + const Parameters_ & parameters, + std::shared_ptr > flags, + std::shared_ptr > obserr) + : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) +{ + oops::Log::trace() << "SatwindInversion contructor starting" << std::endl; + // Get parameters from options + allvars_ += parameters_.obs_pressure; + // Include list of required data from GeoVaLs + allvars_ += Variable("air_temperature@GeoVaLs"); + allvars_ += Variable("relative_humidity@GeoVaLs"); + allvars_ += Variable("air_pressure@GeoVaLs"); +} + +// ----------------------------------------------------------------------------- + +SatwindInversionCorrection::~SatwindInversionCorrection() { + oops::Log::trace() << "SatwindInversion destructed" << std::endl; +} + +// ----------------------------------------------------------------------------- +/*! \brief A filter that modifies the assigned pressure of AMV observations if a + * temperature inversion is detected in the model profile and defined criteria + * are met. + * + * \details The model profile is searched for the presence of a temperature + * inversion. Where there are multiple temperature inversions, only the lowest one is found. + * This is intended only for use on low level AMVs, typically below 700 hPa height. + * + * The pressure of the AMV is corrected downwards in height if the following conditions are true: + * a) Originally assigned pressure is greater than or equal to min_pressure (Pa). + * b) AMV derived from IR and visible channels only. + * c) Temperature inversion is present in the model profile for pressures less than or equal to + * max_pressure (Pa). + * d) In order to be considered significant, the temperature difference across the top and base of + * the inversion must be greater than or equal to the inversion_temperature (K) value. + * e) Relative humidity at the top of the inversion is less than the rh_threshold value. + * f) AMV has been assigned above the height of the inversion base. + * + * The AMV is then re-assigned to the base of the inversion. + * + * Reference for initial implementation: + * Cotton, J., Forsythe, M., Warrick, F., (2016). Towards improved height assignment and + * quality control of AMVs in Met Office NWP. Proceedings for the 13th International Winds + * Workshop 27 June - 1 July 2016, Monterey, California, USA. + * + * Example: + * \code{.unparsed} + * obs filters: + * - filter: Satwind Inversion Correction + * observation pressure: + * name: air_pressure_levels@MetaData + * RH threshold: 50 + * maximum pressure: 96000 + * \endcode + * + * \author J.Cotton (Met Office) + * + * \date 07/05/2021: Created + */ + +void SatwindInversionCorrection::applyFilter(const std::vector & apply, + const Variables & filtervars, + std::vector> & flagged) const { + oops::Log::trace() << "SatwindInversionCorrection priorFilter" << std::endl; + print(oops::Log::trace()); + + const float missing = util::missingValue(missing); + const size_t nlocs = obsdb_.nlocs(); + +// Get parameters from options. + const float rh_threshold = parameters_.rh_threshold.value(); + const float min_pressure = parameters_.min_pressure.value(); + const float max_pressure = parameters_.max_pressure.value(); + const float inversion_temperature = parameters_.inversion_temperature.value(); +// get names of GeoVal variables + const std::string model_temp_name = "air_temperature"; + const std::string model_rh_name = "relative_humidity"; + const std::string model_vcoord_name = "air_pressure"; + +// Get variables from ObsSpace +// Get the observation pressure + std::vector obs_pressure; + data_.get(parameters_.obs_pressure, obs_pressure); +// Get wind computation method + std::vector comp_method(obsdb_.nlocs()); + obsdb_.get_db("MetaData", "wind_computation_method", comp_method); +// Get flags + std::vector u_flags(obsdb_.nlocs()); + std::vector v_flags(obsdb_.nlocs()); + if (obsdb_.has("QCFlags", "eastward_wind") && obsdb_.has("QCFlags", "northward_wind")) { + obsdb_.get_db("QCFlags", "eastward_wind", u_flags); + obsdb_.get_db("QCFlags", "northward_wind", v_flags); + } else { + throw eckit::Exception("eastward_wind@QCFlags or northward_wind@QCFlags not initialised"); + } +// Get GeoVaLs + const ufo::GeoVaLs * gvals = data_.getGeoVaLs(); +// Get number of vertical levels in GeoVaLs + const size_t nlevs = data_.nlevs(Variable(model_vcoord_name+"@GeoVaLs")); +// Vectors storing GeoVaL column for current location. + std::vector model_temp_profile(gvals->nlevs(model_temp_name), 0.0); + std::vector model_rh_profile(gvals->nlevs(model_rh_name), 0.0); + std::vector model_vcoord_profile(gvals->nlevs(model_vcoord_name), 0.0); + +// Vector to store original pressure + std::vector original_pressure(obs_pressure); + +// diagnostic variables to be summed over all processors at the end of the routine + std::unique_ptr> countAccumulator = + obsdb_.distribution()->createAccumulator(); + enum {PDIFF}; + std::unique_ptr>> totalsAccumulator = + obsdb_.distribution()->createAccumulator(nlocs); + +// Loop through locations + for (size_t iloc=0; iloc < nlocs; ++iloc) { + if (apply[iloc]) { + // only consider low level AMVs from infrared or visible channels + if (obs_pressure[iloc] >= min_pressure && + (comp_method[iloc] == CloudMotionMethod::infrared || + comp_method[iloc] == CloudMotionMethod::visible)) { + // Get GeoVaLs at the specified location. + gvals->getAtLocation(model_temp_profile, model_temp_name, iloc); + gvals->getAtLocation(model_rh_profile, model_rh_name, iloc); + gvals->getAtLocation(model_vcoord_profile, model_vcoord_name, iloc); + // --------------------------------------------------------------------------- + // Search for inversion and if present find T and P of base and top + // --------------------------------------------------------------------------- + bool inversion = false; + bool firsttime = true; + float inversion_base = std::numeric_limits::max(); + float inversion_top = std::numeric_limits::max(); + float temp_inversion_base = std::numeric_limits::max(); + float temp_inversion_top = std::numeric_limits::max(); + // loop over levels starting from lowest + for (int ilev = 0; ilev < nlevs-2 ; ++ilev) { + // if haven't found inversion and pressure is less than min_pressure Pa then exit + if (inversion == false && model_vcoord_profile[ilev] < min_pressure) { + break; + } + // if model pressure is greater than max_pressure Pa then skip to next level + if (model_vcoord_profile[ilev] > max_pressure) { + continue; + } + // if haven't found inversion, check for increase in temperature + if (inversion == false && model_temp_profile[ilev+1] > model_temp_profile[ilev]) { + // T of level above is greater so take base at current level + inversion_base = model_vcoord_profile[ilev]; + temp_inversion_base = model_temp_profile[ilev]; + inversion = true; + } + // if inversion found, then detect level the temperature starts to decrease again above + // the inversion base + if (inversion && + model_temp_profile[ilev+1] < model_temp_profile[ilev] && + model_vcoord_profile[ilev] < inversion_base && + firsttime) { + // Check humidity of inversion top + if (model_rh_profile[ilev] < rh_threshold) { + inversion_top = model_vcoord_profile[ilev]; + temp_inversion_top = model_temp_profile[ilev]; + firsttime = false; + } else { + // otherwise discard and keep looking for a new base/top higher up + inversion = false; + } + } + } // level loop + // + //--------------------------------------------------------------------------- + // Correct pressure if inversion found and conditions are met + //--------------------------------------------------------------------------- + if (inversion && + (temp_inversion_top - temp_inversion_base) >= inversion_temperature && + obs_pressure[iloc] < inversion_base) { + // re-assign to base of inversion + countAccumulator->addTerm(iloc, 1); + totalsAccumulator->addTerm(iloc, PDIFF, inversion_base - obs_pressure[iloc]); + obs_pressure[iloc] = inversion_base; + // set flag + u_flags[iloc] |= ufo::MetOfficeQCFlags::SatWind::SatwindInversionFlag; + v_flags[iloc] |= ufo::MetOfficeQCFlags::SatWind::SatwindInversionFlag; + } + } + } // apply + } // location loop + // write back corrected pressure, updated flags and original pressure + obsdb_.put_db("MetaData", "air_pressure_levels", obs_pressure); + obsdb_.put_db("QCFlags", "eastward_wind", u_flags); + obsdb_.put_db("QCFlags", "northward_wind", v_flags); + obsdb_.put_db("MetaData", "air_pressure_original", original_pressure); + + // sum number corrected and pressure differences + const std::size_t count = countAccumulator->computeResult(); + const std::vector totals = totalsAccumulator->computeResult(); + if (count) { + oops::Log::info() << "Satwind Inversion: "<< count + << " observations with modified pressure" << std::endl; + oops::Log::info() << "Satwind Inversion: "<< totals[PDIFF] / count + << " Pa mean pressure difference" << std::endl; + } +} + +// ----------------------------------------------------------------------------- + +void SatwindInversionCorrection::print(std::ostream & os) const { + os << "SatwindInversionCorrection: config = " << parameters_ << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/filters/SatwindInversionCorrection.h b/src/ufo/filters/SatwindInversionCorrection.h new file mode 100644 index 000000000..c58e3b49f --- /dev/null +++ b/src/ufo/filters/SatwindInversionCorrection.h @@ -0,0 +1,95 @@ +/* ----------------------------------------------------------------------------- + * (C) British Crown Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_FILTERS_SATWINDINVERSIONCORRECTION_H_ +#define UFO_FILTERS_SATWINDINVERSIONCORRECTION_H_ + +#include +#include +#include +#include + +#include "oops/util/ObjectCounter.h" +#include "oops/util/parameters/RequiredParameter.h" + +#include "ufo/filters/FilterBase.h" +#include "ufo/filters/QCflags.h" +#include "ufo/filters/Variable.h" +#include "ufo/utils/parameters/ParameterTraitsVariable.h" + +namespace eckit { + class Configuration; +} + +namespace ioda { + template class ObsDataVector; + class ObsSpace; +} + +namespace ufo { + +// Define cloud motion methods +enum CloudMotionMethod { + infrared = 1, /// Motion observed in the infrared channel + visible = 2, /// Motion observed in the visible channel + vapourcloud = 3, /// Motion observed in the water vapour channel in cloud + combination = 4, /// Motion observed in a combination of spectral channels + vapourclear = 5, /// Motion observed in the water vapour channel in clear air + ozone = 6, /// Motion observed in the ozone channel + vapour = 7 /// Motion observed in the water vapour channel (cloud or clear) +}; + +/// \brief Parameters controlling the operation of the SatwindInversionCorrection filter. +class SatwindInversionCorrectionParameters : public FilterParametersBase { + OOPS_CONCRETE_PARAMETERS(SatwindInversionCorrectionParameters, FilterParametersBase) + + public: + /// Name of the observation pressure variable to correct + oops::RequiredParameter obs_pressure{"observation pressure", this}; + /// Relative humidity (%) threshold value + oops::RequiredParameter rh_threshold{"RH threshold", this}; + /// Minimum AMV pressure (Pa) to consider for correction - set default + oops::Parameter min_pressure{"minimum pressure", 70000.0, this}; + /// Maximum model pressure (Pa) to consider - set default + oops::Parameter max_pressure{"maximum pressure", 105000.0, this}; + /// Temperature difference (K) between inversion base and top - set default + oops::Parameter inversion_temperature{"inversion temperature", 2.0, this}; +}; + +// ----------------------------------------------------------------------------- + +/// \brief A filter that modifies the assigned pressure of AMV observations if a +/// temperature inversion is detected in the model profile and defined criteria are met. +/// +/// See SatwindInversionCorrectionParameters for the documentation of the parameters controlling +/// this filter. +class SatwindInversionCorrection : public FilterBase, + private util::ObjectCounter { + public: + /// The type of parameters accepted by the constructor of this filter. + /// This typedef is used by the FilterFactory. + typedef SatwindInversionCorrectionParameters Parameters_; + + static const std::string classname() {return "ufo::SatwindInversionCorrection";} + + SatwindInversionCorrection(ioda::ObsSpace &, const Parameters_ &, + std::shared_ptr >, + std::shared_ptr >); + ~SatwindInversionCorrection(); + + private: + void print(std::ostream &) const override; + void applyFilter(const std::vector &, const Variables &, + std::vector> &) const override; + int qcFlag() const override {return QCflags::pass;} + + Parameters_ parameters_; +}; + +} // namespace ufo + +#endif // UFO_FILTERS_SATWINDINVERSIONCORRECTION_H_ diff --git a/src/ufo/instantiateObsFilterFactory.h b/src/ufo/instantiateObsFilterFactory.h index 9562fb5b5..a60f085cf 100644 --- a/src/ufo/instantiateObsFilterFactory.h +++ b/src/ufo/instantiateObsFilterFactory.h @@ -37,6 +37,7 @@ #include "ufo/filters/ProfileFewObsCheck.h" #include "ufo/filters/QCmanager.h" #include "ufo/filters/SatName.h" +#include "ufo/filters/SatwindInversionCorrection.h" #include "ufo/filters/StuckCheck.h" #include "ufo/filters/TemporalThinning.h" #include "ufo/filters/Thinning.h" @@ -128,6 +129,8 @@ template void instantiateObsFilterFactory() { ProbabilityGrossErrorWholeReportMaker("Bayesian Whole Report"); static oops::FilterMaker > ImpactHeightCheckMaker("GNSSRO Impact Height Check"); + static oops::FilterMaker > + SatwindInversionCorrectionMaker("Satwind Inversion Correction"); // Only include this filter if rttov is present #if defined(RTTOV_FOUND) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 29fb1e199..eab980a9c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -209,6 +209,7 @@ list( APPEND ufo_test_input testinput/satname.yaml testinput/sattcwv.yaml testinput/satwind.yaml + testinput/satwind_inversion_correction.yaml testinput/satwind_metoffice.yaml testinput/sbuv2_n19.yaml testinput/sbuv2_n19_flipz.yaml @@ -1598,6 +1599,13 @@ ecbuild_add_test( TARGET test_ufo_satname DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_satwind_inversion_correction + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x + ARGS "testinput/satwind_inversion_correction.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFilters.x + TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) + ecbuild_add_test( TARGET test_ufo_qc_gen_boundscheck COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x ARGS "testinput/qc_boundscheck.yaml" diff --git a/test/testinput/satwind_inversion_correction.yaml b/test/testinput/satwind_inversion_correction.yaml new file mode 100644 index 000000000..e7fea4e69 --- /dev/null +++ b/test/testinput/satwind_inversion_correction.yaml @@ -0,0 +1,32 @@ +window begin: 2020-10-01T03:00:00Z +window end: 2020-10-01T09:00:00Z + +observations: +- obs space: + name: Satwind + obsdatain: + obsfile: Data/ufo/testinput_tier_1/satwind_obs_1d_2020100106_yesinv.nc4 + obsdataout: + obsfile: Data/satwind_obs_1d_2020100106_yesinv_invcorr_out.nc4 + simulated variables: [eastward_wind, northward_wind] + geovals: + filename: Data/ufo/testinput_tier_1/satwind_geoval_20201001T0600Z_noinv.nc4 + obs filters: + - filter: Variable Assignment + assignments: + - name: eastward_wind@QCFlags + type: int + value: 0 + - name: northward_wind@QCFlags + type: int + value: 0 + - filter: Satwind Inversion Correction + observation pressure: + name: air_pressure_levels@MetaData + RH threshold: 54 + compareVariables: # test output matches precalculated values + - reference: + name: air_pressure_levels@TestReference # corrected pressures from OPS + test: + name: air_pressure_levels@MetaData # corrected pressures in ufo + absTol: 1.0e-5 # tolerance in Pa From b9099a8ab2c59946ed6dc85808d9e83780739cf3 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Thu, 17 Jun 2021 07:22:48 -0600 Subject: [PATCH 07/98] remove ObsBias parameter from LinearObsOperatorBase::setTrajectory (#1219) * remove ObsBias parameter from LinObsOpBase::setTraj * missed one subclass + override warnings + unused includes Co-authored-by: Ryan Honeyager --- src/ufo/LinearObsOperator.cc | 2 +- src/ufo/LinearObsOperatorBase.h | 3 +-- src/ufo/aerosols/AOP/ObsAodExtTLAD.cc | 4 +--- src/ufo/aerosols/AOP/ObsAodExtTLAD.h | 3 +-- src/ufo/aerosols/MR/ObsAodGeos.h | 4 ++-- src/ufo/aerosols/MR/ObsAodGeosTLAD.cc | 3 +-- src/ufo/aerosols/MR/ObsAodGeosTLAD.h | 4 +--- src/ufo/atmvertinterp/ObsAtmVertInterpTLAD.cc | 4 +--- src/ufo/atmvertinterp/ObsAtmVertInterpTLAD.h | 3 +-- src/ufo/atmvertinterplay/ObsAtmVertInterpLayTLAD.cc | 4 +--- src/ufo/atmvertinterplay/ObsAtmVertInterpLayTLAD.h | 3 +-- src/ufo/avgkernel/ObsAvgKernelTLAD.cc | 4 +--- src/ufo/avgkernel/ObsAvgKernelTLAD.h | 3 +-- src/ufo/compositeoper/ObsCompositeTLAD.cc | 6 ++---- src/ufo/compositeoper/ObsCompositeTLAD.h | 3 +-- src/ufo/crtm/ObsAodCRTMTLAD.cc | 4 +--- src/ufo/crtm/ObsAodCRTMTLAD.h | 3 +-- src/ufo/crtm/ObsAodLUTsTLAD.cc | 4 +--- src/ufo/crtm/ObsAodLUTsTLAD.h | 3 +-- src/ufo/crtm/ObsRadianceCRTMTLAD.cc | 4 +--- src/ufo/crtm/ObsRadianceCRTMTLAD.h | 3 +-- src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.cc | 3 +-- src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.h | 3 +-- src/ufo/gnssro/BndNBAM/ObsGnssroBndNBAMTLAD.cc | 3 +-- src/ufo/gnssro/BndNBAM/ObsGnssroBndNBAMTLAD.h | 3 +-- src/ufo/gnssro/BndROPP1D/ObsGnssroBndROPP1DTLAD.cc | 3 +-- src/ufo/gnssro/BndROPP1D/ObsGnssroBndROPP1DTLAD.h | 3 +-- src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2D.cc | 1 - src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2DTLAD.cc | 3 +-- src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2DTLAD.h | 3 +-- src/ufo/gnssro/Ref/ObsGnssroRefTLAD.cc | 3 +-- src/ufo/gnssro/Ref/ObsGnssroRefTLAD.h | 3 +-- .../ZenithTotalDelayMetOffice/ObsGroundgnssMetOfficeTLAD.cc | 3 +-- .../ZenithTotalDelayMetOffice/ObsGroundgnssMetOfficeTLAD.h | 3 +-- src/ufo/identity/ObsIdentityTLAD.cc | 4 +--- src/ufo/identity/ObsIdentityTLAD.h | 3 +-- src/ufo/marine/adt/ObsADTTLAD.cc | 4 +--- src/ufo/marine/adt/ObsADTTLAD.h | 3 +-- src/ufo/marine/coolskin/ObsCoolSkinTLAD.cc | 4 +--- src/ufo/marine/coolskin/ObsCoolSkinTLAD.h | 3 +-- .../marine/insitutemperature/ObsInsituTemperatureTLAD.cc | 4 +--- src/ufo/marine/insitutemperature/ObsInsituTemperatureTLAD.h | 3 +-- src/ufo/marine/marinevertinterp/ObsMarineVertInterp.cc | 1 - src/ufo/marine/marinevertinterp/ObsMarineVertInterp.h | 1 - src/ufo/marine/marinevertinterp/ObsMarineVertInterpTLAD.cc | 4 +--- src/ufo/marine/marinevertinterp/ObsMarineVertInterpTLAD.h | 3 +-- src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc | 4 +--- src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.h | 3 +-- src/ufo/marine/seaicethickness/ObsSeaIceThicknessTLAD.cc | 4 +--- src/ufo/marine/seaicethickness/ObsSeaIceThicknessTLAD.h | 3 +-- src/ufo/radarradialvelocity/ObsRadarRadialVelocityTLAD.cc | 3 +-- src/ufo/radarradialvelocity/ObsRadarRadialVelocityTLAD.h | 3 +-- src/ufo/rttov/ObsRadianceRTTOV.cc | 1 - src/ufo/rttov/ObsRadianceRTTOVTLAD.cc | 4 +--- src/ufo/rttov/ObsRadianceRTTOVTLAD.h | 3 +-- src/ufo/rttovcpp/ObsRadianceRTTOVCPP.cc | 1 - src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc | 4 +--- src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.h | 3 +-- src/ufo/sattcwv/SatTCWVTLAD.cc | 3 +-- src/ufo/sattcwv/SatTCWVTLAD.h | 3 +-- src/ufo/timeoper/ObsTimeOperTLAD.cc | 5 +---- src/ufo/timeoper/ObsTimeOperTLAD.h | 3 +-- tools/new_obsop/example/ObsExampleTLAD.cc | 4 +--- tools/new_obsop/example/ObsExampleTLAD.h | 3 +-- 64 files changed, 61 insertions(+), 144 deletions(-) diff --git a/src/ufo/LinearObsOperator.cc b/src/ufo/LinearObsOperator.cc index 5199e0588..50228dcf9 100644 --- a/src/ufo/LinearObsOperator.cc +++ b/src/ufo/LinearObsOperator.cc @@ -47,7 +47,7 @@ void LinearObsOperator::setTrajectory(const GeoVaLs & gvals, const ObsBias & bia odb_.get_db("MetaData", "longitude", lons); odb_.get_db("MetaData", "datetime", times); ObsDiagnostics ydiags(odb_, Locations(lons, lats, times, odb_.distribution()), vars); - oper_->setTrajectory(gvals, bias, ydiags); + oper_->setTrajectory(gvals, ydiags); if (bias) { biasoper_.reset(new LinearObsBiasOperator(odb_)); biasoper_->setTrajectory(gvals, bias, ydiags); diff --git a/src/ufo/LinearObsOperatorBase.h b/src/ufo/LinearObsOperatorBase.h index ba0bd1b1e..9961268ba 100644 --- a/src/ufo/LinearObsOperatorBase.h +++ b/src/ufo/LinearObsOperatorBase.h @@ -25,7 +25,6 @@ class ObsVector; namespace ufo { class GeoVaLs; -class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -39,7 +38,7 @@ class LinearObsOperatorBase : public util::Printable, virtual ~LinearObsOperatorBase() {} /// Obs Operator - virtual void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) = 0; + virtual void setTrajectory(const GeoVaLs &, ObsDiagnostics &) = 0; virtual void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const = 0; virtual void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const = 0; diff --git a/src/ufo/aerosols/AOP/ObsAodExtTLAD.cc b/src/ufo/aerosols/AOP/ObsAodExtTLAD.cc index d52c13362..2fe2ba719 100644 --- a/src/ufo/aerosols/AOP/ObsAodExtTLAD.cc +++ b/src/ufo/aerosols/AOP/ObsAodExtTLAD.cc @@ -14,7 +14,6 @@ #include "oops/base/Variables.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" namespace ufo { @@ -42,8 +41,7 @@ ObsAodExtTLAD::~ObsAodExtTLAD() { // ----------------------------------------------------------------------------- -void ObsAodExtTLAD::setTrajectory(const GeoVaLs & geovals, - const ObsBias & bias, ObsDiagnostics &) { +void ObsAodExtTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { oops::Log::trace() << "ObsAodExtTLAD:trajectory entering" < &component : components_) - component->setTrajectory(geovals, bias, ydiags); + component->setTrajectory(geovals, ydiags); oops::Log::trace() << "ObsCompositeTLAD: setTrajectory exit " << std::endl; } diff --git a/src/ufo/compositeoper/ObsCompositeTLAD.h b/src/ufo/compositeoper/ObsCompositeTLAD.h index b49fe4553..372e72d53 100644 --- a/src/ufo/compositeoper/ObsCompositeTLAD.h +++ b/src/ufo/compositeoper/ObsCompositeTLAD.h @@ -30,7 +30,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -43,7 +42,7 @@ class ObsCompositeTLAD : public LinearObsOperatorBase, ObsCompositeTLAD(const ioda::ObsSpace &, const eckit::Configuration &); ~ObsCompositeTLAD() override; - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/crtm/ObsAodCRTMTLAD.cc b/src/ufo/crtm/ObsAodCRTMTLAD.cc index 5b45cf301..5342caf21 100644 --- a/src/ufo/crtm/ObsAodCRTMTLAD.cc +++ b/src/ufo/crtm/ObsAodCRTMTLAD.cc @@ -18,7 +18,6 @@ #include "oops/util/IntSetParser.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" namespace ufo { @@ -50,8 +49,7 @@ ObsAodCRTMTLAD::~ObsAodCRTMTLAD() { // ----------------------------------------------------------------------------- -void ObsAodCRTMTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsAodCRTMTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_aodcrtm_tlad_settraj_f90(keyOperAodCRTM_, geovals.toFortran(), obsspace()); } diff --git a/src/ufo/crtm/ObsAodCRTMTLAD.h b/src/ufo/crtm/ObsAodCRTMTLAD.h index 262d4dbdd..14a84d8fb 100644 --- a/src/ufo/crtm/ObsAodCRTMTLAD.h +++ b/src/ufo/crtm/ObsAodCRTMTLAD.h @@ -30,7 +30,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -43,7 +42,7 @@ class ObsAodCRTMTLAD : public LinearObsOperatorBase, virtual ~ObsAodCRTMTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/crtm/ObsAodLUTsTLAD.cc b/src/ufo/crtm/ObsAodLUTsTLAD.cc index cd8b332e7..ac1400851 100644 --- a/src/ufo/crtm/ObsAodLUTsTLAD.cc +++ b/src/ufo/crtm/ObsAodLUTsTLAD.cc @@ -18,7 +18,6 @@ #include "oops/util/IntSetParser.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" namespace ufo { @@ -50,8 +49,7 @@ ObsAodLUTsTLAD::~ObsAodLUTsTLAD() { // ----------------------------------------------------------------------------- -void ObsAodLUTsTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsAodLUTsTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_aodluts_tlad_settraj_f90(keyOperAodLUTs_, geovals.toFortran(), obsspace()); } diff --git a/src/ufo/crtm/ObsAodLUTsTLAD.h b/src/ufo/crtm/ObsAodLUTsTLAD.h index 14b36a010..77abba8a5 100644 --- a/src/ufo/crtm/ObsAodLUTsTLAD.h +++ b/src/ufo/crtm/ObsAodLUTsTLAD.h @@ -30,7 +30,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -43,7 +42,7 @@ class ObsAodLUTsTLAD : public LinearObsOperatorBase, virtual ~ObsAodLUTsTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/crtm/ObsRadianceCRTMTLAD.cc b/src/ufo/crtm/ObsRadianceCRTMTLAD.cc index ed5aa2eeb..7dd83ac19 100644 --- a/src/ufo/crtm/ObsRadianceCRTMTLAD.cc +++ b/src/ufo/crtm/ObsRadianceCRTMTLAD.cc @@ -18,7 +18,6 @@ #include "oops/util/IntSetParser.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" #include "ufo/ObsDiagnostics.h" namespace ufo { @@ -51,8 +50,7 @@ ObsRadianceCRTMTLAD::~ObsRadianceCRTMTLAD() { // ----------------------------------------------------------------------------- -void ObsRadianceCRTMTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics & ydiags) { +void ObsRadianceCRTMTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics & ydiags) { ufo_radiancecrtm_tlad_settraj_f90(keyOperRadianceCRTM_, geovals.toFortran(), obsspace(), ydiags.toFortran()); oops::Log::trace() << "ObsRadianceCRTMTLAD::setTrajectory done" << std::endl; diff --git a/src/ufo/crtm/ObsRadianceCRTMTLAD.h b/src/ufo/crtm/ObsRadianceCRTMTLAD.h index a97c508ba..120ef83fd 100644 --- a/src/ufo/crtm/ObsRadianceCRTMTLAD.h +++ b/src/ufo/crtm/ObsRadianceCRTMTLAD.h @@ -29,7 +29,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -43,7 +42,7 @@ class ObsRadianceCRTMTLAD : public LinearObsOperatorBase, virtual ~ObsRadianceCRTMTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.cc b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.cc index a56358bfe..aa099f09b 100644 --- a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.cc +++ b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.cc @@ -52,8 +52,7 @@ ObsGnssroBendMetOfficeTLAD::~ObsGnssroBendMetOfficeTLAD() { // ----------------------------------------------------------------------------- -void ObsGnssroBendMetOfficeTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsGnssroBendMetOfficeTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_gnssro_bendmetoffice_tlad_settraj_f90(keyOperGnssroBendMetOffice_, geovals.toFortran(), obsspace()); } diff --git a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.h b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.h index 7c215a185..c6d7d35b6 100644 --- a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.h +++ b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.h @@ -29,7 +29,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -43,7 +42,7 @@ class ObsGnssroBendMetOfficeTLAD : public LinearObsOperatorBase, virtual ~ObsGnssroBendMetOfficeTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/gnssro/BndNBAM/ObsGnssroBndNBAMTLAD.cc b/src/ufo/gnssro/BndNBAM/ObsGnssroBndNBAMTLAD.cc index e89974b64..dda3e49c1 100644 --- a/src/ufo/gnssro/BndNBAM/ObsGnssroBndNBAMTLAD.cc +++ b/src/ufo/gnssro/BndNBAM/ObsGnssroBndNBAMTLAD.cc @@ -62,8 +62,7 @@ ObsGnssroBndNBAMTLAD::~ObsGnssroBndNBAMTLAD() { // ----------------------------------------------------------------------------- -void ObsGnssroBndNBAMTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsGnssroBndNBAMTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_gnssro_bndnbam_tlad_settraj_f90(keyOperGnssroBndNBAM_, geovals.toFortran(), obsspace()); } diff --git a/src/ufo/gnssro/BndNBAM/ObsGnssroBndNBAMTLAD.h b/src/ufo/gnssro/BndNBAM/ObsGnssroBndNBAMTLAD.h index ea7968bb8..6a34c1766 100644 --- a/src/ufo/gnssro/BndNBAM/ObsGnssroBndNBAMTLAD.h +++ b/src/ufo/gnssro/BndNBAM/ObsGnssroBndNBAMTLAD.h @@ -29,7 +29,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -44,7 +43,7 @@ class ObsGnssroBndNBAMTLAD : public LinearObsOperatorBase, virtual ~ObsGnssroBndNBAMTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/gnssro/BndROPP1D/ObsGnssroBndROPP1DTLAD.cc b/src/ufo/gnssro/BndROPP1D/ObsGnssroBndROPP1DTLAD.cc index 2e0621f7c..fdf0967fe 100644 --- a/src/ufo/gnssro/BndROPP1D/ObsGnssroBndROPP1DTLAD.cc +++ b/src/ufo/gnssro/BndROPP1D/ObsGnssroBndROPP1DTLAD.cc @@ -49,8 +49,7 @@ ObsGnssroBndROPP1DTLAD::~ObsGnssroBndROPP1DTLAD() { // ----------------------------------------------------------------------------- -void ObsGnssroBndROPP1DTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsGnssroBndROPP1DTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_gnssro_bndropp1d_tlad_settraj_f90(keyOperGnssroBndROPP1D_, geovals.toFortran(), obsspace()); } diff --git a/src/ufo/gnssro/BndROPP1D/ObsGnssroBndROPP1DTLAD.h b/src/ufo/gnssro/BndROPP1D/ObsGnssroBndROPP1DTLAD.h index fff42f694..279a2825c 100644 --- a/src/ufo/gnssro/BndROPP1D/ObsGnssroBndROPP1DTLAD.h +++ b/src/ufo/gnssro/BndROPP1D/ObsGnssroBndROPP1DTLAD.h @@ -29,7 +29,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -43,7 +42,7 @@ class ObsGnssroBndROPP1DTLAD : public LinearObsOperatorBase, virtual ~ObsGnssroBndROPP1DTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2D.cc b/src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2D.cc index 441f029b9..117447cf7 100644 --- a/src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2D.cc +++ b/src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2D.cc @@ -18,7 +18,6 @@ #include "ufo/GeoVaLs.h" #include "ufo/Locations.h" -#include "ufo/ObsBias.h" #include "ufo/ObsDiagnostics.h" namespace ufo { diff --git a/src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2DTLAD.cc b/src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2DTLAD.cc index 20644315f..60297aa44 100644 --- a/src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2DTLAD.cc +++ b/src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2DTLAD.cc @@ -49,8 +49,7 @@ ObsGnssroBndROPP2DTLAD::~ObsGnssroBndROPP2DTLAD() { // ----------------------------------------------------------------------------- -void ObsGnssroBndROPP2DTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsGnssroBndROPP2DTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_gnssro_bndropp2d_tlad_settraj_f90(keyOperGnssroBndROPP2D_, geovals.toFortran(), obsspace()); } diff --git a/src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2DTLAD.h b/src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2DTLAD.h index 005998826..459d40a60 100644 --- a/src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2DTLAD.h +++ b/src/ufo/gnssro/BndROPP2D/ObsGnssroBndROPP2DTLAD.h @@ -29,7 +29,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -43,7 +42,7 @@ class ObsGnssroBndROPP2DTLAD : public LinearObsOperatorBase, virtual ~ObsGnssroBndROPP2DTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.cc b/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.cc index 8196e6166..4668db6bb 100644 --- a/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.cc +++ b/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.cc @@ -48,8 +48,7 @@ ObsGnssroRefTLAD::~ObsGnssroRefTLAD() { // ----------------------------------------------------------------------------- -void ObsGnssroRefTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsGnssroRefTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_gnssro_ref_tlad_settraj_f90(keyOperGnssroRef_, geovals.toFortran(), obsspace()); } diff --git a/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.h b/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.h index ce03eccce..c8be1e27d 100644 --- a/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.h +++ b/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.h @@ -29,7 +29,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -43,7 +42,7 @@ class ObsGnssroRefTLAD : public LinearObsOperatorBase, virtual ~ObsGnssroRefTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/groundgnss/ZenithTotalDelayMetOffice/ObsGroundgnssMetOfficeTLAD.cc b/src/ufo/groundgnss/ZenithTotalDelayMetOffice/ObsGroundgnssMetOfficeTLAD.cc index b63d7f2c5..8a3f3f1c4 100644 --- a/src/ufo/groundgnss/ZenithTotalDelayMetOffice/ObsGroundgnssMetOfficeTLAD.cc +++ b/src/ufo/groundgnss/ZenithTotalDelayMetOffice/ObsGroundgnssMetOfficeTLAD.cc @@ -52,8 +52,7 @@ ObsGroundgnssMetOfficeTLAD::~ObsGroundgnssMetOfficeTLAD() { // ----------------------------------------------------------------------------- -void ObsGroundgnssMetOfficeTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsGroundgnssMetOfficeTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_groundgnss_metoffice_tlad_settraj_f90(keyOperGroundgnssMetOffice_, geovals.toFortran(), obsspace()); } diff --git a/src/ufo/groundgnss/ZenithTotalDelayMetOffice/ObsGroundgnssMetOfficeTLAD.h b/src/ufo/groundgnss/ZenithTotalDelayMetOffice/ObsGroundgnssMetOfficeTLAD.h index 2131b7911..0e38f26fc 100644 --- a/src/ufo/groundgnss/ZenithTotalDelayMetOffice/ObsGroundgnssMetOfficeTLAD.h +++ b/src/ufo/groundgnss/ZenithTotalDelayMetOffice/ObsGroundgnssMetOfficeTLAD.h @@ -29,7 +29,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -43,7 +42,7 @@ class ObsGroundgnssMetOfficeTLAD : public LinearObsOperatorBase, virtual ~ObsGroundgnssMetOfficeTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/identity/ObsIdentityTLAD.cc b/src/ufo/identity/ObsIdentityTLAD.cc index aec347669..b3bfdecb8 100644 --- a/src/ufo/identity/ObsIdentityTLAD.cc +++ b/src/ufo/identity/ObsIdentityTLAD.cc @@ -17,7 +17,6 @@ #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" #include "ufo/utils/OperatorUtils.h" // for getOperatorVariables namespace ufo { @@ -50,8 +49,7 @@ ObsIdentityTLAD::~ObsIdentityTLAD() { // ----------------------------------------------------------------------------- -void ObsIdentityTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsIdentityTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_identity_tlad_settraj_f90(keyOperObsIdentity_, geovals.toFortran(), obsspace()); oops::Log::trace() << "ObsIdentityTLAD: trajectory set" << std::endl; } diff --git a/src/ufo/identity/ObsIdentityTLAD.h b/src/ufo/identity/ObsIdentityTLAD.h index 19c5eedbb..163e0db04 100644 --- a/src/ufo/identity/ObsIdentityTLAD.h +++ b/src/ufo/identity/ObsIdentityTLAD.h @@ -29,7 +29,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -43,7 +42,7 @@ class ObsIdentityTLAD : public LinearObsOperatorBase, virtual ~ObsIdentityTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/marine/adt/ObsADTTLAD.cc b/src/ufo/marine/adt/ObsADTTLAD.cc index 7252b4cf8..65fb9ebf0 100644 --- a/src/ufo/marine/adt/ObsADTTLAD.cc +++ b/src/ufo/marine/adt/ObsADTTLAD.cc @@ -16,7 +16,6 @@ #include "oops/base/Variables.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" namespace ufo { @@ -42,8 +41,7 @@ ObsADTTLAD::~ObsADTTLAD() { // ----------------------------------------------------------------------------- -void ObsADTTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsADTTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_adt_tlad_settraj_f90(keyOper_, geovals.toFortran(), obsspace()); oops::Log::trace() << "ObsADTTLAD: trajectory set" << std::endl; } diff --git a/src/ufo/marine/adt/ObsADTTLAD.h b/src/ufo/marine/adt/ObsADTTLAD.h index e46a77342..4a4e28960 100644 --- a/src/ufo/marine/adt/ObsADTTLAD.h +++ b/src/ufo/marine/adt/ObsADTTLAD.h @@ -30,7 +30,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -44,7 +43,7 @@ class ObsADTTLAD : public LinearObsOperatorBase, virtual ~ObsADTTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/marine/coolskin/ObsCoolSkinTLAD.cc b/src/ufo/marine/coolskin/ObsCoolSkinTLAD.cc index af2fc45ec..6ef9cd482 100644 --- a/src/ufo/marine/coolskin/ObsCoolSkinTLAD.cc +++ b/src/ufo/marine/coolskin/ObsCoolSkinTLAD.cc @@ -16,7 +16,6 @@ #include "oops/base/Variables.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" namespace ufo { @@ -47,8 +46,7 @@ ObsCoolSkinTLAD::~ObsCoolSkinTLAD() { // ----------------------------------------------------------------------------- -void ObsCoolSkinTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsCoolSkinTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_CoolSkin_tlad_settraj_f90(keyOper_, geovals.toFortran(), obsspace()); oops::Log::trace() << "ObsCoolSkinTLAD: trajectory set" << std::endl; } diff --git a/src/ufo/marine/coolskin/ObsCoolSkinTLAD.h b/src/ufo/marine/coolskin/ObsCoolSkinTLAD.h index 025980dd1..eb011fadf 100644 --- a/src/ufo/marine/coolskin/ObsCoolSkinTLAD.h +++ b/src/ufo/marine/coolskin/ObsCoolSkinTLAD.h @@ -30,7 +30,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -44,7 +43,7 @@ class ObsCoolSkinTLAD : public LinearObsOperatorBase, virtual ~ObsCoolSkinTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/marine/insitutemperature/ObsInsituTemperatureTLAD.cc b/src/ufo/marine/insitutemperature/ObsInsituTemperatureTLAD.cc index d4d25d86b..0b4a01e3b 100644 --- a/src/ufo/marine/insitutemperature/ObsInsituTemperatureTLAD.cc +++ b/src/ufo/marine/insitutemperature/ObsInsituTemperatureTLAD.cc @@ -16,7 +16,6 @@ #include "oops/base/Variables.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" namespace ufo { @@ -46,8 +45,7 @@ ObsInsituTemperatureTLAD::~ObsInsituTemperatureTLAD() { // ----------------------------------------------------------------------------- -void ObsInsituTemperatureTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsInsituTemperatureTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_insitutemperature_tlad_settraj_f90(keyOper_, geovals.toFortran(), obsspace()); oops::Log::trace() << "ObsInsituTemperatureTLAD: trajectory set" << std::endl; } diff --git a/src/ufo/marine/insitutemperature/ObsInsituTemperatureTLAD.h b/src/ufo/marine/insitutemperature/ObsInsituTemperatureTLAD.h index 5d65c50af..d90ca3712 100644 --- a/src/ufo/marine/insitutemperature/ObsInsituTemperatureTLAD.h +++ b/src/ufo/marine/insitutemperature/ObsInsituTemperatureTLAD.h @@ -30,7 +30,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -44,7 +43,7 @@ class ObsInsituTemperatureTLAD : public LinearObsOperatorBase, virtual ~ObsInsituTemperatureTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/marine/marinevertinterp/ObsMarineVertInterp.cc b/src/ufo/marine/marinevertinterp/ObsMarineVertInterp.cc index 31518ed10..8c285a7dd 100644 --- a/src/ufo/marine/marinevertinterp/ObsMarineVertInterp.cc +++ b/src/ufo/marine/marinevertinterp/ObsMarineVertInterp.cc @@ -14,7 +14,6 @@ #include "oops/base/Variables.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" #include "ufo/ObsDiagnostics.h" namespace ufo { diff --git a/src/ufo/marine/marinevertinterp/ObsMarineVertInterp.h b/src/ufo/marine/marinevertinterp/ObsMarineVertInterp.h index e8b77bcec..04683ae0e 100644 --- a/src/ufo/marine/marinevertinterp/ObsMarineVertInterp.h +++ b/src/ufo/marine/marinevertinterp/ObsMarineVertInterp.h @@ -29,7 +29,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- diff --git a/src/ufo/marine/marinevertinterp/ObsMarineVertInterpTLAD.cc b/src/ufo/marine/marinevertinterp/ObsMarineVertInterpTLAD.cc index 9567c60c3..550270fef 100644 --- a/src/ufo/marine/marinevertinterp/ObsMarineVertInterpTLAD.cc +++ b/src/ufo/marine/marinevertinterp/ObsMarineVertInterpTLAD.cc @@ -14,7 +14,6 @@ #include "oops/base/Variables.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" namespace ufo { @@ -40,8 +39,7 @@ ObsMarineVertInterpTLAD::~ObsMarineVertInterpTLAD() { // ----------------------------------------------------------------------------- -void ObsMarineVertInterpTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsMarineVertInterpTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_marinevertinterp_tlad_settraj_f90(keyOper_, geovals.toFortran(), obsspace()); oops::Log::trace() << "ObsMarineVertInterpTLAD: trajectory set" << std::endl; } diff --git a/src/ufo/marine/marinevertinterp/ObsMarineVertInterpTLAD.h b/src/ufo/marine/marinevertinterp/ObsMarineVertInterpTLAD.h index a5625d596..e18051c9b 100644 --- a/src/ufo/marine/marinevertinterp/ObsMarineVertInterpTLAD.h +++ b/src/ufo/marine/marinevertinterp/ObsMarineVertInterpTLAD.h @@ -29,7 +29,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -43,7 +42,7 @@ class ObsMarineVertInterpTLAD : public LinearObsOperatorBase, virtual ~ObsMarineVertInterpTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc b/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc index 02e6cf1bd..e1b90eaa1 100644 --- a/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc +++ b/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc @@ -17,7 +17,6 @@ #include "oops/util/Logger.h" #include "oops/util/missingValues.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" namespace ufo { @@ -46,8 +45,7 @@ ObsSeaIceFractionTLAD::~ObsSeaIceFractionTLAD() { // ----------------------------------------------------------------------------- -void ObsSeaIceFractionTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsSeaIceFractionTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_seaicelinear_settraj_f90(keyOper_, geovals.toFortran(), obsspace()); oops::Log::trace() << "ObsSeaIceFractionTLAD: trajectory set" << std::endl; } diff --git a/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.h b/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.h index ef5891c8c..0a5d20856 100644 --- a/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.h +++ b/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.h @@ -30,7 +30,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -44,7 +43,7 @@ class ObsSeaIceFractionTLAD : public LinearObsOperatorBase, virtual ~ObsSeaIceFractionTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/marine/seaicethickness/ObsSeaIceThicknessTLAD.cc b/src/ufo/marine/seaicethickness/ObsSeaIceThicknessTLAD.cc index aa64f4300..15b189543 100644 --- a/src/ufo/marine/seaicethickness/ObsSeaIceThicknessTLAD.cc +++ b/src/ufo/marine/seaicethickness/ObsSeaIceThicknessTLAD.cc @@ -16,7 +16,6 @@ #include "oops/base/Variables.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" namespace ufo { @@ -44,8 +43,7 @@ ObsSeaIceThicknessTLAD::~ObsSeaIceThicknessTLAD() { // ----------------------------------------------------------------------------- -void ObsSeaIceThicknessTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsSeaIceThicknessTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_seaicethickness_tlad_settraj_f90(keyOper_, geovals.toFortran(), obsspace()); oops::Log::trace() << "ObsSeaIceThicknessTLAD: trajectory set" << std::endl; } diff --git a/src/ufo/marine/seaicethickness/ObsSeaIceThicknessTLAD.h b/src/ufo/marine/seaicethickness/ObsSeaIceThicknessTLAD.h index 7ceae69b2..fde276cb8 100644 --- a/src/ufo/marine/seaicethickness/ObsSeaIceThicknessTLAD.h +++ b/src/ufo/marine/seaicethickness/ObsSeaIceThicknessTLAD.h @@ -30,7 +30,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -44,7 +43,7 @@ class ObsSeaIceThicknessTLAD : public LinearObsOperatorBase, virtual ~ObsSeaIceThicknessTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/radarradialvelocity/ObsRadarRadialVelocityTLAD.cc b/src/ufo/radarradialvelocity/ObsRadarRadialVelocityTLAD.cc index 9d776ee26..5f67609e2 100644 --- a/src/ufo/radarradialvelocity/ObsRadarRadialVelocityTLAD.cc +++ b/src/ufo/radarradialvelocity/ObsRadarRadialVelocityTLAD.cc @@ -16,7 +16,6 @@ #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" namespace ufo { @@ -45,7 +44,7 @@ ObsRadarRadialVelocityTLAD::~ObsRadarRadialVelocityTLAD() { // ----------------------------------------------------------------------------- void ObsRadarRadialVelocityTLAD::setTrajectory(const GeoVaLs & geovals, - const ObsBias & bias, ObsDiagnostics &) { + ObsDiagnostics &) { oops::Log::trace() << "ObsRadarRadialVelocityTLAD::setTrajectory entering" << std::endl; ufo_radarradialvelocity_tlad_settraj_f90(keyOperRadarRadialVelocity_, geovals.toFortran(), diff --git a/src/ufo/radarradialvelocity/ObsRadarRadialVelocityTLAD.h b/src/ufo/radarradialvelocity/ObsRadarRadialVelocityTLAD.h index e79df9891..6b4df92d1 100644 --- a/src/ufo/radarradialvelocity/ObsRadarRadialVelocityTLAD.h +++ b/src/ufo/radarradialvelocity/ObsRadarRadialVelocityTLAD.h @@ -29,7 +29,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -43,7 +42,7 @@ class ObsRadarRadialVelocityTLAD : public LinearObsOperatorBase, virtual ~ObsRadarRadialVelocityTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/rttov/ObsRadianceRTTOV.cc b/src/ufo/rttov/ObsRadianceRTTOV.cc index 006bf62f2..5d99bd194 100644 --- a/src/ufo/rttov/ObsRadianceRTTOV.cc +++ b/src/ufo/rttov/ObsRadianceRTTOV.cc @@ -19,7 +19,6 @@ #include "oops/util/IntSetParser.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" #include "ufo/ObsDiagnostics.h" namespace ufo { diff --git a/src/ufo/rttov/ObsRadianceRTTOVTLAD.cc b/src/ufo/rttov/ObsRadianceRTTOVTLAD.cc index 4f50815d6..0c494d4ce 100644 --- a/src/ufo/rttov/ObsRadianceRTTOVTLAD.cc +++ b/src/ufo/rttov/ObsRadianceRTTOVTLAD.cc @@ -18,7 +18,6 @@ #include "oops/util/IntSetParser.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" #include "ufo/ObsDiagnostics.h" namespace ufo { @@ -51,8 +50,7 @@ ObsRadianceRTTOVTLAD::~ObsRadianceRTTOVTLAD() { // ----------------------------------------------------------------------------- -void ObsRadianceRTTOVTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics & ydiags) { +void ObsRadianceRTTOVTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics & ydiags) { ufo_radiancerttov_tlad_settraj_f90(keyOperRadianceRTTOV_, geovals.toFortran(), obsspace(), ydiags.toFortran()); oops::Log::trace() << "ObsRadianceRTTOVTLAD::setTrajectory done" << std::endl; diff --git a/src/ufo/rttov/ObsRadianceRTTOVTLAD.h b/src/ufo/rttov/ObsRadianceRTTOVTLAD.h index 8b0185d9e..9bfc54171 100644 --- a/src/ufo/rttov/ObsRadianceRTTOVTLAD.h +++ b/src/ufo/rttov/ObsRadianceRTTOVTLAD.h @@ -30,7 +30,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -44,7 +43,7 @@ class ObsRadianceRTTOVTLAD : public LinearObsOperatorBase, virtual ~ObsRadianceRTTOVTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/rttovcpp/ObsRadianceRTTOVCPP.cc b/src/ufo/rttovcpp/ObsRadianceRTTOVCPP.cc index b6a69247b..d961ea054 100644 --- a/src/ufo/rttovcpp/ObsRadianceRTTOVCPP.cc +++ b/src/ufo/rttovcpp/ObsRadianceRTTOVCPP.cc @@ -16,7 +16,6 @@ #include "oops/util/missingValues.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" #include "ufo/ObsDiagnostics.h" #include "ufo/rttovcpp/ObsRadianceRTTOVCPP.h" #include "ufo/rttovcpp/rttovcpp_interface.h" diff --git a/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc b/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc index 45f36f504..13cfb5407 100644 --- a/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc +++ b/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc @@ -18,7 +18,6 @@ #include "oops/util/missingValues.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" #include "ufo/ObsDiagnostics.h" #include "ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.h" #include "ufo/rttovcpp/rttovcpp_interface.h" @@ -63,8 +62,7 @@ ObsRadianceRTTOVCPPTLAD::~ObsRadianceRTTOVCPPTLAD() { // ----------------------------------------------------------------------------- -void ObsRadianceRTTOVCPPTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void ObsRadianceRTTOVCPPTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { // ufo::rttovcpp_interface(geovals, obsspace(), aRttov_, CoefFileName, channels_, nlevels, skip_profile); diff --git a/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.h b/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.h index 19cd5bd4d..3cb775082 100644 --- a/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.h +++ b/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.h @@ -31,7 +31,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -45,7 +44,7 @@ class ObsRadianceRTTOVCPPTLAD : public LinearObsOperatorBase, virtual ~ObsRadianceRTTOVCPPTLAD(); // Calculate Jacobian H(x_g) of obs operator - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; // Calculate dy = H dx void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; // Calculate H^T dy diff --git a/src/ufo/sattcwv/SatTCWVTLAD.cc b/src/ufo/sattcwv/SatTCWVTLAD.cc index 9a4d0489b..6775a49cb 100644 --- a/src/ufo/sattcwv/SatTCWVTLAD.cc +++ b/src/ufo/sattcwv/SatTCWVTLAD.cc @@ -50,8 +50,7 @@ SatTCWVTLAD::~SatTCWVTLAD() { // ----------------------------------------------------------------------------- -void SatTCWVTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics &) { +void SatTCWVTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { ufo_sattcwv_tlad_settraj_f90(keyOperSatTCWV_, geovals.toFortran(), obsspace()); } diff --git a/src/ufo/sattcwv/SatTCWVTLAD.h b/src/ufo/sattcwv/SatTCWVTLAD.h index c07e22766..1a42c257f 100644 --- a/src/ufo/sattcwv/SatTCWVTLAD.h +++ b/src/ufo/sattcwv/SatTCWVTLAD.h @@ -29,7 +29,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -43,7 +42,7 @@ class SatTCWVTLAD : public LinearObsOperatorBase, virtual ~SatTCWVTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/src/ufo/timeoper/ObsTimeOperTLAD.cc b/src/ufo/timeoper/ObsTimeOperTLAD.cc index 70accf786..3e4130c7c 100644 --- a/src/ufo/timeoper/ObsTimeOperTLAD.cc +++ b/src/ufo/timeoper/ObsTimeOperTLAD.cc @@ -20,8 +20,6 @@ #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" -#include "ufo/ObsBiasIncrement.h" #include "ufo/timeoper/ObsTimeOperUtil.h" namespace ufo { @@ -49,7 +47,6 @@ ObsTimeOperTLAD::~ObsTimeOperTLAD() { // ----------------------------------------------------------------------------- void ObsTimeOperTLAD::setTrajectory(const GeoVaLs & geovals, - const ObsBias & bias, ObsDiagnostics & ydiags) { oops::Log::trace() << "ObsTimeOperTLAD::setTrajectory entering" << std::endl; @@ -73,7 +70,7 @@ void ObsTimeOperTLAD::setTrajectory(const GeoVaLs & geovals, oops::Log::debug() << "ObsTimeOperTLAD::setTrajectory final geovals gv1 " << gv1 << std::endl; - actualoperator_->setTrajectory(gv1, bias, ydiags); + actualoperator_->setTrajectory(gv1, ydiags); oops::Log::debug() << gv1; diff --git a/src/ufo/timeoper/ObsTimeOperTLAD.h b/src/ufo/timeoper/ObsTimeOperTLAD.h index e09b1b3cf..6476e5060 100644 --- a/src/ufo/timeoper/ObsTimeOperTLAD.h +++ b/src/ufo/timeoper/ObsTimeOperTLAD.h @@ -32,7 +32,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; class ObsDiagnostics; // ----------------------------------------------------------------------------- @@ -46,7 +45,7 @@ class ObsTimeOperTLAD : public LinearObsOperatorBase, virtual ~ObsTimeOperTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; diff --git a/tools/new_obsop/example/ObsExampleTLAD.cc b/tools/new_obsop/example/ObsExampleTLAD.cc index ac2239bc8..47268f82f 100644 --- a/tools/new_obsop/example/ObsExampleTLAD.cc +++ b/tools/new_obsop/example/ObsExampleTLAD.cc @@ -14,7 +14,6 @@ #include "oops/base/Variables.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" -#include "ufo/ObsBias.h" #include "ufo/ObsDiagnostics.h" namespace ufo { @@ -40,8 +39,7 @@ ObsExampleTLAD::~ObsExampleTLAD() { // ----------------------------------------------------------------------------- -void ObsExampleTLAD::setTrajectory(const GeoVaLs & geovals, const ObsBias & bias, - ObsDiagnostics & ydiags) { +void ObsExampleTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics & ydiags) { ufo_example_tlad_settraj_f90(keyOper_, geovals.toFortran(), obsspace(), ydiags.toFortran()); oops::Log::trace() << "ObsExampleTLAD: trajectory set" << std::endl; } diff --git a/tools/new_obsop/example/ObsExampleTLAD.h b/tools/new_obsop/example/ObsExampleTLAD.h index 2ed1ef9dd..c51f662bf 100644 --- a/tools/new_obsop/example/ObsExampleTLAD.h +++ b/tools/new_obsop/example/ObsExampleTLAD.h @@ -29,7 +29,6 @@ namespace ioda { namespace ufo { class GeoVaLs; - class ObsBias; // ----------------------------------------------------------------------------- /// Example TL/AD observation operator class @@ -42,7 +41,7 @@ class ObsExampleTLAD : public LinearObsOperatorBase, virtual ~ObsExampleTLAD(); // Obs Operators - void setTrajectory(const GeoVaLs &, const ObsBias &, ObsDiagnostics &) override; + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; From cc82515004245fd88ddee5e97dcc38ee25b2a8af Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Thu, 17 Jun 2021 15:13:32 +0100 Subject: [PATCH 08/98] C++ to Fortran indexing for GeoVaLs (#1257) * putAtLocation and associated tests * comment * replace std::cout * coding norms * Modified test values * add getAtLevel, putAtLevel; add test * location -> level * move index + 1 to interface * modify a few more indices * modifications of data.get(geovals) * remove index modification * fix offset * comments * move index change into geovals interface * remove newline at start of file * change indexing for putAtLevel * change indices for ObsFilterData::get * change index of final geoval * indices in sea ice TLAD * index in obs error factor code * change index in RTTOV TLAD code * improve debug output * index in obsdiagnostics save * correct index in RTTOV * add ulps parameter in geovals test * correct indices in dot product * make some values const * modify copy_one to change index internally * move index change to interface Co-authored-by: Ryan Honeyager --- src/ufo/GeoVaLs.cc | 7 ++-- src/ufo/GeoVaLs.interface.F90 | 32 ++++++++++++++----- src/ufo/GeoVaLs.interface.h | 2 +- src/ufo/ObsBiasOperator.cc | 2 +- src/ufo/filters/ImpactHeightCheck.cc | 4 +-- .../filters/obsfunctions/CloudCostFunction.cc | 8 ++--- .../CloudDetectMinResidualAVHRR.cc | 8 ++--- .../obsfunctions/CloudDetectMinResidualIR.cc | 8 ++--- .../filters/obsfunctions/NearSSTRetCheckIR.cc | 4 +-- .../ObsErrorFactorConventional.cc | 2 +- .../obsfunctions/ObsErrorFactorSfcPressure.cc | 4 +-- .../obsfunctions/ObsErrorFactorTopoRad.cc | 2 +- .../ObsErrorFactorTransmitTopRad.cc | 2 +- .../obsfunctions/ObsErrorFactorWavenumIR.cc | 2 +- .../obsfunctions/SatwindIndivErrors.cc | 4 +-- .../obsfunctions/TotalColumnVaporGuess.cc | 7 ++-- src/ufo/gnssro/QC/ROobserror.cc | 2 +- .../marine/chleuzintegr/ObsChlEuzIntegr.cc | 4 +-- .../seaicefraction/ObsSeaIceFraction.cc | 2 +- .../seaicefraction/ObsSeaIceFractionTLAD.cc | 4 +-- src/ufo/predictors/LapseRate.cc | 4 +-- src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc | 12 +++---- src/ufo/rttovcpp/rttovcpp_interface.cc | 6 ++-- test/ufo/GeoVaLs.h | 24 +++++++------- test/ufo/MetOfficeBuddyCheck.h | 2 +- test/ufo/ObsDiagnostics.h | 4 +-- test/ufo/ObsFilterData.h | 8 ++--- test/ufo/ProfileConsistencyChecks.h | 4 +-- 28 files changed, 94 insertions(+), 80 deletions(-) diff --git a/src/ufo/GeoVaLs.cc b/src/ufo/GeoVaLs.cc index 33b73eb69..10d8fc14f 100644 --- a/src/ufo/GeoVaLs.cc +++ b/src/ufo/GeoVaLs.cc @@ -86,8 +86,7 @@ GeoVaLs::GeoVaLs(const GeoVaLs & other, const int & index) { oops::Log::trace() << "GeoVaLs copy one GeoVaLs constructor starting" << std::endl; ufo_geovals_setup_f90(keyGVL_, 1, vars_); - int fort_index = index + 1; // Fortran numbers from 1 - ufo_geovals_copy_one_f90(keyGVL_, other.keyGVL_, fort_index); + ufo_geovals_copy_one_f90(keyGVL_, other.keyGVL_, index); oops::Log::trace() << "GeoVaLs copy one GeoVaLs constructor key = " << keyGVL_ << std::endl; } // ----------------------------------------------------------------------------- @@ -226,8 +225,8 @@ double GeoVaLs::dot_product_with(const GeoVaLs & other) const { assert(nlevs == other.nlevs(vars_[jvar])); // loop over all levels for this variable for (size_t jlev = 0; jlev < nlevs; ++jlev) { - this->getAtLevel(this_values, vars_[jvar], jlev+1); - other.getAtLevel(other_values, vars_[jvar], jlev+1); + this->getAtLevel(this_values, vars_[jvar], jlev); + other.getAtLevel(other_values, vars_[jvar], jlev); // loop over all locations for (size_t jloc = 0; jloc < nlocs; ++jloc) { if ((this_values[jloc] != missing) && (other_values[jloc] != missing)) { diff --git a/src/ufo/GeoVaLs.interface.F90 b/src/ufo/GeoVaLs.interface.F90 index 2c2f7ee0d..e6846c437 100644 --- a/src/ufo/GeoVaLs.interface.F90 +++ b/src/ufo/GeoVaLs.interface.F90 @@ -103,13 +103,17 @@ end subroutine ufo_geovals_copy_c ! ------------------------------------------------------------------------------ !> Copy one GeoVaLs location into another object -subroutine ufo_geovals_copy_one_c(c_key_self, c_key_other, ind) bind(c,name='ufo_geovals_copy_one_f90') +subroutine ufo_geovals_copy_one_c(c_key_self, c_key_other, c_ind) bind(c,name='ufo_geovals_copy_one_f90') implicit none integer(c_int), intent(in) :: c_key_self integer(c_int), intent(in) :: c_key_other -integer(c_int), intent(in) :: ind +integer(c_int), intent(in) :: c_ind type(ufo_geovals), pointer :: self type(ufo_geovals), pointer :: other +integer :: ind + +! Convert location index from the C++ to the Fortran convention. +ind = c_ind + 1 call ufo_geovals_registry%get(c_key_self, self) call ufo_geovals_registry%get(c_key_other, other) @@ -466,14 +470,14 @@ end subroutine ufo_geovals_get2d_c ! ------------------------------------------------------------------------------ -subroutine ufo_geovals_get_c(c_key_self, lvar, c_var, lev, nlocs, values) bind(c, name='ufo_geovals_get_f90') +subroutine ufo_geovals_get_c(c_key_self, lvar, c_var, c_lev, nlocs, values) bind(c, name='ufo_geovals_get_f90') use ufo_vars_mod, only: MAXVARLEN use string_f_c_mod implicit none integer(c_int), intent(in) :: c_key_self integer(c_int), intent(in) :: lvar character(kind=c_char, len=1), intent(in) :: c_var(lvar+1) -integer(c_int), intent(in) :: lev +integer(c_int), intent(in) :: c_lev integer(c_int), intent(in) :: nlocs real(c_float), intent(inout) :: values(nlocs) @@ -481,6 +485,10 @@ subroutine ufo_geovals_get_c(c_key_self, lvar, c_var, lev, nlocs, values) bind(c type(ufo_geoval), pointer :: geoval character(len=MAXVARLEN) :: varname type(ufo_geovals), pointer :: self +integer(c_int) :: lev + +! Convert level index from the C++ to the Fortran convention. +lev = c_lev + 1 call c_f_string(c_var, varname) call ufo_geovals_registry%get(c_key_self, self) @@ -544,7 +552,7 @@ end subroutine ufo_geovals_get_loc_c ! ------------------------------------------------------------------------------ -subroutine ufo_geovals_getdouble_c(c_key_self, lvar, c_var, lev, nlocs, values)& +subroutine ufo_geovals_getdouble_c(c_key_self, lvar, c_var, c_lev, nlocs, values)& bind(c, name='ufo_geovals_getdouble_f90') use ufo_vars_mod, only: MAXVARLEN use string_f_c_mod @@ -552,13 +560,17 @@ subroutine ufo_geovals_getdouble_c(c_key_self, lvar, c_var, lev, nlocs, values)& integer(c_int), intent(in) :: c_key_self integer(c_int), intent(in) :: lvar character(kind=c_char, len=1), intent(in) :: c_var(lvar+1) -integer(c_int), intent(in) :: lev +integer(c_int), intent(in) :: c_lev integer(c_int), intent(in) :: nlocs real(c_double), intent(inout) :: values(nlocs) type(ufo_geoval), pointer :: geoval character(len=MAXVARLEN) :: varname type(ufo_geovals), pointer :: self +integer(c_int) :: lev + +! Convert level index from the C++ to the Fortran convention. +lev = c_lev + 1 call c_f_string(c_var, varname) call ufo_geovals_registry%get(c_key_self, self) @@ -569,19 +581,23 @@ end subroutine ufo_geovals_getdouble_c ! ------------------------------------------------------------------------------ -subroutine ufo_geovals_putdouble_c(c_key_self, lvar, c_var, lev, nlocs, values) bind(c, name='ufo_geovals_putdouble_f90') +subroutine ufo_geovals_putdouble_c(c_key_self, lvar, c_var, c_lev, nlocs, values) bind(c, name='ufo_geovals_putdouble_f90') use ufo_vars_mod, only: MAXVARLEN use string_f_c_mod integer(c_int), intent(in) :: c_key_self integer(c_int), intent(in) :: lvar character(kind=c_char, len=1), intent(in) :: c_var(lvar+1) -integer(c_int), intent(in) :: lev +integer(c_int), intent(in) :: c_lev integer(c_int), intent(in) :: nlocs real(c_double), intent(in) :: values(nlocs) type(ufo_geoval), pointer :: geoval character(len=MAXVARLEN) :: varname type(ufo_geovals), pointer :: self +integer(c_int) :: lev + +! Convert level index from the C++ to the Fortran convention. +lev = c_lev + 1 call c_f_string(c_var, varname) call ufo_geovals_registry%get(c_key_self, self) diff --git a/src/ufo/GeoVaLs.interface.h b/src/ufo/GeoVaLs.interface.h index 41f50e581..604f20c2b 100644 --- a/src/ufo/GeoVaLs.interface.h +++ b/src/ufo/GeoVaLs.interface.h @@ -39,7 +39,7 @@ extern "C" { void ufo_geovals_allocate_f90(F90goms &, const size_t &, const oops::Variables &); void ufo_geovals_delete_f90(F90goms &); void ufo_geovals_copy_f90(const F90goms &, F90goms &); - void ufo_geovals_copy_one_f90(F90goms &, const F90goms &, int &); + void ufo_geovals_copy_one_f90(F90goms &, const F90goms &, const int &); void ufo_geovals_zero_f90(const F90goms &); void ufo_geovals_reorderzdir_f90(const F90goms &, const int &, const char *, const int &, const char *); diff --git a/src/ufo/ObsBiasOperator.cc b/src/ufo/ObsBiasOperator.cc index 095138e26..3158dcaab 100644 --- a/src/ufo/ObsBiasOperator.cc +++ b/src/ufo/ObsBiasOperator.cc @@ -86,7 +86,7 @@ void ObsBiasOperator::computeObsBias(const GeoVaLs & geovals, ioda::ObsVector & const std::string varname = predictors[jp]->name() + "_" + predictorSuffix; if (ydiags.has(varname)) { ydiags.allocate(1, oops::Variables({varname})); - ydiags.save(biasTerm, varname, 1); + ydiags.save(biasTerm, varname, 0); } else { oops::Log::error() << varname << " is not reserved in ydiags !" << std::endl; ABORT("ObsBiasOperatorTerm variable is not reserved in ydiags"); diff --git a/src/ufo/filters/ImpactHeightCheck.cc b/src/ufo/filters/ImpactHeightCheck.cc index 9fd7c4855..e0682666d 100644 --- a/src/ufo/filters/ImpactHeightCheck.cc +++ b/src/ufo/filters/ImpactHeightCheck.cc @@ -65,7 +65,7 @@ void ImpactHeightCheck::applyFilter(const std::vector & apply, const size_t nRefLevels = data_.nlevs(refractivityVariable); std::vector> refractivity; - for (size_t iLevel = 1; iLevel < nRefLevels+1; ++iLevel) { + for (size_t iLevel = 0; iLevel < nRefLevels; ++iLevel) { std::vector inputData; data_.get(refractivityVariable, static_cast(iLevel), inputData); refractivity.push_back(inputData); @@ -90,7 +90,7 @@ void ImpactHeightCheck::applyFilter(const std::vector & apply, std::vector> modelHeights; // Read the heights of the refractivity levels - for (size_t iLevel = 1; iLevel < nRefLevels+1; ++iLevel) { + for (size_t iLevel = 0; iLevel < nRefLevels; ++iLevel) { std::vector inputData; data_.get(modelHeightsVariable, static_cast(iLevel), inputData); modelHeights.push_back(inputData); diff --git a/src/ufo/filters/obsfunctions/CloudCostFunction.cc b/src/ufo/filters/obsfunctions/CloudCostFunction.cc index ab11430aa..d64ef96fe 100755 --- a/src/ufo/filters/obsfunctions/CloudCostFunction.cc +++ b/src/ufo/filters/obsfunctions/CloudCostFunction.cc @@ -82,8 +82,8 @@ void CloudCostFunction::compute(const ObsFilterData & in, // Determine if pressure is ascending or descending (B-matrix assumption) size_t np = in.nlevs(Variable("air_pressure@GeoVaLs")); std::vector gv_pres_1(nlocs), gv_pres_N(nlocs); - in.get(Variable("air_pressure@GeoVaLs"), 1, gv_pres_1); - in.get(Variable("air_pressure@GeoVaLs"), np, gv_pres_N); + in.get(Variable("air_pressure@GeoVaLs"), 0, gv_pres_1); + in.get(Variable("air_pressure@GeoVaLs"), np - 1, gv_pres_N); const float missing = util::missingValue(missing); ASSERT(gv_pres_1[0] != missing); ASSERT(gv_pres_N[0] != missing); @@ -102,8 +102,8 @@ void CloudCostFunction::compute(const ObsFilterData & in, size_t nlevs = in.nlevs(Variable(jac_name, channels_)[0]); std::vector jac_store(nlocs); for (size_t ilev = 0; ilev < nlevs; ++ilev) { - int level_gv = (p_ascending ? ilev+1 : nlevs-ilev); - int level_jac = (options_.reverse_Jacobian.value() ? nlevs-level_gv+1 : level_gv); + const int level_gv = (p_ascending ? ilev : nlevs-ilev-1); + const int level_jac = (options_.reverse_Jacobian.value() ? nlevs-level_gv-1 : level_gv); if (fields_[ifield] == "specific_humidity" && options_.qtotal_lnq_gkg.value()) { in.get(Variable("air_pressure@GeoVaLs"), level_gv, gv_pres); in.get(Variable("air_temperature@GeoVaLs"), level_gv, gv_temp); diff --git a/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.cc b/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.cc index f0a4b9674..6a0dffec3 100755 --- a/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.cc +++ b/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.cc @@ -110,7 +110,7 @@ void CloudDetectMinResidualAVHRR::compute(const ObsFilterData & in, dbtdt(nchans, std::vector>(nlevs, std::vector(nlocs))); for (size_t ichan = 0; ichan < nchans; ++ichan) { for (size_t ilev = 0; ilev < nlevs; ++ilev) { - int level = nlevs - ilev; + const int level = nlevs - ilev - 1; in.get(Variable("brightness_temperature_jacobian_air_temperature@ObsDiag", channels_)[ichan], level, dbtdt[ichan][ilev]); } @@ -121,7 +121,7 @@ void CloudDetectMinResidualAVHRR::compute(const ObsFilterData & in, tao(nchans, std::vector>(nlevs, std::vector(nlocs))); for (size_t ichan = 0; ichan < nchans; ++ichan) { for (size_t ilev = 0; ilev < nlevs; ++ilev) { - int level = nlevs - ilev; + const int level = nlevs - ilev - 1; in.get(Variable("transmittances_of_atmosphere_layer@ObsDiag", channels_)[ichan], level, tao[ichan][ilev]); } @@ -224,14 +224,14 @@ void CloudDetectMinResidualAVHRR::compute(const ObsFilterData & in, // Get air pressure [Pa] std::vector> prsl(nlevs, std::vector(nlocs)); for (size_t ilev = 0; ilev < nlevs; ++ilev) { - size_t level = nlevs - ilev; + const size_t level = nlevs - ilev - 1; in.get(Variable("air_pressure@GeoVaLs"), level, prsl[ilev]); } // Get air temperature std::vector> tair(nlevs, std::vector(nlocs)); for (size_t ilev = 0; ilev < nlevs; ++ilev) { - size_t level = nlevs - ilev; + const size_t level = nlevs - ilev - 1; in.get(Variable("air_temperature@GeoVaLs"), level, tair[ilev]); } diff --git a/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.cc b/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.cc index 9fb13974a..597b698bd 100755 --- a/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.cc +++ b/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.cc @@ -107,7 +107,7 @@ void CloudDetectMinResidualIR::compute(const ObsFilterData & in, dbtdt(nchans, std::vector>(nlevs, std::vector(nlocs))); for (size_t ichan = 0; ichan < nchans; ++ichan) { for (size_t ilev = 0; ilev < nlevs; ++ilev) { - int level = nlevs - ilev; + const int level = nlevs - ilev - 1; in.get(Variable("brightness_temperature_jacobian_air_temperature@ObsDiag", channels_)[ichan], level, dbtdt[ichan][ilev]); } @@ -118,7 +118,7 @@ void CloudDetectMinResidualIR::compute(const ObsFilterData & in, tao(nchans, std::vector>(nlevs, std::vector(nlocs))); for (size_t ichan = 0; ichan < nchans; ++ichan) { for (size_t ilev = 0; ilev < nlevs; ++ilev) { - int level = nlevs - ilev; + const int level = nlevs - ilev - 1; in.get(Variable("transmittances_of_atmosphere_layer@ObsDiag", channels_)[ichan], level, tao[ichan][ilev]); } @@ -218,14 +218,14 @@ void CloudDetectMinResidualIR::compute(const ObsFilterData & in, // Get air pressure [Pa] std::vector> prsl(nlevs, std::vector(nlocs)); for (size_t ilev = 0; ilev < nlevs; ++ilev) { - size_t level = nlevs - ilev; + const size_t level = nlevs - ilev - 1; in.get(Variable("air_pressure@GeoVaLs"), level, prsl[ilev]); } // Get air temperature std::vector> tair(nlevs, std::vector(nlocs)); for (size_t ilev = 0; ilev < nlevs; ++ilev) { - size_t level = nlevs - ilev; + const size_t level = nlevs - ilev - 1; in.get(Variable("air_temperature@GeoVaLs"), level, tair[ilev]); } diff --git a/src/ufo/filters/obsfunctions/NearSSTRetCheckIR.cc b/src/ufo/filters/obsfunctions/NearSSTRetCheckIR.cc index 9b5c629d6..0af48685e 100755 --- a/src/ufo/filters/obsfunctions/NearSSTRetCheckIR.cc +++ b/src/ufo/filters/obsfunctions/NearSSTRetCheckIR.cc @@ -99,7 +99,7 @@ void NearSSTRetCheckIR::compute(const ObsFilterData & in, dbtdt(nchans, std::vector>(nlevs, std::vector(nlocs))); for (size_t ichan = 0; ichan < nchans; ++ichan) { for (size_t ilev = 0; ilev < nlevs; ++ilev) { - int level = nlevs - ilev; + const int level = nlevs - ilev - 1; in.get(Variable("brightness_temperature_jacobian_air_temperature@ObsDiag", channels_)[ichan], level, dbtdt[ichan][ilev]); } @@ -110,7 +110,7 @@ void NearSSTRetCheckIR::compute(const ObsFilterData & in, dbtdq(nchans, std::vector>(nlevs, std::vector(nlocs))); for (size_t ichan = 0; ichan < nchans; ++ichan) { for (size_t ilev = 0; ilev < nlevs; ++ilev) { - int level = nlevs - ilev; + const int level = nlevs - ilev - 1; in.get(Variable("brightness_temperature_jacobian_humidity_mixing_ratio@ObsDiag", channels_)[ichan], level, dbtdq[ichan][ilev]); } diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorConventional.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorConventional.cc index 168cf3623..6dab885c6 100755 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorConventional.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorConventional.cc @@ -109,7 +109,7 @@ void ObsErrorFactorConventional::compute(const ObsFilterData & data, // Get GeoVals of air pressure [Pa] in vertical column std::vector> prsl(nlevs, std::vector(nlocs)); for (size_t ilev = 0; ilev < nlevs; ++ilev) { - data.get(Variable("air_pressure@GeoVaLs"), ilev+1, prsl[ilev]); + data.get(Variable("air_pressure@GeoVaLs"), ilev, prsl[ilev]); } for (size_t iv = 0; iv < varsize; ++iv) { // Variable loop diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.cc index 1cf186e34..871a1b5be 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.cc @@ -108,13 +108,13 @@ void ObsErrorFactorSfcPressure::compute(const ObsFilterData & data, // Get GeoVals of air pressure [Pa] and temperature in vertical column. std::vector> prsl(nlevs, std::vector(nlocs)); for (size_t ilev = 0; ilev < nlevs; ++ilev) { - size_t level = nlevs - ilev; + const size_t level = nlevs - ilev - 1; data.get(Variable("air_pressure@GeoVaLs"), level, prsl[ilev]); } std::vector> tair(nlevs, std::vector(nlocs)); const std::string geovar_temp = options_->geovar_temp.value(); for (size_t ilev = 0; ilev < nlevs; ++ilev) { - size_t level = nlevs - ilev; + const size_t level = nlevs - ilev - 1; data.get(Variable(geovar_temp + "@GeoVaLs"), level, tair[ilev]); } diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.cc index 736c831a8..e47c8c989 100755 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.cc @@ -95,7 +95,7 @@ void ObsErrorFactorTopoRad::compute(const ObsFilterData & in, std::vector tao_sfc(nlocs); for (size_t ich = 0; ich < nchans; ++ich) { in.get(Variable("transmittances_of_atmosphere_layer@ObsDiag", channels_)[ich], - nlevs, tao_sfc); + nlevs - 1, tao_sfc); for (size_t iloc = 0; iloc < nlocs; ++iloc) { out[ich][iloc] = 1.0; if (zsges[iloc] > 2000.0) { diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorTransmitTopRad.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorTransmitTopRad.cc index 6a58d0e6e..0617e8813 100755 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorTransmitTopRad.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorTransmitTopRad.cc @@ -57,7 +57,7 @@ void ObsErrorFactorTransmitTopRad::compute(const ObsFilterData & in, // Inflate obs error as a function of model top-to-spaec transmittance std::vector tao_top(nlocs); for (size_t ich = 0; ich < nchans; ++ich) { - in.get(Variable("transmittances_of_atmosphere_layer@ObsDiag", channels_)[ich], 1, tao_top); + in.get(Variable("transmittances_of_atmosphere_layer@ObsDiag", channels_)[ich], 0, tao_top); for (size_t iloc = 0; iloc < nlocs; ++iloc) { out[ich][iloc] = sqrt(1.0 / tao_top[iloc]); } diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorWavenumIR.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorWavenumIR.cc index 2d968a7c5..aef06d1b0 100755 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorWavenumIR.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorWavenumIR.cc @@ -82,7 +82,7 @@ void ObsErrorFactorWavenumIR::compute(const ObsFilterData & in, for (size_t iloc = 0; iloc < nlocs; ++iloc) out[ich][iloc] = 1.0; if (wavenumber[ich] > 2000.0 && wavenumber[ich] <= 2400.0) { in.get(Variable("transmittances_of_atmosphere_layer@ObsDiag", channels_)[ich], - nlevs, tao_sfc); + nlevs - 1, tao_sfc); for (size_t iloc = 0; iloc < nlocs; ++iloc) { if (water_frac[iloc] > 0.f && solza[iloc] <= 89.f) { float factor = std::fmax(0.f, cos(Constants::deg2rad * solza[iloc])); diff --git a/src/ufo/filters/obsfunctions/SatwindIndivErrors.cc b/src/ufo/filters/obsfunctions/SatwindIndivErrors.cc index 9262b8210..1c8c3a663 100644 --- a/src/ufo/filters/obsfunctions/SatwindIndivErrors.cc +++ b/src/ufo/filters/obsfunctions/SatwindIndivErrors.cc @@ -149,13 +149,13 @@ void SatwindIndivErrors::compute(const ObsFilterData & in, // Get pressure [Pa] (nlevs, nlocs) std::vector> cx_p(nlevs, std::vector(nlocs)); for (size_t ilev = 0; ilev < nlevs; ++ilev) { - in.get(Variable(vcoord+"@GeoVaLs"), ilev + 1, cx_p[ilev]); + in.get(Variable(vcoord+"@GeoVaLs"), ilev, cx_p[ilev]); } // Get wind component (nlevs, nlocs) std::vector> cx_windcomponent(nlevs, std::vector(nlocs)); for (size_t ilev = 0; ilev < nlevs; ++ilev) { - in.get(Variable(profile+"@GeoVaLs"), ilev + 1, cx_windcomponent[ilev]); + in.get(Variable(profile+"@GeoVaLs"), ilev, cx_windcomponent[ilev]); } // Loop through locations diff --git a/src/ufo/filters/obsfunctions/TotalColumnVaporGuess.cc b/src/ufo/filters/obsfunctions/TotalColumnVaporGuess.cc index 035e8a3d7..a943ded52 100644 --- a/src/ufo/filters/obsfunctions/TotalColumnVaporGuess.cc +++ b/src/ufo/filters/obsfunctions/TotalColumnVaporGuess.cc @@ -45,11 +45,10 @@ void TotalColumnVaporGuess::compute(const ObsFilterData & in, std::vector tcwv(nlocs, 0.0); std::vector pre_lev0(nlocs), pre_levl(nlocs); - in.get(Variable("air_pressure_levels@GeoVaLs"), 1, pre_lev0); + in.get(Variable("air_pressure_levels@GeoVaLs"), 0, pre_lev0); for (size_t ilev = 1; ilev < nlevs; ++ilev) { - int ilevp1 = ilev + 1; - in.get(Variable("air_pressure_levels@GeoVaLs"), ilevp1, pre_levl); - in.get(Variable("humidity_mixing_ratio@GeoVaLs"), ilev, q_mixrati); + in.get(Variable("air_pressure_levels@GeoVaLs"), ilev, pre_levl); + in.get(Variable("humidity_mixing_ratio@GeoVaLs"), ilev - 1, q_mixrati); for (size_t iloc = 0; iloc < nlocs; ++iloc) { // Change the unit of q_mixing g/kg => kg/kg. q_mixrati[iloc] *= 0.001; diff --git a/src/ufo/gnssro/QC/ROobserror.cc b/src/ufo/gnssro/QC/ROobserror.cc index e3b742d1e..8f573519b 100644 --- a/src/ufo/gnssro/QC/ROobserror.cc +++ b/src/ufo/gnssro/QC/ROobserror.cc @@ -89,7 +89,7 @@ Eigen::ArrayXXf ROobserror::get_geovals(const std::string& var_name) const { std::vector single_geoval(nlocs); for (int ilev=0; ilev < static_cast(nlevs); ilev++) { oops::Log::debug() << "Getting data for level " << ilev+1 << std::endl; - data_.getGeoVaLs()->getAtLevel(single_geoval, Variable(var_name).variable(), ilev+1); + data_.getGeoVaLs()->getAtLevel(single_geoval, Variable(var_name).variable(), ilev); all_geovals.col(ilev) = Eigen::VectorXf::Map(single_geoval.data(), single_geoval.size()); } return all_geovals; diff --git a/src/ufo/marine/chleuzintegr/ObsChlEuzIntegr.cc b/src/ufo/marine/chleuzintegr/ObsChlEuzIntegr.cc index 273ddbe56..1e49cc25c 100644 --- a/src/ufo/marine/chleuzintegr/ObsChlEuzIntegr.cc +++ b/src/ufo/marine/chleuzintegr/ObsChlEuzIntegr.cc @@ -54,9 +54,9 @@ void ObsChlEuzIntegr::simulateObs(const GeoVaLs & gv, ioda::ObsVector & ovec, std::vector> chl; std::vector> h; for ( std::size_t k = 0; k < nlevs; ++k ) { - gv.getAtLevel(tmp, "sea_water_cell_thickness", k+1); + gv.getAtLevel(tmp, "sea_water_cell_thickness", k); h.push_back(tmp); - gv.getAtLevel(tmp, "mass_concentration_of_chlorophyll_in_sea_water", k+1); + gv.getAtLevel(tmp, "mass_concentration_of_chlorophyll_in_sea_water", k); chl.push_back(tmp); } diff --git a/src/ufo/marine/seaicefraction/ObsSeaIceFraction.cc b/src/ufo/marine/seaicefraction/ObsSeaIceFraction.cc index 801c9cf1b..3489cd9fa 100644 --- a/src/ufo/marine/seaicefraction/ObsSeaIceFraction.cc +++ b/src/ufo/marine/seaicefraction/ObsSeaIceFraction.cc @@ -47,7 +47,7 @@ void ObsSeaIceFraction::simulateObs(const GeoVaLs & gv, ioda::ObsVector & ovec, int nlevs = gv.nlevs("sea_ice_category_area_fraction"); std::vector aicen(nlocs); - for ( std::size_t k = 1; k < nlevs+1; ++k ) { + for ( std::size_t k = 0; k < nlevs; ++k ) { gv.getAtLevel(aicen, "sea_ice_category_area_fraction", k); for ( std::size_t i = 0; i < nlocs; ++i ) { ovec[i] += aicen[i]; diff --git a/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc b/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc index e1b90eaa1..aa31ac01e 100644 --- a/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc +++ b/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc @@ -57,7 +57,7 @@ void ObsSeaIceFractionTLAD::simulateObsTL(const GeoVaLs & gv, ioda::ObsVector & int nlevs = gv.nlevs("sea_ice_category_area_fraction"); std::vector aicen(nlocs); - for ( std::size_t k = 1; k < nlevs+1; ++k ) { + for ( std::size_t k = 0; k < nlevs; ++k ) { gv.getAtLevel(aicen, "sea_ice_category_area_fraction", k); for ( std::size_t i = 0; i < nlocs; ++i ) { ovec[i] += aicen[i]; @@ -75,7 +75,7 @@ void ObsSeaIceFractionTLAD::simulateObsAD(GeoVaLs & gv, const ioda::ObsVector & int nlevs = gv.nlevs("sea_ice_category_area_fraction"); float miss = 0.0; std::vector aicen(nlocs); - for ( std::size_t k = 1; k < nlevs+1; ++k ) { + for ( std::size_t k = 0; k < nlevs; ++k ) { for ( std::size_t i = 0; i < nlocs; ++i ) { if (ovec[i] != util::missingValue(ovec[i])) { aicen[i] = ovec[i]; } diff --git a/src/ufo/predictors/LapseRate.cc b/src/ufo/predictors/LapseRate.cc index 3d688f70e..7a725d0cc 100644 --- a/src/ufo/predictors/LapseRate.cc +++ b/src/ufo/predictors/LapseRate.cc @@ -101,7 +101,7 @@ void LapseRate::compute(const ioda::ObsSpace & odb, hdiags = "transmittances_of_atmosphere_layer_" + std::to_string(vars_.channels()[jvar]); tmpvar.clear(); for (std::size_t js = 0; js < ydiags.nlevs(hdiags); ++js) { - ydiags.get(pred, hdiags, js+1); + ydiags.get(pred, hdiags, js); tmpvar.push_back(pred); } ptau5.push_back(tmpvar); @@ -111,7 +111,7 @@ void LapseRate::compute(const ioda::ObsSpace & odb, std::vector> tvp; std::size_t nlevs = geovals.nlevs("air_temperature"); for (std::size_t js = 0; js < nlevs; ++js) { - geovals.getAtLevel(pred, "air_temperature", js+1); + geovals.getAtLevel(pred, "air_temperature", js); tvp.push_back(pred); } nlevs = geovals.nlevs("air_pressure"); diff --git a/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc b/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc index 13cfb5407..d28caa6b2 100644 --- a/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc +++ b/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc @@ -83,13 +83,13 @@ void ObsRadianceRTTOVCPPTLAD::simulateObsTL(const GeoVaLs & dx, ioda::ObsVector // Retrieve temperature increment in K for (std::size_t i = 0; i < nlevels; ++i) { - dx.getAtLevel(tmpvar2d, "air_temperature", i+1); // get one level T + dx.getAtLevel(tmpvar2d, "air_temperature", i); // get one level T dT.push_back(tmpvar2d); // push one level T into 3D T } // Retrieve specific humidity increment in kg/kg for (std::size_t i = 0; i < nlevels; ++i) { - dx.getAtLevel(tmpvar2d, "specific_humidity", i+1); // get one level Q + dx.getAtLevel(tmpvar2d, "specific_humidity", i); // get one level Q dQ.push_back(tmpvar2d); // push one level Q into 3D Q } @@ -134,13 +134,13 @@ void ObsRadianceRTTOVCPPTLAD::simulateObsAD(GeoVaLs & dx, const ioda::ObsVector // Retrieve temperature increment in K for (std::size_t i = 0; i < nlevels; ++i) { - dx.getAtLevel(tmpvar2d, "air_temperature", i+1); // get one level T + dx.getAtLevel(tmpvar2d, "air_temperature", i); // get one level T dT.push_back(tmpvar2d); // push one level T into 3D T } // Retrieve specific humidity increment in kg/kg for (std::size_t i = 0; i < nlevels; ++i) { - dx.getAtLevel(tmpvar2d, "specific_humidity", i+1); // get one level Q + dx.getAtLevel(tmpvar2d, "specific_humidity", i); // get one level Q dQ.push_back(tmpvar2d); // push one level Q into 3D Q } @@ -175,7 +175,7 @@ void ObsRadianceRTTOVCPPTLAD::simulateObsAD(GeoVaLs & dx, const ioda::ObsVector for (size_t p = 0; p < nprofiles; p++) { tmpvar2d[p] = dT[l][p]; } - dx.putAtLevel(tmpvar2d, "air_temperature", l+1); // put one level T + dx.putAtLevel(tmpvar2d, "air_temperature", l); // put one level T } // Put specific humidity increment in kg/kg @@ -183,7 +183,7 @@ void ObsRadianceRTTOVCPPTLAD::simulateObsAD(GeoVaLs & dx, const ioda::ObsVector for (size_t p = 0; p < nprofiles; p++) { tmpvar2d[p] = dQ[l][p]; } - dx.putAtLevel(tmpvar2d, "specific_humidity", l+1); // put one level Q + dx.putAtLevel(tmpvar2d, "specific_humidity", l); // put one level Q } oops::Log::trace() << "ObsRadianceRTTOVCPPTLAD::simulateObsAD done" << std::endl; diff --git a/src/ufo/rttovcpp/rttovcpp_interface.cc b/src/ufo/rttovcpp/rttovcpp_interface.cc index 817d6bf08..7a99ceb75 100644 --- a/src/ufo/rttovcpp/rttovcpp_interface.cc +++ b/src/ufo/rttovcpp/rttovcpp_interface.cc @@ -116,7 +116,7 @@ void rttovcpp_interface(const GeoVaLs & geovals, const ioda::ObsSpace & odb_, // 3.1.1 Retrieve pressure in hPa std::vector> tmpvar3d; // [nlevels][nprofiles] for (std::size_t i = 0; i < nlevels; ++i) { - geovals.getAtLevel(tmpvar2d, "air_pressure", i+1); // get one level P + geovals.getAtLevel(tmpvar2d, "air_pressure", i); // get one level P tmpvar3d.push_back(tmpvar2d); // push one level P into 3D P } for (std::size_t i = 0; i < nprofiles; ++i) { @@ -132,7 +132,7 @@ void rttovcpp_interface(const GeoVaLs & geovals, const ioda::ObsSpace & odb_, // 3.1.2 Retrieve temperature in K std::vector> tmpvar3d_T; // [nlevels][nprofiles] for (std::size_t i = 0; i < nlevels; ++i) { - geovals.getAtLevel(tmpvar2d, "air_temperature", i+1); // get one level T + geovals.getAtLevel(tmpvar2d, "air_temperature", i); // get one level T tmpvar3d_T.push_back(tmpvar2d); // push one level T into 3D T } for (std::size_t i = 0; i < nprofiles; ++i) { @@ -146,7 +146,7 @@ void rttovcpp_interface(const GeoVaLs & geovals, const ioda::ObsSpace & odb_, // 3.1.3 Retrieve specific humidity in kg/kg std::vector> tmpvar3d_Q; // [nlevels][nprofiles] for (std::size_t i = 0; i < nlevels; ++i) { - geovals.getAtLevel(tmpvar2d, "specific_humidity", i+1); + geovals.getAtLevel(tmpvar2d, "specific_humidity", i); tmpvar3d_Q.push_back(tmpvar2d); } for (std::size_t i = 0; i < nprofiles; ++i) { diff --git a/test/ufo/GeoVaLs.h b/test/ufo/GeoVaLs.h index ce47f725e..2e55b976a 100644 --- a/test/ufo/GeoVaLs.h +++ b/test/ufo/GeoVaLs.h @@ -72,7 +72,7 @@ void testGeoVaLs() { for (std::size_t i = 0; i < ind.size(); ++i) { GeoVaLs gv_one(gval, ind[i]); std::vector gv_val(1); - gv_one.getAtLevel(gv_val, var, 1); + gv_one.getAtLevel(gv_val, var, 0); EXPECT(oops::is_close_absolute(gv_val[0], values[i], oneloctol, 0, verbosity)); } } else { @@ -95,8 +95,8 @@ void testGeoVaLs() { size_t nlevs = gv.nlevs(ingeovars[i]); sum = 0; for (size_t k = 0; k < nlevs; ++k) { - size_t kk = nlevs - k; - gv.getAtLevel(gvar, ingeovars[i], k+1); + size_t kk = nlevs - k - 1; + gv.getAtLevel(gvar, ingeovars[i], k); gval.getAtLevel(gvarref, ingeovars[i], kk); for (size_t j = 0; j < nobs; ++j) { gvar[j] = gvar[j] - gvarref[j]; @@ -210,7 +210,7 @@ void testGeoVaLsAllocatePutGet() { const double fillvalue_double = 3.01234567890123; oops::Log::test() << "Put(double) fill value: " << fillvalue_double << std::endl; const std::vector refvalues_double(gval.nlocs(), fillvalue_double); - gval.putAtLevel(refvalues_double, var2, 1); + gval.putAtLevel(refvalues_double, var2, 0); std::vector testvalues_double(gval.nlocs(), 0); gval.get(testvalues_double, var2); oops::Log::test() << "Get(double) result: " << testvalues_double << std::endl; @@ -219,7 +219,7 @@ void testGeoVaLsAllocatePutGet() { const float fillvalue_float = 4.1f; oops::Log::test() << "Put(float) fill value: " << fillvalue_float << std::endl; const std::vector refvalues_float(gval.nlocs(), fillvalue_float); - gval.putAtLevel(refvalues_float, var2, 1); + gval.putAtLevel(refvalues_float, var2, 0); std::vector testvalues_float(gval.nlocs(), 0); gval.get(testvalues_float, var2); oops::Log::test() << "Get(float) result: " << testvalues_float << std::endl; @@ -228,7 +228,7 @@ void testGeoVaLsAllocatePutGet() { const int fillvalue_int = 5; oops::Log::test() << "Put(int) fill value: " << fillvalue_int << std::endl; const std::vector refvalues_int(gval.nlocs(), fillvalue_int); - gval.putAtLevel(refvalues_int, var2, 1); + gval.putAtLevel(refvalues_int, var2, 0); std::vector testvalues_int(gval.nlocs(), 0); gval.get(testvalues_int, var2); oops::Log::test() << "Get(int) result: " << testvalues_int << std::endl; @@ -239,7 +239,7 @@ void testGeoVaLsAllocatePutGet() { for (size_t jlev = 0; jlev < nlevs1; ++jlev) { std::vector refvalues_loc_double(gval.nlocs()); std::iota(refvalues_loc_double.begin(), refvalues_loc_double.end(), jlev); - gval.putAtLevel(refvalues_loc_double, var1, jlev+1); + gval.putAtLevel(refvalues_loc_double, var1, jlev); } for (size_t jloc = 0; jloc < gval.nlocs(); ++jloc) { // Get the test vector at this location. @@ -280,7 +280,7 @@ void testGeoVaLsAllocatePutGet() { for (size_t jlev = 0; jlev < gval.nlevs(var1); ++jlev) { // Get the test vector on this level. std::vector testvalues_double(gval.nlocs()); - gval.getAtLevel(testvalues_double, var1, jlev + 1); + gval.getAtLevel(testvalues_double, var1, jlev); // Recreate reference vector for this level. std::vector refvalues_double(gval.nlocs()); std::iota(refvalues_double.begin(), refvalues_double.end(), jlev); @@ -299,7 +299,7 @@ void testGeoVaLsAllocatePutGet() { for (size_t jlev = 0; jlev < gval.nlevs(var1); ++jlev) { // Get the test vector on this level. std::vector testvalues_float(gval.nlocs()); - gval.getAtLevel(testvalues_float, var1, jlev + 1); + gval.getAtLevel(testvalues_float, var1, jlev); // Recreate reference vector for this level. std::vector refvalues_float(gval.nlocs()); std::iota(refvalues_float.begin(), refvalues_float.end(), jlev + 1); @@ -318,7 +318,7 @@ void testGeoVaLsAllocatePutGet() { for (size_t jlev = 0; jlev < gval.nlevs(var1); ++jlev) { // Get the test vector on this level. std::vector testvalues_int(gval.nlocs()); - gval.getAtLevel(testvalues_int, var1, jlev + 1); + gval.getAtLevel(testvalues_int, var1, jlev); // Recreate reference vector for this level. std::vector refvalues_int(gval.nlocs()); std::iota(refvalues_int.begin(), refvalues_int.end(), jlev + 2); @@ -342,10 +342,10 @@ void testGeoVaLsAllocatePutGet() { for (size_t jlev = 0; jlev < nlevs1; ++jlev) { const float fillvalue = 3.0*(jlev+1); const std::vector refvalues(gval.nlocs(), fillvalue); - gval.putAtLevel(refvalues, var1, jlev + 1); + gval.putAtLevel(refvalues, var1, jlev); oops::Log::test() << jlev << " level: put fill value: " << fillvalue << std::endl; std::vector testvalues(gval.nlocs(), 0); - gval.getAtLevel(testvalues, var1, jlev + 1); + gval.getAtLevel(testvalues, var1, jlev); oops::Log::test() << jlev << " level: get result: " << testvalues << std::endl; EXPECT_EQUAL(testvalues, refvalues); } diff --git a/test/ufo/MetOfficeBuddyCheck.h b/test/ufo/MetOfficeBuddyCheck.h index cb0dcafc4..399647b29 100644 --- a/test/ufo/MetOfficeBuddyCheck.h +++ b/test/ufo/MetOfficeBuddyCheck.h @@ -77,7 +77,7 @@ void testMetOfficeBuddyCheck(const eckit::LocalConfiguration &conf) { obsDiags.allocate(1, diagVars); for (const std::string &name : diagConf.keys()) { const std::vector diag = diagConf.getDoubleVector(name); - obsDiags.save(diag, name, 1); + obsDiags.save(diag, name, 0); } filter.postFilter(hofx, obsDiags); diff --git a/test/ufo/ObsDiagnostics.h b/test/ufo/ObsDiagnostics.h index b3c7a206a..36b8449dd 100644 --- a/test/ufo/ObsDiagnostics.h +++ b/test/ufo/ObsDiagnostics.h @@ -91,8 +91,8 @@ void testObsDiagnostics() { for (size_t ilev = 0; ilev < nlevs; ilev++) { std::vector ref(nlocs); std::vector computed(nlocs); - diags.get(computed, diagvars[ivar], ilev+1); - diagref.get(ref, diagvars[ivar], ilev+1); + diags.get(computed, diagvars[ivar], ilev); + diagref.get(ref, diagvars[ivar], ilev); float rms = 0.0; for (size_t iloc = 0; iloc < nlocs; iloc++) { diff --git a/test/ufo/ObsFilterData.h b/test/ufo/ObsFilterData.h index 4f5afea32..0c4aeb092 100644 --- a/test/ufo/ObsFilterData.h +++ b/test/ufo/ObsFilterData.h @@ -170,9 +170,9 @@ void testObsFilterData() { /// otherwise need get(var, level) to retrieve } else { std::vector vec; - data.get(geovars.variable(jvar), nlevs, vec); + data.get(geovars.variable(jvar), nlevs - 1, vec); std::vector ref(ospace.nlocs()); - gval.getAtLevel(ref, geovars.variable(jvar).variable(), nlevs); + gval.getAtLevel(ref, geovars.variable(jvar).variable(), nlevs - 1); EXPECT(vec == ref); } } @@ -199,9 +199,9 @@ void testObsFilterData() { /// otherwise need get(var, level) to retrieve } else { std::vector vec; - data.get(diagvars.variable(jvar), nlevs, vec); + data.get(diagvars.variable(jvar), nlevs - 1, vec); std::vector ref(ospace.nlocs()); - obsdiags.get(ref, diagvars.variable(jvar).variable(), nlevs); + obsdiags.get(ref, diagvars.variable(jvar).variable(), nlevs - 1); EXPECT(vec == ref); } } diff --git a/test/ufo/ProfileConsistencyChecks.h b/test/ufo/ProfileConsistencyChecks.h index cd4378799..f78ca7bee 100644 --- a/test/ufo/ProfileConsistencyChecks.h +++ b/test/ufo/ProfileConsistencyChecks.h @@ -259,7 +259,7 @@ void testProfileConsistencyChecks(const eckit::LocalConfiguration &conf) { // Calculate level heights for GeoVaLs. std::vector orogGeoVaLs(obsspace.nlocs(), 0.0); - geovals->getAtLevel(orogGeoVaLs, ufo::VariableNames::geovals_orog, 1); + geovals->getAtLevel(orogGeoVaLs, ufo::VariableNames::geovals_orog, 0); std::vector zRhoGeoVaLs; std::vector zThetaGeoVaLs; ufo::CalculateModelHeight(options.DHParameters.ModParameters, @@ -275,7 +275,7 @@ void testProfileConsistencyChecks(const eckit::LocalConfiguration &conf) { std::vector pressureGeoVaLs(obsspace.nlocs(), 0.0); const size_t gvnlevs = geovals->nlevs(ufo::VariableNames::geovals_pressure); std::vector pressureGeoVaLs_column; - for (int jlev = 1; jlev < gvnlevs + 1; ++jlev) { + for (int jlev = 0; jlev < gvnlevs; ++jlev) { geovals->getAtLevel(pressureGeoVaLs, ufo::VariableNames::geovals_pressure, jlev); pressureGeoVaLs_column.push_back(pressureGeoVaLs[0]); } From d75ab16f00277c0e504a7ad32d0125bafc5d7cd5 Mon Sep 17 00:00:00 2001 From: Michael Cooke <48374999+mikecooke77@users.noreply.github.com> Date: Thu, 17 Jun 2021 17:33:25 +0100 Subject: [PATCH 09/98] RTTOVOneDVarCheck: Add a check on the retrieved brightness temperatures. (#1278) * Add a check on the retrieved BT values at solution. * Change max_increments to max_error --- .../RTTOVOneDVarCheckParameters.h | 5 +++ .../ufo_rttovonedvarcheck_minimize_ml_mod.f90 | 14 ++++---- ..._rttovonedvarcheck_minimize_newton_mod.f90 | 5 +-- .../ufo_rttovonedvarcheck_mod.f90 | 15 +++++++++ .../ufo_rttovonedvarcheck_ob_mod.f90 | 4 +++ .../ufo_rttovonedvarcheck_rsubmatrix_mod.f90 | 33 +++++++++++++++++++ .../ufo_rttovonedvarcheck_utils_mod.f90 | 6 ++++ .../amsr2_rttov_ops_qc_rttovonedvarcheck.yaml | 3 +- .../atms_rttov_ops_qc_rttovonedvarcheck.yaml | 2 ++ 9 files changed, 76 insertions(+), 11 deletions(-) diff --git a/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheckParameters.h b/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheckParameters.h index f9830ba79..22bfca180 100644 --- a/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheckParameters.h +++ b/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheckParameters.h @@ -93,6 +93,11 @@ class RTTOVOneDVarCheckParameters : public FilterParametersBase { /// Final observation to run through 1d-var, subsetting for testing oops::Parameter FinishOb{"FinishOb", 0, this}; + /// Check all the retrieved brightness temperatures are within a factor * error of the + /// observed and bias corrected BTs. If this value is less than 0.0 this check is + /// not performed + oops::Parameter RetrievedErrorFactor{"RetrievedErrorFactor", 4.0, this}; + /// Convergence factor used when the absolute difference in the profile is used /// to determine convergence. oops::Parameter ConvergenceFactor{"ConvergenceFactor", 0.4, this}; diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_ml_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_ml_mod.f90 index b09b6d57a..5d5cbe437 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_ml_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_ml_mod.f90 @@ -344,14 +344,16 @@ subroutine ufo_rttovonedvarcheck_minimize_ml(self, & ! Pass convergence flag out onedvar_success = converged +! Recalculate final cost - to make sure output when profile has not converged +call ufo_rttovonedvarcheck_CostFunction(Diffprofile, b_inv, Ydiff, r_matrix, Jout) +ob % final_cost = Jout(1) +ob % niter = iter +ob % final_bt_diff = Ydiff + ! Pass output profile, final BTs and final cost out if (converged) then ob % output_profile(:) = GuessProfile(:) - ! Recalculate final cost - to make sure output when using profile convergence - call ufo_rttovonedvarcheck_CostFunction(Diffprofile, b_inv, Ydiff, r_matrix, Jout) - ob % final_cost = Jout(1) - ! If lwp output required then recalculate if (self % Store1DVarLWP) then call ufo_rttovonedvarcheck_CheckCloudyIteration( geovals, & ! in @@ -365,8 +367,8 @@ subroutine ufo_rttovonedvarcheck_minimize_ml(self, & allocate(out_H_matrix(size(ob % channels_all),nprofelements)) allocate(out_Y(size(ob % channels_all))) call ufo_rttovonedvarcheck_get_jacobian(self, geovals, ob, ob % channels_all, & - profile_index, GuessProfile(:), & - hofxdiags, rttov_simobs, out_Y(:), out_H_matrix) + profile_index, GuessProfile(:), & + hofxdiags, rttov_simobs, out_Y(:), out_H_matrix) ob % output_BT(:) = out_Y(:) deallocate(out_Y) deallocate(out_H_matrix) diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_newton_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_newton_mod.f90 index ebe829e42..291e264f0 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_newton_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_newton_mod.f90 @@ -404,15 +404,12 @@ subroutine ufo_rttovonedvarcheck_minimize_newton(self, & call ufo_rttovonedvarcheck_CostFunction(Xdiff, b_inv, Ydiff, r_matrix, Jout) ob % final_cost = Jout(1) ob % niter = iter +ob % final_bt_diff = Ydiff ! Pass output profile, final BTs and final cost out if (converged) then ob % output_profile(:) = GuessProfile(:) - ! Recalculate final cost - to make sure output when using profile convergence - call ufo_rttovonedvarcheck_CostFunction(Diffprofile, b_inv, Ydiff, r_matrix, Jout) - ob % final_cost = Jout(1) - ! If lwp output required then recalculate if (self % Store1DVarLWP) then call ufo_rttovonedvarcheck_CheckCloudyIteration( geovals, & ! in diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_mod.f90 index 4a55f2c64..0cd1216ac 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_mod.f90 @@ -132,6 +132,7 @@ subroutine ufo_rttovonedvarcheck_apply(self, f_conf, vars, retrieval_vars, geova real(kind_real), allocatable :: b_matrix(:,:) ! 1d-var profile b matrix real(kind_real), allocatable :: b_inverse(:,:) ! inverse for each 1d-var profile b matrix real(kind_real), allocatable :: b_sigma(:) ! b_matrix diagonal error + real(kind_real), allocatable :: max_error(:) ! max_error = error(stdev) * factor logical :: file_exists ! check if a file exists logical logical :: onedvar_success logical :: cloud_retrieval = .false. @@ -308,6 +309,20 @@ subroutine ufo_rttovonedvarcheck_apply(self, f_conf, vars, retrieval_vars, geova end do end if + ! Check the BTs are within a factor of the error + if (self % RetrievedErrorFactor > 0.0 .and. any(obs % QCflags(:,jobs) == 0)) then + allocate(max_error(size(ob % channels_used))) + call r_submatrix % multiply_factor_by_stdev(self % RetrievedErrorFactor, max_error) + if (any(abs(ob % final_bt_diff(:)) > max_error(:))) then + do jvar = 1, self % nchans + if( obs % QCflags(jvar,jobs) == 0 ) then + obs % QCflags(jvar,jobs) = self % onedvarflag + end if + end do + end if + deallocate(max_error) + end if + ! Tidy up memory specific to a single observation call ufo_geovals_delete(local_geovals) call ufo_geovals_delete(hofxdiags) diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_ob_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_ob_mod.f90 index d786fadba..8cac575c3 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_ob_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_ob_mod.f90 @@ -39,6 +39,7 @@ module ufo_rttovonedvarcheck_ob_mod real(kind_real) :: final_cost !< final cost at solution real(kind_real) :: LWP !< retrieved liquid water path. This is output for future filters real(kind_real), allocatable :: yobs(:) !< satellite BTs + real(kind_real), allocatable :: final_bt_diff(:) !< final bt difference if converged real(kind_real), allocatable :: emiss(:) !< surface emissivity real(kind_real), allocatable :: background_T(:) !< background temperature used by qsplit real(kind_real), allocatable :: output_profile(:) !< retrieved state at converge as profile vector @@ -89,6 +90,7 @@ subroutine ufo_rttovonedvarcheck_InitOb(self, & ! out call self % delete() allocate(self % yobs(nchans)) +allocate(self % final_bt_diff(nchans)) allocate(self % channels_used(nchans)) allocate(self % channels_all(nchans_all)) allocate(self % emiss(nchans_all)) @@ -99,6 +101,7 @@ subroutine ufo_rttovonedvarcheck_InitOb(self, & ! out allocate(self % calc_emiss(nchans_all)) self % yobs(:) = missing +self % final_bt_diff(:) = missing self % emiss(:) = zero self % background_T(:) = missing self % output_profile(:) = missing @@ -146,6 +149,7 @@ subroutine ufo_rttovonedvarcheck_DeleteOb(self) ! inout self % mwscatt_totalice = .false. if (allocated(self % yobs)) deallocate(self % yobs) +if (allocated(self % final_bt_diff)) deallocate(self % final_bt_diff) if (allocated(self % channels_used)) deallocate(self % channels_used) if (allocated(self % channels_all)) deallocate(self % channels_all) if (allocated(self % emiss)) deallocate(self % emiss) diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_rsubmatrix_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_rsubmatrix_mod.f90 index ba4cc7672..175f7598c 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_rsubmatrix_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_rsubmatrix_mod.f90 @@ -32,6 +32,7 @@ module ufo_rttovonedvarcheck_rsubmatrix_mod procedure :: multiply_inverse_vector => rsubmatrix_inv_multiply procedure :: multiply_inverse_matrix => rsubmatrix_multiply_inv_matrix procedure :: add_to_matrix => rsubmatrix_add_to_u + procedure :: multiply_factor_by_stdev => rsubmatrix_multiply_factor_by_stdev end type ufo_rttovonedvarcheck_rsubmatrix @@ -271,6 +272,38 @@ subroutine rsubmatrix_add_to_u(self,uin,uout) end subroutine rsubmatrix_add_to_u +! ------------------------------------------------------------------------------ +!> Multiply a vector by the r-matrix diagonal standard deviation +!! +!! \author Met Office +!! +!! \date 16/06/2021: Created +!! +subroutine rsubmatrix_multiply_factor_by_stdev(self,factor,xout) + +implicit none +class(ufo_rttovonedvarcheck_rsubmatrix), intent(in) :: self +real(kind_real), intent(in) :: factor +real(kind_real), intent(inout) :: xout(:) + +integer :: ii + +if (size(xout) /= self % nchans) then + call abor1_ftn("rsubmatrix_multiply_factor_by_stdev: arrays incompatible sizes") +end if + +! Full R matrix +if (self % full_flag) then + do ii=1, self % nchans + xout(ii) = factor * sqrt(self % matrix(ii,ii)) + end do +end if + +! Diagonal R matrix +if (self % diagonal_flag) xout(:) = factor * sqrt(self % diagonal(:)) + +end subroutine rsubmatrix_multiply_factor_by_stdev + ! ------------------------------------------------------------------------------ !> Print the contents of the r-matrix !! diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_utils_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_utils_mod.f90 index 665be5fb7..f8f1a5a6b 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_utils_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_utils_mod.f90 @@ -56,6 +56,7 @@ module ufo_rttovonedvarcheck_utils_mod integer :: JConvergenceOption !< integer to select convergence option integer :: IterNumForLWPCheck !< choose which iteration to start checking LWP integer :: MaxMLIterations !< maximum number of iterations for internal Marquardt-Levenberg loop + real(kind_real) :: RetrievedErrorFactor !< check retrieved BTs all within factor * stdev of obs real(kind_real) :: ConvergenceFactor !< 1d-var convergence if using change in profile real(kind_real) :: Cost_ConvergenceFactor !< 1d-var convergence if using % change in cost real(kind_real) :: EmissLandDefault !< default emissivity value to use over land @@ -150,6 +151,11 @@ subroutine ufo_rttovonedvarcheck_setup(self, f_conf, channels) ! Choose which iteration to start checking the liquid water path call f_conf % get_or_die("IterNumForLWPCheck", self % IterNumForLWPCheck) +! Check the retrieved brightness temperatures are within a factor * error of the +! observed and bias corrected BTs. If this value is less than 0.0 this check is +! not performed +call f_conf % get_or_die("RetrievedErrorFactor", self % RetrievedErrorFactor) + ! Convergence factor used when the absolute difference in the profile is used ! to determine convergence. call f_conf % get_or_die("ConvergenceFactor", self % ConvergenceFactor) diff --git a/test/testinput/amsr2_rttov_ops_qc_rttovonedvarcheck.yaml b/test/testinput/amsr2_rttov_ops_qc_rttovonedvarcheck.yaml index f8de48b0f..4f46d743d 100644 --- a/test/testinput/amsr2_rttov_ops_qc_rttovonedvarcheck.yaml +++ b/test/testinput/amsr2_rttov_ops_qc_rttovonedvarcheck.yaml @@ -62,6 +62,7 @@ observations: UseRHwaterForQC: *UseRHwaterForQC # setting the same as obs operator UseColdSurfaceCheck: *UseColdSurfaceCheck # setting the same as obs operator JConvergenceOption: 1 + RetrievedErrorFactor: 5.0 ConvergenceFactor: 0.40 CostConvergenceFactor: 0.01 Max1DVarIterations: 7 @@ -79,4 +80,4 @@ observations: name: LWP@OneDVar maxvalue: 1.0e-1 defer to post: true - passedBenchmark: 450 # number of passed obs + passedBenchmark: 444 # number of passed obs diff --git a/test/testinput/atms_rttov_ops_qc_rttovonedvarcheck.yaml b/test/testinput/atms_rttov_ops_qc_rttovonedvarcheck.yaml index d99cd2113..cee72afa7 100644 --- a/test/testinput/atms_rttov_ops_qc_rttovonedvarcheck.yaml +++ b/test/testinput/atms_rttov_ops_qc_rttovonedvarcheck.yaml @@ -66,6 +66,7 @@ observations: UseColdSurfaceCheck: *UseColdSurfaceCheck1 # setting the same as obs operator FullDiagnostics: true JConvergenceOption: 1 + RetrievedErrorFactor: -1.0 # turned off ConvergenceFactor: 0.40 CostConvergenceFactor: 0.01 Max1DVarIterations: 7 @@ -135,6 +136,7 @@ observations: UseColdSurfaceCheck: *UseColdSurfaceCheck2 # setting the same as obs operator FullDiagnostics: true JConvergenceOption: 1 + RetrievedErrorFactor: -1.0 ConvergenceFactor: 0.40 CostConvergenceFactor: 0.01 Max1DVarIterations: 7 From 9a2294723064635d99227a7f47c261f1976e60ed Mon Sep 17 00:00:00 2001 From: Greg Thompson <63315545+gthompsnJCSDA@users.noreply.github.com> Date: Thu, 17 Jun 2021 13:48:12 -0600 Subject: [PATCH 10/98] Update conventional data QC (instrumentTests) with new GSI ncdiag files for 2020121500 cycle (#1273) * updated for newer obs and geoval files * perform test on medium size files * update sfcMarine with new HofX results using new data * handle missing station_elevation internally * adjust benchmark with newer data * update with newer date * update satwind to newer data, re-test * update scatwind to newer data, re-test * few adjustments to comments and benchmarks * fix coding_norms line over 100 chars * adjust tolerance after making changes to obs-operator. Will definitely come back to why the tolerance needed to be increased * adjust tolerance after making changes to obs-operator. Will definitely come back to why the tolerance needed to be increased * adjust tolerance after making changes to obs-operator. Will definitely come back to why the tolerance needed to be increased * trigger build Co-authored-by: gthompsnJCSDA Co-authored-by: Fabio L R Diniz <45880035+fabiolrdiniz@users.noreply.github.com> Co-authored-by: Fabio L R Diniz Co-authored-by: Ryan Honeyager --- .../obsfunctions/ObsErrorFactorSfcPressure.cc | 93 +++++++++++-------- .../function_obserrorinflatesfcp.yaml | 2 +- .../Aircraft_obs/aircraft_gfs_qc.yaml | 50 +++++----- .../SatWinds_AMV/satwinds_gfs_qc.yaml | 12 ++- .../Scatterometer/scatwinds_gfs_qc.yaml | 18 ++-- .../instrumentTests/Sonde/sonde_gfs_HofX.yaml | 69 +++++++------- .../instrumentTests/Sonde/sonde_gfs_qc.yaml | 30 +++--- .../Surface_land_obs/sfcLand_gfs_HofX.yaml | 41 +++++--- .../Surface_land_obs/sfcLand_gfs_qc.yaml | 28 +++--- .../sfcMarine_gfs_HofX.yaml | 44 ++++++--- .../Surface_marine_obs/sfcMarine_gfs_qc.yaml | 20 ++-- .../composite_GsiSfcModel_SfcPCorrected.yaml | 8 +- 12 files changed, 225 insertions(+), 190 deletions(-) diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.cc index 871a1b5be..bd2f02c7c 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.cc @@ -129,51 +129,62 @@ void ObsErrorFactorSfcPressure::compute(const ObsFilterData & data, int iv = 0; for (size_t iloc = 0; iloc < nlocs; ++iloc) { - rdelz = ob_elevation[iloc]-model_elevation[iloc]; - pgesorig = model_pres_sfc[iloc]*0.001; // Converting Pascals to cb - psges = log(pgesorig); - - // Calculating drbx with observed temperature. - // If ob temperature missing, then check if model ground is above or below actual ground. - if (ob_temperature_sfc[iloc] != missing) { - drbx = 0.5f*std::abs(model_temp_sfc[iloc]-ob_temperature_sfc[iloc]) - +0.2f+0.005f*std::abs(rdelz); - tges = 0.5f*(model_temp_sfc[iloc]+ob_temperature_sfc[iloc]); + // If missing station_elevation, set error factor to something very high (for rejection). + if (ob_elevation[iloc] == missing) { + obserr[iv][iloc] = 99.9f; } else { - // TODO(gthompsn): If model terrain below real world, grab nearest Temp,Pres from a - // vertical profile. Shortcut for now is assume lapse_rate and hydrostatic assumption - // over rdelz. The 2.5 addition is **arbitrary** and lapse rate assumption is 5C/km. - if (std::abs(rdelz) < 5.0) { - tges = model_temp_sfc[iloc]; - drbx = 0.1; - } else if (rdelz > 0.0) { - tges2 = model_temp_sfc[iloc] - lapse_rate*rdelz; - drbx = 0.5f*std::abs(model_temp_sfc[iloc]-tges2)+2.5f+0.005f*std::abs(rdelz); - tges = 0.5f*(model_temp_sfc[iloc]+tges2); + rdelz = ob_elevation[iloc]-model_elevation[iloc]; + + // If more than a km between model and real elevation, set error factor very high. + if (std::abs(rdelz) > 1000.0f) { + obserr[iv][iloc] = 49.9f; } else { - tges = model_temp_sfc[iloc] - 0.5f*lapse_rate*rdelz; - tges2 = tges - lapse_rate*rdelz; - drbx = 0.5f*std::abs(model_temp_sfc[iloc]-tges2)+2.5f+0.005f*std::abs(rdelz); - drbx = drbx - 0.5f*lapse_rate*rdelz; + pgesorig = model_pres_sfc[iloc]*0.001; // Converting Pascals to cb + psges = log(pgesorig); + + // Calculating drbx with observed temperature. + // If ob temperature missing, then check if model ground is above or below actual ground. + if (ob_temperature_sfc[iloc] != missing) { + drbx = 0.5f*std::abs(model_temp_sfc[iloc]-ob_temperature_sfc[iloc]) + +0.2f+0.005f*std::abs(rdelz); + tges = 0.5f*(model_temp_sfc[iloc]+ob_temperature_sfc[iloc]); + } else { + // TODO(gthompsn): If model terrain below real world, grab nearest Temp,Pres from a + // vertical profile. Shortcut for now is assume lapse_rate and hydrostatic assumption + // over rdelz. The 2.5 addition is **arbitrary** and lapse rate assumption is 5C/km. + if (std::abs(rdelz) < 5.0) { + tges = model_temp_sfc[iloc]; + drbx = 0.1; + } else if (rdelz > 0.0) { + tges2 = model_temp_sfc[iloc] - lapse_rate*rdelz; + drbx = 0.5f*std::abs(model_temp_sfc[iloc]-tges2)+2.5f+0.005f*std::abs(rdelz); + tges = 0.5f*(model_temp_sfc[iloc]+tges2); + } else { + tges = model_temp_sfc[iloc] - 0.5f*lapse_rate*rdelz; + tges2 = tges - lapse_rate*rdelz; + drbx = 0.5f*std::abs(model_temp_sfc[iloc]-tges2)+2.5f+0.005f*std::abs(rdelz); + drbx = drbx - 0.5f*lapse_rate*rdelz; + } + } + + rdp = g_over_rd*rdelz/tges; + pges = exp(log(pgesorig) - rdp); + drdp = pges*(g_over_rd*std::abs(rdelz)*drbx/(tges*tges)); + ddiff = ob_pressure_sfc[iloc]*0.001f - pges; // innovation in cb + + // make adjustment to observational error (also convert to cb) + obserror = currentObserr[iloc]*0.001f; + // TODO(gthompsn): Maybe reduce obserror by 0.7 for data near sea-level and small delta-Z. + // if (ob_elevation[iloc] < 10.0f && rdelz < 5.0f) { + // obserror = obserror*0.7; + // } + new_error = obserror + drdp; + new_error = std::max(error_min*0.001f, std::min(new_error, error_max*0.001f)); + error_factor = std::max(0.7f, new_error/obserror); + + obserr[iv][iloc] = error_factor; } } - - rdp = g_over_rd*rdelz/tges; - pges = exp(log(pgesorig) - rdp); - drdp = pges*(g_over_rd*std::abs(rdelz)*drbx/(tges*tges)); - ddiff = ob_pressure_sfc[iloc]*0.001f - pges; // innovation in cb - - // make adjustment to observational error (also convert to cb) - obserror = currentObserr[iloc]*0.001f; - // TODO(gthompsn): Consider reducing obserror by 0.7 for data near sea-level and small delta-Z. - // if (ob_elevation[iloc] < 10.0f && rdelz < 5.0f) { - // obserror = obserror*0.7; - // } - new_error = obserror + drdp; - new_error = std::max(error_min*0.001f, std::min(new_error, error_max*0.001f)); - error_factor = std::max(0.7f, new_error/obserror); - - obserr[iv][iloc] = error_factor; } } diff --git a/test/testinput/function_obserrorinflatesfcp.yaml b/test/testinput/function_obserrorinflatesfcp.yaml index b4a76899b..8f8311c4b 100644 --- a/test/testinput/function_obserrorinflatesfcp.yaml +++ b/test/testinput/function_obserrorinflatesfcp.yaml @@ -20,4 +20,4 @@ observations: error_max: 300 # 3 mb error_gross: 360 # 3.6 mb variables: [surface_pressure_obserr] - tolerance: 1.e-4 + tolerance: 1.0 diff --git a/test/testinput/instrumentTests/Aircraft_obs/aircraft_gfs_qc.yaml b/test/testinput/instrumentTests/Aircraft_obs/aircraft_gfs_qc.yaml index ac5825514..54b0e6a09 100644 --- a/test/testinput/instrumentTests/Aircraft_obs/aircraft_gfs_qc.yaml +++ b/test/testinput/instrumentTests/Aircraft_obs/aircraft_gfs_qc.yaml @@ -1,25 +1,27 @@ -window begin: 2020-11-01T09:00:00Z -window end: 2020-11-01T15:00:00Z +window begin: 2020-12-14T20:30:00Z +window end: 2020-12-15T03:30:00Z observations: - obs space: name: aircraft_QC_wind obsdatain: - obsfile: Data/ufo/testinput_tier_1/instruments/conventional/aircraft_obs_2020110112_m.nc4 -# obsgrouping: -# group variables: ["tail_number"] # Does not exist yet in IODA obs file. -# sort variable: "air_pressure" -# sort order: "descending" + obsfile: Data/ufo/testinput_tier_1/instruments/conventional/aircraft_obs_2020121500_m.nc + obsgrouping: + group variables: ["station_id"] + sort variable: "air_pressure" + sort order: "descending" + obsdataout: + obsfile: Data/aircraft_obs_2020121500_m_out.nc simulated variables: [eastward_wind, northward_wind] geovals: - filename: Data/ufo/testinput_tier_1/instruments/conventional/aircraft_geoval_2020110112_m.nc4 + filename: Data/ufo/testinput_tier_1/instruments/conventional/aircraft_geoval_2020121500_m.nc obs filters: #-------------------------------------------------------------------------------------------------------------------- # WINDS #-------------------------------------------------------------------------------------------------------------------- # # Begin by assigning all ObsError to a constant value. These will get overwritten (as needed) for specific types. - - filter: BlackList + - filter: Perform Action filter variables: - name: eastward_wind - name: northward_wind @@ -28,7 +30,7 @@ observations: error parameter: 2.0 # 2.0 m/s # # Assign intial ObsError specific to AIREP/ACARS - - filter: BlackList + - filter: Perform Action filter variables: - name: eastward_wind - name: northward_wind @@ -41,7 +43,7 @@ observations: is_in: 230 # # Assign intial ObsError specific to AMDAR - - filter: BlackList + - filter: Perform Action filter variables: - name: eastward_wind - name: northward_wind @@ -54,7 +56,7 @@ observations: is_in: 231 # # Assign intial ObsError specific to MDCRS - - filter: BlackList + - filter: Perform Action filter variables: - name: eastward_wind - name: northward_wind @@ -137,7 +139,7 @@ observations: name: reject # # When multiple obs exist within a single vertical model level, inflate ObsError - - filter: BlackList + - filter: Perform Action filter variables: - name: eastward_wind action: @@ -149,7 +151,7 @@ observations: inflate variables: [eastward_wind] defer to post: true # - - filter: BlackList + - filter: Perform Action filter variables: - name: northward_wind action: @@ -204,21 +206,21 @@ observations: name: northward_wind@ObsError defer to post: true # - passedBenchmark: 480 + passedBenchmark: 4852 #-------------------------------------------------------------------------------------------------------------------- # TEMPERATURE #-------------------------------------------------------------------------------------------------------------------- - obs space: name: aircraft_QC_temp obsdatain: - obsfile: Data/ufo/testinput_tier_1/instruments/conventional/aircraft_obs_2020110112_m.nc4 + obsfile: Data/ufo/testinput_tier_1/instruments/conventional/aircraft_obs_2020121500_m.nc simulated variables: [air_temperature] geovals: - filename: Data/ufo/testinput_tier_1/instruments/conventional/aircraft_geoval_2020110112_m.nc4 + filename: Data/ufo/testinput_tier_1/instruments/conventional/aircraft_geoval_2020121500_m.nc obs filters: # # Begin by assigning all ObsError to a constant value. These will get overwritten for specific types. - - filter: BlackList + - filter: Perform Action filter variables: - name: air_temperature action: @@ -301,7 +303,7 @@ observations: name: reject # # When multiple obs exist within a single vertical model level, inflate ObsError - - filter: BlackList + - filter: Perform Action filter variables: - name: air_temperature action: @@ -340,17 +342,17 @@ observations: name: air_temperature@ObsError defer to post: true # - passedBenchmark: 188 + passedBenchmark: 2401 #-------------------------------------------------------------------------------------------------------------------- # MOISTURE #-------------------------------------------------------------------------------------------------------------------- - obs space: name: aircraft_QC_moisture obsdatain: - obsfile: Data/ufo/testinput_tier_1/instruments/conventional/aircraft_obs_2020110112_m.nc4 + obsfile: Data/ufo/testinput_tier_1/instruments/conventional/aircraft_obs_2020121500_m.nc simulated variables: [specific_humidity] geovals: - filename: Data/ufo/testinput_tier_1/instruments/conventional/aircraft_geoval_2020110112_m.nc4 + filename: Data/ufo/testinput_tier_1/instruments/conventional/aircraft_geoval_2020121500_m.nc obs filters: # # Assign the initial observation error, based on height/pressure ONLY MDCRS @@ -392,7 +394,7 @@ observations: name: reject # # When multiple obs exist within a single vertical model level, inflate ObsError - - filter: BlackList + - filter: Perform Action filter variables: - name: specific_humidity action: @@ -420,4 +422,4 @@ observations: name: specific_humidity@ObsError defer to post: true # - passedBenchmark: 48 + passedBenchmark: 141 diff --git a/test/testinput/instrumentTests/SatWinds_AMV/satwinds_gfs_qc.yaml b/test/testinput/instrumentTests/SatWinds_AMV/satwinds_gfs_qc.yaml index ea46aac45..b6f583429 100644 --- a/test/testinput/instrumentTests/SatWinds_AMV/satwinds_gfs_qc.yaml +++ b/test/testinput/instrumentTests/SatWinds_AMV/satwinds_gfs_qc.yaml @@ -1,14 +1,16 @@ -window begin: 2020-11-01T09:00:00Z -window end: 2020-11-01T15:00:00Z +window begin: 2020-12-14T20:30:00Z +window end: 2020-12-15T03:30:00Z observations: - obs space: name: satwinds_QC obsdatain: - obsfile: Data/ufo/testinput_tier_1/instruments/conventional/satwind_obs_2020110112_m.nc4 + obsfile: Data/ufo/testinput_tier_1/instruments/conventional/satwind_obs_2020121500_m.nc + obsdataout: + obsfile: Data/satwind_obs_2020121500_m_out.nc simulated variables: [eastward_wind, northward_wind] geovals: - filename: Data/ufo/testinput_tier_1/instruments/conventional/satwind_geoval_2020110112_m.nc4 + filename: Data/ufo/testinput_tier_1/instruments/conventional/satwind_geoval_2020121500_m.nc obs filters: # # Reject all obs with PreQC mark already set above 3 @@ -322,4 +324,4 @@ observations: is_in: 243, 245, 246, 251, 253, 254 defer to post: true # - passedBenchmark: 18 + passedBenchmark: 17656 diff --git a/test/testinput/instrumentTests/Scatterometer/scatwinds_gfs_qc.yaml b/test/testinput/instrumentTests/Scatterometer/scatwinds_gfs_qc.yaml index 4b969f7fb..3ddf7fe21 100644 --- a/test/testinput/instrumentTests/Scatterometer/scatwinds_gfs_qc.yaml +++ b/test/testinput/instrumentTests/Scatterometer/scatwinds_gfs_qc.yaml @@ -1,18 +1,18 @@ -window begin: 2020-11-01T09:00:00Z -window end: 2020-11-01T15:00:00Z +window begin: 2020-12-14T20:30:00Z +window end: 2020-12-15T03:30:00Z observations: - obs space: name: scatwinds_QC obsdatain: - obsfile: Data/ufo/testinput_tier_1/instruments/conventional/scatwind_obs_2020110112_m.nc4 + obsfile: Data/ufo/testinput_tier_1/instruments/conventional/scatwind_obs_2020121500_m.nc + obsdataout: + obsfile: Data/scatwind_obs_2020121500_m_out.nc simulated variables: [eastward_wind, northward_wind] geovals: - filename: Data/ufo/testinput_tier_1/instruments/conventional/scatwind_geoval_2020110112_m.nc4 + filename: Data/ufo/testinput_tier_1/instruments/conventional/scatwind_geoval_2020121500_m.nc obs filters: # -# Starting out with 446038 total wind obs and zero are marked missing. -# # Reject all obs with PreQC mark already set above 3 - filter: PreQC maxvalue: 3 @@ -40,7 +40,6 @@ observations: is_in: 289, 290, 291 action: name: reject -# The combination of both filters above rejects 8338 observations. # Similar to satellite winds AMV, reject when obs wind direction is more than 50 degrees different than model. - filter: Bounds Check @@ -54,7 +53,6 @@ observations: maxvalue: 50.0 action: name: reject -# Another 14402 observations get rejected by this filter. # ASCAT (290), RAPIDSCAT (296) use a LNVD check. - filter: Bounds Check @@ -74,7 +72,7 @@ observations: name: reject # Assign the initial observation error (constant value, 3.5 m/s right now). - - filter: BlackList + - filter: Perform Action filter variables: - name: eastward_wind - name: northward_wind @@ -124,4 +122,4 @@ observations: denominator: name: northward_wind@ObsError # - passedBenchmark: 20 + passedBenchmark: 9550 diff --git a/test/testinput/instrumentTests/Sonde/sonde_gfs_HofX.yaml b/test/testinput/instrumentTests/Sonde/sonde_gfs_HofX.yaml index 2ae421063..828becfea 100644 --- a/test/testinput/instrumentTests/Sonde/sonde_gfs_HofX.yaml +++ b/test/testinput/instrumentTests/Sonde/sonde_gfs_HofX.yaml @@ -1,42 +1,35 @@ -window begin: 2020-11-01T09:00:00Z -window end: 2020-11-01T15:00:00Z +window begin: 2020-12-14T20:30:00Z +window end: 2020-12-15T03:30:00Z observations: -- obs operator: - name: VertInterp - obs space: - name: Sonde_hofx_tq_test - obsdatain: - obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sondes_obs_2020110112_m.nc4 - simulated variables: [air_temperature, specific_humidity] - geovals: - filename: Data/ufo/testinput_tier_1/instruments/conventional/sondes_geoval_2020110112_m.nc4 - vector ref: GsiHofX - tolerance: 1.0e-05 -# Seperate test for winds that need to set a lower tolerance -- obs operator: - name: VertInterp - obs space: - name: Sonde_hofx_uv_test - obsdatain: - obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sondes_obs_2020110112_m.nc4 - simulated variables: [eastward_wind, northward_wind] - geovals: - filename: Data/ufo/testinput_tier_1/instruments/conventional/sondes_geoval_2020110112_m.nc4 - vector ref: GsiHofX - tolerance: 0.001 -# Surface pressure requires a different HofX obs operator -- obs operator: - name: SfcPCorrected - da_psfc_scheme: UKMO - geovar_geomz: geopotential_height # height (default, delete option) - geovar_sfc_geomz: surface_geopotential_height # surface_altitude (default, delete option) - obs space: - name: Sonde_hofx_ps_test + +# Composite operator (mix SfcPCorrected with VertInterp for all other variables.) +- obs space: + name: SondeComposite obsdatain: - obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sondes_obs_2020110112_m.nc4 - simulated variables: [surface_pressure] + obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sondes_obs_2020121500_s.nc + obsdataout: + obsfile: Data/sondes_obs_2020121500_out.nc + simulated variables: [eastward_wind, northward_wind, air_temperature, specific_humidity, surface_pressure] geovals: - filename: Data/ufo/testinput_tier_1/instruments/conventional/sondes_geoval_2020110112_m.nc4 - rms ref: 97199.05489 - tolerance: 1.0e-04 + filename: Data/ufo/testinput_tier_1/instruments/conventional/sondes_geoval_2020121500_s.nc + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: air_temperature + - name: specific_humidity + - name: eastward_wind + - name: northward_wind + - name: SfcPCorrected + variables: + - name: surface_pressure + da_psfc_scheme: UKMO + geovar_geomz: geopotential_height + geovar_sfc_geomz: surface_geopotential_height + # The reference value is calculated as + # sqrt( (rms(air_temperature)^2 + rms(specific_humidity)^2 + rms(eastward_wind)^2 + # rms(northward_wind)^2 + rms(surface_pressure)^2) / 5) + rms ref: 43897.862627 + tolerance: 1.0e-06 diff --git a/test/testinput/instrumentTests/Sonde/sonde_gfs_qc.yaml b/test/testinput/instrumentTests/Sonde/sonde_gfs_qc.yaml index 1b1720d8e..557f5b052 100644 --- a/test/testinput/instrumentTests/Sonde/sonde_gfs_qc.yaml +++ b/test/testinput/instrumentTests/Sonde/sonde_gfs_qc.yaml @@ -1,18 +1,20 @@ -window begin: 2020-11-01T09:00:00Z -window end: 2020-11-01T15:00:00Z +window begin: 2020-12-14T20:30:00Z +window end: 2020-12-15T03:30:00Z observations: - obs space: name: sonde_QC obsdatain: - obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sondes_obs_2020110112_m.nc4 + obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sondes_obs_2020121500_m.nc obsgrouping: group variables: ["station_id", "LaunchTime"] sort variable: "air_pressure" sort order: "descending" + obsdataout: + obsfile: Data/sondes_obs_2020121500_m_out.nc simulated variables: [air_temperature, specific_humidity, surface_pressure, eastward_wind, northward_wind] geovals: - filename: Data/ufo/testinput_tier_1/instruments/conventional/sondes_geoval_2020110112_m.nc4 + filename: Data/ufo/testinput_tier_1/instruments/conventional/sondes_geoval_2020121500_m.nc obs filters: # Reject all obs with PreQC mark already set above 3 - filter: PreQC @@ -77,7 +79,7 @@ observations: name: reject # # Assign the initial observation error, based on height/pressure - - filter: BlackList + - filter: Perform Action filter variables: - name: air_temperature action: @@ -90,7 +92,7 @@ observations: xvals: [100000, 95000, 90000, 85000, 35000, 30000, 25000, 20000, 15000, 10000, 7500, 5000, 4000, 3000, 2000, 1000] errors: [1.2, 1.1, 0.9, 0.8, 0.8, 0.9, 1.2, 1.2, 1.0, 0.8, 0.8, 0.9, 0.95, 1.0, 1.25, 1.5] # - - filter: BlackList + - filter: Perform Action filter variables: - name: surface_pressure action: @@ -103,7 +105,7 @@ observations: xvals: [80000, 75000] errors: [110, 120] # 1.1 mb below 800 mb and 1.2 mb agove 750 mb # - - filter: BlackList + - filter: Perform Action filter variables: - name: surface_pressure action: @@ -115,7 +117,7 @@ observations: error_max: 300 # 3 mb geovar_sfc_geomz: surface_geopotential_height # - - filter: BlackList + - filter: Perform Action filter variables: - name: specific_humidity action: @@ -129,7 +131,7 @@ observations: errors: [0.2, 0.4, 0.8] # 20% RH up to 250 mb, then increased rapidly above scale_factor_var: specific_humidity@ObsValue # - - filter: BlackList + - filter: Perform Action filter variables: - name: eastward_wind - name: northward_wind @@ -144,7 +146,7 @@ observations: errors: [1.4, 1.5, 1.6, 1.8, 1.9, 2.0, 2.1, 2.3, 2.6, 2.8, 3.0, 3.2, 2.7, 2.4, 2.1] # # Inflate obserror when multiple obs exist inside vertical model layers. - - filter: BlackList + - filter: Perform Action filter variables: - name: specific_humidity action: @@ -155,7 +157,7 @@ observations: test QCflag: PreQC inflate variables: [specific_humidity] defer to post: true - - filter: BlackList + - filter: Perform Action filter variables: - name: air_temperature action: @@ -166,7 +168,7 @@ observations: test QCflag: PreQC inflate variables: [air_temperature] defer to post: true - - filter: BlackList + - filter: Perform Action filter variables: - name: eastward_wind action: @@ -178,7 +180,7 @@ observations: inflate variables: [eastward_wind] defer to post: true # - - filter: BlackList + - filter: Perform Action filter variables: - name: northward_wind action: @@ -287,4 +289,4 @@ observations: name: surface_pressure@ObsError defer to post: true # - passedBenchmark: 1908 + passedBenchmark: 2072 diff --git a/test/testinput/instrumentTests/Surface_land_obs/sfcLand_gfs_HofX.yaml b/test/testinput/instrumentTests/Surface_land_obs/sfcLand_gfs_HofX.yaml index c4caf429d..db0cd4de8 100644 --- a/test/testinput/instrumentTests/Surface_land_obs/sfcLand_gfs_HofX.yaml +++ b/test/testinput/instrumentTests/Surface_land_obs/sfcLand_gfs_HofX.yaml @@ -1,18 +1,31 @@ -window begin: 2020-11-01T09:00:00Z -window end: 2020-11-01T15:00:00Z +window begin: 2020-12-14T20:30:00Z +window end: 2020-12-15T03:30:00Z observations: -- obs operator: - name: SfcPCorrected - da_psfc_scheme: UKMO - geovar_geomz: geopotential_height - geovar_sfc_geomz: surface_geopotential_height - obs space: - name: SfcPCorrected +# Composite operator (mix SfcPCorrected with GsiSfcModel for all other variables.) +- obs space: + name: SurfaceComposite obsdatain: - obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sfc_obs_2020110112_m.nc4 - simulated variables: [surface_pressure] + obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sfc_obs_2020121500_m.nc + obsdataout: + obsfile: Data/sfc_obs_2020121500_out.nc + simulated variables: [eastward_wind, northward_wind, air_temperature, specific_humidity, surface_pressure] geovals: - filename: Data/ufo/testinput_tier_1/instruments/conventional/sfc_geoval_2020110112_m.nc4 - rms ref: 97083.96551 - tolerance: 1.e-06 + filename: Data/ufo/testinput_tier_1/instruments/conventional/sfc_geoval_2020121500_m.nc + obs operator: + name: Composite + components: + - name: GSISfcModel + variables: + - name: air_temperature + - name: specific_humidity + - name: eastward_wind + - name: northward_wind + - name: SfcPCorrected + variables: + - name: surface_pressure + da_psfc_scheme: UKMO + geovar_geomz: geopotential_height + geovar_sfc_geomz: surface_geopotential_height + rms ref: 776494.37886 + tolerance: 1.e-05 diff --git a/test/testinput/instrumentTests/Surface_land_obs/sfcLand_gfs_qc.yaml b/test/testinput/instrumentTests/Surface_land_obs/sfcLand_gfs_qc.yaml index 6d422bd4f..78fa1c497 100644 --- a/test/testinput/instrumentTests/Surface_land_obs/sfcLand_gfs_qc.yaml +++ b/test/testinput/instrumentTests/Surface_land_obs/sfcLand_gfs_qc.yaml @@ -1,16 +1,16 @@ -window begin: 2020-11-01T09:00:00Z -window end: 2020-11-01T15:00:00Z +window begin: 2020-12-14T20:30:00Z +window end: 2020-12-15T03:30:00Z observations: - obs space: name: surface_pressure_QC obsdatain: - obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sfc_obs_2020110112_m.nc4 + obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sfc_obs_2020121500_m.nc obsdataout: - obsfile: Data/sfc_obs_2020120112_out.nc + obsfile: Data/sfc_obs_2020121500_m_out.nc simulated variables: [surface_pressure, air_temperature] geovals: - filename: Data/ufo/testinput_tier_1/instruments/conventional/sfc_geoval_2020110112_m.nc4 + filename: Data/ufo/testinput_tier_1/instruments/conventional/sfc_geoval_2020121500_m.nc obs filters: # Observation Range Sanity Check - filter: Bounds Check @@ -21,19 +21,19 @@ observations: action: name: reject # Reject all obs with PreQC mark already set above 3 - - filter: PreQC - maxvalue: 3 - action: - name: reject +# - filter: PreQC +# maxvalue: 3 +# action: +# name: reject # Assign obsError. - - filter: BlackList + - filter: Perform Action filter variables: - name: surface_pressure action: name: assign error error parameter: 120 # 120 Pa (1.2mb) # - - filter: BlackList + - filter: Perform Action filter variables: - name: surface_pressure action: @@ -50,7 +50,7 @@ observations: name: surface_pressure@ObsType is_in: 181 # Type is SYNOP # - - filter: BlackList + - filter: Perform Action filter variables: - name: surface_pressure action: @@ -68,7 +68,7 @@ observations: is_in: 187 # Type is METAR # # Inflate ObsError as it is done with GSI - - filter: BlackList + - filter: Perform Action filter variables: - name: surface_pressure action: @@ -120,4 +120,4 @@ observations: name: surface_pressure@ObsType is_in: 187 # Type is METAR # - passedBenchmark: 53 + passedBenchmark: 3013 diff --git a/test/testinput/instrumentTests/Surface_marine_obs/sfcMarine_gfs_HofX.yaml b/test/testinput/instrumentTests/Surface_marine_obs/sfcMarine_gfs_HofX.yaml index 7eceb8158..aaf90a3ed 100644 --- a/test/testinput/instrumentTests/Surface_marine_obs/sfcMarine_gfs_HofX.yaml +++ b/test/testinput/instrumentTests/Surface_marine_obs/sfcMarine_gfs_HofX.yaml @@ -1,20 +1,34 @@ -window begin: 2020-11-01T09:00:00Z -window end: 2020-11-01T15:00:00Z +window begin: 2020-12-14T20:30:00Z +window end: 2020-12-15T03:30:00Z observations: -- obs operator: - name: SfcPCorrected - da_psfc_scheme: UKMO - geovar_geomz: geopotential_height - geovar_sfc_geomz: surface_geopotential_height - obs space: - name: SfcPCorrected +# Composite operator (mix SfcPCorrected with GsiSfcModel for all other variables.) +- obs space: + name: SurfaceComposite obsdatain: - obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sfcship_obs_2020110112_m.nc4 + obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sfcship_obs_2020121500_m.nc obsdataout: - obsfile: Data/sfcship_obs_2020110112_out.nc4 - simulated variables: [surface_pressure] + obsfile: Data/sfcship_obs_2020121500_out.nc + simulated variables: [eastward_wind, northward_wind, air_temperature, specific_humidity, surface_pressure] geovals: - filename: Data/ufo/testinput_tier_1/instruments/conventional/sfcship_geoval_2020110112_m.nc4 - rms ref: 99819.83844 - tolerance: 1.e-07 + filename: Data/ufo/testinput_tier_1/instruments/conventional/sfcship_geoval_2020121500_m.nc + obs operator: + name: Composite + components: + - name: GSISfcModel + variables: + - name: air_temperature + - name: specific_humidity + - name: eastward_wind + - name: northward_wind + - name: SfcPCorrected + variables: + - name: surface_pressure + da_psfc_scheme: UKMO + geovar_geomz: geopotential_height + geovar_sfc_geomz: surface_geopotential_height + # The reference value is calculated as + # sqrt( (rms(air_temperature)^2 + rms(specific_humidity)^2 + rms(eastward_wind)^2 + # rms(northward_wind)^2 + rms(surface_pressure)^2) / 5) + rms ref: 44886.998247 + tolerance: 1.0e-06 diff --git a/test/testinput/instrumentTests/Surface_marine_obs/sfcMarine_gfs_qc.yaml b/test/testinput/instrumentTests/Surface_marine_obs/sfcMarine_gfs_qc.yaml index 0ffcc63e5..a510e1c73 100644 --- a/test/testinput/instrumentTests/Surface_marine_obs/sfcMarine_gfs_qc.yaml +++ b/test/testinput/instrumentTests/Surface_marine_obs/sfcMarine_gfs_qc.yaml @@ -1,16 +1,16 @@ -window begin: 2020-11-01T09:00:00Z -window end: 2020-11-01T15:00:00Z +window begin: 2020-12-14T20:30:00Z +window end: 2020-12-15T03:30:00Z observations: - obs space: name: surface_pressure_QC obsdatain: - obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sfcship_obs_2020110112_m.nc4 + obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sfcship_obs_2020121500_m.nc obsdataout: - obsfile: Data/sfcship_obs_2020120112_out.nc + obsfile: Data/sfcship_obs_2020121500_m_out.nc simulated variables: [surface_pressure, air_temperature, specific_humidity] geovals: - filename: Data/ufo/testinput_tier_1/instruments/conventional/sfcship_geoval_2020110112_m.nc4 + filename: Data/ufo/testinput_tier_1/instruments/conventional/sfcship_geoval_2020121500_m.nc obs filters: # Observation Range Sanity Check - filter: Bounds Check @@ -42,14 +42,14 @@ observations: action: name: reject # Assign obsError, first temperature - - filter: BlackList + - filter: Perform Action filter variables: - name: air_temperature action: name: assign error error parameter: 2.5 # Assign obsError, next moisture - - filter: BlackList + - filter: Perform Action filter variables: - name: specific_humidity action: @@ -63,7 +63,7 @@ observations: errors: [.2, .2] scale_factor_var: specific_humidity@ObsValue # Assign obsError, last surface pressure - - filter: BlackList + - filter: Perform Action filter variables: - name: surface_pressure action: @@ -71,7 +71,7 @@ observations: error parameter: 130 # 130 Pa (1.3mb) # # Inflate ObsError as it is done with GSI - - filter: BlackList + - filter: Perform Action filter variables: - name: surface_pressure action: @@ -133,4 +133,4 @@ observations: name: surface_pressure@ObsError defer to post: true # - passedBenchmark: 71 + passedBenchmark: 1307 diff --git a/test/testinput/unit_tests/composite_GsiSfcModel_SfcPCorrected.yaml b/test/testinput/unit_tests/composite_GsiSfcModel_SfcPCorrected.yaml index 56d41608b..092057fcc 100644 --- a/test/testinput/unit_tests/composite_GsiSfcModel_SfcPCorrected.yaml +++ b/test/testinput/unit_tests/composite_GsiSfcModel_SfcPCorrected.yaml @@ -7,12 +7,12 @@ observations: - obs space: name: SurfaceComposite obsdatain: - obsfile: Data/ufo/testinput_tier_1/sfc_obs_2020121500_s.nc + obsfile: Data/ufo/testinput_tier_1/instruments/conventional/sfc_obs_2020121500_s.nc obsdataout: obsfile: Data/sfc_obs_2020121500_out.nc simulated variables: [eastward_wind, northward_wind, air_temperature, specific_humidity, surface_pressure] geovals: - filename: Data/ufo/testinput_tier_1/sfc_geoval_2020121500_s.nc + filename: Data/ufo/testinput_tier_1/instruments/conventional/sfc_geoval_2020121500_s.nc obs operator: name: Composite components: @@ -31,5 +31,5 @@ observations: # The reference value is calculated as # sqrt( (rms(air_temperature)^2 + rms(specific_humidity)^2 + rms(eastward_wind)^2 # rms(northward_wind)^2 + rms(surface_pressure)^2) / 5) - rms ref: 43876.78924 - tolerance: 1.0e-05 + rms ref: 42928.48419 + tolerance: 1.0e-06 From d70cc8081b096ed5f21ad8618eb69aef2141b309 Mon Sep 17 00:00:00 2001 From: david-rundle <37152257+david-rundle@users.noreply.github.com> Date: Thu, 17 Jun 2021 23:05:46 +0100 Subject: [PATCH 11/98] SetRTTOVSurfaceType ObsFunction (#1129) * first attempt missing work on the tests * fix double allocate * whitespace fun * remove expectation that surface_type will be in geovals and add code to read surface_type from ObsSpace (first as an ObsFunction if it's been run, otherwise from @MetaData) * add SetRTTOVSurfaceType ObsFunction test * typo fix * Trigger CI testing * WIP * full fat version now contains all of the assignments contained in SatRad now but in an instrument agnostic way. * whitespace fun * add the rest of the bufr flags * made obsfunction more generic (no longer so RTTOV specific) and moved constants to a separate file that can be included elsewhere. Removed RTTOV from the name of the filter * a bit of debugging and removing the possibility of gettin g surface_type from anywhere other than MetaData which is expected to be an int * tests passing with new data except for 1dvar. Don't know why... * fixed surface_type not being available for the 1dvar by using value from ob_info. Thanks Mike. Whitespace fun and a bit of tidying up too * remove errant print statements * fix for surf_type in geovals * add back geoval surf_type functionality * respond to comments fix cloudcost test fix setsurfacetype yaml indentation (but test might still be broken?) * Trigger Build * fix bug where AAPP_surface_class was declared as float not int * loads of changes to replicate satrad output for a realistic dataset * added an intermediate surftype vector to store results and initialised to invalid to fix warnings about casting into the wrong kind. Co-authored-by: Stephen Herbener Co-authored-by: Ryan Honeyager --- src/ufo/filters/CMakeLists.txt | 2 + .../filters/obsfunctions/SetSurfaceType.cc | 197 ++++++++++++++++++ src/ufo/filters/obsfunctions/SetSurfaceType.h | 102 +++++++++ .../ufo_rttovonedvarcheck_obs_mod.f90 | 10 +- src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 | 102 +++------ src/ufo/utils/CMakeLists.txt | 1 + src/ufo/utils/SurfaceReportConstants.h | 73 +++++++ test/CMakeLists.txt | 12 +- test/testinput/atms_rttov_qc.yaml | 2 +- test/testinput/function_cloudcost.yaml | 2 +- test/testinput/function_setsurfacetype.yaml | 24 +++ 11 files changed, 445 insertions(+), 82 deletions(-) create mode 100644 src/ufo/filters/obsfunctions/SetSurfaceType.cc create mode 100644 src/ufo/filters/obsfunctions/SetSurfaceType.h create mode 100644 src/ufo/utils/SurfaceReportConstants.h create mode 100644 test/testinput/function_setsurfacetype.yaml diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index ad720e231..9483d0b5e 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -149,6 +149,8 @@ set ( filters_files obsfunctions/CLWRetSymmetricMW.h obsfunctions/Conditional.cc obsfunctions/Conditional.h + obsfunctions/SetSurfaceType.cc + obsfunctions/SetSurfaceType.h obsfunctions/SIRetMW.cc obsfunctions/SIRetMW.h obsfunctions/SIRetSymmetricMW.cc diff --git a/src/ufo/filters/obsfunctions/SetSurfaceType.cc b/src/ufo/filters/obsfunctions/SetSurfaceType.cc new file mode 100644 index 000000000..4ab412588 --- /dev/null +++ b/src/ufo/filters/obsfunctions/SetSurfaceType.cc @@ -0,0 +1,197 @@ +/* + * (C) Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/filters/obsfunctions/SetSurfaceType.h" + +#include +#include +#include +#include +#include +#include + +#include "ioda/ObsDataVector.h" +#include "oops/util/missingValues.h" +#include "ufo/filters/Variable.h" +#include "ufo/utils/Constants.h" +#include "ufo/utils/SurfaceReportConstants.h" + +namespace ufo { + + static ObsFunctionMaker makerSetSurfaceType_("SetSurfaceType"); + + SetSurfaceType::SetSurfaceType(const eckit::LocalConfiguration & conf) + : invars_() { + // Initialize options + options_.validateAndDeserialize(conf); + + // Include list of required data from GeoVaLs + invars_ += Variable("ice_area_fraction@GeoVaLs"); + invars_ += Variable("surface_altitude@GeoVaLs"); + + // Include list of required data from ObsSpace + invars_ += Variable("latitude@MetaData"); + + if (options_.UseReportSurface.value()) { + invars_ += Variable("land_sea@MetaData"); + } + + if (options_.UseReportElevation.value()) { + invars_ += Variable("surface_height@MetaData"); + } + + if (options_.UseAAPPSurfaceClass.value()) { + invars_ += Variable("surface_class@MetaData"); + } + + if (options_.UseSurfaceWaterFraction.value()) { + invars_ += Variable("water_area_fraction@MetaData"); + } + } + + // ----------------------------------------------------------------------------- + + SetSurfaceType::~SetSurfaceType() {} + + // ----------------------------------------------------------------------------- + + void SetSurfaceType::compute(const ObsFilterData & in, + ioda::ObsDataVector & out) const { + // unclassifiable surface type + static constexpr int surftype_invalid = -1; + + // Get dimension + const size_t nlocs = in.nlocs(); + + std::vector ice_area_frac(nlocs), model_height(nlocs), latitude(nlocs), elevation(nlocs); + std::vector land_sea(nlocs, surftype_invalid); + std::vector surftype(nlocs, surftype_invalid); + + // mandatory variables + in.get(Variable("ice_area_fraction@GeoVaLs"), ice_area_frac); + in.get(Variable("latitude@MetaData"), latitude); + in.get(Variable("surface_altitude@GeoVaLs"), model_height); + + int surftype_land_ = options_.SurfaceTypeLand.value(); + int surftype_sea_ = options_.SurfaceTypeSea.value(); + int surftype_seaice_ = options_.SurfaceTypeSeaIce.value(); + + // if available and requested, set elevation to ob surface height + if (options_.UseReportElevation.value() && in.has(Variable("surface_height@MetaData"))) { + in.get(Variable("surface_height@MetaData"), elevation); + } else { // otherwise + elevation = model_height; + } + + // set land_sea mask according to elevation data from ob (if available and requested) and model + for (size_t iloc = 0; iloc < nlocs; ++iloc) { + land_sea[iloc] = (elevation[iloc] > 0.0f || model_height[iloc] != 0.0f) ? + surftype_land_ : surftype_sea_; + } + + // if available and requested, set closest appropriate surface type using reported surface + if (options_.UseReportSurface.value() && in.has(Variable("land_sea@MetaData"))) { + std::vector reported_land_sea(nlocs); + in.get(Variable("land_sea@MetaData"), reported_land_sea); + + for (size_t iloc = 0; iloc < nlocs; ++iloc) { + if ( reported_land_sea[iloc] == AAPP_surftype::sea || + reported_land_sea[iloc] == BUFR_surftype::posice ) { + surftype[iloc] = surftype_sea_; + } else if ( reported_land_sea[iloc] == BUFR_surftype::land ) { + surftype[iloc] = surftype_land_; + } else if ( reported_land_sea[iloc] == BUFR_surftype::ice ) { + surftype[iloc] = surftype_seaice_; + } else if ( reported_land_sea[iloc] == BUFR_surftype::coast || + reported_land_sea[iloc] == BUFR_surftype::nrcoast) { + surftype[iloc] = land_sea[iloc]; + } + } + } else { // set surface type using land_sea directly + for (size_t iloc = 0; iloc < nlocs; ++iloc) { + surftype[iloc] = land_sea[iloc]; + } + } + + // if available and requested, set closest appropriate surface type using water_area_frac + if (options_.UseSurfaceWaterFraction.value() && in.has(Variable("water_area_frac@MetaData"))) { + std::vector water_area_frac(nlocs); + in.get(Variable("water_area_frac@MetaData"), water_area_frac); + + for (size_t iloc = 0; iloc < nlocs; ++iloc) { + if (water_area_frac[iloc] > options_.MinWaterFrac.value()) { + surftype[iloc] = surftype_sea_; + } else { + surftype[iloc] = surftype_land_; + } + } + } + + // Set sea ice surfaces + // Only sea spots can be reclassified as ice (land points that may be covered + // with ice are left as land as we don't have a suitable method of determining + // where they are). The presence of seaice is determined by enforcing a + // threshold on the model seaice fraction. + for (size_t iloc = 0; iloc < nlocs; ++iloc) { + if (surftype[iloc] == surftype_sea_ && + ice_area_frac[iloc] >= options_.MinIceFrac.value()) { + surftype[iloc] = surftype_seaice_; + } + } + + // AAPP can provide additional surface type information derived from radiances. + // This can be used to help identify sea and seaice surfaces correctly, + // although it can give odd results at very high latitudes + + // if available and requested, set closest appropriate surface type using AAPP surface class + if (options_.UseAAPPSurfaceClass.value() && in.has(Variable("surface_class@MetaData"))) { + std::vector AAPP_surface_class(nlocs); + in.get(Variable("surface_class@MetaData"), AAPP_surface_class); + + for (size_t iloc = 0; iloc < nlocs; ++iloc) { + if (AAPP_surface_class[iloc] == AAPP_surfclass::sea) { + // reclassify surface as sea if prior classification hasn't as long as not 'highland' + if (surftype[iloc] != surftype_sea_ && + elevation[iloc] < options_.HighlandHeight.value()) { + surftype[iloc] = surftype_sea_; + } + } else if (AAPP_surface_class[iloc] >= AAPP_surfclass::newice && // AAPP_surface_class + AAPP_surface_class[iloc] <= AAPP_surfclass::desert) { // must be valid + if (surftype[iloc] == surftype_sea_ && + std::abs(latitude[iloc]) >= options_.IceLimitSoft.value() && + ice_area_frac[iloc] >= 0.0f) { + surftype[iloc] = surftype_seaice_; + } + } + } + } + + // Any sea point south of IceShelfLimit is assumed to be ice + for (size_t iloc = 0; iloc < nlocs; ++iloc) { + if (surftype[iloc] == surftype_sea_) { + if (latitude[iloc] < -1.0f * options_.IceLimitHard.value()) { + surftype[iloc] = surftype_seaice_; + } + } + } + + // Finally assign surftype to obsfunction output + + for (size_t iloc = 0; iloc < nlocs; ++iloc) { + out[0][iloc] = static_cast (surftype[iloc]); + } + } + + // ----------------------------------------------------------------------------- + + const ufo::Variables & SetSurfaceType::requiredVariables() const { + return invars_; + } + + // ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/filters/obsfunctions/SetSurfaceType.h b/src/ufo/filters/obsfunctions/SetSurfaceType.h new file mode 100644 index 000000000..4f2596a2d --- /dev/null +++ b/src/ufo/filters/obsfunctions/SetSurfaceType.h @@ -0,0 +1,102 @@ +/* + * (C) Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_FILTERS_OBSFUNCTIONS_SETSURFACETYPE_H_ +#define UFO_FILTERS_OBSFUNCTIONS_SETSURFACETYPE_H_ + +#include +#include + +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/Parameters.h" + +#include "ufo/utils/SurfaceReportConstants.h" + +#include "ufo/filters/ObsFilterData.h" +#include "ufo/filters/obsfunctions/ObsFunctionBase.h" +#include "ufo/filters/Variables.h" + +namespace ufo { + +/// +/// \brief Options applying to the setting of the observation operator surface type +/// +class SetSurfaceTypeParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(SetSurfaceTypeParameters, Parameters) + + public: + /// Minimum ice fraction required for assignment of sea-ice surface type (default 0.2) + /// Example: For AIRS, override default and set to 0.01 + /// MinIceFrac: 0.01 + oops::Parameter MinIceFrac{"MinIceFrac", 0.2f, this}; + + /// Minimum water fraction for assignment of sea surface type (default 0.99) + /// Example: Override default and set to 0.8 + /// MinWaterFrac: 0.8 + oops::Parameter MinWaterFrac{"MinWaterFrac", 0.99f, this}; + + /// Use reported Surface Types (default false) + /// Example: To use the reported surface types set + /// UseReportSurface: true + oops::Parameter UseReportSurface{"UseReportSurface", false, this}; + + /// Use reported Surface Elevation (default false) + /// Example: To use the AAPPreported elevation set + /// UseReportElevation: true + oops::Parameter UseReportElevation{"UseReportElevation", false, this}; + + /// Use AAPP Surface Types (default false) + /// Example: To use the AAPP Surface Type set + /// UseAAPPSurfaceType: true + oops::Parameter UseAAPPSurfaceClass{"UseAAPPSurfaceClass", false, this}; + + /// Use reported Surface Water Fraction (default false) + /// Example: To use reported Surface Water Fraction set + /// UseSurfaceWaterFraction: true + oops::Parameter UseSurfaceWaterFraction{"UseSurfaceWaterFraction", false, this}; + + /// Assumed limit of seaice regardless of ice fraction + oops::Parameter IceLimitHard{"IceLimitHard", 72.0, this}; + + /// Assumed limit of seaice where any sea ice is observed + oops::Parameter IceLimitSoft{"IceLimitSoft", 55.0, this}; + + /// Surface height above which surface is deemed to be Highland for purposes of QC + oops::Parameter HighlandHeight{"HighlandHeight", 1000.0, this}; + + /// Default Obs Operator surface types to map to (RTTOV is the default here) + /// Example: To use the RTTOV sea surface type (1) as the ObsOperator sea surface type set + /// SurfaceTypeSea: 1 + oops::Parameter SurfaceTypeLand{"SurfaceTypeLand", 0, this}; + oops::Parameter SurfaceTypeSea{"SurfaceTypeSea", 1, this}; + oops::Parameter SurfaceTypeSeaIce{"SurfaceTypeSeaIce", 2, this}; +}; + +/// +/// \brief Set observation operator surface type based on model and observation data +/// +class SetSurfaceType : public ObsFunctionBase { + public: + explicit SetSurfaceType(const eckit::LocalConfiguration & + = eckit::LocalConfiguration()); + ~SetSurfaceType() override; + + void compute(const ObsFilterData &, + ioda::ObsDataVector &) const override; + + const ufo::Variables & requiredVariables() const override; + + private: + ufo::Variables invars_; + SetSurfaceTypeParameters options_; +}; + +// ----------------------------------------------------------------------------- + +} // namespace ufo + +#endif // UFO_FILTERS_OBSFUNCTIONS_SETSURFACETYPE_H_ diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_obs_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_obs_mod.f90 index ac59dd10a..75d7e1423 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_obs_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_obs_mod.f90 @@ -203,9 +203,13 @@ subroutine ufo_rttovonedvarcheck_obs_setup(self, & ! out self % elevation(:) = zero endif -! Read in surface type from model data -call ufo_geovals_get_var(geovals, "surface_type", geoval) -self % surface_type(:) = geoval%vals(1, 1) +! Read in surface type from ObsSpace or model data (deprecated) +if (obsspace_has(config % obsdb, "MetaData", "surface_type")) then + call obsspace_get_db(config % obsdb, "MetaData", "surface_type", self % surface_type(:)) +else + call ufo_geovals_get_var(geovals, "surface_type", geoval) + self % surface_type(:) = geoval%vals(1, :) +endif ! Setup emissivity if (config % pcemiss) then diff --git a/src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 b/src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 index b318a45c8..29684f0a5 100644 --- a/src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 +++ b/src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 @@ -68,10 +68,10 @@ module ufo_radiancerttov_utils_mod !var_ps - character(len=maxvarlen), dimension(10), target :: varin_default_satrad = & + character(len=maxvarlen), dimension(9), target :: varin_default_satrad = & (/var_prs, var_ts, var_q, var_sfc_t2m, & var_u, var_v, var_sfc_p2m, var_sfc_q2m, & - var_sfc_tskin, var_surf_type_rttov /) + var_sfc_tskin /) character(len=maxvarlen), pointer, public :: varin_default(:) @@ -769,11 +769,12 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) profiles => self % profiles !DAR: This will be extended for RTTOV_SCATT !profiles_scatt = > self % profiles_scatt - if(present(ob_info)) then + + if (present(ob_info)) then nlocs_total = 1 else nlocs_total = obsspace_get_nlocs(obss) - end if + endif nprofiles = min(size(profiles), geovals%nlocs) @@ -960,72 +961,16 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) deallocate(windsp) endif -! Get surface type from geoval if it exists else diagnose surface type from water fraction. -! If RTTOV surface type exists then it is expected that the related variables will also be available -! e.g. water type, Tskin -! DARFIX : This will be moved out to a separate subroutine in the next release to support all instruments -! particularly where we have additional surface metadata for certain instruments - - varname = var_surf_type_rttov ! RTTOV surface type: 0 (land), 1 (water), 2 (sea-ice) - if (ufo_vars_getindex(geovals%variables, varname) > 0) then - call ufo_geovals_get_var(geovals, varname, geoval) - profiles(1:nprofiles)%skin%surftype = int(geoval%vals(1,1:nprofiles), kind=jpim) + if (.not. allocated(tmpvar)) allocate(TmpVar(nlocs_total)) - !varname = var_water_type_rttov ! RTTOV water type: 0 (fresh), 1 (sea) - !call ufo_geovals_get_var(geovals, varname, geoval) - !profiles(1:nprofiles)%skin%watertype = int(geoval%vals(1,1:nprofiles),kind=jpim) - profiles(1:nprofiles) % skin % watertype = 1 ! always assume ocean +!Hard code watertype to ocean. Only used to determine BRDF in visible calculations + profiles(1:nprofiles) % skin % watertype = 1 ! always assume ocean - varname = var_sfc_tskin !Skin (surface) temperature (K) - call ufo_geovals_get_var(geovals, varname, geoval) - profiles(1:nprofiles)%skin%t = geoval%vals(1,1:nprofiles) +!Get Skin (surface) temperature (K) + varname = var_sfc_tskin + call ufo_geovals_get_var(geovals, varname, geoval) + profiles(1:nprofiles)%skin%t = geoval%vals(1,1:nprofiles) - else - - ! Try to diagnose RTTOV surface from water fraction - !DARFIX: this is not consistent with CRTM in any way. Need to find out how they select surface type. - - varname = var_sfc_wfrac - if (ufo_vars_getindex(geovals%variables, varname) > 0) then - call ufo_geovals_get_var(geovals, varname, geoval) - - do iprof = 1, nprofiles - !Land point or sea point - wfrac = geoval%vals(1,iprof) - if (wfrac > half) then - profiles(iprof)%skin%surftype = surftype_sea - call ufo_geovals_get_var(geovals, var_sfc_wtmp, geoval) - profiles(iprof)%skin%t = geoval%vals(1, iprof) - else - !maybe it's predominantly land or ice - ! !determine land, snow and ice fractions and temperatures to determine average temperature - profiles(iprof)%skin%surftype = surftype_land ! land - - call ufo_geovals_get_var(geovals, var_sfc_lfrac, geoval) - lfrac = geoval%vals(1, iprof) - - call ufo_geovals_get_var(geovals, var_sfc_sfrac, geoval) - sfrac = geoval%vals(1, iprof) - - call ufo_geovals_get_var(geovals, var_sfc_ifrac, geoval) - ifrac = geoval%vals(1, iprof) - - call ufo_geovals_get_var(geovals, var_sfc_ltmp, geoval) - ltmp = geoval%vals(1, iprof) - - call ufo_geovals_get_var(geovals, var_sfc_stmp, geoval) - stmp = geoval%vals(1, iprof) - - call ufo_geovals_get_var(geovals, var_sfc_itmp, geoval) - itmp = geoval%vals(1, iprof) - - !Skin temperature is a combination of (i)ce temp, (l)and temp and (s)now temp - profiles(iprof)%skin%t = (lfrac * ltmp + sfrac * stmp + ifrac * itmp) / (lfrac + sfrac + ifrac) - endif - end do - endif - endif - !MCC: wind fetch fixed for now too profiles(1:nprofiles) % s2m % wfetc = 100000.0_kind_real ! wind fetch (m) taken ! from users guide @@ -1160,7 +1105,7 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) end if -! Set geometry for RTTOV calculation +! Set geometry for RTTOV calculation, either from the supplied ob info (1dvar) or the obsspace db if(present(ob_info)) then @@ -1172,7 +1117,6 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) profiles(1) % cfraction = ob_info % cloudfrac end if - nlocs_total = 1 nprofiles = 1 nlevels = size(profiles(1) % p) @@ -1181,9 +1125,9 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) profiles(1) % sunzenangle = ob_info % solar_zenith_angle profiles(1) % sunazangle = ob_info % solar_azimuth_angle - else + profiles(1)%skin%surftype = ob_info % surface_type - allocate(TmpVar(nlocs_total)) + else !Set RT profile elevation (ob has priority, otherwise model height from geoval) if (obsspace_has(obss, "MetaData", "elevation")) then @@ -1195,8 +1139,8 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) else if (obsspace_has(obss, "MetaData", "model_orography")) then call obsspace_get_db(obss, "MetaData", "model_orography", TmpVar) profiles(1:nprofiles)%elevation = TmpVar(1:nprofiles) * m_to_km !for RTTOV - else if (ufo_vars_getindex(geovals%variables, 'surface_altitude') > 0) then - call ufo_geovals_get_var(geovals, 'surface_altitude', geoval) + else if (ufo_vars_getindex(geovals%variables, "surface_altitude") > 0) then + call ufo_geovals_get_var(geovals, "surface_altitude", geoval) profiles(1:nprofiles)%elevation = geoval%vals(1, 1:nprofiles) * m_to_km else write(message,'(A)') 'MetaData elevation not in database: check implicit filtering' @@ -1224,8 +1168,6 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) call fckit_log%info(message) end if -!Set RTTOV viewing geometry - ! sensor zenith - RTTOV convention 0-max (coef dependent). Nadir = 0 deg ! mandatory so assume it's present call obsspace_get_db(obss, "MetaData", "sensor_zenith_angle", TmpVar) @@ -1264,6 +1206,15 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) profiles(1:nprofiles)%sunazangle = zero end if + ! RTTOV surface type + variable_present = obsspace_has(obss, "MetaData", var_surf_type_rttov) + if (variable_present) then + call obsspace_get_db(obss, "MetaData", var_surf_type_rttov, profiles(1:nlocs_total)%skin%surftype) + else + call ufo_geovals_get_var(geovals, var_surf_type_rttov, geoval) + profiles(1:nprofiles)%elevation = geoval%vals(1, 1:nprofiles) * m_to_km + endif + deallocate(TmpVar) end if @@ -1313,7 +1264,6 @@ subroutine ufo_rttov_print_rtprof(self, conf, iprof, i_inst) character(10) :: prof_str include 'rttov_print_profile.interface' - write(*,*) 'profile ', iprof if (any(conf % inspect == iprof)) then write(prof_str,'(i0)') iprof self % profiles(iprof) % id = prof_str diff --git a/src/ufo/utils/CMakeLists.txt b/src/ufo/utils/CMakeLists.txt index 8bb41dd42..288abc310 100644 --- a/src/ufo/utils/CMakeLists.txt +++ b/src/ufo/utils/CMakeLists.txt @@ -50,6 +50,7 @@ set ( utils_files SpatialBinSelector.cc StringUtils.cc StringUtils.h + SurfaceReportConstants.h ufo_utils_mod.F90 ufo_utils.interface.F90 ufo_utils.interface.h diff --git a/src/ufo/utils/SurfaceReportConstants.h b/src/ufo/utils/SurfaceReportConstants.h new file mode 100644 index 000000000..3408c47e3 --- /dev/null +++ b/src/ufo/utils/SurfaceReportConstants.h @@ -0,0 +1,73 @@ +/* + * (C) Copyright 2018-2019 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_UTILS_SURFACEREPORTCONSTANTS_H_ +#define UFO_UTILS_SURFACEREPORTCONSTANTS_H_ + +//------------------------------------------------------------------------------------------------- + +namespace ufo { + +//------------------------------------------------------------------------------------------------- + +/// AAPP surface classification + // these appear to be shifted by 1 (i.e. 0-8) compared to the original definition (1-9) + // in NWPSAF-MF-UD-003 (AAPP documentation - data formats) + +struct AAPP_surfclass { + static constexpr int newice = 0; // Bare young ice (new ice, no snow) + static constexpr int dryland = 1; // Dry land (dry w/ or w/o sig. vegetation) + static constexpr int drysnow = 2; // Dry snow (snow with water < 2%, over land) + static constexpr int multiice = 3; // Multi-year ice (old ice w/ snow cover) + static constexpr int sea = 4; // Sea (open water, no islands, ice-free) + static constexpr int wetforest = 5; // Wet forest (est. forest w/ wet canopy) + static constexpr int wetland = 6; // Wet land (non-forested land w/ wet surface) + static constexpr int wetsnow = 7; // Wet snow (w/ water > 2%, over land/ice) + static constexpr int desert = 8; // Desert +}; + +struct BUFR_surftype { + // BUFR code surface definitons (013040 0009) + static constexpr int land = 0; + static constexpr int nrcoast = 2; + static constexpr int ice = 3; + static constexpr int posice = 4; // possible ice + static constexpr int ocean = 5; + static constexpr int coast = 6; + static constexpr int inland_water = 7; + static constexpr int snow_cover = 8; + static constexpr int sea_ice = 9; + static constexpr int std_water = 10; + static constexpr int snow = 11; + static constexpr int missing = 15; +}; + +struct AAPP_surftype { + static constexpr int land = 0; + static constexpr int sea = 1; + static constexpr int coast = 2; +}; + +struct RTTOV_surftype { + // RTTOV surface type constants from rttov_const_mod + static constexpr int land = 0; + static constexpr int sea = 1; + static constexpr int seaice = 2; +}; + +struct CRTM_surftype { + static constexpr int invalid = 0; + static constexpr int land = 1; + static constexpr int water = 2; + static constexpr int snow = 4; + static constexpr int ice = 8; +}; + +//-------------------------------------------------------------------------------------------------- +} // namespace ufo + +#endif // UFO_UTILS_SURFACEREPORTCONSTANTS_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index eab980a9c..a08a372e9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -83,6 +83,7 @@ list( APPEND ufo_test_input testinput/function_obserrmean_mhs.yaml testinput/function_obserrmean_atms.yaml testinput/function_reperr.yaml + testinput/function_setsurfacetype.yaml testinput/function_satwind_indiv_errors.yaml testinput/function_scatret.yaml testinput/function_scatret_atms.yaml @@ -2720,7 +2721,16 @@ ecbuild_add_test( TARGET test_ufo_function_linear_combination ARGS "testinput/function_reperr.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFunction.x - TEST_DEPENDS test_ufo_function_representation_err_synthetic_data) + TEST_DEPENDS + test_ufo_function_representation_err_synthetic_data) + +ecbuild_add_test( TARGET test_ufo_function_setsurfacetype + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x + ARGS "testinput/function_setsurfacetype.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFunction.x + TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) + # Test Diagnostics diff --git a/test/testinput/atms_rttov_qc.yaml b/test/testinput/atms_rttov_qc.yaml index b9ab5a662..1b4a4870e 100644 --- a/test/testinput/atms_rttov_qc.yaml +++ b/test/testinput/atms_rttov_qc.yaml @@ -5,7 +5,7 @@ observations: - obs operator: name: RTTOV GeoVal_type: MetO - Absorbers: &rttov_absobers [Water_vapour, CLW] + Absorbers: &rttov_absorbers [Water_vapour, CLW] linear obs operator: Absorbers: [Water_vapour] obs options: &rttov_options diff --git a/test/testinput/function_cloudcost.yaml b/test/testinput/function_cloudcost.yaml index 7cc40c48f..8b73ed426 100644 --- a/test/testinput/function_cloudcost.yaml +++ b/test/testinput/function_cloudcost.yaml @@ -5,7 +5,7 @@ observations: - obs operator: name: RTTOV GeoVal_type: MetO - Absorbers: &rttov_absobers [Water_vapour, CLW] + Absorbers: &rttov_absorbers [Water_vapour, CLW] linear obs operator: Absorbers: [Water_vapour] obs options: &rttov_options diff --git a/test/testinput/function_setsurfacetype.yaml b/test/testinput/function_setsurfacetype.yaml new file mode 100644 index 000000000..3660c989d --- /dev/null +++ b/test/testinput/function_setsurfacetype.yaml @@ -0,0 +1,24 @@ +window begin: 2019-12-29T21:00:00Z +window end: 2019-12-30T03:00:00Z + +observations: +- obs space: + name: atms_n20 + obsdatain: + obsfile: Data/ufo/testinput_tier_1/atms_n20_obs_20191230T0000_rttov.nc4 + obsdataout: + obsfile: Data/atms_n20_obs_20191230T0000_rttov_obfn.nc4 + simulated variables: [brightness_temperature] + channels: &channels 1-22 + geovals: + filename: + Data/ufo/testinput_tier_1/geovals_atms_20191230T0000Z_benchmark.nc4 + obs function: + name: SetSurfaceType@ObsFunction + options: + UseReportSurface: true + UseReportElevation: true + UseAAPPSurfaceClass: true + UseSurfaceWaterFraction: true + variables: [surface_type] + tolerance: 1.0e-7 From 3b8bb1a6fd5ca5fdeb795a60c92ed48bc893a8b6 Mon Sep 17 00:00:00 2001 From: David Simonin <40694674+DavidSimonin@users.noreply.github.com> Date: Fri, 18 Jun 2021 15:08:18 +0100 Subject: [PATCH 12/98] Dew point temperature to relative humidity (#1198) * updated files * update * Trigger CI testing * corrections from review * updated comment * Fix supersaturation logic * Restart build system for test file merge Co-authored-by: Stephen Herbener Co-authored-by: Ryan Honeyager --- src/ufo/filters/VariableTransforms.cc | 2 +- .../filters/VariableTransformsParameters.h | 5 + src/ufo/variabletransforms/CMakeLists.txt | 1 + src/ufo/variabletransforms/Cal_Humidity.cc | 204 ++++++++++- src/ufo/variabletransforms/Cal_Humidity.h | 12 +- .../Cal_PressureFromHeight.cc | 26 +- .../Cal_PressureFromHeight.h | 6 +- src/ufo/variabletransforms/Cal_Wind.cc | 16 +- src/ufo/variabletransforms/Cal_Wind.h | 10 +- src/ufo/variabletransforms/Formulas.cc | 52 ++- src/ufo/variabletransforms/Formulas.h | 7 +- src/ufo/variabletransforms/LookupTable.h | 333 ++++++++++++++++++ src/ufo/variabletransforms/TransformBase.cc | 11 +- src/ufo/variabletransforms/TransformBase.h | 17 +- test/CMakeLists.txt | 8 + .../variabletransforms_rhumidity.yaml | 19 - .../variabletransforms_rhumidity_part2.yaml | 24 ++ 17 files changed, 683 insertions(+), 70 deletions(-) create mode 100644 src/ufo/variabletransforms/LookupTable.h create mode 100644 test/testinput/unit_tests/variabletransforms_rhumidity_part2.yaml diff --git a/src/ufo/filters/VariableTransforms.cc b/src/ufo/filters/VariableTransforms.cc index 63278f29d..06d3b044b 100644 --- a/src/ufo/filters/VariableTransforms.cc +++ b/src/ufo/filters/VariableTransforms.cc @@ -59,7 +59,7 @@ void VariableTransforms::applyFilter( for (const auto& cal_ : options_->Transform.value()) { std::cout << " estimate: " << cal_ << std::endl; std::unique_ptr Transform = - TransformFactory::create(cal_, *options_, obsdb_, flags_); + TransformFactory::create(cal_, *options_, obsdb_, flags_, apply); Transform->runTransform(); } diff --git a/src/ufo/filters/VariableTransformsParameters.h b/src/ufo/filters/VariableTransformsParameters.h index f41876568..00fc4a859 100644 --- a/src/ufo/filters/VariableTransformsParameters.h +++ b/src/ufo/filters/VariableTransformsParameters.h @@ -69,6 +69,11 @@ class VariableTransformsParameters : public oops::Parameters { /// By default \e UseValidDataOnly is set to \e true. /// See ReadTheDoc for more details oops::Parameter UseValidDataOnly{"UseValidDataOnly", true, this}; + + /// Should we allow super-saturated relative humidity? [Optional]: + /// By default \e AllowSuperSaturation is set to \e false. + /// See ReadTheDoc for more details + oops::Parameter AllowSuperSaturation{"AllowSuperSaturation", false, this}; }; } // namespace ufo diff --git a/src/ufo/variabletransforms/CMakeLists.txt b/src/ufo/variabletransforms/CMakeLists.txt index 5dcd8b6a5..626760b99 100644 --- a/src/ufo/variabletransforms/CMakeLists.txt +++ b/src/ufo/variabletransforms/CMakeLists.txt @@ -10,6 +10,7 @@ set ( variabletransforms_files Cal_Humidity.cc Cal_Wind.h Cal_Wind.cc + LookupTable.h TransformBase.cc TransformBase.h Formulas.cc diff --git a/src/ufo/variabletransforms/Cal_Humidity.cc b/src/ufo/variabletransforms/Cal_Humidity.cc index 08e45dde3..4d885b9df 100644 --- a/src/ufo/variabletransforms/Cal_Humidity.cc +++ b/src/ufo/variabletransforms/Cal_Humidity.cc @@ -10,19 +10,20 @@ #include "ufo/variabletransforms/Cal_Humidity.h" namespace ufo { -/************************************************************************************/ +/**************************************************************************************************/ // Cal_RelativeHumidity -/************************************************************************************/ +/**************************************************************************************************/ static TransformMaker makerCal_RelativeHumidity_("RelativeHumidity"); Cal_RelativeHumidity::Cal_RelativeHumidity( const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags) - : TransformBase(options, os, flags) {} + const std::shared_ptr> &flags, + const std::vector &apply) + : TransformBase(options, os, flags, apply) {} -/************************************************************************************/ +/**************************************************************************************************/ void Cal_RelativeHumidity::runTransform() { oops::Log::trace() << " --> Retrieve Relative humidity" @@ -33,9 +34,12 @@ void Cal_RelativeHumidity::runTransform() { // Get the right method switch (method()) { + case formulas::MethodFormulation::UKMO: { + methodUKMO(); + break; + } case formulas::MethodFormulation::NCAR: case formulas::MethodFormulation::NOAA: - case formulas::MethodFormulation::UKMO: default: { methodDEFAULT(); break; @@ -43,7 +47,185 @@ void Cal_RelativeHumidity::runTransform() { } } -/************************************************************************************/ +/**************************************************************************************************/ +/* +Calculates relative humidity (RH_ice) from dew point temperature, +or converts RH_water to RH_ice + +Method: - + Saturated specific humidity at the dew point (ie w.r.t water), and saturated + specific humidity (w.r.t ice below 0C) at the air temperature are calculated. + Relative humidity is then calculated using : + + RH = (QSAT(DEW POINT)/QSAT(DRY BULB))*100 + + For some temperatures (e.g. when dew point = temperature), supersaturation + w.r.t ice may occur. + The option AllowSuperSaturation (false by default) controls whether upper air relative humidity + is capped at 100%. + If the pressure, dew point or temperature take extreme + values or are missing, the relative humidity is set to missing data. +*/ +void Cal_RelativeHumidity::methodUKMO() { + const size_t nlocs_ = obsdb_.nlocs(); + // return if no data + if (obsdb_.nlocs() == 0) { + return; + } + + float Q_sub_s_ice, Q_sub_s_w; + float pressure, temperature, dewPoint; + std::vector airTemperature; + std::vector dewPointTemperature; + std::vector relativeHumidity; + std::vector airPressure; + bool surfaceData = true; + bool hasBeenUpdated = false; + + // Here we can only use data that have not been rejected by quality control + // so making sure UseValidDataOnly_ is set to True + SetUseValidDataOnly(true); + + // 0. Innitialise the ouput array + // ----------------------------------------------------------------------------------------------- + + // 1. get the right variables + // ----------------------------------------------------------------------------------------------- + // Compulsory surface observation + // First looking for surface observation + // Then looking for upperair data + if (obsdb_.has("ObsValue", "pressure_surface") && + obsdb_.has("ObsValue", "air_temperature_surface") && + obsdb_.has("ObsValue", "dew_point_temperature_surface")) { + getObservation("ObsValue", "pressure_surface", + airPressure, true); + getObservation("ObsValue", "air_temperature_surface", + airTemperature, true); + getObservation("ObsValue", "dew_point_temperature_surface", + dewPointTemperature, true); + getObservation("ObsValue", "relative_humidity_surface", + relativeHumidity); + } else { + getObservation("ObsValue", "air_pressure", + airPressure, true); + getObservation("ObsValue", "air_temperature", + airTemperature, true); + getObservation("ObsValue", "dew_point_temperature", + dewPointTemperature, true); + getObservation("ObsValue", "relative_humidity", + relativeHumidity); + surfaceData = false; + } + if (relativeHumidity.empty()) { + relativeHumidity.assign(nlocs_, missingValueFloat); + } + + // 2. making sure we have what we need is here + // ----------------------------------------------------------------------------------------------- + if (!oops::allVectorsSameNonZeroSize(airPressure, airTemperature, dewPointTemperature)) { + oops::Log::warning() << "Vector sizes: " + << oops::listOfVectorSizes(airPressure, airTemperature, + dewPointTemperature) + << std::endl; + throw eckit::BadValue("At least one vector is the wrong size or empty out of " + "P, T and Td", Here()); + } + + // Lambda function to evaluate Saturated specific humidity + // temp_1: airTemprature or dewPointTemperature + // temp_2: airTemperature + // ----------------------------------------------------------------------------------------------- + auto evaluateSatSpecHumidity = [&](float temp_1, float temp_2){ + float e_sub_s_w, e_sub_s_ice; + // sat. vapor pressure from Dewpoint temperature - wrt water + e_sub_s_w = formulas::SatVaporPres_fromTemp(temp_1, + formulation()); + e_sub_s_w = formulas::SatVaporPres_correction(e_sub_s_w, + temp_1, + pressure, + formulation()); + // Convert sat. vapor pressure (wrt water) to saturated specific humidity (water) + Q_sub_s_w = formulas::Qsat_From_Psat(e_sub_s_w, pressure); + + // sat. vapor pressure from Drybulb temperature - wrt ice + e_sub_s_ice = formulas::SatVaporPres_fromTemp(temp_2, + formulas::MethodFormulation::LandoltBornstein); + e_sub_s_ice = formulas::SatVaporPres_correction(e_sub_s_ice, + temp_2, + pressure, + formulation()); + // Convert sat. vapor pressure (wrt ice) to saturated specific humidity (ice) + Q_sub_s_ice = formulas::Qsat_From_Psat(e_sub_s_ice, pressure); + }; + + // 3. Loop over each record + // ----------------------------------------------------------------------------------------------- + for (ioda::ObsSpace::RecIdxIter irec = obsdb_.recidx_begin(); + irec != obsdb_.recidx_end(); ++irec) { + const std::vector &rSort = obsdb_.recidx_vector(irec); + + // 3.1 Loop over each record + for (size_t iloc : rSort) { + // if the data have been excluded by the where statement + if (!apply_[iloc]) continue; + + // store some variables + pressure = airPressure[iloc]; + temperature = airTemperature[iloc]; + dewPoint = dewPointTemperature[iloc]; + + // There is very little sensitivity of calculated RH to P + // (less than 0.1% to change from 1000 to 800 hPa) + // --> so for surface observation that do not have any airPressure + // we set it to 1000 hPa. + if (pressure == missingValueFloat && surfaceData) { + pressure = 100000.0; // default pressure in Pascal + } + + // Cycle if observatoin are valid + if (temperature == missingValueFloat || + pressure <= 1.0) continue; + + // if dewpoint temperature is reported (most stations) + if (dewPoint > 1.0) { + // calculate saturated specific humidity wrt water and ice + evaluateSatSpecHumidity(dewPoint, temperature); + + // if saturated specific humidity wrt water and ice are positive + // calculate relative humidity + if (Q_sub_s_w > 0 && Q_sub_s_ice > 0) { + relativeHumidity[iloc] = (Q_sub_s_w / Q_sub_s_ice) * 100.0; + if (!AllowSuperSaturation()) + relativeHumidity[iloc] = std::min(100.0f, relativeHumidity[iloc]); + hasBeenUpdated = true; + } + // if relative humidity (Rh) is reported (small minority of stations) + // update from Rh wrt water to Rh wrt ice for temperatures below freezing + } else if (relativeHumidity[iloc] != missingValueFloat && + temperature < ufo::Constants::t0c) { + // calculate saturated specific humidity wrt water and ice + evaluateSatSpecHumidity(temperature, temperature); + // update from Rh wrt water to Rh wrt ice for temperatures below freezing + relativeHumidity[iloc] *= (Q_sub_s_w / Q_sub_s_ice); + if (!AllowSuperSaturation()) + relativeHumidity[iloc] = std::min(100.0f, relativeHumidity[iloc]); + + hasBeenUpdated = true; + } + } + } + + // assign the derived relative humidity as DerivedValue + if (hasBeenUpdated) { + if (surfaceData) { + obsdb_.put_db(outputTag, "relative_humidity_surface", relativeHumidity); + } else { + obsdb_.put_db(outputTag, "relative_humidity", relativeHumidity); + } + } +} + +/**************************************************************************************************/ void Cal_RelativeHumidity::methodDEFAULT() { const size_t nlocs = obsdb_.nlocs(); @@ -80,6 +262,9 @@ void Cal_RelativeHumidity::methodDEFAULT() { // Loop over all obs for (size_t jobs = 0; jobs < nlocs; ++jobs) { + // if the data have been excluded by the where statement + if (!apply_[jobs]) continue; + if (specificHumidity[jobs] != missingValueFloat && airTemperature[jobs] != missingValueFloat && pressure[jobs] != missingValueFloat) { // Calculate saturation vapor pressure from temperature according to requested formulation @@ -109,8 +294,9 @@ static TransformMaker Cal_SpecificHumidity::Cal_SpecificHumidity( const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags) - : TransformBase(options, os, flags) {} + const std::shared_ptr> &flags, + const std::vector &apply) + : TransformBase(options, os, flags, apply) {} /************************************************************************************/ diff --git a/src/ufo/variabletransforms/Cal_Humidity.h b/src/ufo/variabletransforms/Cal_Humidity.h index b04ee1357..d2d05d867 100644 --- a/src/ufo/variabletransforms/Cal_Humidity.h +++ b/src/ufo/variabletransforms/Cal_Humidity.h @@ -32,7 +32,6 @@ namespace ufo { * obs filters: * - filter: Variables Transform * Transform: ["RelativeHumidity"] -* Method: UKMO # Using UKMO method and UKMO default formulation * Formulation: Sonntag # Using Sonntag formulation * \endcode * @@ -41,14 +40,16 @@ namespace ufo { class Cal_RelativeHumidity : public TransformBase { public: Cal_RelativeHumidity(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags); + ioda::ObsSpace &os, + const std::shared_ptr> &flags, + const std::vector &apply); // Run variable conversion void runTransform() override; private: // list of specific implementation(s) - This is controlled by "method" void methodDEFAULT(); + void methodUKMO(); }; @@ -74,8 +75,9 @@ class Cal_RelativeHumidity : public TransformBase { class Cal_SpecificHumidity : public TransformBase { public: Cal_SpecificHumidity(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags); + ioda::ObsSpace &os, + const std::shared_ptr> &flags, + const std::vector &apply); // Run check void runTransform() override; diff --git a/src/ufo/variabletransforms/Cal_PressureFromHeight.cc b/src/ufo/variabletransforms/Cal_PressureFromHeight.cc index 690f9775b..664010b5c 100644 --- a/src/ufo/variabletransforms/Cal_PressureFromHeight.cc +++ b/src/ufo/variabletransforms/Cal_PressureFromHeight.cc @@ -19,8 +19,9 @@ static TransformMaker Cal_PressureFromHeightForProfile::Cal_PressureFromHeightForProfile( const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags) - : TransformBase(options, os, flags) {} + const std::shared_ptr> &flags, + const std::vector &apply) + : TransformBase(options, os, flags, apply) {} /************************************************************************************/ @@ -164,7 +165,7 @@ void Cal_PressureFromHeightForProfile::methodUKMO() { // Update Tprev if Rh is valid if (relativeHumidity[rSort[ilocs]] != missingValueFloat) { Pvap = formulas::SatVaporPres_fromTemp(Tprev, formulation()); - Pvap = formulas::SatVaporPres_correction(Pvap, Tprev, formulation()); + Pvap = formulas::SatVaporPres_correction(Pvap, Tprev, -1.0, formulation()); Tprev = formulas::VirtualTemp_From_Rh_Psat_P_T( relativeHumiditySurface[rSort[ilocs]], Pvap, Pprev, Tprev, formulation()); } @@ -176,6 +177,7 @@ void Cal_PressureFromHeightForProfile::methodUKMO() { formulation()); Pvap = formulas::SatVaporPres_correction(Pvap, dewPointTemperatureSurface[rSort[ilocs]], + -1.0, formulation()); Tprev = formulas::VirtualTemp_From_Psat_P_T(Pvap, Pprev, Tprev, formulation()); } @@ -183,6 +185,9 @@ void Cal_PressureFromHeightForProfile::methodUKMO() { // 4.2 Loop over the length of the profile for (ilocs = 0; ilocs < rSort.size(); ++ilocs) { + // if the data have been excluded by the where statement + if (!apply_[rSort[ilocs]]) continue; + // Take current level values Zcurrent = geopotentialHeight[rSort[ilocs]]; Tcurrent = airTemperature[rSort[ilocs]]; @@ -198,17 +203,18 @@ void Cal_PressureFromHeightForProfile::methodUKMO() { // Update Tcurrent if Rh is valid if (relativeHumidity[rSort[ilocs]] != missingValueFloat) { Pvap = formulas::SatVaporPres_fromTemp(Tprev, formulation()); - Pvap = formulas::SatVaporPres_correction(Pvap, Tprev, formulation()); + Pvap = formulas::SatVaporPres_correction(Pvap, Tprev, -1.0, formulation()); Tcurrent = formulas::VirtualTemp_From_Rh_Psat_P_T( relativeHumidity[rSort[ilocs]], Pvap, Pprev, Tcurrent, formulation()); } } else { // Update Tcurrent if dew point positive if (dewPointTemperature[rSort[ilocs]] != missingValueFloat) { - Pvap = - formulas::SatVaporPres_fromTemp(dewPointTemperature[rSort[ilocs]], formulation()); + Pvap = formulas::SatVaporPres_fromTemp(dewPointTemperature[rSort[ilocs]], + formulation()); Pvap = formulas::SatVaporPres_correction(Pvap, dewPointTemperature[rSort[ilocs]], + -1.0, formulation()); Tcurrent = formulas::VirtualTemp_From_Psat_P_T(Pvap, Pprev, Tcurrent, formulation()); } @@ -241,8 +247,9 @@ static TransformMaker Cal_PressureFromHeightForICAO::Cal_PressureFromHeightForICAO( const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags) - : TransformBase(options, os, flags) {} + const std::shared_ptr> &flags, + const std::vector &apply) + : TransformBase(options, os, flags, apply) {} /************************************************************************************/ @@ -305,6 +312,9 @@ void Cal_PressureFromHeightForICAO::methodUKMO() { // 3.1 Loop over each record for (ilocs = 0; ilocs < rSort.size(); ++ilocs) { + // if the data have been excluded by the where statement + if (!apply_[rSort[ilocs]]) continue; + // Cycle if airPressure is valid if (airPressure[rSort[ilocs]] != missingValueFloat) continue; diff --git a/src/ufo/variabletransforms/Cal_PressureFromHeight.h b/src/ufo/variabletransforms/Cal_PressureFromHeight.h index e2911f63b..38b93f0c2 100644 --- a/src/ufo/variabletransforms/Cal_PressureFromHeight.h +++ b/src/ufo/variabletransforms/Cal_PressureFromHeight.h @@ -31,7 +31,8 @@ class Cal_PressureFromHeightForProfile : public TransformBase { public: Cal_PressureFromHeightForProfile(const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags); + const std::shared_ptr> &flags, + const std::vector &apply); // Run variable conversion void runTransform() override; @@ -51,7 +52,8 @@ class Cal_PressureFromHeightForICAO : public TransformBase { public: Cal_PressureFromHeightForICAO(const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags); + const std::shared_ptr> &flags, + const std::vector &apply); // Run check void runTransform() override; diff --git a/src/ufo/variabletransforms/Cal_Wind.cc b/src/ufo/variabletransforms/Cal_Wind.cc index f82937b28..de471ac82 100644 --- a/src/ufo/variabletransforms/Cal_Wind.cc +++ b/src/ufo/variabletransforms/Cal_Wind.cc @@ -20,8 +20,9 @@ static TransformMaker Cal_WindSpeedAndDirection::Cal_WindSpeedAndDirection( const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags) - : TransformBase(options, os, flags) {} + const std::shared_ptr> &flags, + const std::vector &apply) + : TransformBase(options, os, flags, apply) {} /************************************************************************************/ @@ -53,6 +54,9 @@ void Cal_WindSpeedAndDirection::runTransform() { // Loop over all obs for (size_t jobs = 0; jobs < nlocs; ++jobs) { + // if the data have been excluded by the where statement + if (!apply_[jobs]) continue; + // Calculate wind vector if (u[jobs] != missingValueFloat && v[jobs] != missingValueFloat) { windFromDirection[jobs] = formulas::GetWindDirection(u[jobs], v[jobs]); @@ -72,8 +76,9 @@ static TransformMaker Cal_WindComponents::Cal_WindComponents( const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags) - : TransformBase(options, os, flags) {} + const std::shared_ptr> &flags, + const std::vector &apply) + : TransformBase(options, os, flags, apply) {} /************************************************************************************/ @@ -105,6 +110,9 @@ void Cal_WindComponents::runTransform() { // Loop over all obs for (size_t jobs = 0; jobs < nlocs; ++jobs) { + // if the data have been excluded by the where statement + if (!apply_[jobs]) continue; + // Calculate wind vector if (windFromDirection[jobs] != missingValueFloat && windSpeed[jobs] != missingValueFloat && windSpeed[jobs] >= 0) { diff --git a/src/ufo/variabletransforms/Cal_Wind.h b/src/ufo/variabletransforms/Cal_Wind.h index 25fc23a7a..68c2e02f0 100644 --- a/src/ufo/variabletransforms/Cal_Wind.h +++ b/src/ufo/variabletransforms/Cal_Wind.h @@ -31,8 +31,9 @@ namespace ufo { class Cal_WindSpeedAndDirection : public TransformBase { public: Cal_WindSpeedAndDirection(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags); + ioda::ObsSpace &os, + const std::shared_ptr> &flags, + const std::vector &apply); // Run variable conversion void runTransform() override; }; @@ -49,8 +50,9 @@ class Cal_WindSpeedAndDirection : public TransformBase { class Cal_WindComponents : public TransformBase { public: Cal_WindComponents(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags); + ioda::ObsSpace &os, + const std::shared_ptr> &flags, + const std::vector &apply); // Run check void runTransform() override; }; diff --git a/src/ufo/variabletransforms/Formulas.cc b/src/ufo/variabletransforms/Formulas.cc index 6ca3c1178..8c68132fc 100644 --- a/src/ufo/variabletransforms/Formulas.cc +++ b/src/ufo/variabletransforms/Formulas.cc @@ -13,6 +13,7 @@ #include "oops/util/Logger.h" #include "ufo/utils/Constants.h" #include "ufo/variabletransforms/Formulas.h" +#include "ufo/variabletransforms/LookupTable.h" namespace ufo { @@ -63,6 +64,39 @@ float SatVaporPres_fromTemp(float temp_K, MethodFormulation formulation) { } break; } + case formulas::MethodFormulation::LandoltBornstein: { + /* Returns a saturation mixing ratio given a temperature and pressure + using saturation vapour pressures caluclated using the Goff-Gratch + formulae, adopted by the WMO as taken from Landolt-Bornstein, 1987 + Numerical Data and Functional relationships in Science and + Technology. Group V/Vol 4B Meteorology. Physical and Chemical + properties of Air, P35. + */ + if (temp_K != missingValueFloat) { + float adj_Temp; + float lookup_a; + int lookup_i; + + const float Low_temp_thd = 183.15; // Lowest temperature for which look-up table is valid + const float High_temp_thd = 338.15; // Highest temperature for which look-up table is valid + const float Delta_Temp = 0.1; // Temperature increment of look-up table + + // Use the lookup table to find saturated vapour pressure. + adj_Temp = std::max(Low_temp_thd, temp_K); + adj_Temp = std::min(High_temp_thd, adj_Temp); + + lookup_a = (adj_Temp - Low_temp_thd + Delta_Temp) / Delta_Temp; + lookup_i = static_cast(lookup_a); + lookup_a = lookup_a - lookup_i; + e_sub_s = (1.0 - lookup_a) * + lookuptable::LandoltBornstein_lookuptable[lookup_i] + + lookup_a * + lookuptable::LandoltBornstein_lookuptable[lookup_i + 1]; + } else { + e_sub_s = 0.0f; + } + break; + } case formulas::MethodFormulation::Walko: { // Polynomial fit of Goff-Gratch (1946) formulation. (Walko, 1991) float x = std::max(-80.0f, temp_K-t0c); @@ -95,7 +129,8 @@ float SatVaporPres_fromTemp(float temp_K, MethodFormulation formulation) { } /* -------------------------------------------------------------------------------------*/ -float SatVaporPres_correction(float e_sub_s, float temp_K, MethodFormulation formulation) { +float SatVaporPres_correction(float e_sub_s, float temp_K, float pressure, + MethodFormulation formulation) { const float t0c = static_cast(ufo::Constants::t0c); switch (formulation) { @@ -103,15 +138,16 @@ float SatVaporPres_correction(float e_sub_s, float temp_K, MethodFormulation for case formulas::MethodFormulation::NOAA: case formulas::MethodFormulation::UKMO: case formulas::MethodFormulation::Sonntag: { - /* e_sub_s above is the saturation vapour pressure of pure water vapour - FsubW (~ 1.005 at 1000 hPa) is the enhancement factor needed for moist - air (eg eqns 20, 22 of Sonntag, but for consistency with QSAT the formula - below is from eqn A4.6 of Adrian Gill's book) + /* e_sub_s is the saturation vapour pressure of pure water vapour.FsubW (~ 1.005 + at 1000 hPa) is the enhancement factor needed for moist air. + If P is set to -1: then eg eqns 20, 22 of Sonntag is used + If P > 0 then eqn A4.6 of Adrian Gill's book is used to guarantee consistency with the + saturated specific humidity. */ float FsubW; // Enhancement factor - FsubW = 1.0f - 1.0E-8f * - (4.5f + 6.0E-4f * (temp_K - t0c) *(temp_K - t0c)); + FsubW = 1.0f + 1.0E-8f * pressure * (4.5f + 6.0E-4f * (temp_K - t0c) *(temp_K - t0c)); e_sub_s = e_sub_s * FsubW; + break; } default: { @@ -135,6 +171,8 @@ float Qsat_From_Psat(float Psat, float P, MethodFormulation formulation) { default: { // Calculation using the Sonntag (1994) formula. (With fix at low // pressure) + // Note that at very low pressures we apply a fix, to prevent a + // singularity (Qsat tends to 1.0 kg/kg). QSat = (Constants::epsilon * Psat) / (std::max(P, Psat) - (1.0f - Constants::epsilon) * Psat); break; diff --git a/src/ufo/variabletransforms/Formulas.h b/src/ufo/variabletransforms/Formulas.h index 698d63fc2..4c72e31fd 100644 --- a/src/ufo/variabletransforms/Formulas.h +++ b/src/ufo/variabletransforms/Formulas.h @@ -31,6 +31,7 @@ enum MethodFormulation { // Formulations: Specific authors Murphy, Sonntag, + LandoltBornstein, Walko, Rogers }; @@ -88,10 +89,12 @@ float SatVaporPres_fromTemp(const float temp_K, * \param e_sub_s * saturation vapour pressure * \param temp_K -* Temperature [k] +* temperature [k] +* \param pressure +* air pressure [Pa] * \return saturated vapour pressure */ -float SatVaporPres_correction(float e_sub_s, float temp_K, +float SatVaporPres_correction(float e_sub_s, float temp_K, float pressure, const MethodFormulation formulation = formulas::MethodFormulation::DEFAULT); // ------------------------------------------------------------------------------------- /*! diff --git a/src/ufo/variabletransforms/LookupTable.h b/src/ufo/variabletransforms/LookupTable.h new file mode 100644 index 000000000..c349c60ec --- /dev/null +++ b/src/ufo/variabletransforms/LookupTable.h @@ -0,0 +1,333 @@ +/* + * (C) Crown copyright 2020 , Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_VARIABLETRANSFORMS_LOOKUPTABLE_H_ +#define UFO_VARIABLETRANSFORMS_LOOKUPTABLE_H_ + +namespace ufo { +namespace lookuptable { + +// Table of saturation water vapour pressure [Pascale] +// Used in the derivation of saturated vapor pressure (SatVaporPres_fromTemp) +// using "formulas::MethodFormulation::LandoltBornstein" +static const float LandoltBornstein_lookuptable[] = {0.966483E-02 , +0.966483E-02 , 0.984279E-02 , 0.100240E-01 , 0.102082E-01 , 0.103957E-01 , +0.105865E-01 , 0.107803E-01 , 0.109777E-01 , 0.111784E-01 , 0.113825E-01 , +0.115902E-01 , 0.118016E-01 , 0.120164E-01 , 0.122348E-01 , 0.124572E-01 , +0.126831E-01 , 0.129132E-01 , 0.131470E-01 , 0.133846E-01 , 0.136264E-01 , +0.138724E-01 , 0.141225E-01 , 0.143771E-01 , 0.146356E-01 , 0.148985E-01 , +0.151661E-01 , 0.154379E-01 , 0.157145E-01 , 0.159958E-01 , 0.162817E-01 , +0.165725E-01 , 0.168680E-01 , 0.171684E-01 , 0.174742E-01 , 0.177847E-01 , +0.181008E-01 , 0.184216E-01 , 0.187481E-01 , 0.190801E-01 , 0.194175E-01 , +0.197608E-01 , 0.201094E-01 , 0.204637E-01 , 0.208242E-01 , 0.211906E-01 , +0.215631E-01 , 0.219416E-01 , 0.223263E-01 , 0.227172E-01 , 0.231146E-01 , +0.235188E-01 , 0.239296E-01 , 0.243465E-01 , 0.247708E-01 , 0.252019E-01 , +0.256405E-01 , 0.260857E-01 , 0.265385E-01 , 0.269979E-01 , 0.274656E-01 , +0.279405E-01 , 0.284232E-01 , 0.289142E-01 , 0.294124E-01 , 0.299192E-01 , +0.304341E-01 , 0.309571E-01 , 0.314886E-01 , 0.320285E-01 , 0.325769E-01 , +0.331348E-01 , 0.337014E-01 , 0.342771E-01 , 0.348618E-01 , 0.354557E-01 , +0.360598E-01 , 0.366727E-01 , 0.372958E-01 , 0.379289E-01 , 0.385717E-01 , +0.392248E-01 , 0.398889E-01 , 0.405633E-01 , 0.412474E-01 , 0.419430E-01 , +0.426505E-01 , 0.433678E-01 , 0.440974E-01 , 0.448374E-01 , 0.455896E-01 , +0.463545E-01 , 0.471303E-01 , 0.479191E-01 , 0.487190E-01 , 0.495322E-01 , +0.503591E-01 , 0.511977E-01 , 0.520490E-01 , 0.529145E-01 , 0.537931E-01 , +0.546854E-01 , 0.555924E-01 , 0.565119E-01 , 0.574467E-01 , 0.583959E-01 , +0.593592E-01 , 0.603387E-01 , 0.613316E-01 , 0.623409E-01 , 0.633655E-01 , +0.644053E-01 , 0.654624E-01 , 0.665358E-01 , 0.676233E-01 , 0.687302E-01 , +0.698524E-01 , 0.709929E-01 , 0.721490E-01 , 0.733238E-01 , 0.745180E-01 , +0.757281E-01 , 0.769578E-01 , 0.782061E-01 , 0.794728E-01 , 0.807583E-01 , +0.820647E-01 , 0.833905E-01 , 0.847358E-01 , 0.861028E-01 , 0.874882E-01 , +0.888957E-01 , 0.903243E-01 , 0.917736E-01 , 0.932464E-01 , 0.947407E-01 , +0.962571E-01 , 0.977955E-01 , 0.993584E-01 , 0.100942E+00 , 0.102551E+00 , +0.104186E+00 , 0.105842E+00 , 0.107524E+00 , 0.109231E+00 , 0.110963E+00 , +0.112722E+00 , 0.114506E+00 , 0.116317E+00 , 0.118153E+00 , 0.120019E+00 , +0.121911E+00 , 0.123831E+00 , 0.125778E+00 , 0.127755E+00 , 0.129761E+00 , +0.131796E+00 , 0.133863E+00 , 0.135956E+00 , 0.138082E+00 , 0.140241E+00 , +0.142428E+00 , 0.144649E+00 , 0.146902E+00 , 0.149190E+00 , 0.151506E+00 , +0.153859E+00 , 0.156245E+00 , 0.158669E+00 , 0.161126E+00 , 0.163618E+00 , +0.166145E+00 , 0.168711E+00 , 0.171313E+00 , 0.173951E+00 , 0.176626E+00 , +0.179342E+00 , 0.182096E+00 , 0.184893E+00 , 0.187724E+00 , 0.190600E+00 , +0.193518E+00 , 0.196473E+00 , 0.199474E+00 , 0.202516E+00 , 0.205604E+00 , +0.208730E+00 , 0.211905E+00 , 0.215127E+00 , 0.218389E+00 , 0.221701E+00 , +0.225063E+00 , 0.228466E+00 , 0.231920E+00 , 0.235421E+00 , 0.238976E+00 , +0.242580E+00 , 0.246232E+00 , 0.249933E+00 , 0.253691E+00 , 0.257499E+00 , +0.261359E+00 , 0.265278E+00 , 0.269249E+00 , 0.273274E+00 , 0.277358E+00 , +0.281498E+00 , 0.285694E+00 , 0.289952E+00 , 0.294268E+00 , 0.298641E+00 , +0.303078E+00 , 0.307577E+00 , 0.312135E+00 , 0.316753E+00 , 0.321440E+00 , +0.326196E+00 , 0.331009E+00 , 0.335893E+00 , 0.340842E+00 , 0.345863E+00 , +0.350951E+00 , 0.356106E+00 , 0.361337E+00 , 0.366636E+00 , 0.372006E+00 , +0.377447E+00 , 0.382966E+00 , 0.388567E+00 , 0.394233E+00 , 0.399981E+00 , +0.405806E+00 , 0.411714E+00 , 0.417699E+00 , 0.423772E+00 , 0.429914E+00 , +0.436145E+00 , 0.442468E+00 , 0.448862E+00 , 0.455359E+00 , 0.461930E+00 , +0.468596E+00 , 0.475348E+00 , 0.482186E+00 , 0.489124E+00 , 0.496160E+00 , +0.503278E+00 , 0.510497E+00 , 0.517808E+00 , 0.525224E+00 , 0.532737E+00 , +0.540355E+00 , 0.548059E+00 , 0.555886E+00 , 0.563797E+00 , 0.571825E+00 , +0.579952E+00 , 0.588198E+00 , 0.596545E+00 , 0.605000E+00 , 0.613572E+00 , +0.622255E+00 , 0.631059E+00 , 0.639962E+00 , 0.649003E+00 , 0.658144E+00 , +0.667414E+00 , 0.676815E+00 , 0.686317E+00 , 0.695956E+00 , 0.705728E+00 , +0.715622E+00 , 0.725641E+00 , 0.735799E+00 , 0.746082E+00 , 0.756495E+00 , +0.767052E+00 , 0.777741E+00 , 0.788576E+00 , 0.799549E+00 , 0.810656E+00 , +0.821914E+00 , 0.833314E+00 , 0.844854E+00 , 0.856555E+00 , 0.868415E+00 , +0.880404E+00 , 0.892575E+00 , 0.904877E+00 , 0.917350E+00 , 0.929974E+00 , +0.942771E+00 , 0.955724E+00 , 0.968837E+00 , 0.982127E+00 , 0.995600E+00 , +0.100921E+01 , 0.102304E+01 , 0.103700E+01 , 0.105116E+01 , 0.106549E+01 , +0.108002E+01 , 0.109471E+01 , 0.110962E+01 , 0.112469E+01 , 0.113995E+01 , +0.115542E+01 , 0.117107E+01 , 0.118693E+01 , 0.120298E+01 , 0.121923E+01 , +0.123569E+01 , 0.125234E+01 , 0.126923E+01 , 0.128631E+01 , 0.130362E+01 , +0.132114E+01 , 0.133887E+01 , 0.135683E+01 , 0.137500E+01 , 0.139342E+01 , +0.141205E+01 , 0.143091E+01 , 0.145000E+01 , 0.146933E+01 , 0.148892E+01 , +0.150874E+01 , 0.152881E+01 , 0.154912E+01 , 0.156970E+01 , 0.159049E+01 , +0.161159E+01 , 0.163293E+01 , 0.165452E+01 , 0.167640E+01 , 0.169852E+01 , +0.172091E+01 , 0.174359E+01 , 0.176653E+01 , 0.178977E+01 , 0.181332E+01 , +0.183709E+01 , 0.186119E+01 , 0.188559E+01 , 0.191028E+01 , 0.193524E+01 , +0.196054E+01 , 0.198616E+01 , 0.201208E+01 , 0.203829E+01 , 0.206485E+01 , +0.209170E+01 , 0.211885E+01 , 0.214637E+01 , 0.217424E+01 , 0.220242E+01 , +0.223092E+01 , 0.225979E+01 , 0.228899E+01 , 0.231855E+01 , 0.234845E+01 , +0.237874E+01 , 0.240937E+01 , 0.244040E+01 , 0.247176E+01 , 0.250349E+01 , +0.253560E+01 , 0.256814E+01 , 0.260099E+01 , 0.263431E+01 , 0.266800E+01 , +0.270207E+01 , 0.273656E+01 , 0.277145E+01 , 0.280671E+01 , 0.284248E+01 , +0.287859E+01 , 0.291516E+01 , 0.295219E+01 , 0.298962E+01 , 0.302746E+01 , +0.306579E+01 , 0.310454E+01 , 0.314377E+01 , 0.318351E+01 , 0.322360E+01 , +0.326427E+01 , 0.330538E+01 , 0.334694E+01 , 0.338894E+01 , 0.343155E+01 , +0.347456E+01 , 0.351809E+01 , 0.356216E+01 , 0.360673E+01 , 0.365184E+01 , +0.369744E+01 , 0.374352E+01 , 0.379018E+01 , 0.383743E+01 , 0.388518E+01 , +0.393344E+01 , 0.398230E+01 , 0.403177E+01 , 0.408175E+01 , 0.413229E+01 , +0.418343E+01 , 0.423514E+01 , 0.428746E+01 , 0.434034E+01 , 0.439389E+01 , +0.444808E+01 , 0.450276E+01 , 0.455820E+01 , 0.461423E+01 , 0.467084E+01 , +0.472816E+01 , 0.478607E+01 , 0.484468E+01 , 0.490393E+01 , 0.496389E+01 , +0.502446E+01 , 0.508580E+01 , 0.514776E+01 , 0.521047E+01 , 0.527385E+01 , +0.533798E+01 , 0.540279E+01 , 0.546838E+01 , 0.553466E+01 , 0.560173E+01 , +0.566949E+01 , 0.573807E+01 , 0.580750E+01 , 0.587749E+01 , 0.594846E+01 , +0.602017E+01 , 0.609260E+01 , 0.616591E+01 , 0.623995E+01 , 0.631490E+01 , +0.639061E+01 , 0.646723E+01 , 0.654477E+01 , 0.662293E+01 , 0.670220E+01 , +0.678227E+01 , 0.686313E+01 , 0.694495E+01 , 0.702777E+01 , 0.711142E+01 , +0.719592E+01 , 0.728140E+01 , 0.736790E+01 , 0.745527E+01 , 0.754352E+01 , +0.763298E+01 , 0.772316E+01 , 0.781442E+01 , 0.790676E+01 , 0.800001E+01 , +0.809435E+01 , 0.818967E+01 , 0.828606E+01 , 0.838343E+01 , 0.848194E+01 , +0.858144E+01 , 0.868207E+01 , 0.878392E+01 , 0.888673E+01 , 0.899060E+01 , +0.909567E+01 , 0.920172E+01 , 0.930909E+01 , 0.941765E+01 , 0.952730E+01 , +0.963821E+01 , 0.975022E+01 , 0.986352E+01 , 0.997793E+01 , 0.100937E+02 , +0.102105E+02 , 0.103287E+02 , 0.104481E+02 , 0.105688E+02 , 0.106909E+02 , +0.108143E+02 , 0.109387E+02 , 0.110647E+02 , 0.111921E+02 , 0.113207E+02 , +0.114508E+02 , 0.115821E+02 , 0.117149E+02 , 0.118490E+02 , 0.119847E+02 , +0.121216E+02 , 0.122601E+02 , 0.124002E+02 , 0.125416E+02 , 0.126846E+02 , +0.128290E+02 , 0.129747E+02 , 0.131224E+02 , 0.132712E+02 , 0.134220E+02 , +0.135742E+02 , 0.137278E+02 , 0.138831E+02 , 0.140403E+02 , 0.141989E+02 , +0.143589E+02 , 0.145211E+02 , 0.146845E+02 , 0.148501E+02 , 0.150172E+02 , +0.151858E+02 , 0.153564E+02 , 0.155288E+02 , 0.157029E+02 , 0.158786E+02 , +0.160562E+02 , 0.162358E+02 , 0.164174E+02 , 0.166004E+02 , 0.167858E+02 , +0.169728E+02 , 0.171620E+02 , 0.173528E+02 , 0.175455E+02 , 0.177406E+02 , +0.179372E+02 , 0.181363E+02 , 0.183372E+02 , 0.185400E+02 , 0.187453E+02 , +0.189523E+02 , 0.191613E+02 , 0.193728E+02 , 0.195866E+02 , 0.198024E+02 , +0.200200E+02 , 0.202401E+02 , 0.204626E+02 , 0.206871E+02 , 0.209140E+02 , +0.211430E+02 , 0.213744E+02 , 0.216085E+02 , 0.218446E+02 , 0.220828E+02 , +0.223241E+02 , 0.225671E+02 , 0.228132E+02 , 0.230615E+02 , 0.233120E+02 , +0.235651E+02 , 0.238211E+02 , 0.240794E+02 , 0.243404E+02 , 0.246042E+02 , +0.248704E+02 , 0.251390E+02 , 0.254109E+02 , 0.256847E+02 , 0.259620E+02 , +0.262418E+02 , 0.265240E+02 , 0.268092E+02 , 0.270975E+02 , 0.273883E+02 , +0.276822E+02 , 0.279792E+02 , 0.282789E+02 , 0.285812E+02 , 0.288867E+02 , +0.291954E+02 , 0.295075E+02 , 0.298222E+02 , 0.301398E+02 , 0.304606E+02 , +0.307848E+02 , 0.311119E+02 , 0.314424E+02 , 0.317763E+02 , 0.321133E+02 , +0.324536E+02 , 0.327971E+02 , 0.331440E+02 , 0.334940E+02 , 0.338475E+02 , +0.342050E+02 , 0.345654E+02 , 0.349295E+02 , 0.352975E+02 , 0.356687E+02 , +0.360430E+02 , 0.364221E+02 , 0.368042E+02 , 0.371896E+02 , 0.375790E+02 , +0.379725E+02 , 0.383692E+02 , 0.387702E+02 , 0.391744E+02 , 0.395839E+02 , +0.399958E+02 , 0.404118E+02 , 0.408325E+02 , 0.412574E+02 , 0.416858E+02 , +0.421188E+02 , 0.425551E+02 , 0.429962E+02 , 0.434407E+02 , 0.438910E+02 , +0.443439E+02 , 0.448024E+02 , 0.452648E+02 , 0.457308E+02 , 0.462018E+02 , +0.466775E+02 , 0.471582E+02 , 0.476428E+02 , 0.481313E+02 , 0.486249E+02 , +0.491235E+02 , 0.496272E+02 , 0.501349E+02 , 0.506479E+02 , 0.511652E+02 , +0.516876E+02 , 0.522142E+02 , 0.527474E+02 , 0.532836E+02 , 0.538266E+02 , +0.543737E+02 , 0.549254E+02 , 0.554839E+02 , 0.560456E+02 , 0.566142E+02 , +0.571872E+02 , 0.577662E+02 , 0.583498E+02 , 0.589392E+02 , 0.595347E+02 , +0.601346E+02 , 0.607410E+02 , 0.613519E+02 , 0.619689E+02 , 0.625922E+02 , +0.632204E+02 , 0.638550E+02 , 0.644959E+02 , 0.651418E+02 , 0.657942E+02 , +0.664516E+02 , 0.671158E+02 , 0.677864E+02 , 0.684624E+02 , 0.691451E+02 , +0.698345E+02 , 0.705293E+02 , 0.712312E+02 , 0.719398E+02 , 0.726542E+02 , +0.733754E+02 , 0.741022E+02 , 0.748363E+02 , 0.755777E+02 , 0.763247E+02 , +0.770791E+02 , 0.778394E+02 , 0.786088E+02 , 0.793824E+02 , 0.801653E+02 , +0.809542E+02 , 0.817509E+02 , 0.825536E+02 , 0.833643E+02 , 0.841828E+02 , +0.850076E+02 , 0.858405E+02 , 0.866797E+02 , 0.875289E+02 , 0.883827E+02 , +0.892467E+02 , 0.901172E+02 , 0.909962E+02 , 0.918818E+02 , 0.927760E+02 , +0.936790E+02 , 0.945887E+02 , 0.955071E+02 , 0.964346E+02 , 0.973689E+02 , +0.983123E+02 , 0.992648E+02 , 0.100224E+03 , 0.101193E+03 , 0.102169E+03 , +0.103155E+03 , 0.104150E+03 , 0.105152E+03 , 0.106164E+03 , 0.107186E+03 , +0.108217E+03 , 0.109256E+03 , 0.110303E+03 , 0.111362E+03 , 0.112429E+03 , +0.113503E+03 , 0.114588E+03 , 0.115684E+03 , 0.116789E+03 , 0.117903E+03 , +0.119028E+03 , 0.120160E+03 , 0.121306E+03 , 0.122460E+03 , 0.123623E+03 , +0.124796E+03 , 0.125981E+03 , 0.127174E+03 , 0.128381E+03 , 0.129594E+03 , +0.130822E+03 , 0.132058E+03 , 0.133306E+03 , 0.134563E+03 , 0.135828E+03 , +0.137109E+03 , 0.138402E+03 , 0.139700E+03 , 0.141017E+03 , 0.142338E+03 , +0.143676E+03 , 0.145025E+03 , 0.146382E+03 , 0.147753E+03 , 0.149133E+03 , +0.150529E+03 , 0.151935E+03 , 0.153351E+03 , 0.154783E+03 , 0.156222E+03 , +0.157678E+03 , 0.159148E+03 , 0.160624E+03 , 0.162117E+03 , 0.163621E+03 , +0.165142E+03 , 0.166674E+03 , 0.168212E+03 , 0.169772E+03 , 0.171340E+03 , +0.172921E+03 , 0.174522E+03 , 0.176129E+03 , 0.177755E+03 , 0.179388E+03 , +0.181040E+03 , 0.182707E+03 , 0.184382E+03 , 0.186076E+03 , 0.187782E+03 , +0.189503E+03 , 0.191240E+03 , 0.192989E+03 , 0.194758E+03 , 0.196535E+03 , +0.198332E+03 , 0.200141E+03 , 0.201963E+03 , 0.203805E+03 , 0.205656E+03 , +0.207532E+03 , 0.209416E+03 , 0.211317E+03 , 0.213236E+03 , 0.215167E+03 , +0.217121E+03 , 0.219087E+03 , 0.221067E+03 , 0.223064E+03 , 0.225080E+03 , +0.227113E+03 , 0.229160E+03 , 0.231221E+03 , 0.233305E+03 , 0.235403E+03 , +0.237520E+03 , 0.239655E+03 , 0.241805E+03 , 0.243979E+03 , 0.246163E+03 , +0.248365E+03 , 0.250593E+03 , 0.252830E+03 , 0.255093E+03 , 0.257364E+03 , +0.259667E+03 , 0.261979E+03 , 0.264312E+03 , 0.266666E+03 , 0.269034E+03 , +0.271430E+03 , 0.273841E+03 , 0.276268E+03 , 0.278722E+03 , 0.281185E+03 , +0.283677E+03 , 0.286190E+03 , 0.288714E+03 , 0.291266E+03 , 0.293834E+03 , +0.296431E+03 , 0.299045E+03 , 0.301676E+03 , 0.304329E+03 , 0.307006E+03 , +0.309706E+03 , 0.312423E+03 , 0.315165E+03 , 0.317930E+03 , 0.320705E+03 , +0.323519E+03 , 0.326350E+03 , 0.329199E+03 , 0.332073E+03 , 0.334973E+03 , +0.337897E+03 , 0.340839E+03 , 0.343800E+03 , 0.346794E+03 , 0.349806E+03 , +0.352845E+03 , 0.355918E+03 , 0.358994E+03 , 0.362112E+03 , 0.365242E+03 , +0.368407E+03 , 0.371599E+03 , 0.374802E+03 , 0.378042E+03 , 0.381293E+03 , +0.384588E+03 , 0.387904E+03 , 0.391239E+03 , 0.394604E+03 , 0.397988E+03 , +0.401411E+03 , 0.404862E+03 , 0.408326E+03 , 0.411829E+03 , 0.415352E+03 , +0.418906E+03 , 0.422490E+03 , 0.426095E+03 , 0.429740E+03 , 0.433398E+03 , +0.437097E+03 , 0.440827E+03 , 0.444570E+03 , 0.448354E+03 , 0.452160E+03 , +0.455999E+03 , 0.459870E+03 , 0.463765E+03 , 0.467702E+03 , 0.471652E+03 , +0.475646E+03 , 0.479674E+03 , 0.483715E+03 , 0.487811E+03 , 0.491911E+03 , +0.496065E+03 , 0.500244E+03 , 0.504448E+03 , 0.508698E+03 , 0.512961E+03 , +0.517282E+03 , 0.521617E+03 , 0.525989E+03 , 0.530397E+03 , 0.534831E+03 , +0.539313E+03 , 0.543821E+03 , 0.548355E+03 , 0.552938E+03 , 0.557549E+03 , +0.562197E+03 , 0.566884E+03 , 0.571598E+03 , 0.576351E+03 , 0.581131E+03 , +0.585963E+03 , 0.590835E+03 , 0.595722E+03 , 0.600663E+03 , 0.605631E+03 , +0.610641E+03 , 0.615151E+03 , 0.619625E+03 , 0.624140E+03 , 0.628671E+03 , +0.633243E+03 , 0.637845E+03 , 0.642465E+03 , 0.647126E+03 , 0.651806E+03 , +0.656527E+03 , 0.661279E+03 , 0.666049E+03 , 0.670861E+03 , 0.675692E+03 , +0.680566E+03 , 0.685471E+03 , 0.690396E+03 , 0.695363E+03 , 0.700350E+03 , +0.705381E+03 , 0.710444E+03 , 0.715527E+03 , 0.720654E+03 , 0.725801E+03 , +0.730994E+03 , 0.736219E+03 , 0.741465E+03 , 0.746756E+03 , 0.752068E+03 , +0.757426E+03 , 0.762819E+03 , 0.768231E+03 , 0.773692E+03 , 0.779172E+03 , +0.784701E+03 , 0.790265E+03 , 0.795849E+03 , 0.801483E+03 , 0.807137E+03 , +0.812842E+03 , 0.818582E+03 , 0.824343E+03 , 0.830153E+03 , 0.835987E+03 , +0.841871E+03 , 0.847791E+03 , 0.853733E+03 , 0.859727E+03 , 0.865743E+03 , +0.871812E+03 , 0.877918E+03 , 0.884046E+03 , 0.890228E+03 , 0.896433E+03 , +0.902690E+03 , 0.908987E+03 , 0.915307E+03 , 0.921681E+03 , 0.928078E+03 , +0.934531E+03 , 0.941023E+03 , 0.947539E+03 , 0.954112E+03 , 0.960708E+03 , +0.967361E+03 , 0.974053E+03 , 0.980771E+03 , 0.987545E+03 , 0.994345E+03 , +0.100120E+04 , 0.100810E+04 , 0.101502E+04 , 0.102201E+04 , 0.102902E+04 , +0.103608E+04 , 0.104320E+04 , 0.105033E+04 , 0.105753E+04 , 0.106475E+04 , +0.107204E+04 , 0.107936E+04 , 0.108672E+04 , 0.109414E+04 , 0.110158E+04 , +0.110908E+04 , 0.111663E+04 , 0.112421E+04 , 0.113185E+04 , 0.113952E+04 , +0.114725E+04 , 0.115503E+04 , 0.116284E+04 , 0.117071E+04 , 0.117861E+04 , +0.118658E+04 , 0.119459E+04 , 0.120264E+04 , 0.121074E+04 , 0.121888E+04 , +0.122709E+04 , 0.123534E+04 , 0.124362E+04 , 0.125198E+04 , 0.126036E+04 , +0.126881E+04 , 0.127731E+04 , 0.128584E+04 , 0.129444E+04 , 0.130307E+04 , +0.131177E+04 , 0.132053E+04 , 0.132931E+04 , 0.133817E+04 , 0.134705E+04 , +0.135602E+04 , 0.136503E+04 , 0.137407E+04 , 0.138319E+04 , 0.139234E+04 , +0.140156E+04 , 0.141084E+04 , 0.142015E+04 , 0.142954E+04 , 0.143896E+04 , +0.144845E+04 , 0.145800E+04 , 0.146759E+04 , 0.147725E+04 , 0.148694E+04 , +0.149672E+04 , 0.150655E+04 , 0.151641E+04 , 0.152635E+04 , 0.153633E+04 , +0.154639E+04 , 0.155650E+04 , 0.156665E+04 , 0.157688E+04 , 0.158715E+04 , +0.159750E+04 , 0.160791E+04 , 0.161836E+04 , 0.162888E+04 , 0.163945E+04 , +0.165010E+04 , 0.166081E+04 , 0.167155E+04 , 0.168238E+04 , 0.169325E+04 , +0.170420E+04 , 0.171522E+04 , 0.172627E+04 , 0.173741E+04 , 0.174859E+04 , +0.175986E+04 , 0.177119E+04 , 0.178256E+04 , 0.179402E+04 , 0.180552E+04 , +0.181711E+04 , 0.182877E+04 , 0.184046E+04 , 0.185224E+04 , 0.186407E+04 , +0.187599E+04 , 0.188797E+04 , 0.190000E+04 , 0.191212E+04 , 0.192428E+04 , +0.193653E+04 , 0.194886E+04 , 0.196122E+04 , 0.197368E+04 , 0.198618E+04 , +0.199878E+04 , 0.201145E+04 , 0.202416E+04 , 0.203698E+04 , 0.204983E+04 , +0.206278E+04 , 0.207580E+04 , 0.208887E+04 , 0.210204E+04 , 0.211525E+04 , +0.212856E+04 , 0.214195E+04 , 0.215538E+04 , 0.216892E+04 , 0.218249E+04 , +0.219618E+04 , 0.220994E+04 , 0.222375E+04 , 0.223766E+04 , 0.225161E+04 , +0.226567E+04 , 0.227981E+04 , 0.229399E+04 , 0.230829E+04 , 0.232263E+04 , +0.233708E+04 , 0.235161E+04 , 0.236618E+04 , 0.238087E+04 , 0.239560E+04 , +0.241044E+04 , 0.242538E+04 , 0.244035E+04 , 0.245544E+04 , 0.247057E+04 , +0.248583E+04 , 0.250116E+04 , 0.251654E+04 , 0.253204E+04 , 0.254759E+04 , +0.256325E+04 , 0.257901E+04 , 0.259480E+04 , 0.261073E+04 , 0.262670E+04 , +0.264279E+04 , 0.265896E+04 , 0.267519E+04 , 0.269154E+04 , 0.270794E+04 , +0.272447E+04 , 0.274108E+04 , 0.275774E+04 , 0.277453E+04 , 0.279137E+04 , +0.280834E+04 , 0.282540E+04 , 0.284251E+04 , 0.285975E+04 , 0.287704E+04 , +0.289446E+04 , 0.291198E+04 , 0.292954E+04 , 0.294725E+04 , 0.296499E+04 , +0.298288E+04 , 0.300087E+04 , 0.301890E+04 , 0.303707E+04 , 0.305529E+04 , +0.307365E+04 , 0.309211E+04 , 0.311062E+04 , 0.312927E+04 , 0.314798E+04 , +0.316682E+04 , 0.318577E+04 , 0.320477E+04 , 0.322391E+04 , 0.324310E+04 , +0.326245E+04 , 0.328189E+04 , 0.330138E+04 , 0.332103E+04 , 0.334073E+04 , +0.336058E+04 , 0.338053E+04 , 0.340054E+04 , 0.342069E+04 , 0.344090E+04 , +0.346127E+04 , 0.348174E+04 , 0.350227E+04 , 0.352295E+04 , 0.354369E+04 , +0.356458E+04 , 0.358559E+04 , 0.360664E+04 , 0.362787E+04 , 0.364914E+04 , +0.367058E+04 , 0.369212E+04 , 0.371373E+04 , 0.373548E+04 , 0.375731E+04 , +0.377929E+04 , 0.380139E+04 , 0.382355E+04 , 0.384588E+04 , 0.386826E+04 , +0.389081E+04 , 0.391348E+04 , 0.393620E+04 , 0.395910E+04 , 0.398205E+04 , +0.400518E+04 , 0.402843E+04 , 0.405173E+04 , 0.407520E+04 , 0.409875E+04 , +0.412246E+04 , 0.414630E+04 , 0.417019E+04 , 0.419427E+04 , 0.421840E+04 , +0.424272E+04 , 0.426715E+04 , 0.429165E+04 , 0.431634E+04 , 0.434108E+04 , +0.436602E+04 , 0.439107E+04 , 0.441618E+04 , 0.444149E+04 , 0.446685E+04 , +0.449241E+04 , 0.451810E+04 , 0.454385E+04 , 0.456977E+04 , 0.459578E+04 , +0.462197E+04 , 0.464830E+04 , 0.467468E+04 , 0.470127E+04 , 0.472792E+04 , +0.475477E+04 , 0.478175E+04 , 0.480880E+04 , 0.483605E+04 , 0.486336E+04 , +0.489087E+04 , 0.491853E+04 , 0.494623E+04 , 0.497415E+04 , 0.500215E+04 , +0.503034E+04 , 0.505867E+04 , 0.508707E+04 , 0.511568E+04 , 0.514436E+04 , +0.517325E+04 , 0.520227E+04 , 0.523137E+04 , 0.526068E+04 , 0.529005E+04 , +0.531965E+04 , 0.534939E+04 , 0.537921E+04 , 0.540923E+04 , 0.543932E+04 , +0.546965E+04 , 0.550011E+04 , 0.553064E+04 , 0.556139E+04 , 0.559223E+04 , +0.562329E+04 , 0.565449E+04 , 0.568577E+04 , 0.571727E+04 , 0.574884E+04 , +0.578064E+04 , 0.581261E+04 , 0.584464E+04 , 0.587692E+04 , 0.590924E+04 , +0.594182E+04 , 0.597455E+04 , 0.600736E+04 , 0.604039E+04 , 0.607350E+04 , +0.610685E+04 , 0.614036E+04 , 0.617394E+04 , 0.620777E+04 , 0.624169E+04 , +0.627584E+04 , 0.631014E+04 , 0.634454E+04 , 0.637918E+04 , 0.641390E+04 , +0.644887E+04 , 0.648400E+04 , 0.651919E+04 , 0.655467E+04 , 0.659021E+04 , +0.662599E+04 , 0.666197E+04 , 0.669800E+04 , 0.673429E+04 , 0.677069E+04 , +0.680735E+04 , 0.684415E+04 , 0.688104E+04 , 0.691819E+04 , 0.695543E+04 , +0.699292E+04 , 0.703061E+04 , 0.706837E+04 , 0.710639E+04 , 0.714451E+04 , +0.718289E+04 , 0.722143E+04 , 0.726009E+04 , 0.729903E+04 , 0.733802E+04 , +0.737729E+04 , 0.741676E+04 , 0.745631E+04 , 0.749612E+04 , 0.753602E+04 , +0.757622E+04 , 0.761659E+04 , 0.765705E+04 , 0.769780E+04 , 0.773863E+04 , +0.777975E+04 , 0.782106E+04 , 0.786246E+04 , 0.790412E+04 , 0.794593E+04 , +0.798802E+04 , 0.803028E+04 , 0.807259E+04 , 0.811525E+04 , 0.815798E+04 , +0.820102E+04 , 0.824427E+04 , 0.828757E+04 , 0.833120E+04 , 0.837493E+04 , +0.841895E+04 , 0.846313E+04 , 0.850744E+04 , 0.855208E+04 , 0.859678E+04 , +0.864179E+04 , 0.868705E+04 , 0.873237E+04 , 0.877800E+04 , 0.882374E+04 , +0.886979E+04 , 0.891603E+04 , 0.896237E+04 , 0.900904E+04 , 0.905579E+04 , +0.910288E+04 , 0.915018E+04 , 0.919758E+04 , 0.924529E+04 , 0.929310E+04 , +0.934122E+04 , 0.938959E+04 , 0.943804E+04 , 0.948687E+04 , 0.953575E+04 , +0.958494E+04 , 0.963442E+04 , 0.968395E+04 , 0.973384E+04 , 0.978383E+04 , +0.983412E+04 , 0.988468E+04 , 0.993534E+04 , 0.998630E+04 , 0.100374E+05 , +0.100888E+05 , 0.101406E+05 , 0.101923E+05 , 0.102444E+05 , 0.102966E+05 , +0.103492E+05 , 0.104020E+05 , 0.104550E+05 , 0.105082E+05 , 0.105616E+05 , +0.106153E+05 , 0.106693E+05 , 0.107234E+05 , 0.107779E+05 , 0.108325E+05 , +0.108874E+05 , 0.109425E+05 , 0.109978E+05 , 0.110535E+05 , 0.111092E+05 , +0.111653E+05 , 0.112217E+05 , 0.112782E+05 , 0.113350E+05 , 0.113920E+05 , +0.114493E+05 , 0.115070E+05 , 0.115646E+05 , 0.116228E+05 , 0.116809E+05 , +0.117396E+05 , 0.117984E+05 , 0.118574E+05 , 0.119167E+05 , 0.119762E+05 , +0.120360E+05 , 0.120962E+05 , 0.121564E+05 , 0.122170E+05 , 0.122778E+05 , +0.123389E+05 , 0.124004E+05 , 0.124619E+05 , 0.125238E+05 , 0.125859E+05 , +0.126484E+05 , 0.127111E+05 , 0.127739E+05 , 0.128372E+05 , 0.129006E+05 , +0.129644E+05 , 0.130285E+05 , 0.130927E+05 , 0.131573E+05 , 0.132220E+05 , +0.132872E+05 , 0.133526E+05 , 0.134182E+05 , 0.134842E+05 , 0.135503E+05 , +0.136168E+05 , 0.136836E+05 , 0.137505E+05 , 0.138180E+05 , 0.138854E+05 , +0.139534E+05 , 0.140216E+05 , 0.140900E+05 , 0.141588E+05 , 0.142277E+05 , +0.142971E+05 , 0.143668E+05 , 0.144366E+05 , 0.145069E+05 , 0.145773E+05 , +0.146481E+05 , 0.147192E+05 , 0.147905E+05 , 0.148622E+05 , 0.149341E+05 , +0.150064E+05 , 0.150790E+05 , 0.151517E+05 , 0.152250E+05 , 0.152983E+05 , +0.153721E+05 , 0.154462E+05 , 0.155205E+05 , 0.155952E+05 , 0.156701E+05 , +0.157454E+05 , 0.158211E+05 , 0.158969E+05 , 0.159732E+05 , 0.160496E+05 , +0.161265E+05 , 0.162037E+05 , 0.162811E+05 , 0.163589E+05 , 0.164369E+05 , +0.165154E+05 , 0.165942E+05 , 0.166732E+05 , 0.167526E+05 , 0.168322E+05 , +0.169123E+05 , 0.169927E+05 , 0.170733E+05 , 0.171543E+05 , 0.172356E+05 , +0.173173E+05 , 0.173993E+05 , 0.174815E+05 , 0.175643E+05 , 0.176471E+05 , +0.177305E+05 , 0.178143E+05 , 0.178981E+05 , 0.179826E+05 , 0.180671E+05 , +0.181522E+05 , 0.182377E+05 , 0.183232E+05 , 0.184093E+05 , 0.184955E+05 , +0.185823E+05 , 0.186695E+05 , 0.187568E+05 , 0.188447E+05 , 0.189326E+05 , +0.190212E+05 , 0.191101E+05 , 0.191991E+05 , 0.192887E+05 , 0.193785E+05 , +0.194688E+05 , 0.195595E+05 , 0.196503E+05 , 0.197417E+05 , 0.198332E+05 , +0.199253E+05 , 0.200178E+05 , 0.201105E+05 , 0.202036E+05 , 0.202971E+05 , +0.203910E+05 , 0.204853E+05 , 0.205798E+05 , 0.206749E+05 , 0.207701E+05 , +0.208659E+05 , 0.209621E+05 , 0.210584E+05 , 0.211554E+05 , 0.212524E+05 , +0.213501E+05 , 0.214482E+05 , 0.215465E+05 , 0.216452E+05 , 0.217442E+05 , +0.218439E+05 , 0.219439E+05 , 0.220440E+05 , 0.221449E+05 , 0.222457E+05 , +0.223473E+05 , 0.224494E+05 , 0.225514E+05 , 0.226542E+05 , 0.227571E+05 , +0.228606E+05 , 0.229646E+05 , 0.230687E+05 , 0.231734E+05 , 0.232783E+05 , +0.233839E+05 , 0.234898E+05 , 0.235960E+05 , 0.237027E+05 , 0.238097E+05 , +0.239173E+05 , 0.240254E+05 , 0.241335E+05 , 0.242424E+05 , 0.243514E+05 , +0.244611E+05 , 0.245712E+05 , 0.246814E+05 , 0.247923E+05 , 0.249034E+05 , +0.250152E+05 , 0.250152E+05}; + +} // namespace lookuptable +} // namespace ufo + +#endif // UFO_VARIABLETRANSFORMS_LOOKUPTABLE_H_ diff --git a/src/ufo/variabletransforms/TransformBase.cc b/src/ufo/variabletransforms/TransformBase.cc index 0f2a5a300..66fd37080 100644 --- a/src/ufo/variabletransforms/TransformBase.cc +++ b/src/ufo/variabletransforms/TransformBase.cc @@ -14,12 +14,14 @@ namespace ufo { TransformBase::TransformBase(const VariableTransformsParameters& options, ioda::ObsSpace& os, - const std::shared_ptr>& flags) - : options_(options), obsdb_(os) , flags_(*flags) { + const std::shared_ptr>& flags, + const std::vector &apply) + : options_(options), obsdb_(os) , flags_(*flags), apply_(apply) { method_ = formulas::resolveMethods(options.Method.value()); formulation_ = formulas::resolveFormulations(options.Formulation.value(), options.Method.value()); UseValidDataOnly_ = options.UseValidDataOnly.value(); + AllowSuperSaturation_ = options.AllowSuperSaturation.value(); obsName_ = os.obsname(); } @@ -70,7 +72,8 @@ TransformFactory::TransformFactory(const std::string& name) { std::unique_ptr TransformFactory::create( const std::string& name, const VariableTransformsParameters& options, - ioda::ObsSpace& os, const std::shared_ptr>& flags) { + ioda::ObsSpace& os, const std::shared_ptr>& flags, + const std::vector& apply) { oops::Log::trace() << " --> TransformFactory::create" << std::endl; oops::Log::trace() << " --> name: " << name << std::endl; @@ -94,7 +97,7 @@ std::unique_ptr TransformFactory::create( Here()); } - std::unique_ptr ptr = jloc->second->make(options, os, flags); + std::unique_ptr ptr = jloc->second->make(options, os, flags, apply); oops::Log::trace() << "TransformBase::create done" << std::endl; return ptr; diff --git a/src/ufo/variabletransforms/TransformBase.h b/src/ufo/variabletransforms/TransformBase.h index 5279219f8..28919fec6 100644 --- a/src/ufo/variabletransforms/TransformBase.h +++ b/src/ufo/variabletransforms/TransformBase.h @@ -50,7 +50,8 @@ class TransformBase { public: TransformBase(const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr>& flags); + const std::shared_ptr>& flags, + const std::vector& apply); /// Destructor virtual ~TransformBase() {} /// Run variable conversion @@ -62,6 +63,7 @@ class TransformBase { formulas::MethodFormulation method_; formulas::MethodFormulation formulation_; bool UseValidDataOnly_; + bool AllowSuperSaturation_; /// The observation name std::string obsName_; @@ -72,6 +74,7 @@ class TransformBase { formulas::MethodFormulation method() const { return method_; } formulas::MethodFormulation formulation() const { return formulation_; } bool UseValidDataOnly() const { return UseValidDataOnly_; } + bool AllowSuperSaturation() const { return AllowSuperSaturation_; } void SetUseValidDataOnly(bool t) {UseValidDataOnly_ = t; } /// subclasses to access the observation name std::string obsName() const { return obsName_; } @@ -80,6 +83,7 @@ class TransformBase { /// Observation space ioda::ObsSpace &obsdb_; const ioda::ObsDataVector &flags_; + const std::vector &apply_; /// Missing value (float) const float missingValueFloat = util::missingValue(1.0f); /// output tag for derived parameters @@ -91,7 +95,8 @@ class TransformFactory { public: static std::unique_ptr create( const std::string &, const VariableTransformsParameters &, - ioda::ObsSpace &, const std::shared_ptr> &); + ioda::ObsSpace &, const std::shared_ptr> &, + const std::vector &); virtual ~TransformFactory() = default; protected: @@ -100,7 +105,8 @@ class TransformFactory { private: virtual std::unique_ptr make( const VariableTransformsParameters &, ioda::ObsSpace &, - const std::shared_ptr> &) = 0; + const std::shared_ptr> &, + const std::vector &) = 0; static std::map &getMakers() { static std::map makers_; return makers_; @@ -112,8 +118,9 @@ template class TransformMaker : public TransformFactory { virtual std::unique_ptr make( const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags) { - return std::unique_ptr(new T(options, os, flags)); + const std::shared_ptr> &flags, + const std::vector &apply) { + return std::unique_ptr(new T(options, os, flags, apply)); } public: diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a08a372e9..7d492b888 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -280,6 +280,7 @@ list( APPEND ufo_test_input testinput/unit_tests/function_satwindsSPDBcheck.yaml testinput/unit_tests/function_wdirdiff.yaml testinput/unit_tests/variabletransforms_rhumidity.yaml + testinput/unit_tests/variabletransforms_rhumidity_part2.yaml testinput/unit_tests/variabletransforms_shumidity.yaml testinput/unit_tests/variabletransforms_SondePressureFromHeight.yaml testinput/unit_tests/variabletransforms_SondePressureFromHeightICAO.yaml @@ -2441,6 +2442,13 @@ ecbuild_add_test( TARGET test_ufo_variabletransforms_humidityrelative DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_variabletransforms_humidityrelative_part2 + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x + ARGS "testinput/unit_tests/variabletransforms_rhumidity_part2.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFilters.x + TEST_DEPENDS ufo_get_ioda_test_data ) + ecbuild_add_test( TARGET test_ufo_variabletransforms_humidityspecific COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x ARGS "testinput/unit_tests/variabletransforms_shumidity.yaml" diff --git a/test/testinput/unit_tests/variabletransforms_rhumidity.yaml b/test/testinput/unit_tests/variabletransforms_rhumidity.yaml index 9e0beb029..4747a43ad 100644 --- a/test/testinput/unit_tests/variabletransforms_rhumidity.yaml +++ b/test/testinput/unit_tests/variabletransforms_rhumidity.yaml @@ -28,25 +28,6 @@ observations: obs filters: - filter: Variable Transforms Transform: ["RelativeHumidity"] - Method: UKMO # Using UKMO method and UKMO default formulation - compareVariables: # test output matches precalculated values - - reference: - name: relative_humidity_UKMO@TestReference - test: - name: relative_humidity@DerivedValue - absTol: 1.0e-5 - passedBenchmark: 99 - -# Exercise the method using UKMO saturation vapor pressure method. -- obs space: - name: test_relative_humidity2 - obsdatain: - obsfile: Data/ufo/testinput_tier_1/sfc_obs_2018041500_metars_small.nc - simulated variables: [specific_humidity, air_temperature, surface_pressure] - obs filters: - - filter: Variable Transforms - Transform: ["RelativeHumidity"] - Method: UKMO # Using UKMO method Formulation: Sonntag # Using Sonntag formulation (whihc is the default formulation for UKMO method) compareVariables: # test output matches precalculated values - reference: diff --git a/test/testinput/unit_tests/variabletransforms_rhumidity_part2.yaml b/test/testinput/unit_tests/variabletransforms_rhumidity_part2.yaml new file mode 100644 index 000000000..0a2aa51af --- /dev/null +++ b/test/testinput/unit_tests/variabletransforms_rhumidity_part2.yaml @@ -0,0 +1,24 @@ +window begin: 2018-06-18T21:00:00Z +window end: 2018-06-19T03:00:00Z + +observations: +- obs space: + name: test_relative_humidity1 + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_conversion_td2Rh_surface.nc4 + obsgrouping: + group variables: ["station_id"] + sort variable: "datetime" + sort order: "descending" + simulated variables: [dew_point_temperature_surface, air_temperature_surface, pressure_surface] + obs filters: + - filter: Variable Transforms + Transform: ["RelativeHumidity"] + Method: UKMO + AllowSuperSaturation: true + compareVariables: + - reference: + name: relative_humidity_reference@ObsValue + test: + name: relative_humidity_surface@DerivedValue + relTol: 1.0e-5 From 202ec54f960bb09873d81b75fe2d8b3f0ead3502 Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Fri, 18 Jun 2021 08:09:17 -0600 Subject: [PATCH 13/98] change the order (#1281) --- test/CMakeLists.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7d492b888..f79ed878f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -325,15 +325,17 @@ function(ADD_DOWNLOAD_TEST repo test_files_dirname output_data_path) # If REPO branch is being built set GIT_BRANCH_FUNC to REPO's current branch. # If a tagged version of REPO is being built set GIT_TAG_FUNC to REPO's current tag. find_branch_name(REPO_DIR_NAME ${repo}) + + if ( ${repo} STREQUAL "crtm" ) + set( GIT_TAG_FUNC ${CRTM_COEFFS_BRANCH} ) + endif() + if( DEFINED GIT_BRANCH_FUNC ) set( REPO_GIT_BRANCH ${GIT_BRANCH_FUNC} ) elseif( DEFINED GIT_TAG_FUNC ) set( REPO_GIT_BRANCH ${GIT_TAG_FUNC} ) endif() - if ( ${repo} STREQUAL "crtm" ) - set( GIT_TAG_FUNC ${CRTM_COEFFS_BRANCH} ) - endif() # When env veriable LOCAL_PATH_JEDI_TESTFILES is set, simply link test files # to build directory. get_${repo}_test_data checks the existence of test file directory. From 8925df2fa098c12007f035464efa251332f1026d Mon Sep 17 00:00:00 2001 From: Neill Bowler <45592137+neill-b@users.noreply.github.com> Date: Fri, 18 Jun 2021 17:15:27 +0100 Subject: [PATCH 14/98] Add Met Office's refractivity operator (#1171) * First version of new refractivity calculation. * First version of unified refractivity code. * Integration of forward operator with new code First version of changes to allow forward operator to work with the unified refractivity code. * First working version Runs through 1DVar, and gives same results as current routines. * Adjusting to allow for a unfified refractivity. * First version which has added new directory * Changes to make use of the unified refractivity code * Adjusting the alignments. * More aligning lines. * Move NBAM refractivity operator to a differently-named directory. First working version of forward model for Met Office refractivity. * Add optional arguments needed for ground GNSS interaction, and remove duplicate QC test for bending angle. * Minor bug fixes * Mostly white-space changes - this version compares with the updated version on develop. * Remove TL and AD code, since this has not yet been implemented. Change name of header guards to fit in with naming convention. * Moved obs file for gnssro refractivity from ioda to ufo. * Remove redundant code. * Remove Met Office refractivity operator, since this is separate to the original set of changes. * Bug fixes from removing refractivity operator. * Bug fix - make sure that nstate is initialised * Various minor changes in response to Chris's review. * Revert "Merge branch 'feature/unify_refractivity' into feature/gnssro_refractivity_operator" This reverts commit 794d6dc1a01b03f4c1c47beeac5f915d908fdc10, reversing changes made to 0f818775e18773eb3cee1ac0d88f7d8d8df8f4b0. * Bug fix and review comments from unify_refractivity branch. * Minor changes from previous PR review. * Various changes following review. * Fix typo. * Remove unnecessary declarations in header files. * Remove print which is called in normal running, and add second test. * Testing build system * Refreshed testing data Co-authored-by: orlewis Co-authored-by: Stephen Herbener Co-authored-by: Ryan Honeyager --- .../BendMetOffice/ObsGnssroBendMetOffice.h | 2 - src/ufo/gnssro/CMakeLists.txt | 21 +- src/ufo/gnssro/RefMetOffice/CMakeLists.txt | 13 + .../RefMetOffice/ObsGnssroRefMetOffice.cc | 66 +++ .../RefMetOffice/ObsGnssroRefMetOffice.h | 64 +++ .../ObsGnssroRefMetOffice.interface.F90 | 99 +++++ .../ObsGnssroRefMetOffice.interface.h | 35 ++ .../ufo_gnssro_refmetoffice_mod.F90 | 393 ++++++++++++++++++ .../gnssro/{Ref => RefNBAM}/CMakeLists.txt | 2 +- .../gnssro/{Ref => RefNBAM}/ObsGnssroRef.cc | 2 +- .../gnssro/{Ref => RefNBAM}/ObsGnssroRef.h | 8 +- .../ObsGnssroRef.interface.F90 | 0 .../{Ref => RefNBAM}/ObsGnssroRef.interface.h | 6 +- .../{Ref => RefNBAM}/ObsGnssroRefTLAD.cc | 2 +- .../{Ref => RefNBAM}/ObsGnssroRefTLAD.h | 8 +- .../ObsGnssroRefTLAD.interface.F90 | 0 .../ObsGnssroRefTLAD.interface.h | 6 +- .../{Ref => RefNBAM}/ufo_gnssro_ref_mod.F90 | 0 .../ufo_gnssro_ref_tlad_mod.F90 | 0 test/CMakeLists.txt | 8 + test/testinput/gnssrorefmetoffice.yaml | 54 +++ 21 files changed, 761 insertions(+), 28 deletions(-) create mode 100644 src/ufo/gnssro/RefMetOffice/CMakeLists.txt create mode 100644 src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc create mode 100644 src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h create mode 100644 src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.F90 create mode 100644 src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.h create mode 100644 src/ufo/gnssro/RefMetOffice/ufo_gnssro_refmetoffice_mod.F90 rename src/ufo/gnssro/{Ref => RefNBAM}/CMakeLists.txt (95%) rename src/ufo/gnssro/{Ref => RefNBAM}/ObsGnssroRef.cc (97%) rename src/ufo/gnssro/{Ref => RefNBAM}/ObsGnssroRef.h (88%) rename src/ufo/gnssro/{Ref => RefNBAM}/ObsGnssroRef.interface.F90 (100%) rename src/ufo/gnssro/{Ref => RefNBAM}/ObsGnssroRef.interface.h (86%) rename src/ufo/gnssro/{Ref => RefNBAM}/ObsGnssroRefTLAD.cc (98%) rename src/ufo/gnssro/{Ref => RefNBAM}/ObsGnssroRefTLAD.h (89%) rename src/ufo/gnssro/{Ref => RefNBAM}/ObsGnssroRefTLAD.interface.F90 (100%) rename src/ufo/gnssro/{Ref => RefNBAM}/ObsGnssroRefTLAD.interface.h (88%) rename src/ufo/gnssro/{Ref => RefNBAM}/ufo_gnssro_ref_mod.F90 (100%) rename src/ufo/gnssro/{Ref => RefNBAM}/ufo_gnssro_ref_tlad_mod.F90 (100%) create mode 100644 test/testinput/gnssrorefmetoffice.yaml diff --git a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.h b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.h index 4fb2c733e..3e9d63d92 100644 --- a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.h +++ b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.h @@ -55,8 +55,6 @@ class ObsGnssroBendMetOffice : public ObsOperatorBase, F90hop keyOperGnssroBendMetOffice_; const ioda::ObsSpace& odb_; std::unique_ptr varin_; - bool vert_interp_ops; // Whether to use interpolation in ln(p) or exner - bool pseudo_ops; // Whether to use pseudo levels }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/gnssro/CMakeLists.txt b/src/ufo/gnssro/CMakeLists.txt index c561d69d7..d058f0570 100644 --- a/src/ufo/gnssro/CMakeLists.txt +++ b/src/ufo/gnssro/CMakeLists.txt @@ -1,5 +1,6 @@ ########################## -add_subdirectory( Ref ) +add_subdirectory( RefNBAM ) +add_subdirectory( RefMetOffice ) add_subdirectory( BndNBAM ) add_subdirectory( BndROPP1D ) add_subdirectory( BndROPP2D ) @@ -7,16 +8,18 @@ add_subdirectory( utils ) add_subdirectory( QC ) add_subdirectory( BendMetOffice ) -PREPEND( _p_ref_files "gnssro/Ref" ${ref_src_files} ) -PREPEND( _p_bndnbam_files "gnssro/BndNBAM" ${bndnbam_src_files} ) -PREPEND( _p_bndropp1d_files "gnssro/BndROPP1D" ${bndropp1d_src_files} ) -PREPEND( _p_bndropp2d_files "gnssro/BndROPP2D" ${bndropp2d_src_files} ) -PREPEND( _p_utils_files "gnssro/utils" ${utils_src_files} ) -PREPEND( _p_qc_files "gnssro/QC" ${qc_src_files} ) -PREPEND( _p_metoffice_files "gnssro/BendMetOffice" ${bendmetoffice_src_files} ) +PREPEND( _p_refnbam_files "gnssro/RefNBAM" ${refnbam_src_files} ) +PREPEND( _p_refmetoffice_files "gnssro/RefMetOffice" ${refmetoffice_src_files} ) +PREPEND( _p_bndnbam_files "gnssro/BndNBAM" ${bndnbam_src_files} ) +PREPEND( _p_bndropp1d_files "gnssro/BndROPP1D" ${bndropp1d_src_files} ) +PREPEND( _p_bndropp2d_files "gnssro/BndROPP2D" ${bndropp2d_src_files} ) +PREPEND( _p_utils_files "gnssro/utils" ${utils_src_files} ) +PREPEND( _p_qc_files "gnssro/QC" ${qc_src_files} ) +PREPEND( _p_metoffice_files "gnssro/BendMetOffice" ${bendmetoffice_src_files} ) set ( gnssro_src_files - ${_p_ref_files} + ${_p_refnbam_files} + ${_p_refmetoffice_files} ${_p_bndnbam_files} ${_p_bndropp1d_files} ${_p_bndropp2d_files} diff --git a/src/ufo/gnssro/RefMetOffice/CMakeLists.txt b/src/ufo/gnssro/RefMetOffice/CMakeLists.txt new file mode 100644 index 000000000..29852e138 --- /dev/null +++ b/src/ufo/gnssro/RefMetOffice/CMakeLists.txt @@ -0,0 +1,13 @@ +# (C) British Crown Copyright 2020 Met Office +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +set ( refmetoffice_src_files + ObsGnssroRefMetOffice.h + ObsGnssroRefMetOffice.cc + ObsGnssroRefMetOffice.interface.h + ObsGnssroRefMetOffice.interface.F90 + ufo_gnssro_refmetoffice_mod.F90 +PARENT_SCOPE +) diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc new file mode 100644 index 000000000..c77c08445 --- /dev/null +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc @@ -0,0 +1,66 @@ +/* + * (C) British Crown Copyright 2020 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h" + +#include +#include +#include + +#include "ioda/ObsVector.h" + +#include "oops/base/Variables.h" +#include "oops/util/Logger.h" + +#include "ufo/GeoVaLs.h" +#include "ufo/ObsDiagnostics.h" + +namespace ufo { + +// ----------------------------------------------------------------------------- +static ObsOperatorMaker makerGnssroRefMetOffice_("GnssroRefMetOffice"); +// ----------------------------------------------------------------------------- + +ObsGnssroRefMetOffice::ObsGnssroRefMetOffice(const ioda::ObsSpace & odb, + const eckit::Configuration & config) + : ObsOperatorBase(odb, config), keyOperGnssroRefMetOffice_(0), odb_(odb), varin_() +{ + const std::vector vv{"air_pressure_levels", "specific_humidity", + "geopotential_height", "geopotential_height_levels"}; + varin_.reset(new oops::Variables(vv)); + + const eckit::LocalConfiguration obsOptions(config, "obs options"); + const eckit::Configuration *configc = &obsOptions; + ufo_gnssro_refmetoffice_setup_f90(keyOperGnssroRefMetOffice_, &configc); + + oops::Log::trace() << "ObsGnssroRefMetOffice created." << std::endl; +} + +// ----------------------------------------------------------------------------- + +ObsGnssroRefMetOffice::~ObsGnssroRefMetOffice() { + ufo_gnssro_refmetoffice_delete_f90(keyOperGnssroRefMetOffice_); + oops::Log::trace() << "ObsGnssroRefMetOffice destructed" << std::endl; +} + +// ----------------------------------------------------------------------------- + +void ObsGnssroRefMetOffice::simulateObs(const GeoVaLs & gom, ioda::ObsVector & ovec, + ObsDiagnostics & ydiags) const { + ufo_gnssro_refmetoffice_simobs_f90(keyOperGnssroRefMetOffice_, gom.toFortran(), odb_, + ovec.size(), ovec.toFortran(), ydiags.toFortran()); +} + +// ----------------------------------------------------------------------------- + +void ObsGnssroRefMetOffice::print(std::ostream & os) const { + os << "ObsGnssroRefMetOffice::print not implemented"; +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h new file mode 100644 index 000000000..869958aa4 --- /dev/null +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h @@ -0,0 +1,64 @@ +/* + * (C) British Crown Copyright 2020 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICE_H_ +#define UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICE_H_ + +#include +#include +#include + +#include "oops/base/Variables.h" +#include "oops/util/ObjectCounter.h" +#include "ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.h" +#include "ufo/ObsOperatorBase.h" + +namespace eckit { + class Configuration; +} + +namespace ioda { + class ObsSpace; + class ObsVector; +} + +namespace ufo { + class GeoVaLs; + class ObsDiagnostics; + +// ----------------------------------------------------------------------------- + +/// GnssroRefMetOffice observation operator +class ObsGnssroRefMetOffice : public ObsOperatorBase, + private util::ObjectCounter { + public: + static const std::string classname() {return "ufo::ObsGnssroRefMetOffice";} + + ObsGnssroRefMetOffice(const ioda::ObsSpace &, const eckit::Configuration &); + virtual ~ObsGnssroRefMetOffice(); + +// Obs Operator + void simulateObs(const GeoVaLs &, ioda::ObsVector &, ObsDiagnostics &) const override; + +// Other + const oops::Variables & requiredVars() const override {return *varin_;} + + int & toFortran() {return keyOperGnssroRefMetOffice_;} + const int & toFortran() const {return keyOperGnssroRefMetOffice_;} + + private: + void print(std::ostream &) const override; + F90hop keyOperGnssroRefMetOffice_; + const ioda::ObsSpace& odb_; + std::unique_ptr varin_; +}; + +// ----------------------------------------------------------------------------- + +} // namespace ufo + +#endif // UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICE_H_ diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.F90 b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.F90 new file mode 100644 index 000000000..f4fdbd411 --- /dev/null +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.F90 @@ -0,0 +1,99 @@ +! (C) British Crown Copyright 2020 Met Office +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +!> Fortran module to handle gnssro observations - refractivity Met Office 1d operator + +module ufo_gnssro_refmetoffice_mod_c + + use fckit_configuration_module, only: fckit_configuration + use fckit_log_module, only : fckit_log + use iso_c_binding + use ufo_gnssro_refmetoffice_mod + use ufo_geovals_mod + use ufo_geovals_mod_c, only: ufo_geovals_registry + + implicit none + private + +#define LISTED_TYPE ufo_gnssro_RefMetOffice + + !> Linked list interface - defines registry_t type +#include "oops/util/linkedList_i.f" + + !> Global registry + type(registry_t) :: ufo_gnssro_RefMetOffice_registry + + ! ------------------------------------------------------------------------------ +contains + ! ------------------------------------------------------------------------------ + !> Linked list implementation +#include "oops/util/linkedList_c.f" + +! ------------------------------------------------------------------------------ + +subroutine ufo_gnssro_refmetoffice_setup_c(c_key_self, c_conf) bind(c,name='ufo_gnssro_refmetoffice_setup_f90') +implicit none +integer(c_int), intent(inout) :: c_key_self +type(c_ptr), intent(in) :: c_conf + +type(ufo_gnssro_RefMetOffice), pointer :: self +type(fckit_configuration) :: f_conf + +call ufo_gnssro_refmetoffice_registry%setup(c_key_self, self) +f_conf = fckit_configuration(c_conf) + +call self%setup(f_conf) + +end subroutine ufo_gnssro_refmetoffice_setup_c + +! ------------------------------------------------------------------------------ + +subroutine ufo_gnssro_refmetoffice_delete_c(c_key_self) bind(c,name='ufo_gnssro_refmetoffice_delete_f90') +implicit none +integer(c_int), intent(inout) :: c_key_self + +type(ufo_gnssro_RefMetOffice), pointer :: self + +call ufo_gnssro_RefMetOffice_registry%delete(c_key_self,self) + +end subroutine ufo_gnssro_refmetoffice_delete_c + +! ------------------------------------------------------------------------------ + +subroutine ufo_gnssro_refmetoffice_simobs_c(c_key_self, c_key_geovals, c_obsspace, & + c_nobs, c_hofx, c_key_obs_diags) & + bind(c,name='ufo_gnssro_refmetoffice_simobs_f90') + +implicit none +integer(c_int), intent(in) :: c_key_self ! Key giving pointer to self object +integer(c_int), intent(in) :: c_key_geovals ! Key giving pointer to geovals object +type(c_ptr), value, intent(in) :: c_obsspace ! Pointer to obs-space object +integer(c_int), intent(in) :: c_nobs ! Number of observations +real(c_double), intent(inout) :: c_hofx(c_nobs) ! Array of calculated H(x) object +integer(c_int), intent(in) :: c_key_obs_diags ! Key giving pointer to obs diagnostics object + +type(ufo_gnssro_RefMetOffice), pointer :: self ! Self object +type(ufo_geovals), pointer :: obs_diags ! Observations diagnostics +type(ufo_geovals), pointer :: geovals ! Geovals object +character(len=*), parameter :: myname_="ufo_gnssro_refmetoffice_simobs_c" +character(len=200) :: output_message ! Message to be output + +write(output_message, *) 'TRACE: Beginning interface', c_key_obs_diags, c_key_geovals, c_key_self +call fckit_log % info(output_message) + +call ufo_gnssro_RefMetOffice_registry % get(c_key_self, self) +call ufo_geovals_registry % get(c_key_obs_diags, obs_diags) +call ufo_geovals_registry % get(c_key_geovals, geovals) + +call self%simobs(geovals, c_obsspace, c_hofx, obs_diags) + +write(output_message, *) 'TRACE: Finishing interface' +call fckit_log % info(output_message) + +end subroutine ufo_gnssro_refmetoffice_simobs_c + +! ------------------------------------------------------------------------------ + +end module ufo_gnssro_refmetoffice_mod_c diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.h b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.h new file mode 100644 index 000000000..99b1a7647 --- /dev/null +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.h @@ -0,0 +1,35 @@ +/* + * (C) British Crown Copyright 2020 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICE_INTERFACE_H_ +#define UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICE_INTERFACE_H_ + +#include "ioda/ObsSpace.h" +#include "ufo/Fortran.h" + +namespace ufo { + +/// Interface to Fortran UFO routines +/*! + * The core of the UFO is coded in Fortran. + * Here we define the interfaces to the Fortran code. + */ + +extern "C" { +// ----------------------------------------------------------------------------- +// Gnssro refractivity observation operators - (Met Office 1D) +// ----------------------------------------------------------------------------- + void ufo_gnssro_refmetoffice_setup_f90(F90hop &, const eckit::Configuration * const *); + void ufo_gnssro_refmetoffice_delete_f90(F90hop &); + void ufo_gnssro_refmetoffice_simobs_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, + const int &, double &, const F90goms &); +// ----------------------------------------------------------------------------- + +} // extern C + +} // namespace ufo +#endif // UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICE_INTERFACE_H_ diff --git a/src/ufo/gnssro/RefMetOffice/ufo_gnssro_refmetoffice_mod.F90 b/src/ufo/gnssro/RefMetOffice/ufo_gnssro_refmetoffice_mod.F90 new file mode 100644 index 000000000..20ceecada --- /dev/null +++ b/src/ufo/gnssro/RefMetOffice/ufo_gnssro_refmetoffice_mod.F90 @@ -0,0 +1,393 @@ +!------------------------------------------------------------------------------- +! (C) British Crown Copyright 2020 Met Office +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +!------------------------------------------------------------------------------- + +!> Fortran module for gnssro refractivity Met Office forward operator + +module ufo_gnssro_refmetoffice_mod + +use iso_c_binding +use kinds +use ufo_vars_mod +use ufo_geovals_mod +use ufo_geovals_mod_c, only: ufo_geovals_registry +use vert_interp_mod +use lag_interp_mod, only: lag_interp_const, lag_interp_smthWeights +use obsspace_mod +use missing_values_mod +use ufo_utils_refractivity_calculator, only: ufo_calculate_refractivity +use fckit_log_module, only : fckit_log +use ufo_constants_mod, only: & + rd, & ! Gas constant for dry air + grav, & ! Gravitational field strength + mw_ratio, & ! Ratio of molecular weights of water and dry air + c_virtual, & ! Related to mw_ratio + n_alpha, & ! Refractivity constant a + n_beta ! Refractivity constant b + + +implicit none +public :: ufo_gnssro_refmetoffice +private + + !> Fortran derived type for gnssro trajectory +type :: ufo_gnssro_RefMetOffice + logical :: vert_interp_ops + logical :: pseudo_ops + real(kind_real) :: min_temp_grad + contains + procedure :: setup => ufo_gnssro_refmetoffice_setup + procedure :: simobs => ufo_gnssro_refmetoffice_simobs +end type ufo_gnssro_RefMetOffice + +contains + +! ------------------------------------------------------------------------------ +! Get the optional settings for the forward model, and save them in the object +! so that they can be used in the code. +! ------------------------------------------------------------------------------ +subroutine ufo_gnssro_refmetoffice_setup(self, f_conf) + +use fckit_configuration_module, only: fckit_configuration +implicit none +class(ufo_gnssro_RefMetOffice), intent(inout) :: self +type(fckit_configuration), intent(in) :: f_conf + +call f_conf%get_or_die("vert_interp_ops", self % vert_interp_ops) +call f_conf%get_or_die("pseudo_ops", self % pseudo_ops) +call f_conf%get_or_die("min_temp_grad", self % min_temp_grad) + +end subroutine ufo_gnssro_refmetoffice_setup + +! ------------------------------------------------------------------------------ +! 1-dimensional GNSS-RO forward operator for the Met Office system +! ------------------------------------------------------------------------------ +subroutine ufo_gnssro_refmetoffice_simobs(self, geovals_in, obss, hofx, obs_diags) + + implicit none + + ! Arguments to this routine + class(ufo_gnssro_RefMetOffice), intent(in) :: self ! The object in which this operator is contained + type(ufo_geovals), intent(in) :: geovals_in ! The model values, interpolated to the observation locations + real(kind_real), intent(inout) :: hofx(:) ! The model forecast of the observations + type(c_ptr), value, intent(in) :: obss ! The observations, and meta-data for those observations + type(ufo_geovals), intent(inout) :: obs_diags ! Observation diagnostics + + character(len=*), parameter :: myname_ = "ufo_gnssro_refmetoffice_simobs" + integer, parameter :: max_string = 800 + + character(max_string) :: err_msg ! Error message for output + character(max_string) :: message ! General message for output + integer :: nobs ! Number of observations + integer :: ilev ! Loop variable, level number + integer :: iobs ! Loop variable, observation number + type(ufo_geoval), pointer :: q ! Model background values of specific humidity + type(ufo_geoval), pointer :: prs ! Model background values of air pressure + type(ufo_geoval), pointer :: theta_heights ! Model heights of levels containing specific humidity + type(ufo_geoval), pointer :: rho_heights ! Model heights of levels containing air pressure + real(kind_real), allocatable :: obsLat(:) ! Latitude of the observation + real(kind_real), allocatable :: obsLon(:) ! Longitude of the observation + real(kind_real), allocatable :: obs_height(:) ! Geopotential height of the observation + logical :: BAErr ! Was there an error in the calculation? + integer :: iVar ! Loop variable, obs diagnostics variable number + real(kind_real), allocatable :: refractivity(:) ! Refractivity on various model levels + real(kind_real), allocatable :: model_heights(:) ! Geopotential heights that refractivity is calculated on + type(ufo_geovals) :: geovals ! The model values, interpolated to the observation locations + ! and flipped in the vertical (if required) + + write(err_msg,*) "TRACE: ufo_gnssro_refmetoffice_simobs: begin" + call fckit_log%info(err_msg) + + ! If output to refractivity (and heights of the refractivity levels) is needed, + ! then use nval as a way to check whether the array has been initialised (since + ! it is called in a loop). + DO iVar = 1, obs_diags % nvar + IF (obs_diags % variables(ivar) == "refractivity" .OR. & + obs_diags % variables(ivar) == "model_heights") THEN + write(err_msg,*) "TRACE: ufo_gnssro_refmetoffice_simobs: initialising obs_diags for " // & + obs_diags % variables(ivar) + call fckit_log%info(err_msg) + obs_diags % geovals(iVar) % nval = 0 + END IF + END DO + +! check if nlocs is consistent in geovals & hofx + if (geovals_in%nlocs /= size(hofx)) then + write(err_msg,*) myname_, ' error: nlocs inconsistent!' + call abor1_ftn(err_msg) + endif + +! make sure that the geovals are in the correct vertical order (surface first) + call ufo_geovals_copy(geovals_in, geovals) ! dont want to change geovals_in + call ufo_geovals_reorderzdir(geovals, var_prsi, "bottom2top") + + write(message, *) myname_, ' Running Met Office GNSS-RO forward operator with' + call fckit_log%info(message) + write(message, *) 'vert_interp_ops =', self % vert_interp_ops, & + 'pseudo_ops =', self % pseudo_ops + call fckit_log%info(message) + +! get variables from geovals + call ufo_geovals_get_var(geovals, var_q, q) ! specific humidity + call ufo_geovals_get_var(geovals, var_prsi, prs) ! pressure + call ufo_geovals_get_var(geovals, var_z, theta_heights) ! Geopotential height of the normal model levels + call ufo_geovals_get_var(geovals, var_zi, rho_heights) ! Geopotential height of the pressure levels + + write(message, '(A,10I6)') 'Q: ', q%nval, q%nlocs, shape(q%vals) + call fckit_log%info(message) + write(message, '(A,10I6)') 'Pressure: ', prs%nval, prs%nlocs, shape(prs%vals) + call fckit_log%info(message) + + nobs = obsspace_get_nlocs(obss) + +! set obs space struture + allocate(obsLon(nobs)) + allocate(obsLat(nobs)) + allocate(obs_height(nobs)) + + call obsspace_get_db(obss, "MetaData", "longitude", obsLon) + call obsspace_get_db(obss, "MetaData", "latitude", obsLat) + call obsspace_get_db(obss, "MetaData", "obs_height", obs_height) + + obs_loop: do iobs = 1, nobs + + call RefMetOffice_ForwardModel(prs % nval, & + q % nval, & + rho_heights % vals(:,iobs), & + theta_heights % vals(:,iobs), & + prs % vals(:,iobs), & + q % vals(:,iobs), & + self % pseudo_ops, & + self % vert_interp_ops, & + self % min_temp_grad, & + 1, & + obs_height(iobs:iobs), & + obsLat(iobs), & + hofx(iobs:iobs), & + BAErr, & + refractivity, & + model_heights) + + if (BAErr) then + write(err_msg,*) "Error with observation processing ", iobs + call fckit_log % info(err_msg) + end if + + ! If required, then save the refractivity and model heights to the obs diagnostics. + ! Allocate the output arrays on the first iteration. + DO iVar = 1, obs_diags % nvar + IF (obs_diags % variables(ivar) == "refractivity") THEN + IF (iobs == 1) THEN + obs_diags % geovals(iVar) % nval = SIZE(refractivity) + ALLOCATE(obs_diags % geovals(iVar) % vals(SIZE(refractivity), obs_diags % nlocs)) + END IF + + IF (BAerr) THEN + obs_diags % geovals(iVar) % vals(:,iobs) = missing_value(obs_diags % geovals(iVar) % vals(1,1)) + ELSE + obs_diags % geovals(iVar) % vals(:,iobs) = refractivity(:) + END IF + END IF + + IF (obs_diags % variables(ivar) == "model_heights") THEN + IF (iobs == 1) THEN + obs_diags % geovals(iVar) % nval = SIZE(model_heights) + ALLOCATE(obs_diags % geovals(iVar) % vals(SIZE(model_heights), obs_diags % nlocs)) + END IF + + IF (BAerr) THEN + obs_diags % geovals(iVar) % vals(:,iobs) = missing_value(obs_diags % geovals(iVar) % vals(1,1)) + ELSE + obs_diags % geovals(iVar) % vals(:,iobs) = model_heights(:) + END IF + END IF + END DO + end do obs_loop + + deallocate(obsLat) + deallocate(obsLon) + deallocate(obs_height) + call ufo_geovals_delete(geovals) + + write(err_msg,*) "TRACE: ufo_gnssro_refmetoffice_simobs: completed" + call fckit_log%info(err_msg) + +end subroutine ufo_gnssro_refmetoffice_simobs +! ------------------------------------------------------------------------------ + + +SUBROUTINE RefMetOffice_ForwardModel(nlevp, & + nlevq, & + za, & + zb, & + pressure, & + humidity, & + GPSRO_pseudo_ops, & + GPSRO_vert_interp_ops, & + GPSRO_min_temp_grad, & + nobs, & + zobs, & + Latitude, & + ycalc, & + BAErr, & + refractivity, & + model_heights) + +INTEGER, INTENT(IN) :: nlevp ! no. of p levels in state vec. +INTEGER, INTENT(IN) :: nlevq ! no. of theta levels +REAL(kind_real), INTENT(IN) :: za(1:nlevp) ! heights of rho levs +REAL(kind_real), INTENT(IN) :: zb(1:nlevq) ! heights of theta levs +REAL(kind_real), INTENT(IN) :: pressure(1:nlevp) ! Model background pressure +REAL(kind_real), INTENT(IN) :: humidity(1:nlevq) ! Model background specific humidity +LOGICAL, INTENT(IN) :: GPSRO_pseudo_ops ! Option: Use pseudo-levels in vertical interpolation? +LOGICAL, INTENT(IN) :: GPSRO_vert_interp_ops ! Option: Use ln(p) for vertical interpolation? (rather than exner) +REAL(kind_real), INTENT(IN) :: GPSRO_min_temp_grad ! The minimum temperature gradient which is used +INTEGER, INTENT(IN) :: nobs ! Number of observations in the profile +REAL(kind_real), INTENT(IN) :: zobs(1:nobs) ! Impact parameter for the obs +REAL(kind_real), INTENT(IN) :: Latitude ! Latitude of this profile +REAL(kind_real), INTENT(INOUT) :: ycalc(1:nobs) ! Model forecast of the observations +LOGICAL, INTENT(OUT) :: BAErr ! Was an error encountered during the calculation? +REAL(kind_real), INTENT(INOUT), ALLOCATABLE :: refractivity(:) ! Model refractivity on model/pseudo levels +REAL(kind_real), INTENT(INOUT), ALLOCATABLE :: model_heights(:) ! Geopotential heights of the refractivity levels +! +! Things that may need to be output, as they are used by the TL/AD calculation +! +INTEGER :: nRefLevels ! Number of levels in refractivity calculation +REAL(kind_real), ALLOCATABLE :: nr(:) ! Model calculation of impact parameters +! +! Local parameters +! +integer, parameter :: max_string = 800 ! Length of strings +character(len=*), parameter :: myname_ = "Ops_GPSRO_ForwardModel" +! +! Local variables +! +INTEGER :: num_pseudo ! Number of levels, including pseudo levels +REAL(kind_real) :: x(1:nlevp+nlevq) ! state vector +character(max_string) :: err_msg ! Error message to be output +INTEGER :: iObs ! Loop counter, observation number +INTEGER :: iLevel ! Number of model level just above observation +REAL(kind_real) :: temperature(nlevq)! Calculated profile temperature +REAL(kind_real) :: Pb(nlevq) ! Air pressure, interpolated to temperature-levels +REAL(kind_real) :: humidity_ob ! Specific humidity interpolated to observation height +REAL(kind_real) :: T_ob ! Temperature interpolated to observation height +REAL(kind_real) :: P_ob ! Pressure interpolated to observation height +REAL(kind_real) :: log_humid ! Interpolation coefficient for humidity +REAL(kind_real) :: temperature_grad ! Interpolation coefficient for temperature +REAL(kind_real) :: pressure_coeff ! Interpolation coefficient for pressure +REAL(kind_real) :: alpha_interp ! Interpolation coefficient for refractivity +REAL(kind_real) :: beta_interp ! Interpolation coefficient for refractivity +REAL(kind_real) :: model_height_diff ! Difference in height between two model levels +REAL(kind_real) :: obs_height_diff ! Height difference between observation and closest model level + +! The model data must be on a staggered grid, with nlevp = nlevq+1 +IF (nlevp /= nlevq + 1) THEN + write(err_msg,*) myname_ // ':' // ' Data must be on a staggered grid nlevp, nlevq = ', nlevp, nlevq + call fckit_log % warning(err_msg) + write(err_msg,*) myname_ // ':' // ' error: number of levels inconsistent!' + call abor1_ftn(err_msg) +END IF + +BAErr = .FALSE. +ycalc(:) = missing_value(ycalc(1)) + +! Calculate the refractivity on model or pseudo levels +CALL ufo_calculate_refractivity (nlevp, & + nlevq, & + za, & + zb, & + pressure, & + humidity, & + GPSRO_pseudo_ops, & + GPSRO_vert_interp_ops, & + GPSRO_min_temp_grad, & + BAerr, & + nRefLevels, & + refractivity, & + model_heights, & + temperature, & + Pb) + +! Vertically interpolate the model refractivity to the observation locations +DO iObs = 1, nobs + + ! Use separate interpolation of input quantities in refractivity calculation. + ! This does not use the refractivity values calculated above, but uses the + ! calculated temperature, and pressure on temperature-levels. + IF (GPSRO_pseudo_ops) THEN + IF (zb(nlevq) > zobs(iObs) .AND. zb(1) < zobs(iObs)) THEN + ! Find the model level immediately above the observation + iLevel = 1 + DO + IF (zobs(iObs) < zb(iLevel + 1)) EXIT + iLevel = iLevel + 1 + END DO + + model_height_diff = zb(iLevel + 1) - zb(iLevel) + obs_height_diff = zobs(iObs) - zb(iLevel) + + ! Interpolate P,T,q separately + IF (MIN (humidity(iLevel - 1), humidity(iLevel)) > 0.0) THEN + ! humidity varies exponentially with height + log_humid = LOG(humidity(iLevel) / humidity(iLevel + 1)) / model_height_diff + humidity_ob = humidity(iLevel) * EXP(-log_humid * obs_height_diff) + ! Assume linear variation if humidities are -ve + ELSE + humidity_ob = humidity(iLevel) + (humidity(iLevel + 1) - humidity(iLevel)) / & + model_height_diff * obs_height_diff + END IF + + ! T varies linearly with height + temperature_grad = (temperature(iLevel + 1) - temperature(iLevel)) / & + model_height_diff + T_ob = temperature(iLevel) + temperature_grad * obs_height_diff + + ! P varies to maintain hydrostatic balance + IF (ABS (temperature(iLevel + 1) - temperature(iLevel)) > GPSRO_min_temp_grad) THEN + pressure_coeff = ((Pb(iLevel + 1) / Pb(iLevel)) * (temperature(iLevel + 1) / temperature(iLevel)) ** & + (grav / (rd * temperature_grad)) - 1.0) / model_height_diff + P_ob = (Pb(iLevel) * (T_ob / temperature(iLevel)) ** (-grav / (rd * temperature_grad))) * & + (1.0 + pressure_coeff * obs_height_diff) + ELSE + ! If layer is isothermal, assume exponential variation to + ! avoid singularity + P_ob = Pb(iLevel) * EXP (LOG (Pb(iLevel + 1) / Pb(iLevel)) * & + obs_height_diff / model_height_diff) + END IF + + + ! Calculate refractivity + + ycalc(iObs) = n_alpha * P_ob / T_ob + n_beta * P_ob * humidity_ob / (T_ob ** 2 * & + (mw_ratio + (1.0 - mw_ratio) * humidity_ob)) + END IF + + ELSE + + IF (model_heights(nRefLevels) > zobs(iObs) .AND. model_heights(1) < zobs(iObs)) THEN + ! Find the model level immediately above the observation + iLevel = 1 + DO + IF (zobs(iObs) < model_heights(iLevel + 1)) EXIT + iLevel = iLevel + 1 + END DO + + ! Use simple assumption of exponentially varying refractivity + alpha_interp = (model_heights(iLevel + 1) - zobs(iObs)) / & + (model_heights(iLevel + 1) - model_heights(iLevel)) + beta_interp = 1.0 - alpha_interp + ycalc(iObs) = EXP(alpha_interp * LOG (refractivity(iLevel)) + & + beta_interp * LOG (refractivity(iLevel + 1))) + ELSE IF (zobs(iObs) /= missing_value(zobs(iObs))) THEN + PRINT*, 'Height out of range', iObs, nRefLevels, model_heights(nRefLevels), zobs(iObs) + END IF + END IF + +END DO + +END SUBROUTINE RefMetOffice_ForwardModel + +end module ufo_gnssro_refmetoffice_mod diff --git a/src/ufo/gnssro/Ref/CMakeLists.txt b/src/ufo/gnssro/RefNBAM/CMakeLists.txt similarity index 95% rename from src/ufo/gnssro/Ref/CMakeLists.txt rename to src/ufo/gnssro/RefNBAM/CMakeLists.txt index f263d9892..887b2a106 100644 --- a/src/ufo/gnssro/Ref/CMakeLists.txt +++ b/src/ufo/gnssro/RefNBAM/CMakeLists.txt @@ -3,7 +3,7 @@ # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -set ( ref_src_files +set ( refnbam_src_files ObsGnssroRef.h ObsGnssroRef.cc ObsGnssroRefTLAD.h diff --git a/src/ufo/gnssro/Ref/ObsGnssroRef.cc b/src/ufo/gnssro/RefNBAM/ObsGnssroRef.cc similarity index 97% rename from src/ufo/gnssro/Ref/ObsGnssroRef.cc rename to src/ufo/gnssro/RefNBAM/ObsGnssroRef.cc index c0aaba4e2..65151264c 100644 --- a/src/ufo/gnssro/Ref/ObsGnssroRef.cc +++ b/src/ufo/gnssro/RefNBAM/ObsGnssroRef.cc @@ -5,7 +5,7 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#include "ufo/gnssro/Ref/ObsGnssroRef.h" +#include "ufo/gnssro/RefNBAM/ObsGnssroRef.h" #include #include diff --git a/src/ufo/gnssro/Ref/ObsGnssroRef.h b/src/ufo/gnssro/RefNBAM/ObsGnssroRef.h similarity index 88% rename from src/ufo/gnssro/Ref/ObsGnssroRef.h rename to src/ufo/gnssro/RefNBAM/ObsGnssroRef.h index 2f518f19e..5049794a6 100644 --- a/src/ufo/gnssro/Ref/ObsGnssroRef.h +++ b/src/ufo/gnssro/RefNBAM/ObsGnssroRef.h @@ -5,8 +5,8 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef UFO_GNSSRO_REF_OBSGNSSROREF_H_ -#define UFO_GNSSRO_REF_OBSGNSSROREF_H_ +#ifndef UFO_GNSSRO_REFNBAM_OBSGNSSROREF_H_ +#define UFO_GNSSRO_REFNBAM_OBSGNSSROREF_H_ #include #include @@ -14,7 +14,7 @@ #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" -#include "ufo/gnssro/Ref/ObsGnssroRef.interface.h" +#include "ufo/gnssro/RefNBAM/ObsGnssroRef.interface.h" #include "ufo/ObsOperatorBase.h" namespace eckit { @@ -61,4 +61,4 @@ class ObsGnssroRef : public ObsOperatorBase, } // namespace ufo -#endif // UFO_GNSSRO_REF_OBSGNSSROREF_H_ +#endif // UFO_GNSSRO_REFNBAM_OBSGNSSROREF_H_ diff --git a/src/ufo/gnssro/Ref/ObsGnssroRef.interface.F90 b/src/ufo/gnssro/RefNBAM/ObsGnssroRef.interface.F90 similarity index 100% rename from src/ufo/gnssro/Ref/ObsGnssroRef.interface.F90 rename to src/ufo/gnssro/RefNBAM/ObsGnssroRef.interface.F90 diff --git a/src/ufo/gnssro/Ref/ObsGnssroRef.interface.h b/src/ufo/gnssro/RefNBAM/ObsGnssroRef.interface.h similarity index 86% rename from src/ufo/gnssro/Ref/ObsGnssroRef.interface.h rename to src/ufo/gnssro/RefNBAM/ObsGnssroRef.interface.h index 98fb818e0..783e483f0 100644 --- a/src/ufo/gnssro/Ref/ObsGnssroRef.interface.h +++ b/src/ufo/gnssro/RefNBAM/ObsGnssroRef.interface.h @@ -5,8 +5,8 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef UFO_GNSSRO_REF_OBSGNSSROREF_INTERFACE_H_ -#define UFO_GNSSRO_REF_OBSGNSSROREF_INTERFACE_H_ +#ifndef UFO_GNSSRO_REFNBAM_OBSGNSSROREF_INTERFACE_H_ +#define UFO_GNSSRO_REFNBAM_OBSGNSSROREF_INTERFACE_H_ #include "ioda/ObsSpace.h" #include "ufo/Fortran.h" @@ -32,4 +32,4 @@ extern "C" { } // extern C } // namespace ufo -#endif // UFO_GNSSRO_REF_OBSGNSSROREF_INTERFACE_H_ +#endif // UFO_GNSSRO_REFNBAM_OBSGNSSROREF_INTERFACE_H_ diff --git a/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.cc b/src/ufo/gnssro/RefNBAM/ObsGnssroRefTLAD.cc similarity index 98% rename from src/ufo/gnssro/Ref/ObsGnssroRefTLAD.cc rename to src/ufo/gnssro/RefNBAM/ObsGnssroRefTLAD.cc index 4668db6bb..7259f3efc 100644 --- a/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.cc +++ b/src/ufo/gnssro/RefNBAM/ObsGnssroRefTLAD.cc @@ -5,7 +5,7 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#include "ufo/gnssro/Ref/ObsGnssroRefTLAD.h" +#include "ufo/gnssro/RefNBAM/ObsGnssroRefTLAD.h" #include #include diff --git a/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.h b/src/ufo/gnssro/RefNBAM/ObsGnssroRefTLAD.h similarity index 89% rename from src/ufo/gnssro/Ref/ObsGnssroRefTLAD.h rename to src/ufo/gnssro/RefNBAM/ObsGnssroRefTLAD.h index c8be1e27d..a84da7d51 100644 --- a/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.h +++ b/src/ufo/gnssro/RefNBAM/ObsGnssroRefTLAD.h @@ -5,8 +5,8 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef UFO_GNSSRO_REF_OBSGNSSROREFTLAD_H_ -#define UFO_GNSSRO_REF_OBSGNSSROREFTLAD_H_ +#ifndef UFO_GNSSRO_REFNBAM_OBSGNSSROREFTLAD_H_ +#define UFO_GNSSRO_REFNBAM_OBSGNSSROREFTLAD_H_ #include #include @@ -14,7 +14,7 @@ #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" -#include "ufo/gnssro/Ref/ObsGnssroRefTLAD.interface.h" +#include "ufo/gnssro/RefNBAM/ObsGnssroRefTLAD.interface.h" #include "ufo/LinearObsOperatorBase.h" // Forward declarations @@ -61,4 +61,4 @@ class ObsGnssroRefTLAD : public LinearObsOperatorBase, // ----------------------------------------------------------------------------- } // namespace ufo -#endif // UFO_GNSSRO_REF_OBSGNSSROREFTLAD_H_ +#endif // UFO_GNSSRO_REFNBAM_OBSGNSSROREFTLAD_H_ diff --git a/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.interface.F90 b/src/ufo/gnssro/RefNBAM/ObsGnssroRefTLAD.interface.F90 similarity index 100% rename from src/ufo/gnssro/Ref/ObsGnssroRefTLAD.interface.F90 rename to src/ufo/gnssro/RefNBAM/ObsGnssroRefTLAD.interface.F90 diff --git a/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.interface.h b/src/ufo/gnssro/RefNBAM/ObsGnssroRefTLAD.interface.h similarity index 88% rename from src/ufo/gnssro/Ref/ObsGnssroRefTLAD.interface.h rename to src/ufo/gnssro/RefNBAM/ObsGnssroRefTLAD.interface.h index f24dd3e2d..505c10040 100644 --- a/src/ufo/gnssro/Ref/ObsGnssroRefTLAD.interface.h +++ b/src/ufo/gnssro/RefNBAM/ObsGnssroRefTLAD.interface.h @@ -5,8 +5,8 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef UFO_GNSSRO_REF_OBSGNSSROREFTLAD_INTERFACE_H_ -#define UFO_GNSSRO_REF_OBSGNSSROREFTLAD_INTERFACE_H_ +#ifndef UFO_GNSSRO_REFNBAM_OBSGNSSROREFTLAD_INTERFACE_H_ +#define UFO_GNSSRO_REFNBAM_OBSGNSSROREFTLAD_INTERFACE_H_ #include "ioda/ObsSpace.h" #include "ufo/Fortran.h" @@ -35,4 +35,4 @@ extern "C" { } // extern C } // namespace ufo -#endif // UFO_GNSSRO_REF_OBSGNSSROREFTLAD_INTERFACE_H_ +#endif // UFO_GNSSRO_REFNBAM_OBSGNSSROREFTLAD_INTERFACE_H_ diff --git a/src/ufo/gnssro/Ref/ufo_gnssro_ref_mod.F90 b/src/ufo/gnssro/RefNBAM/ufo_gnssro_ref_mod.F90 similarity index 100% rename from src/ufo/gnssro/Ref/ufo_gnssro_ref_mod.F90 rename to src/ufo/gnssro/RefNBAM/ufo_gnssro_ref_mod.F90 diff --git a/src/ufo/gnssro/Ref/ufo_gnssro_ref_tlad_mod.F90 b/src/ufo/gnssro/RefNBAM/ufo_gnssro_ref_tlad_mod.F90 similarity index 100% rename from src/ufo/gnssro/Ref/ufo_gnssro_ref_tlad_mod.F90 rename to src/ufo/gnssro/RefNBAM/ufo_gnssro_ref_tlad_mod.F90 diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f79ed878f..7b1ac229e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -105,6 +105,7 @@ list( APPEND ufo_test_input testinput/gnssrobendmetoffice_obserror.yaml testinput/gnssrobendmetoffice_nopseudo.yaml testinput/gnssrobendmetoffice_qc.yaml + testinput/gnssrorefmetoffice.yaml testinput/gnssrobndropp1d.yaml testinput/gnssrobndropp1d_qc.yaml testinput/gnssrobndropp2d.yaml @@ -1314,6 +1315,13 @@ ecbuild_add_test( TARGET test_ufo_linopr_gnssroBendMetOffice_nopseudo DEPENDS test_ObsOperatorTLAD.x TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_opr_gnssroRefMetOffice_pseudo + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x + ARGS "testinput/gnssrorefmetoffice.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsOperator.x + TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) + if( ${ropp-ufo_FOUND} ) ecbuild_add_test( TARGET test_ufo_opr_gnssroBndROPP1D COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x diff --git a/test/testinput/gnssrorefmetoffice.yaml b/test/testinput/gnssrorefmetoffice.yaml new file mode 100644 index 000000000..416f7ab47 --- /dev/null +++ b/test/testinput/gnssrorefmetoffice.yaml @@ -0,0 +1,54 @@ +window begin: 2019-12-30T03:00:00Z +window end: 2019-12-30T09:00:00Z + +observations: +- obs operator: + name: GnssroRefMetOffice + obs options: + vert_interp_ops: true + pseudo_ops: true + min_temp_grad: 1.0e-6 + obs space: + name: GnssroBnd + obsdatain: + obsfile: Data/ufo/testinput_tier_1/gnssro_obs_2019123006_refractivity.nc4 + simulated variables: [refractivity] + geovals: + filename: Data/ufo/testinput_tier_1/gnssro_geoval_2019123006_refractivity.nc4 + obs filters: + - filter: Background Check + filter variables: + - name: refractivity + threshold: 3.0 + norm ref: MetOfficeHofX + tolerance: 1.0e-5 + linear obs operator test: + coef TL: 1.0e-3 + iterations TL: 10 + tolerance TL: 1.0e-14 + tolerance AD: 1.0e-14 +- obs operator: + name: GnssroRefMetOffice + obs options: + vert_interp_ops: false + pseudo_ops: false + min_temp_grad: 1.0e-6 + obs space: + name: GnssroBnd + obsdatain: + obsfile: Data/ufo/testinput_tier_1/gnssro_obs_2019123006_refractivity_nopseudo.nc4 + simulated variables: [refractivity] + geovals: + filename: Data/ufo/testinput_tier_1/gnssro_geoval_2019123006_refractivity_nopseudo.nc4 + obs filters: + - filter: Background Check + filter variables: + - name: refractivity + threshold: 3.0 + norm ref: MetOfficeHofX + tolerance: 1.0e-5 + linear obs operator test: + coef TL: 1.0e-4 + iterations TL: 10 + tolerance TL: 1.0e-14 + tolerance AD: 1.0e-14 From 1c3fc901453f27e77351e0aefed11ddcdcf48b7d Mon Sep 17 00:00:00 2001 From: Clementine Gas <43183478+cmgas@users.noreply.github.com> Date: Mon, 21 Jun 2021 10:09:23 -0600 Subject: [PATCH 15/98] Add Obs Error Diagonal in UFO (#1228) * Add files for obserrorfactory n ufo, doesn't compile * namespace ufo * COmments Anna * Trying to compile * With the file it's better * Compiles but needs more work * Yannick's comments on update and obserrors * iTry to implement obserrors * Final changes - it's working * Coding norms * Seperate file into .h and .cc * Update obserrors and inverseVariance * Add printable inheritance * Adding test for obserrorcovariance ufo * Comments Co-authored-by: Anna Shlyaeva --- src/CMakeLists.txt | 2 +- src/ufo/CMakeLists.txt | 5 +- src/ufo/errors/CMakeLists.txt | 15 +++++ src/ufo/errors/ObsErrorDiagonal.cc | 87 ++++++++++++++++++++++++++++ src/ufo/errors/ObsErrorDiagonal.h | 78 +++++++++++++++++++++++++ src/ufo/instantiateObsErrorFactory.h | 25 ++++++++ test/CMakeLists.txt | 9 +++ test/mains/TestObsDiagnostics.cc | 4 +- test/mains/TestObsErrorCovariance.cc | 19 ++++++ test/testinput/obserrordiagonal.yaml | 25 ++++++++ 10 files changed, 265 insertions(+), 4 deletions(-) create mode 100644 src/ufo/errors/CMakeLists.txt create mode 100644 src/ufo/errors/ObsErrorDiagonal.cc create mode 100644 src/ufo/errors/ObsErrorDiagonal.h create mode 100644 src/ufo/instantiateObsErrorFactory.h create mode 100644 test/mains/TestObsErrorCovariance.cc create mode 100644 test/testinput/obserrordiagonal.yaml diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f5c3d9c83..4793c0c44 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# (C) Copyright 2017-2019 UCAR. +# (C) Copyright 2017-2021 UCAR. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. diff --git a/src/ufo/CMakeLists.txt b/src/ufo/CMakeLists.txt index 497e4f961..af3573b02 100644 --- a/src/ufo/CMakeLists.txt +++ b/src/ufo/CMakeLists.txt @@ -1,4 +1,4 @@ -# (C) Copyright 2017-2018 UCAR. +# (C) Copyright 2017-2021 UCAR. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -11,6 +11,7 @@ list( APPEND ufo_src_files GeoVaLs.h GeoVaLs.interface.F90 GeoVaLs.interface.h + instantiateObsErrorFactory.h instantiateObsFilterFactory.h instantiateObsLocFactory.h LinearObsBiasOperator.cc @@ -58,6 +59,7 @@ endfunction(PREPEND) add_subdirectory( utils ) add_subdirectory( basis ) add_subdirectory( predictors ) +add_subdirectory( errors ) add_subdirectory( filters ) add_subdirectory( obslocalization ) add_subdirectory( identity ) @@ -92,6 +94,7 @@ list( APPEND ufo_src_files ${utils_src_files} ${basis_src_files} ${predictor_src_files} + ${errors_src_files} ${filters_src_files} ${obslocalization_src_files} ${identity_src_files} diff --git a/src/ufo/errors/CMakeLists.txt b/src/ufo/errors/CMakeLists.txt new file mode 100644 index 000000000..bc13c7168 --- /dev/null +++ b/src/ufo/errors/CMakeLists.txt @@ -0,0 +1,15 @@ +# (C) Copyright 2021 UCAR +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +set ( errors_files + ObsErrorDiagonal.cc + ObsErrorDiagonal.h +) +PREPEND( _p_errors_files "errors" ${errors_files} ) + +set ( errors_src_files + ${_p_errors_files} + PARENT_SCOPE +) diff --git a/src/ufo/errors/ObsErrorDiagonal.cc b/src/ufo/errors/ObsErrorDiagonal.cc new file mode 100644 index 000000000..9dcee1247 --- /dev/null +++ b/src/ufo/errors/ObsErrorDiagonal.cc @@ -0,0 +1,87 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/errors/ObsErrorDiagonal.h" + +#include "eckit/config/Configuration.h" + +#include "oops/util/Logger.h" + + +namespace ufo { + +// ----------------------------------------------------------------------------- + +ObsErrorDiagonal::ObsErrorDiagonal(const eckit::Configuration & conf, ioda::ObsSpace & obsgeom) + : stddev_(obsgeom, "ObsError"), inverseVariance_(obsgeom) +{ + options_.deserialize(conf); + inverseVariance_ = stddev_; + inverseVariance_ *= stddev_; + inverseVariance_.invert(); + oops::Log::trace() << "ObsErrorDiagonal:ObsErrorDiagonal constructed nobs = " + << stddev_.nobs() << std::endl; +} + +// ----------------------------------------------------------------------------- + +void ObsErrorDiagonal::update(const ioda::ObsVector & obserr) { + stddev_ = obserr; + inverseVariance_ = stddev_; + inverseVariance_ *= stddev_; + inverseVariance_.invert(); + oops::Log::info() << "ObsErrorDiagonal covariance updated " << stddev_.nobs() << std::endl; +} + +// ----------------------------------------------------------------------------- + +void ObsErrorDiagonal::multiply(ioda::ObsVector & dy) const { + dy /= inverseVariance_; +} + +// ----------------------------------------------------------------------------- + +void ObsErrorDiagonal::inverseMultiply(ioda::ObsVector & dy) const { + dy *= inverseVariance_; +} + +// ----------------------------------------------------------------------------- + +void ObsErrorDiagonal::randomize(ioda::ObsVector & dy) const { + dy.random(); + dy *= stddev_; + dy *= options_.pert; +} + +// ----------------------------------------------------------------------------- + +void ObsErrorDiagonal::save(const std::string & name) const { + stddev_.save(name); +} + +// ----------------------------------------------------------------------------- + +ioda::ObsVector ObsErrorDiagonal::obserrors() const { + return stddev_; +} + +// ----------------------------------------------------------------------------- + +ioda::ObsVector ObsErrorDiagonal::inverseVariance() const { + return inverseVariance_; +} + +// ----------------------------------------------------------------------------- +void ObsErrorDiagonal::print(std::ostream & os) const { + os << "UFO Diagonal observation error covariance, inverse variances: " + << inverseVariance_ << std::endl; +} + +// ----------------------------------------------------------------------------- + + +} // namespace ufo diff --git a/src/ufo/errors/ObsErrorDiagonal.h b/src/ufo/errors/ObsErrorDiagonal.h new file mode 100644 index 000000000..77b20a2c3 --- /dev/null +++ b/src/ufo/errors/ObsErrorDiagonal.h @@ -0,0 +1,78 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_ERRORS_OBSERRORDIAGONAL_H_ +#define UFO_ERRORS_OBSERRORDIAGONAL_H_ + +#include +#include + +#include "eckit/config/Configuration.h" + +#include "ioda/ObsSpace.h" +#include "ioda/ObsVector.h" + +#include "oops/util/Logger.h" +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/Printable.h" + + +namespace ufo { + +/// \brief Parameters for diagonal obs errors +class ObsErrorDiagonalParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsErrorDiagonalParameters, Parameters) + public: + /// perturbation amplitude multiplier + oops::Parameter pert{"random amplitude", 1.0, this}; +}; + +// ----------------------------------------------------------------------------- +/// \brief Diagonal observation error covariance matrix. +class ObsErrorDiagonal : public util::Printable { + public: + static const std::string classname() {return "ufo::ObsErrorDiagonal";} + + ObsErrorDiagonal(const eckit::Configuration &, ioda::ObsSpace &); + +/// Update after obs errors potentially changed + void update(const ioda::ObsVector &); + +/// Multiply a Departure by \f$R\f$ + void multiply(ioda::ObsVector &) const; + +/// Multiply a Departure by \f$R^{-1}\f$ + void inverseMultiply(ioda::ObsVector &) const; + +/// Generate random perturbation + void randomize(ioda::ObsVector &) const; + +/// Save obs errors + void save(const std::string &) const; + +/// Get mean error for Jo table + double getRMSE() const {return stddev_.rms();} + +/// Get obs errors std deviation + ioda::ObsVector obserrors() const; + +/// Return inverseVariance + ioda::ObsVector inverseVariance() const; + + private: + void print(std::ostream &) const; + ioda::ObsVector stddev_; + ioda::ObsVector inverseVariance_; + ObsErrorDiagonalParameters options_; +}; + +// ----------------------------------------------------------------------------- + +} // namespace ufo + +#endif // UFO_ERRORS_OBSERRORDIAGONAL_H_ diff --git a/src/ufo/instantiateObsErrorFactory.h b/src/ufo/instantiateObsErrorFactory.h new file mode 100644 index 000000000..c9dac7cce --- /dev/null +++ b/src/ufo/instantiateObsErrorFactory.h @@ -0,0 +1,25 @@ +/* + * (C) Copyright 2020 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_INSTANTIATEOBSERRORFACTORY_H_ +#define UFO_INSTANTIATEOBSERRORFACTORY_H_ + +#include "oops/generic/instantiateObsErrorFactory.h" +#include "oops/interface/ObsErrorCovariance.h" + +#include "ufo/errors/ObsErrorDiagonal.h" + +namespace ufo { +template void instantiateObsErrorFactory() { + oops::instantiateObsErrorFactory(); + static oops::ObsErrorMaker > + makerDiagUFO("diagonal ufo"); +} + +} // namespace ufo + +#endif // UFO_INSTANTIATEOBSERRORFACTORY_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7b1ac229e..d0bb3ab5b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -149,6 +149,7 @@ list( APPEND ufo_test_input testinput/obsdiag_crtm_iasi_jacobian.yaml testinput/obsdiag_crtm_iasi_optics.yaml testinput/obserror_assign_unittests.yaml + testinput/obserrordiagonal.yaml testinput/obsfilterdata.yaml testinput/omi_aura.yaml testinput/omi_aura_flipz.yaml @@ -521,6 +522,14 @@ ecbuild_add_test( TARGET test_ufo_locations LIBS ufo TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_obserrordiagonal + SOURCES mains/TestObsErrorCovariance.cc + ARGS "testinput/obserrordiagonal.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + MPI 4 + LIBS ufo + TEST_DEPENDS ufo_get_ufo_test_data ) + # Test UFO-specific classes: ecbuild_add_test( TARGET test_ufo_obsfilterdata diff --git a/test/mains/TestObsDiagnostics.cc b/test/mains/TestObsDiagnostics.cc index 7f51b6dd6..c9d446bca 100644 --- a/test/mains/TestObsDiagnostics.cc +++ b/test/mains/TestObsDiagnostics.cc @@ -1,8 +1,8 @@ /* * (C) Copyright 2019 UCAR - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ #include "../ufo/ObsDiagnostics.h" diff --git a/test/mains/TestObsErrorCovariance.cc b/test/mains/TestObsErrorCovariance.cc new file mode 100644 index 000000000..838445c32 --- /dev/null +++ b/test/mains/TestObsErrorCovariance.cc @@ -0,0 +1,19 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/runs/Run.h" +#include "test/base/ObsErrorCovariance.h" + +#include "ufo/instantiateObsErrorFactory.h" +#include "ufo/ObsTraits.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + ufo::instantiateObsErrorFactory(); + test::ObsErrorCovariance tests; + return run.execute(tests); +} diff --git a/test/testinput/obserrordiagonal.yaml b/test/testinput/obserrordiagonal.yaml new file mode 100644 index 000000000..bb3994c4b --- /dev/null +++ b/test/testinput/obserrordiagonal.yaml @@ -0,0 +1,25 @@ +window begin: 2018-04-14T21:00:00Z +window end: 2018-04-15T03:00:00Z + +observations: +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/sondes_obs_2018041500_s.nc4 + simulated variables: [air_temperature] + obs error: + covariance model: diagonal ufo +- obs space: + name: Aircraft + obsdatain: + obsfile: Data/ufo/testinput_tier_1/aircraft_obs_2018041500_s.nc4 + simulated variables: [air_temperature,specific_humidity] + obs error: + covariance model: diagonal ufo +- obs space: + name: Satwind + obsdatain: + obsfile: Data/ufo/testinput_tier_1/satwind_obs_2018041500_s.nc4 + simulated variables: [eastward_wind, northward_wind] + obs error: + covariance model: diagonal ufo From e32ad0d449f6e87aea0a779583d89b4b077fbf3c Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Thu, 24 Jun 2021 10:33:53 +0300 Subject: [PATCH 16/98] Add GeoVaLs constructor that does allocation (#1162) * Add new constructor. Temporarily disable the other ctor & allocate to test where they are used * try to fix rttovonedvarcheck * uncomment deprecated ctor and allocate method * more comments, more removed simulateObsAD GeoVaLs allocation * remove geovals allocation from all the simulateObsAD * set linit=true in geovals_setup * Add changes needed to use the new ufo_geovals_setup routine (#1180) * remove missed allocate & zero out in aodluts_ad Co-authored-by: Michael Cooke <48374999+mikecooke77@users.noreply.github.com> Co-authored-by: Ryan Honeyager --- src/ufo/GeoVaLs.cc | 49 ++++--- src/ufo/GeoVaLs.h | 21 ++- src/ufo/GeoVaLs.interface.F90 | 48 +++++-- src/ufo/GeoVaLs.interface.h | 20 ++- src/ufo/aerosols/AOP/ufo_aodext_tlad_mod.F90 | 12 +- src/ufo/aerosols/MR/ufo_aodgeos_tlad_mod.F90 | 7 - .../ufo_atmvertinterp_tlad_mod.F90 | 9 -- .../ufo_atmvertinterplay_tlad_mod.F90 | 10 -- src/ufo/avgkernel/ufo_avgkernel_tlad_mod.F90 | 8 -- src/ufo/crtm/ufo_aodcrtm_tlad_mod.F90 | 7 - src/ufo/crtm/ufo_aodluts_tlad_mod.F90 | 6 - src/ufo/crtm/ufo_radiancecrtm_tlad_mod.F90 | 32 ----- ...o_rttovonedvarcheck_minimize_utils_mod.f90 | 45 +++++- .../ufo_rttovonedvarcheck_mod.f90 | 12 +- .../ufo_gnssro_bendmetoffice_tlad_mod.F90 | 16 --- .../BndNBAM/ufo_gnssro_bndnbam_tlad_mod.F90 | 21 --- .../ufo_gnssro_bndropp1d_tlad_mod.F90 | 24 ---- .../ufo_gnssro_bndropp1d_tlad_mod_stub.F90 | 24 ---- .../ufo_gnssro_bndropp2d_tlad_mod.F90 | 24 ---- .../ufo_gnssro_bndropp2d_tlad_mod_stub.F90 | 24 ---- .../RefNBAM/ufo_gnssro_ref_tlad_mod.F90 | 24 ---- .../ufo_groundgnss_metoffice_tlad_mod.F90 | 16 --- src/ufo/identity/ufo_identity_tlad_mod.F90 | 6 - src/ufo/marine/CMakeLists.txt | 3 - src/ufo/marine/adt/ufo_adt_tlad_mod.F90 | 5 - .../marine/coolskin/ufo_coolskin_tlad_mod.F90 | 32 ----- .../ufo_insitutemperature_tlad_mod.F90 | 17 --- .../ufo_marinevertinterp_tlad_mod.F90 | 7 - .../seaicefraction/ObsSeaIceFractionTLAD.cc | 9 +- .../seaicefraction/ObsSeaIceFractionTLAD.h | 5 - .../ufo_seaicethickness_tlad_mod.F90 | 10 -- src/ufo/marine/utils/CMakeLists.txt | 10 -- .../utils/ObsSeaIceLinear.interface.F90 | 128 ------------------ .../marine/utils/ObsSeaIceLinear.interface.h | 33 ----- .../ufo_radarradialvelocity_tlad_mod.F90 | 9 -- src/ufo/rttov/ufo_radiancerttov_tlad_mod.F90 | 51 ------- src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc | 5 - src/ufo/sattcwv/ufo_SatTCWV_tlad_mod.F90 | 8 -- src/ufo/ufo_geovals_mod.F90 | 46 ++++++- test/ufo/GeoVaLs.h | 43 ++++++ 40 files changed, 242 insertions(+), 644 deletions(-) delete mode 100644 src/ufo/marine/utils/CMakeLists.txt delete mode 100644 src/ufo/marine/utils/ObsSeaIceLinear.interface.F90 delete mode 100644 src/ufo/marine/utils/ObsSeaIceLinear.interface.h diff --git a/src/ufo/GeoVaLs.cc b/src/ufo/GeoVaLs.cc index 10d8fc14f..25e719335 100644 --- a/src/ufo/GeoVaLs.cc +++ b/src/ufo/GeoVaLs.cc @@ -28,8 +28,10 @@ namespace ufo { // ----------------------------------------------------------------------------- -/*! \brief Default constructor - does not allocate fields -*/ +/*! \brief Deprecated default constructor - does not allocate fields. + * + * \details Please do not use this constructor in new code. + */ GeoVaLs::GeoVaLs(std::shared_ptr dist, const oops::Variables & vars) : keyGVL_(-1), vars_(vars), dist_(std::move(dist)) @@ -39,22 +41,39 @@ GeoVaLs::GeoVaLs(std::shared_ptr dist, oops::Log::trace() << "GeoVaLs default constructor end" << std::endl; } -// ----------------------------------------------------------------------------- -/*! \brief Constructor given Locations and Variables - * - * \details This ufo::GeoVaLs constructor is typically used to initialize - * GeoVaLs for the full time window (ufo::Locations hold all locations within - * data assimilation window) and all variables (oops::Variables hold all - * variables specified by the ObsOperator as input varialbes. Note that - * nothing is allocated in the constructor currently, and getValues is - * responsible for allocation +/*! \brief Deprecated constructor given Locations and Variables * + * \details Please do not use in any new code. This constructor is currently + * only used for ObsDiagnostics and will be removed soon. Use the + * GeoVaLs(const Locations &, const oops::Variables &, const std::vector &) + * constructor instead. + * This ufo::GeoVaLs constructor is used to initialize GeoVaLs for specified + * ufo::Locations and oops::Variables hold all. Note that nothing is allocated + * when this constructor is called. */ GeoVaLs::GeoVaLs(const Locations & locs, const oops::Variables & vars) : keyGVL_(-1), vars_(vars), dist_(locs.distribution()) { oops::Log::trace() << "GeoVaLs contructor starting" << std::endl; - ufo_geovals_setup_f90(keyGVL_, locs.size(), vars_); + ufo_geovals_partial_setup_f90(keyGVL_, locs.size(), vars_); + oops::Log::trace() << "GeoVaLs contructor key = " << keyGVL_ << std::endl; +} + +// ----------------------------------------------------------------------------- +/*! \brief Allocating constructor for specified Locations \p locs, Variables \p vars + * and number of levels \p nlevs + * + * \details This ufo::GeoVaLs constructor is used in all oops H(x) and DA + * applications. + * Sizes of GeoVaLs for i-th variable at a single location are defined by i-th value + * of \p nlevs. + */ +GeoVaLs::GeoVaLs(const Locations & locs, const oops::Variables & vars, + const std::vector & nlevs) + : keyGVL_(-1), vars_(vars), dist_(locs.distribution()) +{ + oops::Log::trace() << "GeoVaLs contructor starting" << std::endl; + ufo_geovals_setup_f90(keyGVL_, locs.size(), vars_, nlevs.size(), nlevs[0]); oops::Log::trace() << "GeoVaLs contructor key = " << keyGVL_ << std::endl; } @@ -70,7 +89,7 @@ GeoVaLs::GeoVaLs(const eckit::Configuration & config, : keyGVL_(-1), vars_(vars), dist_(obspace.distribution()) { oops::Log::trace() << "GeoVaLs constructor config starting" << std::endl; - ufo_geovals_setup_f90(keyGVL_, 0, vars_); + ufo_geovals_partial_setup_f90(keyGVL_, 0, vars_); // only read if there are variables specified if (vars_.size() > 0) ufo_geovals_read_file_f90(keyGVL_, config, obspace, vars_); oops::Log::trace() << "GeoVaLs contructor config key = " << keyGVL_ << std::endl; @@ -85,7 +104,6 @@ GeoVaLs::GeoVaLs(const GeoVaLs & other, const int & index) : keyGVL_(-1), vars_(other.vars_), dist_(other.dist_) { oops::Log::trace() << "GeoVaLs copy one GeoVaLs constructor starting" << std::endl; - ufo_geovals_setup_f90(keyGVL_, 1, vars_); ufo_geovals_copy_one_f90(keyGVL_, other.keyGVL_, index); oops::Log::trace() << "GeoVaLs copy one GeoVaLs constructor key = " << keyGVL_ << std::endl; } @@ -96,7 +114,6 @@ GeoVaLs::GeoVaLs(const GeoVaLs & other) : keyGVL_(-1), vars_(other.vars_), dist_(other.dist_) { oops::Log::trace() << "GeoVaLs copy constructor starting" << std::endl; - ufo_geovals_setup_f90(keyGVL_, 0, vars_); ufo_geovals_copy_f90(other.keyGVL_, keyGVL_); oops::Log::trace() << "GeoVaLs copy constructor key = " << keyGVL_ << std::endl; } @@ -108,8 +125,6 @@ GeoVaLs::~GeoVaLs() { oops::Log::trace() << "GeoVaLs destructor done" << std::endl; } // ----------------------------------------------------------------------------- -/*! \brief Allocate GeoVaLs for \p vars variables to have \p nlevels levels (number of - * locations is defined in the constructor) */ void GeoVaLs::allocate(const int & nlevels, const oops::Variables & vars) { oops::Log::trace() << "GeoVaLs::allocate starting" << std::endl; diff --git a/src/ufo/GeoVaLs.h b/src/ufo/GeoVaLs.h index 233b7675d..4edc1d276 100644 --- a/src/ufo/GeoVaLs.h +++ b/src/ufo/GeoVaLs.h @@ -42,6 +42,10 @@ class GeoVaLs : public util::Printable, GeoVaLs(std::shared_ptr, const oops::Variables &); GeoVaLs(const Locations &, const oops::Variables &); + + GeoVaLs(const Locations & locs, const oops::Variables & vars, + const std::vector & nlevs); + GeoVaLs(const eckit::Configuration &, const ioda::ObsSpace &, const oops::Variables &); GeoVaLs(const GeoVaLs &, const int &); @@ -59,12 +63,17 @@ class GeoVaLs : public util::Printable, void split(GeoVaLs &, GeoVaLs &) const; void merge(const GeoVaLs &, const GeoVaLs &); - /// \brief Allocate GeoVaLs for \p vars variables with \p nlev number of levels - /// \details Fails if at least one of the \p vars doesn't exist in GeoVaLs. - /// Only allocates variables that haven't been allocated before. - /// Fails if one of \p vars is already allocated with number of levels - /// different than \p nlev; doesn't reallocate variables that are already - /// allocated with \p nlev. + /// \brief Deprecated method. Allocates GeoVaLs for \p vars variables with + /// \p nlev number of levels + /// \details Please do not use in any new code. This method is currently + /// only used for ObsDiagnostics and will be removed soon. Rely on + /// GeoVaLs(const Locations &, const oops::Variables &, const std::vector &) + /// to allocate GeoVaLs correctly. + /// Fails if at least one of the \p vars doesn't exist in GeoVaLs. + /// Only allocates variables that haven't been allocated before. + /// Fails if one of \p vars is already allocated with number of levels + /// different than \p nlev; doesn't reallocate variables that are already + /// allocated with \p nlev. void allocate(const int & nlev, const oops::Variables & vars); void zero(); diff --git a/src/ufo/GeoVaLs.interface.F90 b/src/ufo/GeoVaLs.interface.F90 index e6846c437..dbf7bcfc5 100644 --- a/src/ufo/GeoVaLs.interface.F90 +++ b/src/ufo/GeoVaLs.interface.F90 @@ -45,8 +45,28 @@ subroutine ufo_geovals_default_constr_c(c_key_self) bind(c,name='ufo_geovals_def end subroutine ufo_geovals_default_constr_c +subroutine ufo_geovals_setup_c(c_key_self, c_nlocs, c_vars, c_nvars, c_sizes) bind(c,name='ufo_geovals_setup_f90') +use oops_variables_mod +implicit none +integer(c_int), intent(inout) :: c_key_self +integer(c_int), intent(in) :: c_nlocs, c_nvars +type(c_ptr), value, intent(in) :: c_vars +integer(c_size_t), intent(in) :: c_sizes(c_nvars) + +type(ufo_geovals), pointer :: self +type(oops_variables) :: vars + +call ufo_geovals_registry%init() +call ufo_geovals_registry%add(c_key_self) +call ufo_geovals_registry%get(c_key_self, self) + +vars = oops_variables(c_vars) +call ufo_geovals_setup(self, vars, c_nlocs, c_nvars, c_sizes) + +end subroutine ufo_geovals_setup_c + !> Setup GeoVaLs (store nlocs, variables; don't do allocation yet) -subroutine ufo_geovals_setup_c(c_key_self, c_nlocs, c_vars) bind(c,name='ufo_geovals_setup_f90') +subroutine ufo_geovals_partial_setup_c(c_key_self, c_nlocs, c_vars) bind(c,name='ufo_geovals_partial_setup_f90') use oops_variables_mod implicit none integer(c_int), intent(inout) :: c_key_self @@ -61,9 +81,9 @@ subroutine ufo_geovals_setup_c(c_key_self, c_nlocs, c_vars) bind(c,name='ufo_geo call ufo_geovals_registry%get(c_key_self, self) vars = oops_variables(c_vars) -call ufo_geovals_setup(self, vars, c_nlocs) +call ufo_geovals_partial_setup(self, vars, c_nlocs) -end subroutine ufo_geovals_setup_c +end subroutine ufo_geovals_partial_setup_c !> Allocate GeoVaLs subroutine ufo_geovals_allocate_c(c_key_self, c_nlevels, c_vars) bind(c,name='ufo_geovals_allocate_f90') @@ -88,11 +108,13 @@ end subroutine ufo_geovals_allocate_c subroutine ufo_geovals_copy_c(c_key_self, c_key_other) bind(c,name='ufo_geovals_copy_f90') implicit none -integer(c_int), intent(in) :: c_key_self -integer(c_int), intent(in) :: c_key_other -type(ufo_geovals), pointer :: self -type(ufo_geovals), pointer :: other +integer(c_int), intent(in) :: c_key_self +integer(c_int), intent(inout) :: c_key_other +type(ufo_geovals), pointer :: self +type(ufo_geovals), pointer :: other +call ufo_geovals_registry%init() +call ufo_geovals_registry%add(c_key_other) call ufo_geovals_registry%get(c_key_self, self) call ufo_geovals_registry%get(c_key_other, other) @@ -105,16 +127,18 @@ end subroutine ufo_geovals_copy_c subroutine ufo_geovals_copy_one_c(c_key_self, c_key_other, c_ind) bind(c,name='ufo_geovals_copy_one_f90') implicit none -integer(c_int), intent(in) :: c_key_self -integer(c_int), intent(in) :: c_key_other -integer(c_int), intent(in) :: c_ind -type(ufo_geovals), pointer :: self -type(ufo_geovals), pointer :: other +integer(c_int), intent(inout) :: c_key_self +integer(c_int), intent(in) :: c_key_other +integer(c_int), intent(in) :: c_ind +type(ufo_geovals), pointer :: self +type(ufo_geovals), pointer :: other integer :: ind ! Convert location index from the C++ to the Fortran convention. ind = c_ind + 1 +call ufo_geovals_registry%init() +call ufo_geovals_registry%add(c_key_self) call ufo_geovals_registry%get(c_key_self, self) call ufo_geovals_registry%get(c_key_other, other) diff --git a/src/ufo/GeoVaLs.interface.h b/src/ufo/GeoVaLs.interface.h index 604f20c2b..61d8fa069 100644 --- a/src/ufo/GeoVaLs.interface.h +++ b/src/ufo/GeoVaLs.interface.h @@ -35,8 +35,24 @@ namespace ufo { extern "C" { void ufo_geovals_default_constr_f90(F90goms &); - void ufo_geovals_setup_f90(F90goms &, const size_t &, const oops::Variables &); - void ufo_geovals_allocate_f90(F90goms &, const size_t &, const oops::Variables &); + /// Deprecated, use ufo_geovals_setup_f90 instead. + /// Creates Fortran GeoVaLs with key \p key, \p nlocs number of locations + /// for \p vars variables. Does not allocate internal structures. + void ufo_geovals_partial_setup_f90(F90goms & key, const size_t & nlocs, + const oops::Variables & vars); + /// Creates and allocates Fortran GeoVaLs with key \p key, \p nlocs number of + /// locations for \p vars variables. \p nlevs is a pointer to the first element + /// of array size \p nvars that contains number of values to allocate for each of + /// the variables. + void ufo_geovals_setup_f90(F90goms & key, const size_t & nlocs, + const oops::Variables & vars, + const size_t & nvars, const size_t & nlevs); + /// Deprecated, rely on ufo_geovals_setup_f90 to allocate GeoVaLs instead. + /// Allocates GeoVaLs for \p vars variables with \p nlevels number of levels. + /// If the GeoVaLs for this variable were allocated before with different size, + /// aborts. + void ufo_geovals_allocate_f90(const F90goms &, const size_t & nlevels, + const oops::Variables & vars); void ufo_geovals_delete_f90(F90goms &); void ufo_geovals_copy_f90(const F90goms &, F90goms &); void ufo_geovals_copy_one_f90(F90goms &, const F90goms &, const int &); diff --git a/src/ufo/aerosols/AOP/ufo_aodext_tlad_mod.F90 b/src/ufo/aerosols/AOP/ufo_aodext_tlad_mod.F90 index d448e8ac7..73470ffba 100644 --- a/src/ufo/aerosols/AOP/ufo_aodext_tlad_mod.F90 +++ b/src/ufo/aerosols/AOP/ufo_aodext_tlad_mod.F90 @@ -384,15 +384,9 @@ subroutine ufo_aodext_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) !Get pointer to ext profiles in geovals and put adjoint of extinction profiles into geovals do nch = self%nprofiles, 1, -1 - geovar = self%geovars%variable(nch) - call ufo_geovals_get_var(geovals, geovar, ext_profile) - if(.not. allocated(ext_profile%vals)) then - ext_profile%nlocs = nlocs - ext_profile%nval = self%nlayers - allocate(ext_profile%vals(ext_profile%nval, ext_profile%nlocs)) - ext_profile%vals(:,:) = zero - endif - ext_profile%vals(:,:) = ext_ad(:,:,nch) + geovar = self%geovars%variable(nch) + call ufo_geovals_get_var(geovals, geovar, ext_profile) + ext_profile%vals(:,:) = ext_ad(:,:,nch) enddo deallocate(angstrom_ad, angstrom, aod_bkg_ad, aod_bkg, ext_ad) diff --git a/src/ufo/aerosols/MR/ufo_aodgeos_tlad_mod.F90 b/src/ufo/aerosols/MR/ufo_aodgeos_tlad_mod.F90 index cb8df2ba3..1fca934da 100644 --- a/src/ufo/aerosols/MR/ufo_aodgeos_tlad_mod.F90 +++ b/src/ufo/aerosols/MR/ufo_aodgeos_tlad_mod.F90 @@ -201,13 +201,6 @@ subroutine ufo_aodgeos_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) geovar = self%geovars%variable(iq) !self%geovars contains tracers call ufo_geovals_get_var(geovals, geovar, aer_profile) - if (.not. allocated(aer_profile%vals)) then - aer_profile%nlocs = nlocs - aer_profile%nval = self%nlayers - allocate(aer_profile%vals(aer_profile%nval, aer_profile%nlocs)) - aer_profile%vals(:,:) = 0.0_kind_real - endif - qm_ad(iq,:,:) = qm_ad(iq,:,:) * self%delp / grav aer_profile%vals = qm_ad(iq,:,:) diff --git a/src/ufo/atmvertinterp/ufo_atmvertinterp_tlad_mod.F90 b/src/ufo/atmvertinterp/ufo_atmvertinterp_tlad_mod.F90 index 6660a5fcf..3b357e967 100644 --- a/src/ufo/atmvertinterp/ufo_atmvertinterp_tlad_mod.F90 +++ b/src/ufo/atmvertinterp/ufo_atmvertinterp_tlad_mod.F90 @@ -188,15 +188,6 @@ subroutine atmvertinterp_simobs_ad_(self, geovals, obss, nvars, nlocs, hofx) ! Get pointer to profile for this variable in geovals call ufo_geovals_get_var(geovals, geovar, profile) - ! Allocate geovals profile if not yet allocated - if (.not. allocated(profile%vals)) then - profile%nlocs = self%nlocs - profile%nval = self%nval - allocate(profile%vals(profile%nval, profile%nlocs)) - profile%vals(:,:) = 0.0_kind_real - endif - if (.not. geovals%linit ) geovals%linit=.true. - ! Adjoint of interpolate, from hofx into geovals do iobs = 1, self%nlocs if (hofx(ivar,iobs) /= missing) then diff --git a/src/ufo/atmvertinterplay/ufo_atmvertinterplay_tlad_mod.F90 b/src/ufo/atmvertinterplay/ufo_atmvertinterplay_tlad_mod.F90 index 5d45b8399..c837d46b8 100644 --- a/src/ufo/atmvertinterplay/ufo_atmvertinterplay_tlad_mod.F90 +++ b/src/ufo/atmvertinterplay/ufo_atmvertinterplay_tlad_mod.F90 @@ -190,16 +190,6 @@ subroutine atmvertinterplay_simobs_ad_(self, geovals, obss, nvars, nlocs, hofx) ! Get the name of input variable in geovals geovar = self%geovars%variable(ivar) call ufo_geovals_get_var(geovals, geovar, profile) - ! Allocate geovals profile if not yet allocated - if (.not. allocated(profile%vals)) then - profile%nlocs = self%nlocs - profile%nval = self%nval - allocate(profile%vals(profile%nval, profile%nlocs)) - profile%vals(:,:) = 0.0_kind_real - endif - if (.not. geovals%linit ) geovals%linit=.true. - - nsig = self%nval-1 do iobs = 1, nlocs call vert_interp_lay_apply_ad(profile%vals(:,iobs), hofx(ivar,iobs), self%coefficients(ivar), self%modelpressures(:,iobs), self%botpressure(iobs), self%toppressure(iobs), nsig) diff --git a/src/ufo/avgkernel/ufo_avgkernel_tlad_mod.F90 b/src/ufo/avgkernel/ufo_avgkernel_tlad_mod.F90 index 77c0267b5..26929927a 100644 --- a/src/ufo/avgkernel/ufo_avgkernel_tlad_mod.F90 +++ b/src/ufo/avgkernel/ufo_avgkernel_tlad_mod.F90 @@ -288,14 +288,6 @@ subroutine avgkernel_simobs_ad_(self, geovals_in, obss, nvars, nlocs, hofx) geovar = self%tracervars(ivar) call ufo_geovals_get_var(geovals_in, geovar, tracer) - ! Allocate geovals profile if not yet allocated - if (.not. allocated(tracer%vals)) then - tracer%nlocs = self%nlocs - tracer%nval = self%nval - allocate(tracer%vals(tracer%nval, tracer%nlocs)) - tracer%vals(:,:) = 0.0_kind_real - endif - do iobs = 1, nlocs if (hofx(ivar,iobs) /= missing) then ! take care of missing obs if (self%troposphere) then diff --git a/src/ufo/crtm/ufo_aodcrtm_tlad_mod.F90 b/src/ufo/crtm/ufo_aodcrtm_tlad_mod.F90 index bc5225c83..40c509beb 100644 --- a/src/ufo/crtm/ufo_aodcrtm_tlad_mod.F90 +++ b/src/ufo/crtm/ufo_aodcrtm_tlad_mod.F90 @@ -397,13 +397,6 @@ SUBROUTINE ufo_aodcrtm_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) DO jaero=1,self%conf%n_aerosols CALL ufo_geovals_get_var(geovals, var_aerosols(jaero), var_p) -! allocate if not yet allocated - IF (.NOT. ALLOCATED(var_p%vals)) THEN - var_p%nlocs = self%n_Profiles - var_p%nval = self%n_Layers - ALLOCATE(var_p%vals(var_p%nval,var_p%nlocs)) - var_p%vals = 0.0_kind_real - ENDIF ! Multiply by Jacobian and add to hofx (adjoint) DO jprofile = 1, self%n_Profiles diff --git a/src/ufo/crtm/ufo_aodluts_tlad_mod.F90 b/src/ufo/crtm/ufo_aodluts_tlad_mod.F90 index 066f08ef4..363648690 100644 --- a/src/ufo/crtm/ufo_aodluts_tlad_mod.F90 +++ b/src/ufo/crtm/ufo_aodluts_tlad_mod.F90 @@ -312,12 +312,6 @@ SUBROUTINE ufo_aodluts_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) DO jaero=self%n_aerosols,1,-1 CALL ufo_geovals_get_var(geovals, var_aerosols(jaero), var_p) - IF (.NOT. ALLOCATED(var_p%vals)) THEN - var_p%nlocs = self%n_profiles - var_p%nval = self%n_layers - ALLOCATE(var_p%vals(var_p%nval,var_p%nlocs)) - var_p%vals = 0.0_kind_real - ENDIF qm_ad(jaero,:,:) = qm_ad(jaero,:,:) * self%layer_factors var_p%vals=qm_ad(jaero,:,:) diff --git a/src/ufo/crtm/ufo_radiancecrtm_tlad_mod.F90 b/src/ufo/crtm/ufo_radiancecrtm_tlad_mod.F90 index 992cf581d..d243ef5b0 100644 --- a/src/ufo/crtm/ufo_radiancecrtm_tlad_mod.F90 +++ b/src/ufo/crtm/ufo_radiancecrtm_tlad_mod.F90 @@ -888,14 +888,6 @@ subroutine ufo_radiancecrtm_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) ! Get t from geovals call ufo_geovals_get_var(geovals, var_ts, geoval_d) - ! allocate if not yet allocated - if (.not. allocated(geoval_d%vals)) then - geoval_d%nlocs = self%n_Profiles - geoval_d%nval = self%n_Layers - allocate(geoval_d%vals(geoval_d%nval,geoval_d%nlocs)) - geoval_d%vals = 0.0_kind_real - endif - ! Multiply by Jacobian and add to hofx (adjoint) do jprofile = 1, self%n_Profiles if (.not.self%Skip_Profiles(jprofile)) then @@ -918,14 +910,6 @@ subroutine ufo_radiancecrtm_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) ! Get Absorber from geovals call ufo_geovals_get_var(geovals, self%conf%Absorbers(jspec), geoval_d) - ! allocate if not yet allocated - if (.not. allocated(geoval_d%vals)) then - geoval_d%nlocs = self%n_Profiles - geoval_d%nval = self%n_Layers - allocate(geoval_d%vals(geoval_d%nval,geoval_d%nlocs)) - geoval_d%vals = 0.0_kind_real - endif - ispec = ufo_vars_getindex(self%conf_traj%Absorbers, self%conf%Absorbers(jspec)) ! Multiply by Jacobian and add to hofx (adjoint) @@ -951,14 +935,6 @@ subroutine ufo_radiancecrtm_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) ! Get Cloud from geovals call ufo_geovals_get_var(geovals, self%conf%Clouds(jspec,1), geoval_d) - ! allocate if not yet allocated - if (.not. allocated(geoval_d%vals)) then - geoval_d%nlocs = self%n_Profiles - geoval_d%nval = self%n_Layers - allocate(geoval_d%vals(geoval_d%nval,geoval_d%nlocs)) - geoval_d%vals = 0.0_kind_real - endif - ispec = ufo_vars_getindex(self%conf_traj%Clouds(:,1), self%conf%Clouds(jspec,1)) ! Multiply by Jacobian and add to hofx (adjoint) @@ -984,14 +960,6 @@ subroutine ufo_radiancecrtm_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) ! Get Cloud from geovals call ufo_geovals_get_var(geovals, self%conf%Surfaces(jspec), geoval_d) - ! allocate if not yet allocated - if (.not. allocated(geoval_d%vals)) then - geoval_d%nlocs = self%n_Profiles - geoval_d%nval = self%n_Layers - allocate(geoval_d%vals(geoval_d%nval,geoval_d%nlocs)) - geoval_d%vals = 0.0_kind_real - endif - select case(self%conf%Surfaces(jspec)) case(var_sfc_wtmp) diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_utils_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_utils_mod.f90 index e87930860..faf9b286b 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_utils_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_utils_mod.f90 @@ -7,9 +7,11 @@ module ufo_rttovonedvarcheck_minimize_utils_mod +use fckit_log_module, only : fckit_log +use iso_c_binding use kinds +use oops_variables_mod use ufo_constants_mod, only: grav, zero, t0c, half, one, two, min_q -use fckit_log_module, only : fckit_log use ufo_geovals_mod use ufo_rttovonedvarcheck_constants_mod use ufo_rttovonedvarcheck_ob_mod @@ -30,6 +32,7 @@ module ufo_rttovonedvarcheck_minimize_utils_mod public ufo_rttovonedvarcheck_CheckIteration public ufo_rttovonedvarcheck_CheckCloudyIteration public ufo_rttovonedvarcheck_PrintIterInfo +public ufo_rttovonedvarcheck_hofxdiags_levels character(len=max_string) :: message @@ -1235,4 +1238,44 @@ end subroutine ufo_rttovonedvarcheck_PrintIterInfo ! ---------------------------------------------------------- +subroutine ufo_rttovonedvarcheck_hofxdiags_levels(retrieval_vars, nlevels, ret_nlevs) + +implicit none + +type(oops_variables), intent(in) :: retrieval_vars !< retrieval variables for 1D-Var +integer, intent(in) :: nlevels +integer(c_size_t), intent(inout) :: ret_nlevs(:) !< number of levels for each retreival val + +character(MAXVARLEN), allocatable :: varlist(:) +character(MAXVARLEN) :: varname, message +integer :: i, ss, ee + +ret_nlevs(:) = zero +varlist = retrieval_vars % varlist() +do i = 1, size(varlist) + ss = index(varlist(i), "jacobian_", .false.) + 9 + ee = index(varlist(i), "_", .true.) - 1 + varname = varlist(i)(ss:ee) + if (trim(varname) == trim(var_ts) .or. & + trim(varname) == trim(var_q) .or. & + trim(varname) == trim(var_clw) .or. & + trim(varname) == trim(var_cli)) then + ret_nlevs(i) = nlevels + else if (trim(varname) == trim(var_sfc_t2m) .or. & + trim(varname) == trim(var_sfc_q2m) .or. & + trim(varname) == trim(var_sfc_p2m) .or. & + trim(varname) == trim(var_sfc_tskin) .or. & + trim(varname) == trim(var_u) .or. & + trim(varname) == trim(var_v)) then + ret_nlevs(i) = 1 + else + write(message, *) trim(varlist(i)), "not setup for retrieval yet: aborting" + call abor1_ftn(message) + end if +end do + +end subroutine ufo_rttovonedvarcheck_hofxdiags_levels + +! ---------------------------------------------------------- + end module ufo_rttovonedvarcheck_minimize_utils_mod diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_mod.f90 index 0cd1216ac..e0eb7d592 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_mod.f90 @@ -95,14 +95,14 @@ end subroutine ufo_rttovonedvarcheck_delete !! !! \date 09/06/2020: Created !! -subroutine ufo_rttovonedvarcheck_apply(self, f_conf, vars, retrieval_vars, geovals, apply) +subroutine ufo_rttovonedvarcheck_apply(self, f_conf, vars, hofxdiags_vars, geovals, apply) use ufo_utils_mod, only: cmp_strings implicit none type(ufo_rttovonedvarcheck), intent(inout) :: self !< rttovonedvarcheck main object type(fckit_configuration), intent(in) :: f_conf !< yaml file contents type(oops_variables), intent(in) :: vars !< channels for 1D-Var - type(oops_variables), intent(in) :: retrieval_vars !< retrieval variables for 1D-Var + type(oops_variables), intent(in) :: hofxdiags_vars !< retrieval variables for 1D-Var type(ufo_geovals), intent(in) :: geovals !< model values at observation space logical, intent(in) :: apply(:) !< qc manager flags @@ -137,6 +137,7 @@ subroutine ufo_rttovonedvarcheck_apply(self, f_conf, vars, retrieval_vars, geova logical :: onedvar_success logical :: cloud_retrieval = .false. type(ufo_radiancerttov) :: rttov_simobs + integer(c_size_t), allocatable :: ret_nlevs(:) ! ------------------------------------------ ! 1. Setup @@ -180,6 +181,7 @@ subroutine ufo_rttovonedvarcheck_apply(self, f_conf, vars, retrieval_vars, geova allocate(b_matrix(prof_index % nprofelements,prof_index % nprofelements)) allocate(b_inverse(prof_index % nprofelements,prof_index % nprofelements)) allocate(b_sigma(prof_index % nprofelements)) + allocate(ret_nlevs(hofxdiags_vars % nvars())) ! Decide on loop parameters - testing if (self % StartOb == 0) self % StartOb = 1 @@ -189,6 +191,9 @@ subroutine ufo_rttovonedvarcheck_apply(self, f_conf, vars, retrieval_vars, geova call abor1_ftn(message) end if + ! Calculate hofxdiags levels for each variable + call ufo_rttovonedvarcheck_hofxdiags_levels(hofxdiags_vars, self % nlevels, ret_nlevs) + ! ------------------------------------------ ! 2. Beginning main observation loop ! ------------------------------------------ @@ -264,7 +269,7 @@ subroutine ufo_rttovonedvarcheck_apply(self, f_conf, vars, retrieval_vars, geova call r_submatrix % setup(nchans_used, ob % channels_used, full_rmatrix=full_rmatrix) ! Setup hofxdiags for this retrieval - call ufo_geovals_setup(hofxdiags, retrieval_vars, 1) + call ufo_geovals_setup(hofxdiags, hofxdiags_vars, 1, hofxdiags_vars % nvars(), ret_nlevs) if (self % FullDiagnostics) then call ob % info() @@ -355,6 +360,7 @@ subroutine ufo_rttovonedvarcheck_apply(self, f_conf, vars, retrieval_vars, geova if (allocated(b_matrix)) deallocate(b_matrix) if (allocated(b_inverse)) deallocate(b_inverse) if (allocated(b_sigma)) deallocate(b_sigma) + if (allocated(ret_nlevs)) deallocate(ret_nlevs) call rttov_simobs % delete() end subroutine ufo_rttovonedvarcheck_apply diff --git a/src/ufo/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 b/src/ufo/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 index 1f3681918..08d0f04b6 100644 --- a/src/ufo/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 +++ b/src/ufo/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 @@ -288,22 +288,6 @@ subroutine ufo_gnssro_bendmetoffice_simobs_ad(self, geovals, hofx, obss) call ufo_geovals_get_var(geovals, var_q, q_d) ! specific humidity call ufo_geovals_get_var(geovals, var_prsi, prs_d) ! pressure -! Allocate the output for the air pressure - if (.not. allocated(prs_d%vals)) then - prs_d % nlocs = self % nlocs - prs_d % nval = self % nlevp - allocate(prs_d%vals(prs_d%nval, prs_d%nlocs)) - prs_d % vals = 0.0_kind_real - endif - -! Allocate the output for the specific humidity - if (.not. allocated(q_d%vals)) then - q_d % nlocs = self % nlocs - q_d % nval = self % nlevq - allocate(q_d%vals(q_d%nval, q_d%nlocs)) - q_d % vals = 0.0_kind_real - endif - missing = missing_value(missing) allocate(x_d(1:prs_d%nval + q_d%nval)) diff --git a/src/ufo/gnssro/BndNBAM/ufo_gnssro_bndnbam_tlad_mod.F90 b/src/ufo/gnssro/BndNBAM/ufo_gnssro_bndnbam_tlad_mod.F90 index 8b4d6cdd8..0c2566c9a 100644 --- a/src/ufo/gnssro/BndNBAM/ufo_gnssro_bndnbam_tlad_mod.F90 +++ b/src/ufo/gnssro/BndNBAM/ufo_gnssro_bndnbam_tlad_mod.F90 @@ -596,27 +596,6 @@ subroutine ufo_gnssro_bndnbam_simobs_ad(self, geovals, hofx, obss) call ufo_geovals_get_var(geovals, var_prsi, prs_ad) ! pressure end if -! allocate if not yet allocated - if (.not. allocated(t_ad%vals)) then - t_ad%nlocs = self%nlocs - t_ad%nval = self%nlev - allocate(t_ad%vals(t_ad%nval,t_ad%nlocs)) - t_ad%vals = 0.0_kind_real - endif - if (.not. allocated(prs_ad%vals)) then - prs_ad%nlocs = self%nlocs - prs_ad%nval = self%nlev1 - allocate(prs_ad%vals(prs_ad%nval,prs_ad%nlocs)) - prs_ad%vals = 0.0_kind_real - endif - if (.not. allocated(q_ad%vals)) then - q_ad%nlocs = self%nlocs - q_ad%nval = self%nlev - allocate(q_ad%vals(q_ad%nval,q_ad%nlocs)) - q_ad%vals = 0.0_kind_real - endif - if (.not. geovals%linit ) geovals%linit=.true. - nlocs = self%nlocs nlev = self%nlev nlev1 = self%nlev1 diff --git a/src/ufo/gnssro/BndROPP1D/ufo_gnssro_bndropp1d_tlad_mod.F90 b/src/ufo/gnssro/BndROPP1D/ufo_gnssro_bndropp1d_tlad_mod.F90 index 2cc2327f5..069784169 100755 --- a/src/ufo/gnssro/BndROPP1D/ufo_gnssro_bndropp1d_tlad_mod.F90 +++ b/src/ufo/gnssro/BndROPP1D/ufo_gnssro_bndropp1d_tlad_mod.F90 @@ -272,30 +272,6 @@ subroutine ufo_gnssro_bndropp1d_simobs_ad(self, geovals, hofx, obss) call ufo_geovals_get_var(geovals, var_q, q_d) ! specific humidity call ufo_geovals_get_var(geovals, var_prs, prs_d) ! pressure - ! allocate if not yet allocated - if (.not. allocated(t_d%vals)) then - t_d%nlocs = self%nlocs - t_d%nval = self%nval - allocate(t_d%vals(t_d%nval,t_d%nlocs)) - t_d%vals = 0.0_kind_real - endif - - if (.not. allocated(prs_d%vals)) then - prs_d%nlocs = self%nlocs - prs_d%nval = self%nval - allocate(prs_d%vals(prs_d%nval,prs_d%nlocs)) - prs_d%vals = 0.0_kind_real - endif - - if (.not. allocated(q_d%vals)) then - q_d%nlocs = self%nlocs - q_d%nval = self%nval - allocate(q_d%vals(q_d%nval,q_d%nlocs)) - q_d%vals = 0.0_kind_real - endif - - if (.not. geovals%linit ) geovals%linit=.true. - nlev = self%nval nlocs = self%nlocs diff --git a/src/ufo/gnssro/BndROPP1D/ufo_gnssro_bndropp1d_tlad_mod_stub.F90 b/src/ufo/gnssro/BndROPP1D/ufo_gnssro_bndropp1d_tlad_mod_stub.F90 index 0d9100d7c..8b3d0a25c 100755 --- a/src/ufo/gnssro/BndROPP1D/ufo_gnssro_bndropp1d_tlad_mod_stub.F90 +++ b/src/ufo/gnssro/BndROPP1D/ufo_gnssro_bndropp1d_tlad_mod_stub.F90 @@ -195,30 +195,6 @@ subroutine ufo_gnssro_bndropp1d_simobs_ad(self, geovals, hofx, obss) call ufo_geovals_get_var(geovals, var_q, q_d) ! specific humidity call ufo_geovals_get_var(geovals, var_prs, prs_d) ! pressure -! allocate if not yet allocated - if (.not. allocated(t_d%vals)) then - t_d%nlocs = self%nlocs - t_d%nval = self%nval - allocate(t_d%vals(t_d%nval,t_d%nlocs)) - t_d%vals = 0.0_kind_real - endif - - if (.not. allocated(prs_d%vals)) then - prs_d%nlocs = self%nlocs - prs_d%nval = self%nval - allocate(prs_d%vals(prs_d%nval,prs_d%nlocs)) - prs_d%vals = 0.0_kind_real - endif - - if (.not. allocated(q_d%vals)) then - q_d%nlocs = self%nlocs - q_d%nval = self%nval - allocate(q_d%vals(q_d%nval,q_d%nlocs)) - q_d%vals = 0.0_kind_real - endif - - if (.not. geovals%linit ) geovals%linit=.true. - nlev = self%nval nlocs = self%nlocs diff --git a/src/ufo/gnssro/BndROPP2D/ufo_gnssro_bndropp2d_tlad_mod.F90 b/src/ufo/gnssro/BndROPP2D/ufo_gnssro_bndropp2d_tlad_mod.F90 index 45560847c..3fc38c2da 100755 --- a/src/ufo/gnssro/BndROPP2D/ufo_gnssro_bndropp2d_tlad_mod.F90 +++ b/src/ufo/gnssro/BndROPP2D/ufo_gnssro_bndropp2d_tlad_mod.F90 @@ -384,30 +384,6 @@ subroutine ufo_gnssro_bndropp2d_simobs_ad(self, geovals, hofx, obss) call ufo_geovals_get_var(geovals, var_q, q_d) ! specific humidity call ufo_geovals_get_var(geovals, var_prs, prs_d) ! pressure -! allocate if not yet allocated - if (.not. allocated(t_d%vals)) then - t_d%nlocs = self%nlocs*n_horiz - t_d%nval = self%nval - allocate(t_d%vals(t_d%nval,t_d%nlocs)) - t_d%vals = 0.0_kind_real - endif - - if (.not. allocated(prs_d%vals)) then - prs_d%nlocs = self%nlocs*n_horiz - prs_d%nval = self%nval - allocate(prs_d%vals(prs_d%nval,prs_d%nlocs)) - prs_d%vals = 0.0_kind_real - endif - - if (.not. allocated(q_d%vals)) then - q_d%nlocs = self%nlocs*n_horiz - q_d%nval = self%nval - allocate(q_d%vals(q_d%nval,q_d%nlocs)) - q_d%vals = 0.0_kind_real - endif - - if (.not. geovals%linit ) geovals%linit=.true. - nlev = self%nval nlocs = self%nlocs diff --git a/src/ufo/gnssro/BndROPP2D/ufo_gnssro_bndropp2d_tlad_mod_stub.F90 b/src/ufo/gnssro/BndROPP2D/ufo_gnssro_bndropp2d_tlad_mod_stub.F90 index eba50d257..5f935887e 100755 --- a/src/ufo/gnssro/BndROPP2D/ufo_gnssro_bndropp2d_tlad_mod_stub.F90 +++ b/src/ufo/gnssro/BndROPP2D/ufo_gnssro_bndropp2d_tlad_mod_stub.F90 @@ -218,30 +218,6 @@ subroutine ufo_gnssro_bndropp2d_simobs_ad(self, geovals, hofx, obss) call ufo_geovals_get_var(geovals, var_q, q_d) ! specific humidity call ufo_geovals_get_var(geovals, var_prs, prs_d) ! pressure -! allocate if not yet allocated - if (.not. allocated(t_d%vals)) then - t_d%nlocs = self%nlocs - t_d%nval = self%nval - allocate(t_d%vals(t_d%nval,t_d%nlocs)) - t_d%vals = 0.0_kind_real - endif - - if (.not. allocated(prs_d%vals)) then - prs_d%nlocs = self%nlocs - prs_d%nval = self%nval - allocate(prs_d%vals(prs_d%nval,prs_d%nlocs)) - prs_d%vals = 0.0_kind_real - endif - - if (.not. allocated(q_d%vals)) then - q_d%nlocs = self%nlocs - q_d%nval = self%nval - allocate(q_d%vals(q_d%nval,q_d%nlocs)) - q_d%vals = 0.0_kind_real - endif - - if (.not. geovals%linit ) geovals%linit=.true. - nlev = self%nval nlocs = self%nlocs diff --git a/src/ufo/gnssro/RefNBAM/ufo_gnssro_ref_tlad_mod.F90 b/src/ufo/gnssro/RefNBAM/ufo_gnssro_ref_tlad_mod.F90 index dbd5136fe..9faeddcf9 100644 --- a/src/ufo/gnssro/RefNBAM/ufo_gnssro_ref_tlad_mod.F90 +++ b/src/ufo/gnssro/RefNBAM/ufo_gnssro_ref_tlad_mod.F90 @@ -202,30 +202,6 @@ subroutine ufo_gnssro_ref_simobs_ad(self, geovals, hofx, obss) call ufo_geovals_get_var(geovals, var_ts,t_d) call ufo_geovals_get_var(geovals, var_q, q_d) -! allocate if not yet allocated - if (.not. allocated(t_d%vals)) then - t_d%nlocs = self%nlocs - t_d%nval = self%nval - allocate(t_d%vals(t_d%nval,t_d%nlocs)) - t_d%vals = 0.0_kind_real - endif - - if (.not. allocated(prs_d%vals)) then - prs_d%nlocs = self%nlocs - prs_d%nval = self%nval - allocate(prs_d%vals(prs_d%nval,prs_d%nlocs)) - prs_d%vals = 0.0_kind_real - endif - - if (.not. allocated(q_d%vals)) then - q_d%nlocs = self%nlocs - q_d%nval = self%nval - allocate(q_d%vals(q_d%nval,q_d%nlocs)) - q_d%vals = 0.0_kind_real - endif - - if (.not. geovals%linit ) geovals%linit=.true. - missing = missing_value(missing) do iobs = 1, geovals%nlocs diff --git a/src/ufo/groundgnss/ZenithTotalDelayMetOffice/ufo_groundgnss_metoffice_tlad_mod.F90 b/src/ufo/groundgnss/ZenithTotalDelayMetOffice/ufo_groundgnss_metoffice_tlad_mod.F90 index 5943d5a7b..957c6d0c5 100644 --- a/src/ufo/groundgnss/ZenithTotalDelayMetOffice/ufo_groundgnss_metoffice_tlad_mod.F90 +++ b/src/ufo/groundgnss/ZenithTotalDelayMetOffice/ufo_groundgnss_metoffice_tlad_mod.F90 @@ -317,22 +317,6 @@ subroutine ufo_groundgnss_metoffice_simobs_ad(self, geovals, hofx, obss) call ufo_geovals_get_var(geovals, var_q, q_d) ! specific humidity call ufo_geovals_get_var(geovals, var_prsi, prs_d) ! pressure -! Allocate the output for the air pressure - if (.not. allocated(prs_d%vals)) then - prs_d % nlocs = self % nlocs - prs_d % nval = self % nlevp - allocate(prs_d%vals(prs_d%nval, prs_d%nlocs)) - prs_d % vals = 0.0_kind_real - endif - -! Allocate the output for the specific humidity - if (.not. allocated(q_d%vals)) then - q_d % nlocs = self % nlocs - q_d % nval = self % nlevq - allocate(q_d%vals(q_d%nval, q_d%nlocs)) - q_d % vals = 0.0_kind_real - endif - missing = missing_value(missing) allocate(x_d(1:prs_d%nval + q_d%nval)) allocate(pressure_d(1:prs_d%nval)) diff --git a/src/ufo/identity/ufo_identity_tlad_mod.F90 b/src/ufo/identity/ufo_identity_tlad_mod.F90 index cca0ce4b8..852ed92a0 100644 --- a/src/ufo/identity/ufo_identity_tlad_mod.F90 +++ b/src/ufo/identity/ufo_identity_tlad_mod.F90 @@ -132,12 +132,6 @@ subroutine identity_simobs_ad_(self, geovals, obss, nvars, nlocs, hofx) !> Get profile for this variable from geovals call ufo_geovals_get_var(geovals, geovar, point) - if (.not.(allocated(point%vals))) then - point%nval=1 - allocate(point%vals(1,size(hofx,2))) - point%vals = 0.0 - end if - ! backward obs operator do iobs = 1, nlocs if (hofx(ivar, iobs) /= missing) then diff --git a/src/ufo/marine/CMakeLists.txt b/src/ufo/marine/CMakeLists.txt index 4adb0b749..a6c9d10bc 100644 --- a/src/ufo/marine/CMakeLists.txt +++ b/src/ufo/marine/CMakeLists.txt @@ -11,14 +11,12 @@ add_subdirectory( coolskin ) add_subdirectory( seaicefraction ) add_subdirectory( seaicethickness ) add_subdirectory( chleuzintegr ) -add_subdirectory( utils ) if( ${gsw_FOUND} ) add_subdirectory( insitutemperature ) add_subdirectory( marinevertinterp ) add_subdirectory( obsop ) endif( ${gsw_FOUND} ) -PREPEND( _p_utils_files "marine/utils" ${utils_files} ) PREPEND( _p_adt_files "marine/adt" ${adt_files} ) PREPEND( _p_coolskin_files "marine/coolskin" ${coolskin_files} ) PREPEND( _p_seaicefraction_files "marine/seaicefraction" ${seaicefraction_files} ) @@ -31,7 +29,6 @@ if( ${gsw_FOUND} ) endif( ${gsw_FOUND} ) set ( marine_src_files - ${_p_utils_files} ${_p_adt_files} ${_p_coolskin_files} ${_p_marinevertinterp_files} diff --git a/src/ufo/marine/adt/ufo_adt_tlad_mod.F90 b/src/ufo/marine/adt/ufo_adt_tlad_mod.F90 index b5e79c25a..b7159832a 100644 --- a/src/ufo/marine/adt/ufo_adt_tlad_mod.F90 +++ b/src/ufo/marine/adt/ufo_adt_tlad_mod.F90 @@ -188,11 +188,6 @@ subroutine ufo_adt_simobs_ad(self, geovals, hofx, obss) call f_comm%allreduce(cnt, cnt_glb, fckit_mpi_sum()) offset_hofx = offset_hofx/cnt_glb -if (.not. allocated(geoval_adt%vals)) then - allocate(geoval_adt%vals(1,nlocs)) - geoval_adt%nval = 1 - geoval_adt%vals = 0.0 -end if do iobs = 1, nlocs if (hofx(iobs)/=self%r_miss_val) then geoval_adt%vals(1,iobs) = geoval_adt%vals(1,iobs) + hofx(iobs) - offset_hofx diff --git a/src/ufo/marine/coolskin/ufo_coolskin_tlad_mod.F90 b/src/ufo/marine/coolskin/ufo_coolskin_tlad_mod.F90 index 28edc4f57..ee6e4c65e 100644 --- a/src/ufo/marine/coolskin/ufo_coolskin_tlad_mod.F90 +++ b/src/ufo/marine/coolskin/ufo_coolskin_tlad_mod.F90 @@ -187,38 +187,6 @@ subroutine ufo_coolskin_simobs_ad(self, geovals, hofx, obss) call ufo_geovals_get_var(geovals, var_lw_rad , S_ns ) call ufo_geovals_get_var(geovals, var_sea_fric_vel , u ) -! If called from a model interface, allocate adjoint variables -if (.not. allocated(Td%vals)) then - allocate(Td%vals(1,nobs)) - Td%nval = 1 - Td%vals = 0.0 -end if -if (.not. allocated(R_nl%vals)) then - allocate(R_nl%vals(1,nobs)) - R_nl%nval = 1 - R_nl%vals = 0.0 -end if -if (.not. allocated(H_I%vals)) then - allocate(H_I%vals(1,nobs)) - H_I%nval = 1 - H_I%vals = 0.0 -end if -if (.not. allocated(H_s%vals)) then - allocate(H_s%vals(1,nobs)) - H_s%nval = 1 - H_s%vals = 0.0 -end if -if (.not. allocated(S_ns%vals)) then - allocate(S_ns%vals(1,nobs)) - S_ns%nval = 1 - S_ns%vals = 0.0 - end if -if (.not. allocated(u%vals)) then - allocate(u%vals(1,nobs)) - u%nval = 1 - u%vals = 0.0 -end if - ! Apply adjoint obs operator do iobs = 1, nobs if (hofx(iobs)/=self%r_miss_val) then diff --git a/src/ufo/marine/insitutemperature/ufo_insitutemperature_tlad_mod.F90 b/src/ufo/marine/insitutemperature/ufo_insitutemperature_tlad_mod.F90 index 6a5963733..fc61680cd 100644 --- a/src/ufo/marine/insitutemperature/ufo_insitutemperature_tlad_mod.F90 +++ b/src/ufo/marine/insitutemperature/ufo_insitutemperature_tlad_mod.F90 @@ -291,23 +291,6 @@ subroutine ufo_insitutemperature_simobs_ad(self, geovals, hofx, obss) nlev = self%nval nlocs = self%nlocs - if (.not. allocated(dtemp%vals)) then - allocate(dtemp%vals(nlev, size(hofx,1))) - dtemp%nval = nlev - dtemp%vals = 0.0 - end if - if (.not. allocated(dsalt%vals)) then - allocate(dsalt%vals(nlev, size(hofx,1))) - dsalt%nval = nlev - dsalt%vals = 0.0 - end if - if (.not. allocated(dlayerthick%vals)) then - allocate(dlayerthick%vals(nlev, size(hofx,1))) - dlayerthick%nval = nlev - dlayerthick%vals = 0.0 - ! Layer thickness is not a control variable: it stays zeroed out! - end if - ! backward sea temperature profile obs operator do iobs = 1, size(hofx,1) diff --git a/src/ufo/marine/marinevertinterp/ufo_marinevertinterp_tlad_mod.F90 b/src/ufo/marine/marinevertinterp/ufo_marinevertinterp_tlad_mod.F90 index 6ec691ec1..20ffd8357 100644 --- a/src/ufo/marine/marinevertinterp/ufo_marinevertinterp_tlad_mod.F90 +++ b/src/ufo/marine/marinevertinterp/ufo_marinevertinterp_tlad_mod.F90 @@ -224,13 +224,6 @@ subroutine ufo_marinevertinterp_simobs_ad(self, geovals, hofx, obss) nlev = self%nval nlocs = self%nlocs - if (.not. allocated(dvar%vals)) then - allocate(dvar%vals(nlev, size(hofx,1))) - dvar%nval = nlev - dvar%vals = 0.0 - end if - - ! backward vertical interp do iobs = 1, size(hofx,1) if (hofx(iobs) /= missing) then diff --git a/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc b/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc index aa31ac01e..303c636a3 100644 --- a/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc +++ b/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.cc @@ -26,27 +26,22 @@ static LinearObsOperatorMaker makerSeaIceFractionTL_("Sea ObsSeaIceFractionTLAD::ObsSeaIceFractionTLAD(const ioda::ObsSpace & odb, const eckit::Configuration & config) - : LinearObsOperatorBase(odb), keyOper_(0), varin_() + : LinearObsOperatorBase(odb), varin_() { const std::vector vv{"sea_ice_category_area_fraction"}; varin_.reset(new oops::Variables(vv)); - std::cout << keyOper_ << std::endl; - ufo_seaicelinear_setup_f90(keyOper_, config); - std::cout << keyOper_ << std::endl; oops::Log::trace() << "ObsSeaIceFractionTLAD created" << std::endl; } // ----------------------------------------------------------------------------- ObsSeaIceFractionTLAD::~ObsSeaIceFractionTLAD() { - ufo_seaicelinear_delete_f90(keyOper_); oops::Log::trace() << "ObsSeaIceFractionTLAD destructed" << std::endl; } // ----------------------------------------------------------------------------- void ObsSeaIceFractionTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { - ufo_seaicelinear_settraj_f90(keyOper_, geovals.toFortran(), obsspace()); oops::Log::trace() << "ObsSeaIceFractionTLAD: trajectory set" << std::endl; } @@ -69,8 +64,6 @@ void ObsSeaIceFractionTLAD::simulateObsTL(const GeoVaLs & gv, ioda::ObsVector & // ----------------------------------------------------------------------------- void ObsSeaIceFractionTLAD::simulateObsAD(GeoVaLs & gv, const ioda::ObsVector & ovec) const { - ufo_seaicelinear_alloc_ad_f90(keyOper_, gv.toFortran(), obsspace(), - ovec.size(), ovec.toFortran()); int nlocs = ovec.size(); int nlevs = gv.nlevs("sea_ice_category_area_fraction"); float miss = 0.0; diff --git a/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.h b/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.h index 0a5d20856..82635ac24 100644 --- a/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.h +++ b/src/ufo/marine/seaicefraction/ObsSeaIceFractionTLAD.h @@ -16,7 +16,6 @@ #include "oops/util/ObjectCounter.h" #include "ufo/LinearObsOperatorBase.h" -#include "ufo/marine/utils/ObsSeaIceLinear.interface.h" // Forward declarations namespace eckit { @@ -50,12 +49,8 @@ class ObsSeaIceFractionTLAD : public LinearObsOperatorBase, // Other const oops::Variables & requiredVars() const override {return *varin_;} - int & toFortran() {return keyOper_;} - const int & toFortran() const {return keyOper_;} - private: void print(std::ostream &) const override; - F90hop keyOper_; std::unique_ptr varin_; }; diff --git a/src/ufo/marine/seaicethickness/ufo_seaicethickness_tlad_mod.F90 b/src/ufo/marine/seaicethickness/ufo_seaicethickness_tlad_mod.F90 index 76131b8d7..215d8ee01 100644 --- a/src/ufo/marine/seaicethickness/ufo_seaicethickness_tlad_mod.F90 +++ b/src/ufo/marine/seaicethickness/ufo_seaicethickness_tlad_mod.F90 @@ -217,16 +217,6 @@ subroutine ufo_seaicethickness_simobs_ad(self, geovals, hofx, obss) write(err_msg,*) myname_, ' unknown number of categories' call abor1_ftn(err_msg) endif -if (.not. allocated(icefrac_d%vals)) then - icefrac_d%nval = ncat - allocate(icefrac_d%vals(ncat,size(hofx,1))) - icefrac_d%vals = 0.0 -endif -if (.not. allocated(icethick_d%vals)) then - icethick_d%nval = ncat - allocate(icethick_d%vals(ncat, size(hofx,1))) - icethick_d%vals = 0.0 -endif ! backward sea ice thickness obs operator diff --git a/src/ufo/marine/utils/CMakeLists.txt b/src/ufo/marine/utils/CMakeLists.txt deleted file mode 100644 index 663d08e08..000000000 --- a/src/ufo/marine/utils/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# (C) Copyright 2017-2019 UCAR. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -set ( utils_files - ObsSeaIceLinear.interface.F90 - ObsSeaIceLinear.interface.h - PARENT_SCOPE - ) diff --git a/src/ufo/marine/utils/ObsSeaIceLinear.interface.F90 b/src/ufo/marine/utils/ObsSeaIceLinear.interface.F90 deleted file mode 100644 index 3a1a877fa..000000000 --- a/src/ufo/marine/utils/ObsSeaIceLinear.interface.F90 +++ /dev/null @@ -1,128 +0,0 @@ -! (C) Copyright 2017-2019 UCAR -! -! This software is licensed under the terms of the Apache Licence Version 2.0 -! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -!> Fortran seaicelinear module for functions on the interface between C++ and Fortran -! to handle linearized observation operators - -module ufo_seaicelinear_mod_c - - use fckit_configuration_module, only: fckit_configuration - use iso_c_binding - use ufo_geovals_mod - use ufo_geovals_mod_c, only: ufo_geovals_registry - use ufo_vars_mod - - implicit none - - private - integer, parameter :: max_string=800 - type, public :: ufo_seaicelinear - integer :: ncat = -1 !< number of ice categories - end type ufo_seaicelinear - -#define LISTED_TYPE ufo_seaicelinear - - !> Linked list interface - defines registry_t type -#include "oops/util/linkedList_i.f" - - !> Global registry - type(registry_t) :: ufo_seaicelinear_registry - -contains - - ! ------------------------------------------------------------------------------ - !> Linked list implementation -#include "oops/util/linkedList_c.f" - -! ------------------------------------------------------------------------------ - -subroutine ufo_seaicelinear_setup_c(c_key_self, c_conf) bind(c,name='ufo_seaicelinear_setup_f90') -integer(c_int), intent(inout) :: c_key_self -type(c_ptr), value, intent(in) :: c_conf - -type(ufo_seaicelinear), pointer :: self -type(fckit_configuration) :: f_conf - -call ufo_seaicelinear_registry%setup(c_key_self, self) -f_conf = fckit_configuration(c_conf) -!call self%setup(f_conf) - -end subroutine ufo_seaicelinear_setup_c - -! ------------------------------------------------------------------------------ - -subroutine ufo_seaicelinear_delete_c(c_key_self) bind(c,name='ufo_seaicelinear_delete_f90') -integer(c_int), intent(inout) :: c_key_self - -type(ufo_seaicelinear), pointer :: self - -call ufo_seaicelinear_registry%get(c_key_self, self) -call ufo_seaicelinear_registry%remove(c_key_self) - -end subroutine ufo_seaicelinear_delete_c - -! ------------------------------------------------------------------------------ - -subroutine ufo_seaicelinear_settraj_c(c_key_self, c_key_geovals, c_obsspace)& - bind(c,name='ufo_seaicelinear_settraj_f90') -integer(c_int), intent(in) :: c_key_self -integer(c_int), intent(in) :: c_key_geovals -type(c_ptr), value, intent(in) :: c_obsspace - -type(ufo_seaicelinear), pointer :: self -type(ufo_geovals), pointer :: geovals -type(ufo_geoval), pointer :: geoval - -call ufo_seaicelinear_registry%get(c_key_self, self) -call ufo_geovals_registry%get(c_key_geovals,geovals) - -call ufo_geovals_get_var(geovals, var_seaicefrac, geoval) -self%ncat = geoval%nval - -end subroutine ufo_seaicelinear_settraj_c - -! ------------------------------------------------------------------------------ - -subroutine ufo_seaicelinear_alloc_ad_c(c_key_self, c_key_geovals, c_obsspace, c_nobs, c_hofx)& - bind(c,name='ufo_seaicelinear_alloc_ad_f90') -integer(c_int), intent(in) :: c_key_self -integer(c_int), intent(in) :: c_key_geovals -type(c_ptr), value, intent(in) :: c_obsspace -integer(c_int), intent(in) :: c_nobs -real(c_double), intent(in) :: c_hofx(c_nobs) - -type(ufo_seaicelinear), pointer :: self -type(ufo_geovals), pointer :: geovals -type(ufo_geoval), pointer :: geoval -character(max_string) :: err_msg - -call ufo_seaicelinear_registry%get(c_key_self, self) -call ufo_geovals_registry%get(c_key_geovals,geovals) - -! check if nlocs is consistent in geovals & hofx -if (geovals%nlocs /= size(c_hofx,1)) then - write(err_msg,*) ' error: nlocs inconsistent!' - call abor1_ftn(err_msg) -endif - -if (.not. geovals%linit ) geovals%linit=.true. - -! check if sea ice fraction variables is in geovals and get it -call ufo_geovals_get_var(geovals, var_seaicefrac, geoval) - -if (.not.(allocated(geoval%vals))) then - if (self%ncat < 1) then - write(err_msg,*)' unknown number of categories' - call abor1_ftn(err_msg) - endif - geoval%nval = self%ncat - allocate(geoval%vals(self%ncat,size(c_hofx,1))) - geoval%vals = 0.0 -end if -end subroutine ufo_seaicelinear_alloc_ad_c - -! ------------------------------------------------------------------------------ - -end module ufo_seaicelinear_mod_c diff --git a/src/ufo/marine/utils/ObsSeaIceLinear.interface.h b/src/ufo/marine/utils/ObsSeaIceLinear.interface.h deleted file mode 100644 index ec6fcc458..000000000 --- a/src/ufo/marine/utils/ObsSeaIceLinear.interface.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * (C) Copyright 2017-2019 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef UFO_MARINE_UTILS_OBSSEAICELINEAR_INTERFACE_H_ -#define UFO_MARINE_UTILS_OBSSEAICELINEAR_INTERFACE_H_ - -#include "ioda/ObsSpace.h" - -#include "ufo/Fortran.h" - -namespace ufo { - -/// Interface to Fortran UFO marine/seaicefraction routines - -extern "C" { - -// ----------------------------------------------------------------------------- - - void ufo_seaicelinear_setup_f90(F90hop &, const eckit::Configuration &); - void ufo_seaicelinear_delete_f90(F90hop &); - void ufo_seaicelinear_settraj_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &); - void ufo_seaicelinear_alloc_ad_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &, const double &); -// ----------------------------------------------------------------------------- - -} // extern C - -} // namespace ufo -#endif // UFO_MARINE_UTILS_OBSSEAICELINEAR_INTERFACE_H_ diff --git a/src/ufo/radarradialvelocity/ufo_radarradialvelocity_tlad_mod.F90 b/src/ufo/radarradialvelocity/ufo_radarradialvelocity_tlad_mod.F90 index 100a97cfa..39fd58242 100644 --- a/src/ufo/radarradialvelocity/ufo_radarradialvelocity_tlad_mod.F90 +++ b/src/ufo/radarradialvelocity/ufo_radarradialvelocity_tlad_mod.F90 @@ -210,15 +210,6 @@ subroutine radarradialvelocity_simobs_ad_(self, geovals, obss, nvars, nlocs, hof ! Get profile for this variable from geovals call ufo_geovals_get_var(geovals, geovar, profile) -! Allocate geovals profile if not yet allocated - if (.not. allocated(profile%vals)) then - profile%nlocs = self%nlocs - profile%nval = self%nval - allocate(profile%vals(profile%nval, profile%nlocs)) - profile%vals(:,:) = 0.0_kind_real - endif - if (.not. geovals%linit ) geovals%linit=.true. - ! Interpolate from geovals to observational location into hofx do iobs = 1, nlocs call vert_interp_apply_ad(profile%nval, profile%vals(:,iobs), & diff --git a/src/ufo/rttov/ufo_radiancerttov_tlad_mod.F90 b/src/ufo/rttov/ufo_radiancerttov_tlad_mod.F90 index d4b7b9f86..856e9f37a 100644 --- a/src/ufo/rttov/ufo_radiancerttov_tlad_mod.F90 +++ b/src/ufo/rttov/ufo_radiancerttov_tlad_mod.F90 @@ -502,14 +502,6 @@ subroutine ufo_radiancerttov_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) ! ----------- call ufo_geovals_get_var(geovals, var_ts, geoval_d) ! var_ts = air_temperature - ! allocate if not yet allocated - if (.not. allocated(geoval_d % vals)) then - geoval_d % nlocs = self % nprofiles - geoval_d % nval = self % nlevels - allocate(geoval_d % vals(geoval_d % nval,geoval_d % nlocs)) - geoval_d % vals = zero - end if - do ichan = 1, self % nchan_total, size(self%channels) prof = self % RTprof_K % chanprof(ichan) % prof do jchan = 1, size(self%channels) @@ -527,14 +519,6 @@ subroutine ufo_radiancerttov_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) do jspec = 1, self%conf%ngas call ufo_geovals_get_var(geovals, self%conf%Absorbers(jspec), geoval_d) - ! allocate if not yet allocated - if (.not. allocated(geoval_d % vals)) then - geoval_d % nlocs = self % nprofiles - geoval_d % nval = self % nlevels - allocate(geoval_d % vals(geoval_d % nval,geoval_d % nlocs)) - geoval_d % vals = zero - end if - do ichan = 1, self % nchan_total, size(self%channels) prof = self % RTprof_K % chanprof(ichan) % prof do jchan = 1, size(self%channels) @@ -570,14 +554,6 @@ subroutine ufo_radiancerttov_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) call ufo_geovals_get_var(geovals, var_sfc_t2m, geoval_d) - ! allocate if not yet allocated - if (.not. allocated(geoval_d % vals)) then - geoval_d % nlocs = self % nprofiles - geoval_d % nval = self % nlevels - allocate(geoval_d % vals(geoval_d % nval,geoval_d % nlocs)) ! DARFIX try setting to 1? - geoval_d % vals = zero - end if - do ichan = 1, self % nchan_total, size(self%channels) prof = self % RTprof_K % chanprof(ichan) % prof do jchan = 1, size(self%channels) @@ -590,13 +566,6 @@ subroutine ufo_radiancerttov_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) !q2m call ufo_geovals_get_var(geovals, var_sfc_q2m, geoval_d) - ! allocate if not yet allocated - if (.not. allocated(geoval_d % vals)) then - geoval_d % nlocs = self % nprofiles - geoval_d % nval = self % nlevels - allocate(geoval_d % vals(geoval_d % nval,geoval_d % nlocs)) ! DARFIX try setting to 1? - geoval_d % vals = zero - end if do ichan = 1, self % nchan_total, size(self%channels) prof = self % RTprof_K % chanprof(ichan) % prof @@ -613,18 +582,6 @@ subroutine ufo_radiancerttov_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) call ufo_geovals_get_var(geovals, var_u, geoval_d) call ufo_geovals_get_var(geovals, var_v, geoval_d2) - ! allocate if not yet allocated - if (.not. allocated(geoval_d % vals)) then - geoval_d % nlocs = self % nprofiles - geoval_d % nval = self % nlevels - geoval_d2 % nlocs = self % nprofiles - geoval_d2 % nval = self % nlevels - allocate(geoval_d % vals(geoval_d % nval,geoval_d % nlocs), & - geoval_d2 % vals(geoval_d % nval,geoval_d % nlocs)) ! DARFIX try setting to 1? - geoval_d % vals = zero - geoval_d2 % vals = zero - end if - do ichan = 1, self % nchan_total, size(self%channels) prof = self % RTprof_K % chanprof(ichan) % prof do jchan = 1, size(self%channels) @@ -641,14 +598,6 @@ subroutine ufo_radiancerttov_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) !Tskin call ufo_geovals_get_var(geovals, var_sfc_tskin, geoval_d) - ! allocate if not yet allocated - if (.not. allocated(geoval_d % vals)) then - geoval_d % nlocs = self % nprofiles - geoval_d % nval = self % nlevels - allocate(geoval_d % vals(geoval_d % nval,geoval_d % nlocs)) ! DARFIX try setting to 1? - geoval_d % vals = zero - end if - do ichan = 1, self % nchan_total, size(self%channels) prof = self % RTprof_K % chanprof(ichan) % prof do jchan = 1, size(self%channels) diff --git a/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc b/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc index d28caa6b2..6347f1fc6 100644 --- a/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc +++ b/src/ufo/rttovcpp/ObsRadianceRTTOVCPPTLAD.cc @@ -123,11 +123,6 @@ void ObsRadianceRTTOVCPPTLAD::simulateObsAD(GeoVaLs & dx, const ioda::ObsVector std::size_t nprofiles = dy.nlocs(); std::size_t nchannels = aRttov_.getNchannels(); - if ( dx.nlevs("air_temperature") == 0 ) { // if dx is not allocated - dx.allocate(nlevels, varin_); - dx.zero(); - } - std::vector tmpvar2d(nprofiles, 0.0); // one single level field std::vector> dT; // [nlevels][nprofiles] std::vector> dQ; // [nlevels][nprofiles] diff --git a/src/ufo/sattcwv/ufo_SatTCWV_tlad_mod.F90 b/src/ufo/sattcwv/ufo_SatTCWV_tlad_mod.F90 index 8073b0deb..b286c2d94 100644 --- a/src/ufo/sattcwv/ufo_SatTCWV_tlad_mod.F90 +++ b/src/ufo/sattcwv/ufo_SatTCWV_tlad_mod.F90 @@ -293,14 +293,6 @@ subroutine ufo_sattcwv_simobs_ad(self, geovals, hofx, obss) nlev = self % nlevq ! Number of layers nlocs = self % nlocs ! Number of observations -! Allocate the output for the specific humidity perturbations - if (.not. allocated(q_d%vals)) then - q_d % nlocs = self % nlocs - q_d % nval = self % nlevq - allocate(q_d % vals(q_d % nval, q_d % nlocs)) - q_d % vals = zero - endif -! missing = missing_value(missing) ! Allocate state vector x_d allocate(x_d(1:nlev)) diff --git a/src/ufo/ufo_geovals_mod.F90 b/src/ufo/ufo_geovals_mod.F90 index e524d809f..e5fb35ab9 100644 --- a/src/ufo/ufo_geovals_mod.F90 +++ b/src/ufo/ufo_geovals_mod.F90 @@ -20,9 +20,9 @@ module ufo_geovals_mod public :: ufo_geovals, ufo_geoval public :: ufo_geovals_get_var -public :: ufo_geovals_default_constr, ufo_geovals_setup, ufo_geovals_delete, ufo_geovals_print +public :: ufo_geovals_default_constr, ufo_geovals_setup, ufo_geovals_partial_setup, ufo_geovals_delete public :: ufo_geovals_zero, ufo_geovals_random, ufo_geovals_scalmult -public :: ufo_geovals_allocate +public :: ufo_geovals_allocate, ufo_geovals_print public :: ufo_geovals_profmult public :: ufo_geovals_reorderzdir public :: ufo_geovals_assign, ufo_geovals_add, ufo_geovals_diff, ufo_geovals_abs @@ -75,8 +75,43 @@ subroutine ufo_geovals_default_constr(self) end subroutine ufo_geovals_default_constr +! ------------------------------------------------------------------------------ +!> Initializes and allocates \p self GeoVaLs with \p nlocs number of locations for +!> \p vars variables. \p nvals array contains number of values to allocate for +!> each of the variables +subroutine ufo_geovals_setup(self, vars, nlocs, nvars, nvals) +use oops_variables_mod +implicit none +type(ufo_geovals), intent(inout) :: self +type(oops_variables), intent(in) :: vars +integer, intent(in) :: nlocs, nvars +integer(c_size_t), intent(in) :: nvals(nvars) + +integer :: ivar -subroutine ufo_geovals_setup(self, vars, nlocs) +call ufo_geovals_delete(self) +self%nlocs = nlocs +self%missing_value = missing_value(self%missing_value) + +self%nvar = vars%nvars() +allocate(self%geovals(self%nvar)) +allocate(self%variables(self%nvar)) +do ivar = 1, self%nvar + self%variables(ivar) = vars%variable(ivar) + self%geovals(ivar)%nlocs = nlocs + self%geovals(ivar)%nval = nvals(ivar) + allocate(self%geovals(ivar)%vals(nvals(ivar), nlocs)) + self%geovals(ivar)%vals(:,:) = 0.0 +enddo +self%linit = .true. + +end subroutine ufo_geovals_setup + +! ------------------------------------------------------------------------------ +!> Deprecated, use ufo_geovals_setup instead. +!> Partially initializes \p self GeoVaLs with \p nlocs number of locations +!> \p vars variables. Does not allocate geovals(i)%vals. +subroutine ufo_geovals_partial_setup(self, vars, nlocs) use oops_variables_mod implicit none type(ufo_geovals), intent(inout) :: self @@ -98,9 +133,10 @@ subroutine ufo_geovals_setup(self, vars, nlocs) self%geovals(ivar)%nval = 0 enddo -end subroutine ufo_geovals_setup +end subroutine ufo_geovals_partial_setup ! ------------------------------------------------------------------------------ +!> Deprecated. Rely on ufo_geovals_setup to allocate GeoVaLs instead. !> Allocates GeoVaLs for \p vars variables with \p nlevels number of levels. !> If the GeoVaLs for this variable were allocated before with different size, !> aborts. @@ -1044,7 +1080,7 @@ subroutine ufo_geovals_read_netcdf(self, filename, loc_multiplier, c_obspace, va end if ! allocate geovals structure -call ufo_geovals_setup(self, vars, nlocs) +call ufo_geovals_partial_setup(self, vars, nlocs) do ivar = 1, self%nvar diff --git a/test/ufo/GeoVaLs.h b/test/ufo/GeoVaLs.h index 2e55b976a..75b9c1152 100644 --- a/test/ufo/GeoVaLs.h +++ b/test/ufo/GeoVaLs.h @@ -351,6 +351,47 @@ void testGeoVaLsAllocatePutGet() { } } +/// \brief Tests GeoVaLs(const Locations &, const Variables &, const std::vector &) +/// constructor. Tests that levels get correctly allocated, and that the GeoVaLs are zeroed +/// out. +void testGeoVaLsConstructor() { + const eckit::LocalConfiguration conf(::test::TestEnvironment::config()); + const eckit::LocalConfiguration testconf(conf, "geovals get test"); + + const std::string var1 = "variable1"; + const std::string var2 = "variable2"; + const oops::Variables testvars({var1, var2}); + + /// Setup GeoVaLs that are not filled in or allocated; test that they are not allocated + const Locations locs(testconf, oops::mpi::world()); + const std::vector testnlevs({10, 1}); + GeoVaLs gval(locs, testvars, testnlevs); + oops::Log::test() << "Created and allocated GeoVaLs: " << var1 << "; nlevs(var1) = " << + gval.nlevs(var1) << ", " << var2 << "; nlevs(var2) = " << + gval.nlevs(var2) << std::endl; + EXPECT_EQUAL(gval.nlevs(var1), 10); + EXPECT_EQUAL(gval.nlevs(var2), 1); + + const std::vector refvalues(gval.nlocs(), 0.0); + const std::vector refvalues_float(gval.nlocs(), 0.0); + const std::vector refvalues_int(gval.nlocs(), 0); + + /// check that get method returns zeroes (initial values in GeoVaLs) + std::vector testvalues(gval.nlocs(), 1.0); + gval.get(testvalues, var2); + oops::Log::test() << "Get result: " << testvalues << std::endl; + EXPECT_EQUAL(testvalues, refvalues); + std::vector testvalues_float(gval.nlocs(), 1.0); + gval.get(testvalues_float, var2); + oops::Log::test() << "Get(float) result: " << testvalues_float << std::endl; + EXPECT_EQUAL(testvalues_float, refvalues_float); + std::vector testvalues_int(gval.nlocs(), 1); + gval.get(testvalues_int, var2); + oops::Log::test() << "Get(int) result: " << testvalues_int << std::endl; + EXPECT_EQUAL(testvalues_int, refvalues_int); +} + + // ----------------------------------------------------------------------------- class GeoVaLs : public oops::Test { @@ -367,6 +408,8 @@ class GeoVaLs : public oops::Test { { testGeoVaLs(); }); ts.emplace_back(CASE("ufo/GeoVaLs/testGeoVaLsAllocatePutGet") { testGeoVaLsAllocatePutGet(); }); + ts.emplace_back(CASE("ufo/GeoVaLs/testGeoVaLsConstructor") + { testGeoVaLsConstructor(); }); } void clear() const override {} From 36566c4a151b0fa6a0b67384a0d03d0c6323f78f Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Fri, 25 Jun 2021 18:41:51 +0100 Subject: [PATCH 17/98] TL/AD code for Categorical observation operator (#1224) * initial TL/AD code for categorical operator * putAtLocation and associated tests * comment * replace std::cout * coding norms * correct AD operator * coding norms * ensure compiles with GCC6 * improved debug statement * fix comment * additional comment in description * remove excessive debug output * const auto in for loop * resize instead of allocate vector * move common data and functions into utilities class * coding norms * modifications following review * add ObsCategoricalData * remove ObsBias from TL/AD code Co-authored-by: Ryan Honeyager --- src/ufo/categoricaloper/CMakeLists.txt | 3 + src/ufo/categoricaloper/ObsCategorical.cc | 82 +-------- src/ufo/categoricaloper/ObsCategorical.h | 19 +-- src/ufo/categoricaloper/ObsCategoricalData.h | 160 ++++++++++++++++++ src/ufo/categoricaloper/ObsCategoricalTLAD.cc | 125 ++++++++++++++ src/ufo/categoricaloper/ObsCategoricalTLAD.h | 67 ++++++++ test/CMakeLists.txt | 7 + 7 files changed, 374 insertions(+), 89 deletions(-) create mode 100644 src/ufo/categoricaloper/ObsCategoricalData.h create mode 100644 src/ufo/categoricaloper/ObsCategoricalTLAD.cc create mode 100644 src/ufo/categoricaloper/ObsCategoricalTLAD.h diff --git a/src/ufo/categoricaloper/CMakeLists.txt b/src/ufo/categoricaloper/CMakeLists.txt index 16603f14d..42fa30a40 100644 --- a/src/ufo/categoricaloper/CMakeLists.txt +++ b/src/ufo/categoricaloper/CMakeLists.txt @@ -6,7 +6,10 @@ set ( categoricaloper_files ObsCategorical.h ObsCategorical.cc + ObsCategoricalData.h ObsCategoricalParameters.h + ObsCategoricalTLAD.h + ObsCategoricalTLAD.cc ) PREPEND( _p_categoricaloper_files "categoricaloper" ${categoricaloper_files} ) diff --git a/src/ufo/categoricaloper/ObsCategorical.cc b/src/ufo/categoricaloper/ObsCategorical.cc index 60ad1adc6..27fe67dd4 100644 --- a/src/ufo/categoricaloper/ObsCategorical.cc +++ b/src/ufo/categoricaloper/ObsCategorical.cc @@ -32,59 +32,7 @@ ObsCategorical::ObsCategorical(const ioda::ObsSpace & odb, { oops::Log::trace() << "ObsCategorical constructor starting" << std::endl; - ObsCategoricalParameters parameters; - parameters.validateAndDeserialize(config); - - // Get categorical variable from ObsSpace (and throw an exception if it is not present). - // In the ObsSpace, the categorical variable can be either a vector of strings or - // a vector of integers; if the latter, it is converted to a vector of strings here. - const std::string &categoricalVariableName = parameters.categoricalVariable.value(); - oops::Log::debug() << "categorical variable: " << categoricalVariableName << std::endl; - categoricalVariable_.assign(odb_.nlocs(), ""); - if (odb_.has("MetaData", categoricalVariableName)) { - const ioda::ObsDtype dtype = odb_.dtype("MetaData", categoricalVariableName); - if (dtype == ioda::ObsDtype::String) { - odb_.get_db("MetaData", categoricalVariableName, categoricalVariable_); - } else if (dtype == ioda::ObsDtype::Integer) { - std::vector categoricalVariableInt(odb_.nlocs()); - odb_.get_db("MetaData", categoricalVariableName, categoricalVariableInt); - std::transform(categoricalVariableInt.cbegin(), categoricalVariableInt.cend(), - categoricalVariable_.begin(), [](int i) {return std::to_string(i);}); - } else { - throw eckit::UserError("The categorical variable must be a vector of " - "either strings or integers", Here()); - } - } else { - throw eckit::UserError("The categorical variable " + categoricalVariableName + - " does not exist", Here()); - } - - // Name of fallback operator. - fallbackOperatorName_ = parameters.fallbackOperatorName.value(); - oops::Log::debug() << "Fallback operator: " << fallbackOperatorName_ << std::endl; - - // Map of categorised operator names. - categorisedOperatorNames_ = parameters.categorisedOperatorNames.value(); - oops::Log::debug() << "Categorised operators: " << categorisedOperatorNames_ << std::endl; - - // Create list of component operators. - for (const eckit::LocalConfiguration &operatorConfig : - parameters.operatorConfigurations.value()) { - std::unique_ptr op(ObsOperatorFactory::create(odb, operatorConfig)); - requiredVars_ += op->requiredVars(); - components_.emplace(std::make_pair(operatorConfig.getString("name"), std::move(op))); - } - - // Check the fallback operator has been configured. - if (components_.find(fallbackOperatorName_) == components_.end()) - throw eckit::UserError("The operator " + fallbackOperatorName_ + - " has not been configured", Here()); - - // Check the categorised operators have been configured. - for (const auto &operName : categorisedOperatorNames_) - if (components_.find(operName.second) == components_.end()) - throw eckit::UserError("The operator " + operName.second + - " has not been configured", Here()); + data_.configure(odb, config); oops::Log::trace() << "ObsCategorical constructor finished" << std::endl; } @@ -101,42 +49,28 @@ void ObsCategorical::simulateObs(const GeoVaLs & gv, ioda::ObsVector & ovec, ObsDiagnostics & ydiags) const { oops::Log::trace() << "ObsCategorical: simulateObs entered" << std::endl; + oops::Log::debug() << "Running operators" << std::endl; + // Container of ObsVectors produced by each operator. std::map ovecs; - oops::Log::debug() << "Running operators" << std::endl; // Run each operator and store output in ovecs. - for (const auto& component : components_) { + for (const auto& component : data_.components()) { ioda::ObsVector ovecTemp(ovec); component.second->simulateObs(gv, ovecTemp, ydiags); ovecs.insert({component.first, ovecTemp}); } - // Insert values into ovec according to the categorical variable. - // Use the fallback operator when necessary. oops::Log::debug() << "Producing final ObsVector" << std::endl; - for (size_t jloc = 0; jloc < ovec.nlocs(); ++jloc) { - auto it_operName = categorisedOperatorNames_.find(categoricalVariable_[jloc]); - const auto &operName = (it_operName != categorisedOperatorNames_.end() ? - it_operName->second : - fallbackOperatorName_); - oops::Log::debug() << "Location " << jloc << ": operator name = " << operName << std::endl; - const auto &ovecloc = ovecs.at(operName); - // Loop over each variable at this location. - for (size_t jvar = 0; jvar < ovec.nvars(); ++jvar) { - const size_t idx = jloc * ovec.nvars() + jvar; - ovec[idx] = ovecloc[idx]; - } - } - oops::Log::trace() << "ObsCategorical: simulateObs exit " << std::endl; + data_.fillHofX(ovecs, ovec); + + oops::Log::trace() << "ObsCategorical: simulateObs finished" << std::endl; } // ----------------------------------------------------------------------------- void ObsCategorical::print(std::ostream & os) const { - os << "ObsCategorical operator:" << std::endl; - os << "- Fallback operator: " << fallbackOperatorName_ << std::endl; - os << "- Categorised operators: " << categorisedOperatorNames_ << std::endl; + data_.print(os); } // ----------------------------------------------------------------------------- diff --git a/src/ufo/categoricaloper/ObsCategorical.h b/src/ufo/categoricaloper/ObsCategorical.h index f9f15a371..3ddbbddb3 100644 --- a/src/ufo/categoricaloper/ObsCategorical.h +++ b/src/ufo/categoricaloper/ObsCategorical.h @@ -17,6 +17,7 @@ #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" +#include "ufo/categoricaloper/ObsCategoricalData.h" #include "ufo/ObsOperatorBase.h" /// Forward declarations @@ -91,7 +92,7 @@ class ObsCategorical : public ObsOperatorBase, void simulateObs(const GeoVaLs &, ioda::ObsVector &, ObsDiagnostics &) const override; - const oops::Variables & requiredVars() const override { return requiredVars_; } + const oops::Variables & requiredVars() const override { return data_.requiredVars(); } private: void print(std::ostream &) const override; @@ -100,20 +101,8 @@ class ObsCategorical : public ObsOperatorBase, /// ObsSpace. const ioda::ObsSpace& odb_; - /// Observation operators which are run by the Categorical operator. - std::map> components_; - - /// Required variables. - oops::Variables requiredVars_; - - /// Value of the categorical variable in the ObsSpace. - std::vector categoricalVariable_; - - /// Name of the fallback observation operator. - std::string fallbackOperatorName_; - - /// Names of the categorised observation operators. - std::map categorisedOperatorNames_; + /// Data handler for the Categorical operator and TL/AD code. + ObsCategoricalData data_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/categoricaloper/ObsCategoricalData.h b/src/ufo/categoricaloper/ObsCategoricalData.h new file mode 100644 index 000000000..8bde78843 --- /dev/null +++ b/src/ufo/categoricaloper/ObsCategoricalData.h @@ -0,0 +1,160 @@ +/* + * (C) Copyright 2021 UK Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_CATEGORICALOPER_OBSCATEGORICALDATA_H_ +#define UFO_CATEGORICALOPER_OBSCATEGORICALDATA_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "ioda/ObsVector.h" + +#include "oops/base/Variables.h" + +#include "oops/util/Logger.h" + +#include "ufo/categoricaloper/ObsCategoricalParameters.h" + +/// Forward declarations +namespace eckit { + class Configuration; +} + +namespace ioda { + class ObsSpace; +} + +namespace ufo { +/// \brief Data handler for the Categorical observation operator and TL/AD code. +template +class ObsCategoricalData { + public: + /// Get all information related to the configuration of the Categorical operator and TL/AD code. + void configure(const ioda::ObsSpace & odb, + const eckit::Configuration & config) + { + ObsCategoricalParameters parameters; + parameters.validateAndDeserialize(config); + + // Get categorical variable from ObsSpace (and throw an exception if it is not present). + // In the ObsSpace, the categorical variable can be either a vector of strings or + // a vector of integers; if the latter, it is converted to a vector of strings here. + const std::string &categoricalVariableName = parameters.categoricalVariable.value(); + oops::Log::debug() << "categorical variable: " << categoricalVariableName << std::endl; + categoricalVariable_.assign(odb.nlocs(), ""); + if (odb.has("MetaData", categoricalVariableName)) { + const ioda::ObsDtype dtype = odb.dtype("MetaData", categoricalVariableName); + if (dtype == ioda::ObsDtype::String) { + odb.get_db("MetaData", categoricalVariableName, categoricalVariable_); + } else if (dtype == ioda::ObsDtype::Integer) { + std::vector categoricalVariableInt(odb.nlocs()); + odb.get_db("MetaData", categoricalVariableName, categoricalVariableInt); + std::transform(categoricalVariableInt.cbegin(), categoricalVariableInt.cend(), + categoricalVariable_.begin(), [](int i) {return std::to_string(i);}); + } else { + throw eckit::UserError("The categorical variable must be a vector of " + "either strings or integers", Here()); + } + } else { + throw eckit::UserError("The categorical variable " + categoricalVariableName + + " does not exist", Here()); + } + + // Name of fallback operator. + fallbackOperatorName_ = parameters.fallbackOperatorName.value(); + oops::Log::debug() << "Fallback operator: " << fallbackOperatorName_ << std::endl; + + // Map of categorised operator names. + categorisedOperatorNames_ = parameters.categorisedOperatorNames.value(); + oops::Log::debug() << "Categorised operators: " << categorisedOperatorNames_ << std::endl; + + // Fill vector of operator names. + for (size_t jloc = 0; jloc < odb.nlocs(); ++jloc) { + auto it_operName = categorisedOperatorNames_.find(categoricalVariable_[jloc]); + const auto &operName = (it_operName != categorisedOperatorNames_.end() ? + it_operName->second : + fallbackOperatorName_); + locOperNames_.emplace_back(operName); + } + + // Create list of component operators. + for (const eckit::LocalConfiguration &operatorConfig : + parameters.operatorConfigurations.value()) { + std::unique_ptr op(OPFACTORY::create(odb, operatorConfig)); + requiredVars_ += op->requiredVars(); + components_.emplace(std::make_pair(operatorConfig.getString("name"), std::move(op))); + } + + // Check the fallback operator has been configured. + if (components_.find(fallbackOperatorName_) == components_.end()) + throw eckit::UserError("The operator " + fallbackOperatorName_ + + " has not been configured", Here()); + + // Check the categorised operators have been configured. + for (const auto &operName : categorisedOperatorNames_) + if (components_.find(operName.second) == components_.end()) + throw eckit::UserError("The operator " + operName.second + + " has not been configured", Here()); + } + + /// Return required variables for the operator. + const oops::Variables & requiredVars() const {return requiredVars_;} + + /// Return component operators. + const std::map> & components() const {return components_;} + + /// Return list of operator names to use at each location. + const std::vector & locOperNames() const {return locOperNames_;} + + /// Fill final H(x) vector from a list of components. + void fillHofX(const std::map & ovecs, + ioda::ObsVector & ovec) const { + // Insert values into ovec according to the categorical variable. + // Use the fallback operator when necessary. + for (size_t jloc = 0; jloc < ovec.nlocs(); ++jloc) { + const auto &ovecloc = ovecs.at(locOperNames_[jloc]); + // Loop over each variable at this location. + for (size_t jvar = 0; jvar < ovec.nvars(); ++jvar) { + const size_t idx = jloc * ovec.nvars() + jvar; + ovec[idx] = ovecloc[idx]; + } + } + } + + void print(std::ostream & os) const { + os << "ObsCategorical operator:" << std::endl; + os << "- Fallback operator: " << fallbackOperatorName_ << std::endl; + os << "- Categorised operators: " << categorisedOperatorNames_ << std::endl; + } + + private: + /// Required variables. + oops::Variables requiredVars_; + + /// Observation operators which are run by the Categorical operator. + std::map> components_; + + /// Value of the categorical variable in the ObsSpace. + std::vector categoricalVariable_; + + /// Name of the fallback observation operator. + std::string fallbackOperatorName_; + + /// Names of the categorised observation operators. + std::map categorisedOperatorNames_; + + /// Operator name at each location. + std::vector locOperNames_; +}; + +} // namespace ufo +#endif // UFO_CATEGORICALOPER_OBSCATEGORICALDATA_H_ diff --git a/src/ufo/categoricaloper/ObsCategoricalTLAD.cc b/src/ufo/categoricaloper/ObsCategoricalTLAD.cc new file mode 100644 index 000000000..fc2fa2397 --- /dev/null +++ b/src/ufo/categoricaloper/ObsCategoricalTLAD.cc @@ -0,0 +1,125 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/categoricaloper/ObsCategoricalTLAD.h" + +#include +#include +#include +#include + +#include "ioda/ObsVector.h" + +#include "oops/util/Logger.h" + +#include "ufo/categoricaloper/ObsCategoricalParameters.h" +#include "ufo/GeoVaLs.h" + +namespace ufo { + +// ----------------------------------------------------------------------------- +static LinearObsOperatorMaker makerCategoricalTL_("Categorical"); +// ----------------------------------------------------------------------------- + +ObsCategoricalTLAD::ObsCategoricalTLAD(const ioda::ObsSpace & odb, + const eckit::Configuration & config) + : LinearObsOperatorBase(odb), odb_(odb) +{ + oops::Log::trace() << "ObsCategoricalTLAD constructor starting" << std::endl; + + data_.configure(odb, config); + + oops::Log::trace() << "ObsCategoricalTLAD created." << std::endl; +} + +// ----------------------------------------------------------------------------- + +ObsCategoricalTLAD::~ObsCategoricalTLAD() { + oops::Log::trace() << "ObsCategoricalTLAD destructed" << std::endl; +} + +// ----------------------------------------------------------------------------- + +void ObsCategoricalTLAD::setTrajectory(const GeoVaLs & geovals, + ObsDiagnostics & ydiags) { + oops::Log::trace() << "ObsCategoricalTLAD: setTrajectory entered" << std::endl; + + // Set trajectory for each operator. + for (const auto& component : data_.components()) + component.second->setTrajectory(geovals, ydiags); + + oops::Log::trace() << "ObsCategoricalTLAD: setTrajectory finished" << std::endl; +} + +// ----------------------------------------------------------------------------- + +void ObsCategoricalTLAD::simulateObsTL(const GeoVaLs & geovals, ioda::ObsVector & ovec) const { + oops::Log::trace() << "ObsCategoricalTLAD: simulateObsTL entered" << std::endl; + + oops::Log::debug() << "Running TL operators" << std::endl; + + // Container of ObsVectors produced by each TL operator. + std::map ovecs; + // Run each TL operator and store output in ovecs. + for (const auto& component : data_.components()) { + ioda::ObsVector ovecTemp(ovec); + component.second->simulateObsTL(geovals, ovecTemp); + ovecs.insert({component.first, ovecTemp}); + } + + oops::Log::debug() << "Producing final TL" << std::endl; + + data_.fillHofX(ovecs, ovec); + + oops::Log::trace() << "ObsCategoricalTLAD: simulateObsTL finished" << std::endl; +} + +// ----------------------------------------------------------------------------- + +void ObsCategoricalTLAD::simulateObsAD(GeoVaLs & geovals, const ioda::ObsVector & ovec) const { + oops::Log::trace() << "ObsCategoricalTLAD: simulateObsAD entered" << std::endl; + + oops::Log::debug() << "Running AD operators" << std::endl; + + // Container of GeoVaLs produced by each AD operator. + std::map gvals; + // Run each AD operator and store output in gvals. + for (const auto& component : data_.components()) { + GeoVaLs gvalTemp(geovals); + component.second->simulateObsAD(gvalTemp, ovec); + gvals.insert({component.first, gvalTemp}); + } + + oops::Log::debug() << "Producing final AD" << std::endl; + + // Insert values into geovals according to the categorical variable. + // Use the fallback operator when necessary. + const std::vector &varnames = ovec.varnames().variables(); + std::vector vecgv; + for (size_t jloc = 0; jloc < ovec.nlocs(); ++jloc) { + const auto &operName = data_.locOperNames()[jloc]; + const auto &gvaloper = gvals.at(operName); + // Loop over each variable at this location. + for (const auto& varname : varnames) { + vecgv.resize(gvaloper.nlevs(varname)); + gvaloper.getAtLocation(vecgv, varname, jloc); + geovals.putAtLocation(vecgv, varname, jloc); + } + } + + oops::Log::trace() << "ObsCategoricalTLAD: simulateObsAD finished" << std::endl; +} + +// ----------------------------------------------------------------------------- + +void ObsCategoricalTLAD::print(std::ostream & os) const { + data_.print(os); +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/categoricaloper/ObsCategoricalTLAD.h b/src/ufo/categoricaloper/ObsCategoricalTLAD.h new file mode 100644 index 000000000..b0e44a2ba --- /dev/null +++ b/src/ufo/categoricaloper/ObsCategoricalTLAD.h @@ -0,0 +1,67 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_CATEGORICALOPER_OBSCATEGORICALTLAD_H_ +#define UFO_CATEGORICALOPER_OBSCATEGORICALTLAD_H_ + +#include +#include +#include +#include +#include + +#include "oops/base/Variables.h" +#include "oops/util/ObjectCounter.h" + +#include "ufo/categoricaloper/ObsCategoricalData.h" +#include "ufo/LinearObsOperatorBase.h" + +// Forward declarations +namespace eckit { + class Configuration; +} + +namespace ioda { + class ObsSpace; + class ObsVector; +} + +namespace ufo { + class GeoVaLs; + class ObsDiagnostics; + +/// \brief Categorical observation operator TL/AD code. +/// Please refer to the Categorical observation operator for further documentation. +class ObsCategoricalTLAD : public LinearObsOperatorBase, + private util::ObjectCounter { + public: + static const std::string classname() { return "ufo::ObsCategoricalTLAD"; } + + ObsCategoricalTLAD(const ioda::ObsSpace &, const eckit::Configuration &); + ~ObsCategoricalTLAD() override; + + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; + void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; + void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; + + const oops::Variables & requiredVars() const override { return data_.requiredVars(); } + + private: + void print(std::ostream &) const override; + + private: + /// ObsSpace. + const ioda::ObsSpace& odb_; + + /// Data handler for the Categorical operator and TL/AD code. + ObsCategoricalData data_; +}; + +// ----------------------------------------------------------------------------- + +} // namespace ufo +#endif // UFO_CATEGORICALOPER_OBSCATEGORICALTLAD_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d0bb3ab5b..11b3a59d6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1562,6 +1562,13 @@ ecbuild_add_test( TARGET test_ufo_opr_categorical DEPENDS test_ObsOperator.x TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_linopr_categorical + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperatorTLAD.x + ARGS "testinput/categorical.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsOperatorTLAD.x + TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) + ecbuild_add_test( TARGET test_ufo_opr_profile_average COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x ARGS "testinput/profileconsistencychecks_opr_average.yaml" From fbb7b1f5d85cba9e954c7e8d33542955b2c33fad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Fri, 25 Jun 2021 19:53:06 +0100 Subject: [PATCH 18/98] Parameters subclasses for bias predictors (#1254) * Defined Parameters subclasses for all predictors. * Removed unused YAML options. * Restored previous behavior of the 'order' options. A numeric suffix is appended to predictor names if and only if this option is set (even to its default value, 1). * Final touches. Removed unnecessary includes and forward declarations, added a comment about the Parameters_ typedef. * Removed an unused file. * Replace all identical empty subclasses of PredictorParametersBase with a single class. * Made an error message more readable. * Remove the "options" header above predictor options (#1265) * Flatten predictor options by removing the "options" header. * Adjust YAML files by removing the "options" header. Co-authored-by: Ryan Honeyager --- ewok/jedi-gdas/airs_aqua.yaml | 17 ++-- ewok/jedi-gdas/amsua_aqua.yaml | 19 ++-- ewok/jedi-gdas/amsua_metop-a.yaml | 19 ++-- ewok/jedi-gdas/amsua_metop-b.yaml | 19 ++-- ewok/jedi-gdas/amsua_metop-c.yaml | 19 ++-- ewok/jedi-gdas/amsua_n15.yaml | 19 ++-- ewok/jedi-gdas/amsua_n18.yaml | 19 ++-- ewok/jedi-gdas/amsua_n19.yaml | 19 ++-- ewok/jedi-gdas/atms_n20.yaml | 19 ++-- ewok/jedi-gdas/atms_npp.yaml | 19 ++-- ewok/jedi-gdas/avhrr3_metop-a.yaml | 17 ++-- ewok/jedi-gdas/avhrr3_n18.yaml | 17 ++-- ewok/jedi-gdas/cris-fsr_n20.yaml | 17 ++-- ewok/jedi-gdas/cris-fsr_npp.yaml | 17 ++-- ewok/jedi-gdas/iasi_metop-a.yaml | 17 ++-- ewok/jedi-gdas/iasi_metop-b.yaml | 17 ++-- ewok/jedi-gdas/mhs_metop-b.yaml | 19 ++-- ewok/jedi-gdas/mhs_metop-c.yaml | 19 ++-- ewok/jedi-gdas/mhs_n19.yaml | 19 ++-- ewok/jedi-gdas/seviri_m11.yaml | 26 +++--- ewok/jedi-gdas/ssmis_f17.yaml | 43 ++++------ src/ufo/ObsBias.cc | 14 +-- src/ufo/ObsBias.h | 2 +- src/ufo/ObsBiasCovariance.cc | 6 +- src/ufo/ObsBiasIncrement.cc | 6 +- src/ufo/ObsBiasParameters.h | 17 +++- src/ufo/predictors/CloudLiquidWater.cc | 6 +- src/ufo/predictors/CloudLiquidWater.h | 46 +++++----- src/ufo/predictors/Constant.cc | 4 +- src/ufo/predictors/Constant.h | 10 +-- .../CosineOfLatitudeTimesOrbitNode.cc | 9 +- .../CosineOfLatitudeTimesOrbitNode.h | 14 ++- src/ufo/predictors/Emissivity.cc | 4 +- src/ufo/predictors/Emissivity.h | 6 +- src/ufo/predictors/InterpolateDataFromFile.cc | 10 +-- src/ufo/predictors/InterpolateDataFromFile.h | 16 ++-- src/ufo/predictors/LapseRate.cc | 49 +++++------ src/ufo/predictors/LapseRate.h | 32 +++++-- src/ufo/predictors/Legendre.cc | 12 ++- src/ufo/predictors/Legendre.h | 33 +++++-- src/ufo/predictors/OrbitalAngle.cc | 28 +++--- src/ufo/predictors/OrbitalAngle.h | 57 ++++++++++-- src/ufo/predictors/PredictorBase.cc | 25 ++++-- src/ufo/predictors/PredictorBase.h | 86 ++++++++++++++++--- src/ufo/predictors/ScanAngle.cc | 23 ++--- src/ufo/predictors/ScanAngle.h | 33 +++++-- src/ufo/predictors/SineOfLatitude.cc | 4 +- src/ufo/predictors/SineOfLatitude.h | 10 +-- src/ufo/predictors/Thickness.cc | 7 +- src/ufo/predictors/Thickness.h | 21 ++--- test/testinput/airs_qc_filters.yaml | 17 ++-- test/testinput/amsua_crtm_bc.yaml | 19 ++-- test/testinput/amsua_qc_filters.yaml | 24 ++---- test/testinput/atms_qc_filters.yaml | 17 ++-- test/testinput/bias_coeff.yaml | 21 ++--- test/testinput/bias_coeff_cov.yaml | 19 ++-- test/testinput/cris_qc_filters.yaml | 17 ++-- test/testinput/iasi_qc_filters.yaml | 17 ++-- .../airs/airs_aqua_gfs_HofX_bc.yaml | 17 ++-- .../airs/airs_aqua_gfs_HofX_qc.yaml | 17 ++-- .../amsua/amsua_aqua_gfs_HofX_bc.yaml | 19 ++-- .../amsua/amsua_aqua_gfs_HofX_qc.yaml | 19 ++-- .../amsua/amsua_metop-a_gfs_HofX_bc.yaml | 19 ++-- .../amsua/amsua_metop-a_gfs_HofX_qc.yaml | 19 ++-- .../amsua/amsua_metop-b_gfs_HofX_bc.yaml | 19 ++-- .../amsua/amsua_metop-b_gfs_HofX_qc.yaml | 19 ++-- .../amsua/amsua_metop-c_gfs_HofX_bc.yaml | 19 ++-- .../amsua/amsua_metop-c_gfs_HofX_qc.yaml | 19 ++-- .../amsua/amsua_n15_gfs_HofX_bc.yaml | 19 ++-- .../amsua/amsua_n15_gfs_HofX_qc.yaml | 19 ++-- .../amsua/amsua_n18_gfs_HofX_bc.yaml | 19 ++-- .../amsua/amsua_n18_gfs_HofX_qc.yaml | 19 ++-- .../amsua/amsua_n19_gfs_HofX_bc.yaml | 19 ++-- .../amsua/amsua_n19_gfs_HofX_qc.yaml | 19 ++-- .../atms/atms_n20_gfs_HofX_bc.yaml | 19 ++-- .../atms/atms_n20_gfs_HofX_qc.yaml | 19 ++-- .../atms/atms_npp_gfs_HofX_bc.yaml | 19 ++-- .../atms/atms_npp_gfs_HofX_qc.yaml | 19 ++-- .../avhrr/avhrr_metop-a_gfs_HofX_bc.yaml | 17 ++-- .../avhrr/avhrr_metop-a_gfs_HofX_qc.yaml | 17 ++-- .../avhrr/avhrr_n18_gfs_HofX_bc.yaml | 17 ++-- .../avhrr/avhrr_n18_gfs_HofX_qc.yaml | 17 ++-- .../cris/cris-fsr_n20_gfs_HofX_bc.yaml | 17 ++-- .../cris/cris-fsr_n20_gfs_HofX_qc.yaml | 17 ++-- .../cris/cris-fsr_npp_gfs_HofX_bc.yaml | 17 ++-- .../cris/cris-fsr_npp_gfs_HofX_qc.yaml | 17 ++-- .../iasi/iasi_metop-a_gfs_HofX_bc.yaml | 17 ++-- .../iasi/iasi_metop-a_gfs_HofX_qc.yaml | 17 ++-- .../iasi/iasi_metop-b_gfs_HofX_bc.yaml | 17 ++-- .../iasi/iasi_metop-b_gfs_HofX_qc.yaml | 17 ++-- .../mhs/mhs_metop-b_gfs_HofX_bc.yaml | 19 ++-- .../mhs/mhs_metop-c_gfs_HofX_bc.yaml | 19 ++-- .../mhs/mhs_n19_gfs_HofX_bc.yaml | 19 ++-- .../seviri/seviri_m11_gfs_HofX_bc.yaml | 17 ++-- .../seviri/seviri_m11_gfs_HofX_qc.yaml | 17 ++-- .../ssmis/ssmis_f17_gfs_HofX_bc.yaml | 43 ++++------ .../ssmis/ssmis_f17_gfs_HofX_qc.yaml | 43 ++++------ .../interpolate_data_from_file_predictor.yaml | 44 +++++----- test/testinput/legendre_predictor.yaml | 12 +-- test/testinput/orbital_angle_predictor.yaml | 60 ++++++------- .../ssmis_f17_gfs_backgroundcheck_bc.yaml | 43 ++++------ .../ssmis_f17_gfs_backgroundcheck_nbc.yaml | 43 ++++------ test/testinput/thickness_predictor.yaml | 36 ++++---- 103 files changed, 941 insertions(+), 1211 deletions(-) diff --git a/ewok/jedi-gdas/airs_aqua.yaml b/ewok/jedi-gdas/airs_aqua.yaml index f354657f7..714963842 100644 --- a/ewok/jedi-gdas/airs_aqua.yaml +++ b/ewok/jedi-gdas/airs_aqua.yaml @@ -40,22 +40,17 @@ obs bias: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &airs_aqua_tlapse $(experiment_dir)/{{current_cycle}}/airs_aqua.{{background_time}}.tlapse.txt + order: 2 + tlapse: &airs_aqua_tlapse $(experiment_dir)/{{current_cycle}}/airs_aqua.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *airs_aqua_tlapse + tlapse: *airs_aqua_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/ewok/jedi-gdas/amsua_aqua.yaml b/ewok/jedi-gdas/amsua_aqua.yaml index 7a2bed739..a8e2881cd 100644 --- a/ewok/jedi-gdas/amsua_aqua.yaml +++ b/ewok/jedi-gdas/amsua_aqua.yaml @@ -21,26 +21,19 @@ obs bias: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_aqua_tlapse $(experiment_dir)/{{current_cycle}}/amsua_aqua.{{background_time}}.tlapse.txt + order: 2 + tlapse: &amsua_aqua_tlapse $(experiment_dir)/{{current_cycle}}/amsua_aqua.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *amsua_aqua_tlapse + tlapse: *amsua_aqua_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/ewok/jedi-gdas/amsua_metop-a.yaml b/ewok/jedi-gdas/amsua_metop-a.yaml index 8399b3651..477c7ac96 100644 --- a/ewok/jedi-gdas/amsua_metop-a.yaml +++ b/ewok/jedi-gdas/amsua_metop-a.yaml @@ -21,26 +21,19 @@ obs bias: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_metop-a_tlapse $(experiment_dir)/{{current_cycle}}/amsua_metop-a.{{background_time}}.tlapse.txt + order: 2 + tlapse: &amsua_metop-a_tlapse $(experiment_dir)/{{current_cycle}}/amsua_metop-a.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *amsua_metop-a_tlapse + tlapse: *amsua_metop-a_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/ewok/jedi-gdas/amsua_metop-b.yaml b/ewok/jedi-gdas/amsua_metop-b.yaml index 71d249ada..a63505c9c 100644 --- a/ewok/jedi-gdas/amsua_metop-b.yaml +++ b/ewok/jedi-gdas/amsua_metop-b.yaml @@ -21,26 +21,19 @@ obs bias: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_metop-b_tlapse $(experiment_dir)/{{current_cycle}}/amsua_metop-b.{{background_time}}.tlapse.txt + order: 2 + tlapse: &amsua_metop-b_tlapse $(experiment_dir)/{{current_cycle}}/amsua_metop-b.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *amsua_metop-b_tlapse + tlapse: *amsua_metop-b_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/ewok/jedi-gdas/amsua_metop-c.yaml b/ewok/jedi-gdas/amsua_metop-c.yaml index e7a211aab..0eaa9bff2 100644 --- a/ewok/jedi-gdas/amsua_metop-c.yaml +++ b/ewok/jedi-gdas/amsua_metop-c.yaml @@ -21,26 +21,19 @@ obs bias: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_metop-c_tlapse $(experiment_dir)/{{current_cycle}}/amsua_metop-c.{{background_time}}.tlapse.txt + order: 2 + tlapse: &amsua_metop-c_tlapse $(experiment_dir)/{{current_cycle}}/amsua_metop-c.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *amsua_metop-c_tlapse + tlapse: *amsua_metop-c_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/ewok/jedi-gdas/amsua_n15.yaml b/ewok/jedi-gdas/amsua_n15.yaml index 2ac7a7f68..e7dc2473b 100644 --- a/ewok/jedi-gdas/amsua_n15.yaml +++ b/ewok/jedi-gdas/amsua_n15.yaml @@ -21,26 +21,19 @@ obs bias: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_n15_tlapse $(experiment_dir)/{{current_cycle}}/amsua_n15.{{background_time}}.tlapse.txt + order: 2 + tlapse: &amsua_n15_tlapse $(experiment_dir)/{{current_cycle}}/amsua_n15.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *amsua_n15_tlapse + tlapse: *amsua_n15_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/ewok/jedi-gdas/amsua_n18.yaml b/ewok/jedi-gdas/amsua_n18.yaml index 17af2bb4f..2abcac1dc 100644 --- a/ewok/jedi-gdas/amsua_n18.yaml +++ b/ewok/jedi-gdas/amsua_n18.yaml @@ -21,26 +21,19 @@ obs bias: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_n18_tlapse $(experiment_dir)/{{current_cycle}}/amsua_n18.{{background_time}}.tlapse.txt + order: 2 + tlapse: &amsua_n18_tlapse $(experiment_dir)/{{current_cycle}}/amsua_n18.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *amsua_n18_tlapse + tlapse: *amsua_n18_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/ewok/jedi-gdas/amsua_n19.yaml b/ewok/jedi-gdas/amsua_n19.yaml index 83b6de952..e5f8b5dc3 100644 --- a/ewok/jedi-gdas/amsua_n19.yaml +++ b/ewok/jedi-gdas/amsua_n19.yaml @@ -21,26 +21,19 @@ obs bias: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_n19_tlapse $(experiment_dir)/{{current_cycle}}/amsua_n19.{{background_time}}.tlapse.txt + order: 2 + tlapse: &amsua_n19_tlapse $(experiment_dir)/{{current_cycle}}/amsua_n19.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *amsua_n19_tlapse + tlapse: *amsua_n19_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/ewok/jedi-gdas/atms_n20.yaml b/ewok/jedi-gdas/atms_n20.yaml index f11201406..fea195845 100644 --- a/ewok/jedi-gdas/atms_n20.yaml +++ b/ewok/jedi-gdas/atms_n20.yaml @@ -21,26 +21,19 @@ obs bias: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &atms_n20_tlapse $(experiment_dir)/{{current_cycle}}/atms_n20.{{background_time}}.tlapse.txt + order: 2 + tlapse: &atms_n20_tlapse $(experiment_dir)/{{current_cycle}}/atms_n20.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *atms_n20_tlapse + tlapse: *atms_n20_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/ewok/jedi-gdas/atms_npp.yaml b/ewok/jedi-gdas/atms_npp.yaml index 0172b30f3..7fb792ef7 100644 --- a/ewok/jedi-gdas/atms_npp.yaml +++ b/ewok/jedi-gdas/atms_npp.yaml @@ -21,26 +21,19 @@ obs bias: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &atms_npp_tlapse $(experiment_dir)/{{current_cycle}}/atms_npp.{{background_time}}.tlapse.txt + order: 2 + tlapse: &atms_npp_tlapse $(experiment_dir)/{{current_cycle}}/atms_npp.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *atms_npp_tlapse + tlapse: *atms_npp_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/ewok/jedi-gdas/avhrr3_metop-a.yaml b/ewok/jedi-gdas/avhrr3_metop-a.yaml index 12d530ae0..a037fce04 100644 --- a/ewok/jedi-gdas/avhrr3_metop-a.yaml +++ b/ewok/jedi-gdas/avhrr3_metop-a.yaml @@ -19,22 +19,17 @@ obs bias: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &avhrr3_metop-a_tlapse $(experiment_dir)/{{current_cycle}}/avhrr3_metop-a.{{background_time}}.tlapse.txt + order: 2 + tlapse: &avhrr3_metop-a_tlapse $(experiment_dir)/{{current_cycle}}/avhrr3_metop-a.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *avhrr3_metop-a_tlapse + tlapse: *avhrr3_metop-a_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/ewok/jedi-gdas/avhrr3_n18.yaml b/ewok/jedi-gdas/avhrr3_n18.yaml index 77c294d05..05d86b281 100644 --- a/ewok/jedi-gdas/avhrr3_n18.yaml +++ b/ewok/jedi-gdas/avhrr3_n18.yaml @@ -19,22 +19,17 @@ obs bias: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &avhrr3_n18_tlapse $(experiment_dir)/{{current_cycle}}/avhrr3_n18.{{background_time}}.tlapse.txt + order: 2 + tlapse: &avhrr3_n18_tlapse $(experiment_dir)/{{current_cycle}}/avhrr3_n18.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *avhrr3_n18_tlapse + tlapse: *avhrr3_n18_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/ewok/jedi-gdas/cris-fsr_n20.yaml b/ewok/jedi-gdas/cris-fsr_n20.yaml index 68c6c0dba..0110522d8 100644 --- a/ewok/jedi-gdas/cris-fsr_n20.yaml +++ b/ewok/jedi-gdas/cris-fsr_n20.yaml @@ -50,22 +50,17 @@ obs bias: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &cris-fsr_n20_tlapse $(experiment_dir)/{{current_cycle}}/cris-fsr_n20.{{background_time}}.tlapse.txt + order: 2 + tlapse: &cris-fsr_n20_tlapse $(experiment_dir)/{{current_cycle}}/cris-fsr_n20.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *cris-fsr_n20_tlapse + tlapse: *cris-fsr_n20_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/ewok/jedi-gdas/cris-fsr_npp.yaml b/ewok/jedi-gdas/cris-fsr_npp.yaml index dd2eac237..c9a2311b4 100644 --- a/ewok/jedi-gdas/cris-fsr_npp.yaml +++ b/ewok/jedi-gdas/cris-fsr_npp.yaml @@ -50,22 +50,17 @@ obs bias: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &cris-fsr_npp_tlapse $(experiment_dir)/{{current_cycle}}/cris-fsr_npp.{{background_time}}.tlapse.txt + order: 2 + tlapse: &cris-fsr_npp_tlapse $(experiment_dir)/{{current_cycle}}/cris-fsr_npp.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *cris-fsr_npp_tlapse + tlapse: *cris-fsr_npp_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/ewok/jedi-gdas/iasi_metop-a.yaml b/ewok/jedi-gdas/iasi_metop-a.yaml index 97e455ca8..5fd9dc91b 100644 --- a/ewok/jedi-gdas/iasi_metop-a.yaml +++ b/ewok/jedi-gdas/iasi_metop-a.yaml @@ -67,22 +67,17 @@ obs bias: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &iasi_metop-a_tlapse $(experiment_dir)/{{current_cycle}}/iasi_metop-a.{{background_time}}.tlapse.txt + order: 2 + tlapse: &iasi_metop-a_tlapse $(experiment_dir)/{{current_cycle}}/iasi_metop-a.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *iasi_metop-a_tlapse + tlapse: *iasi_metop-a_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/ewok/jedi-gdas/iasi_metop-b.yaml b/ewok/jedi-gdas/iasi_metop-b.yaml index 814278de2..a96bb47ed 100644 --- a/ewok/jedi-gdas/iasi_metop-b.yaml +++ b/ewok/jedi-gdas/iasi_metop-b.yaml @@ -67,22 +67,17 @@ obs bias: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &iasi_metop-b_tlapse $(experiment_dir)/{{current_cycle}}/iasi_metop-b.{{background_time}}.tlapse.txt + order: 2 + tlapse: &iasi_metop-b_tlapse $(experiment_dir)/{{current_cycle}}/iasi_metop-b.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *iasi_metop-b_tlapse + tlapse: *iasi_metop-b_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/ewok/jedi-gdas/mhs_metop-b.yaml b/ewok/jedi-gdas/mhs_metop-b.yaml index beefcd64f..f24c798f1 100644 --- a/ewok/jedi-gdas/mhs_metop-b.yaml +++ b/ewok/jedi-gdas/mhs_metop-b.yaml @@ -19,24 +19,17 @@ obs bias: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &mhs_metop-b_tlapse $(experiment_dir)/{{current_cycle}}/mhs_metop-b.{{background_time}}.tlapse.txt + order: 2 + tlapse: &mhs_metop-b_tlapse $(experiment_dir)/{{current_cycle}}/mhs_metop-b.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *mhs_metop-b_tlapse + tlapse: *mhs_metop-b_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle diff --git a/ewok/jedi-gdas/mhs_metop-c.yaml b/ewok/jedi-gdas/mhs_metop-c.yaml index bee97fbe8..908da6b53 100644 --- a/ewok/jedi-gdas/mhs_metop-c.yaml +++ b/ewok/jedi-gdas/mhs_metop-c.yaml @@ -19,24 +19,17 @@ obs bias: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &mhs_metop-c_tlapse $(experiment_dir)/{{current_cycle}}/mhs_metop-c.{{background_time}}.tlapse.txt + order: 2 + tlapse: &mhs_metop-c_tlapse $(experiment_dir)/{{current_cycle}}/mhs_metop-c.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *mhs_metop-c_tlapse + tlapse: *mhs_metop-c_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle diff --git a/ewok/jedi-gdas/mhs_n19.yaml b/ewok/jedi-gdas/mhs_n19.yaml index fc3a0b2d0..ac5f24429 100644 --- a/ewok/jedi-gdas/mhs_n19.yaml +++ b/ewok/jedi-gdas/mhs_n19.yaml @@ -19,24 +19,17 @@ obs bias: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &mhs_n19_tlapse $(experiment_dir)/{{current_cycle}}/mhs_n19.{{background_time}}.tlapse.txt + order: 2 + tlapse: &mhs_n19_tlapse $(experiment_dir)/{{current_cycle}}/mhs_n19.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *mhs_n19_tlapse + tlapse: *mhs_n19_tlapse - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle diff --git a/ewok/jedi-gdas/seviri_m11.yaml b/ewok/jedi-gdas/seviri_m11.yaml index 89b27ea56..321a221f7 100644 --- a/ewok/jedi-gdas/seviri_m11.yaml +++ b/ewok/jedi-gdas/seviri_m11.yaml @@ -19,28 +19,22 @@ obs bias: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &seviri_m11_tlapse $(experiment_dir)/{{current_cycle}}/seviri_m11.{{background_time}}.tlapse.txt + order: 2 + tlapse: &seviri_m11_tlapse $(experiment_dir)/{{current_cycle}}/seviri_m11.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *seviri_m11_tlapse + tlapse: *seviri_m11_tlapse - name: emissivity - name: scan_angle - options: - var_name: scan_position - order: 4 + var_name: scan_position + order: 4 - name: scan_angle - options: - var_name: scan_position - order: 3 + var_name: scan_position + order: 3 - name: scan_angle - options: - var_name: scan_position - order: 2 + var_name: scan_position + order: 2 - name: scan_angle - options: - var_name: scan_position + var_name: scan_position obs filters: # Observation Range Sanity Check - filter: Bounds Check diff --git a/ewok/jedi-gdas/ssmis_f17.yaml b/ewok/jedi-gdas/ssmis_f17.yaml index 71dd9e0a9..6cf87fa31 100644 --- a/ewok/jedi-gdas/ssmis_f17.yaml +++ b/ewok/jedi-gdas/ssmis_f17.yaml @@ -19,40 +19,33 @@ obs bias: predictors: - name: constant - name: cloud_liquid_water - options: - satellite: SSMIS - ch19h: 12 - ch19v: 13 - ch22v: 14 - ch37h: 15 - ch37v: 16 - ch91v: 17 - ch91h: 18 + satellite: SSMIS + ch19h: 12 + ch19v: 13 + ch22v: 14 + ch37h: 15 + ch37v: 16 + ch91v: 17 + ch91h: 18 - name: cosine_of_latitude_times_orbit_node - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &ssmis_f17_tlapse $(experiment_dir)/{{current_cycle}}/ssmis_f17.{{background_time}}.tlapse.txt + order: 2 + tlapse: &ssmis_f17_tlapse $(experiment_dir)/{{current_cycle}}/ssmis_f17.{{background_time}}.tlapse.txt - name: lapse_rate - options: - tlapse: *ssmis_f17_tlapse + tlapse: *ssmis_f17_tlapse - name: emissivity - name: scan_angle - options: - var_name: scan_position - order: 4 + var_name: scan_position + order: 4 - name: scan_angle - options: - var_name: scan_position - order: 3 + var_name: scan_position + order: 3 - name: scan_angle - options: - var_name: scan_position - order: 2 + var_name: scan_position + order: 2 - name: scan_angle - options: - var_name: scan_position + var_name: scan_position obs filters: #step1: Gross check (setuprad) - filter: Background Check diff --git a/src/ufo/ObsBias.cc b/src/ufo/ObsBias.cc index 6443eddb3..5c0aa215d 100644 --- a/src/ufo/ObsBias.cc +++ b/src/ufo/ObsBias.cc @@ -34,12 +34,14 @@ ObsBias::ObsBias(ioda::ObsSpace & odb, const ObsBiasParameters & params) oops::Log::trace() << "ObsBias::create starting." << std::endl; // Predictor factory - for (const eckit::LocalConfiguration &conf : params.staticBC.value().predictors.value()) { - initPredictor(conf); + for (const PredictorParametersWrapper &wrapper : + params.staticBC.value().predictors.value()) { + initPredictor(wrapper); ++numStaticPredictors_; } - for (const eckit::LocalConfiguration &conf : params.variationalBC.value().predictors.value()) { - initPredictor(conf); + for (const PredictorParametersWrapper &wrapper : + params.variationalBC.value().predictors.value()) { + initPredictor(wrapper); ++numVariablePredictors_; } @@ -215,8 +217,8 @@ void ObsBias::print(std::ostream & os) const { // ----------------------------------------------------------------------------- -void ObsBias::initPredictor(const eckit::Configuration &predictorConf) { - std::shared_ptr pred(PredictorFactory::create(predictorConf, vars_)); +void ObsBias::initPredictor(const PredictorParametersWrapper ¶ms) { + std::shared_ptr pred(PredictorFactory::create(params.predictorParameters, vars_)); predictors_.push_back(pred); prednames_.push_back(pred->name()); geovars_ += pred->requiredGeovars(); diff --git a/src/ufo/ObsBias.h b/src/ufo/ObsBias.h index 5fa566ef4..6cd82af27 100644 --- a/src/ufo/ObsBias.h +++ b/src/ufo/ObsBias.h @@ -95,7 +95,7 @@ class ObsBias : public util::Printable, /// index in the flattened biascoeffs_ for predictor \p jpred and variable \p jvar size_t index(size_t jpred, size_t jvar) const {return jvar*numVariablePredictors_ + jpred;} - void initPredictor(const eckit::Configuration &predictorConf); + void initPredictor(const PredictorParametersWrapper ¶ms); /// bias correction coefficients (npredictors x nprimitivevariables) Eigen::VectorXd biascoeffs_; diff --git a/src/ufo/ObsBiasCovariance.cc b/src/ufo/ObsBiasCovariance.cc index 324ca84be..6735fcdce 100644 --- a/src/ufo/ObsBiasCovariance.cc +++ b/src/ufo/ObsBiasCovariance.cc @@ -42,8 +42,10 @@ ObsBiasCovariance::ObsBiasCovariance(ioda::ObsSpace & odb, oops::Log::trace() << "ObsBiasCovariance::Constructor starting" << std::endl; // Predictor factory - for (const eckit::LocalConfiguration &conf : params.variationalBC.value().predictors.value()) { - std::shared_ptr pred(PredictorFactory::create(conf, vars_)); + for (const PredictorParametersWrapper &wrapper : + params.variationalBC.value().predictors.value()) { + std::shared_ptr pred(PredictorFactory::create(wrapper.predictorParameters, + vars_)); prednames_.push_back(pred->name()); } diff --git a/src/ufo/ObsBiasIncrement.cc b/src/ufo/ObsBiasIncrement.cc index cec4bd807..7366236c5 100644 --- a/src/ufo/ObsBiasIncrement.cc +++ b/src/ufo/ObsBiasIncrement.cc @@ -25,8 +25,10 @@ ObsBiasIncrement::ObsBiasIncrement(const ioda::ObsSpace & odb, oops::Log::trace() << "ufo::ObsBiasIncrement::create starting." << std::endl; // Predictor factory - for (const eckit::LocalConfiguration &conf : params.variationalBC.value().predictors.value()) { - std::unique_ptr predictor(PredictorFactory::create(conf, vars_)); + for (const PredictorParametersWrapper &wrapper : + params.variationalBC.value().predictors.value()) { + std::unique_ptr predictor(PredictorFactory::create(wrapper.predictorParameters, + vars_)); prednames_.push_back(predictor->name()); } diff --git a/src/ufo/ObsBiasParameters.h b/src/ufo/ObsBiasParameters.h index f44448df6..cec94e455 100644 --- a/src/ufo/ObsBiasParameters.h +++ b/src/ufo/ObsBiasParameters.h @@ -15,15 +15,30 @@ #include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/Parameters.h" #include "oops/util/parameters/RequiredParameter.h" +#include "oops/util/parameters/RequiredPolymorphicParameter.h" + +#include "ufo/predictors/PredictorBase.h" namespace ufo { +/// \brief Contains a polymorphic parameter holding an instance of a subclass of +/// PredictorParametersBase. +class PredictorParametersWrapper : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(PredictorParametersWrapper, Parameters) + public: + /// After deserialization, holds an instance of a subclass of PredictorParametersBase configuring + /// a predictor. The type of the subclass is determined by the value of the "name" key in the + /// Configuration object from which this object is deserialized. + oops::RequiredPolymorphicParameter + predictorParameters{"name", this}; +}; + class StaticOrVariationalBCParameters : public oops::Parameters { OOPS_CONCRETE_PARAMETERS(StaticOrVariationalBCParameters, Parameters) public: /// Each element of this list is used to configure a separate predictor. - oops::Parameter> predictors{"predictors", {}, this}; + oops::Parameter> predictors{"predictors", {}, this}; }; class ObsBiasCovariancePriorInflationParameters : public oops::Parameters { diff --git a/src/ufo/predictors/CloudLiquidWater.cc b/src/ufo/predictors/CloudLiquidWater.cc index 980f0faa8..b498dfe85 100644 --- a/src/ufo/predictors/CloudLiquidWater.cc +++ b/src/ufo/predictors/CloudLiquidWater.cc @@ -25,10 +25,10 @@ static PredictorMaker // ----------------------------------------------------------------------------- -CloudLiquidWater::CloudLiquidWater(const eckit::Configuration & conf, const oops::Variables & vars) - : PredictorBase(conf, vars) { +CloudLiquidWater::CloudLiquidWater(const Parameters_ & parameters, const oops::Variables & vars) + : PredictorBase(parameters, vars) { // Initialize options - options_.deserialize(conf.getSubConfiguration("options")); + options_ = parameters; const std::string &satellite = options_.satellite.value(); // Currently the code is designed only for SSMIS brightness temperatures from diff --git a/src/ufo/predictors/CloudLiquidWater.h b/src/ufo/predictors/CloudLiquidWater.h index 799fe5680..9867d4117 100644 --- a/src/ufo/predictors/CloudLiquidWater.h +++ b/src/ufo/predictors/CloudLiquidWater.h @@ -18,43 +18,43 @@ #include "ufo/predictors/PredictorBase.h" -namespace eckit { - class Configuration; -} - namespace ioda { class ObsSpace; } namespace ufo { -/// -/// \brief Option to override varGroup default of ObsValue for all channels with ObsBias -/// -class CloudLiquidWaterParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(CloudLiquidWaterParameters, Parameters) +// ----------------------------------------------------------------------------- + +/// Configuration parameters of the CloudLiquidWater predictor. +class CloudLiquidWaterParameters : public PredictorParametersBase { + OOPS_CONCRETE_PARAMETERS(CloudLiquidWaterParameters, PredictorParametersBase) public: - /// We must specify a satellite reference name such as SSMIS to know which channels to expect. - oops::RequiredParameter satellite{"satellite", this}; - /// In case we need to override the ObsValue group name with another optional group name. - oops::Parameter varGroup{"varGroup", "ObsValue", this}; - /// List below is solely for SSMIS data, but a different list of channel numbers could be - /// added for a different satellite platform in the future. - oops::OptionalParameter ch19h{"ch19h", this}; - oops::OptionalParameter ch19v{"ch19v", this}; - oops::OptionalParameter ch22v{"ch22v", this}; - oops::OptionalParameter ch37h{"ch37h", this}; - oops::OptionalParameter ch37v{"ch37v", this}; - oops::OptionalParameter ch91h{"ch91h", this}; - oops::OptionalParameter ch91v{"ch91v", this}; + /// We must specify a satellite reference name such as SSMIS to know which channels to expect. + oops::RequiredParameter satellite{"satellite", this}; + /// In case we need to override the ObsValue group name with another optional group name. + oops::Parameter varGroup{"varGroup", "ObsValue", this}; + /// List below is solely for SSMIS data, but a different list of channel numbers could be + /// added for a different satellite platform in the future. + oops::OptionalParameter ch19h{"ch19h", this}; + oops::OptionalParameter ch19v{"ch19v", this}; + oops::OptionalParameter ch22v{"ch22v", this}; + oops::OptionalParameter ch37h{"ch37h", this}; + oops::OptionalParameter ch37v{"ch37v", this}; + oops::OptionalParameter ch91h{"ch91h", this}; + oops::OptionalParameter ch91v{"ch91v", this}; }; // ----------------------------------------------------------------------------- class CloudLiquidWater : public PredictorBase { public: - CloudLiquidWater(const eckit::Configuration &, const oops::Variables &); + /// The type of parameters accepted by the constructor of this predictor. + /// This typedef is used by the PredictorFactory. + typedef CloudLiquidWaterParameters Parameters_; + + CloudLiquidWater(const Parameters_ &, const oops::Variables &); ~CloudLiquidWater() {} void compute(const ioda::ObsSpace &, diff --git a/src/ufo/predictors/Constant.cc b/src/ufo/predictors/Constant.cc index 7e35b360b..7b7bbf24f 100644 --- a/src/ufo/predictors/Constant.cc +++ b/src/ufo/predictors/Constant.cc @@ -17,8 +17,8 @@ static PredictorMaker makerFuncConstant_("constant"); // ----------------------------------------------------------------------------- -Constant::Constant(const eckit::Configuration & conf, const oops::Variables & vars) - : PredictorBase(conf, vars) { +Constant::Constant(const Parameters_ & parameters, const oops::Variables & vars) + : PredictorBase(parameters, vars) { } // ----------------------------------------------------------------------------- diff --git a/src/ufo/predictors/Constant.h b/src/ufo/predictors/Constant.h index 036a586d6..6dc570bf1 100644 --- a/src/ufo/predictors/Constant.h +++ b/src/ufo/predictors/Constant.h @@ -10,10 +10,6 @@ #include "ufo/predictors/PredictorBase.h" -namespace eckit { - class Configuration; -} - namespace oops { class Variables; } @@ -28,7 +24,11 @@ namespace ufo { class Constant : public PredictorBase { public: - Constant(const eckit::Configuration &, const oops::Variables &); + /// The type of parameters accepted by the constructor of this predictor. + /// This typedef is used by the PredictorFactory. + typedef EmptyPredictorParameters Parameters_; + + Constant(const Parameters_ &, const oops::Variables &); void compute(const ioda::ObsSpace &, const GeoVaLs &, diff --git a/src/ufo/predictors/CosineOfLatitudeTimesOrbitNode.cc b/src/ufo/predictors/CosineOfLatitudeTimesOrbitNode.cc index a41172956..a76fb5b19 100644 --- a/src/ufo/predictors/CosineOfLatitudeTimesOrbitNode.cc +++ b/src/ufo/predictors/CosineOfLatitudeTimesOrbitNode.cc @@ -20,12 +20,9 @@ static PredictorMaker // ----------------------------------------------------------------------------- -CosineOfLatitudeTimesOrbitNode::CosineOfLatitudeTimesOrbitNode( - const eckit::Configuration & conf, const oops::Variables & vars) - : PredictorBase(conf, vars) { - // override the preconditioner from options - if (conf.has("options")) - precond_ = conf.getDouble("options.preconditioner"); +CosineOfLatitudeTimesOrbitNode::CosineOfLatitudeTimesOrbitNode(const Parameters_ & parameters, + const oops::Variables & vars) + : PredictorBase(parameters, vars) { } // ----------------------------------------------------------------------------- diff --git a/src/ufo/predictors/CosineOfLatitudeTimesOrbitNode.h b/src/ufo/predictors/CosineOfLatitudeTimesOrbitNode.h index 2e80c85de..756370f48 100644 --- a/src/ufo/predictors/CosineOfLatitudeTimesOrbitNode.h +++ b/src/ufo/predictors/CosineOfLatitudeTimesOrbitNode.h @@ -10,10 +10,6 @@ #include "ufo/predictors/PredictorBase.h" -namespace eckit { - class Configuration; -} - namespace oops { class Variables; } @@ -28,16 +24,16 @@ namespace ufo { class CosineOfLatitudeTimesOrbitNode : public PredictorBase { public: - CosineOfLatitudeTimesOrbitNode(const eckit::Configuration &, const oops::Variables &); + /// The type of parameters accepted by the constructor of this predictor. + /// This typedef is used by the PredictorFactory. + typedef EmptyPredictorParameters Parameters_; + + CosineOfLatitudeTimesOrbitNode(const Parameters_ &, const oops::Variables &); void compute(const ioda::ObsSpace &, const GeoVaLs &, const ObsDiagnostics &, ioda::ObsVector &) const override; - - private: - // default preconditioner for bias terms - double precond_ = 0.01; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/predictors/Emissivity.cc b/src/ufo/predictors/Emissivity.cc index bd876b1f2..c8db552d1 100644 --- a/src/ufo/predictors/Emissivity.cc +++ b/src/ufo/predictors/Emissivity.cc @@ -25,8 +25,8 @@ static PredictorMaker makerFuncEmissivity_("emissivity"); // ----------------------------------------------------------------------------- -Emissivity::Emissivity(const eckit::Configuration & conf, const oops::Variables & vars) - : PredictorBase(conf, vars) { +Emissivity::Emissivity(const Parameters_ & parameters, const oops::Variables & vars) + : PredictorBase(parameters, vars) { // required variables geovars_ += oops::Variables({"water_area_fraction"}); if (vars.size() > 0) { diff --git a/src/ufo/predictors/Emissivity.h b/src/ufo/predictors/Emissivity.h index 11e8ff21d..bf539ba71 100644 --- a/src/ufo/predictors/Emissivity.h +++ b/src/ufo/predictors/Emissivity.h @@ -28,7 +28,11 @@ namespace ufo { class Emissivity : public PredictorBase { public: - Emissivity(const eckit::Configuration &, const oops::Variables &); + /// The type of parameters accepted by the constructor of this predictor. + /// This typedef is used by the PredictorFactory. + typedef EmptyPredictorParameters Parameters_; + + Emissivity(const Parameters_ &, const oops::Variables &); void compute(const ioda::ObsSpace &, const GeoVaLs &, diff --git a/src/ufo/predictors/InterpolateDataFromFile.cc b/src/ufo/predictors/InterpolateDataFromFile.cc index a4f748e45..aadcd5bea 100644 --- a/src/ufo/predictors/InterpolateDataFromFile.cc +++ b/src/ufo/predictors/InterpolateDataFromFile.cc @@ -56,16 +56,12 @@ std::set getVariableNamesWithoutChannels(const oops::Variables &var static PredictorMaker maker("interpolate_data_from_file"); -InterpolateDataFromFile::InterpolateDataFromFile(const eckit::Configuration & conf, +InterpolateDataFromFile::InterpolateDataFromFile(const Parameters_ & parameters, const oops::Variables & vars) - : PredictorBase(conf, vars) { - InterpolateDataFromFileParameters params; - eckit::LocalConfiguration optionsConf(conf, "options"); - params.validateAndDeserialize(optionsConf); - + : PredictorBase(parameters, vars) { const std::set channellessVariables = getVariableNamesWithoutChannels(vars_); - for (const VariableCorrectionParameters & varParams : params.correctedVariables.value()) { + for (const VariableCorrectionParameters & varParams : parameters.correctedVariables.value()) { if (!oops::contains(channellessVariables, varParams.name)) throw eckit::UserError("'" + varParams.name.value() + "' is not in the list of bias-corrected variables", Here()); diff --git a/src/ufo/predictors/InterpolateDataFromFile.h b/src/ufo/predictors/InterpolateDataFromFile.h index 5de258278..3b7608ca3 100644 --- a/src/ufo/predictors/InterpolateDataFromFile.h +++ b/src/ufo/predictors/InterpolateDataFromFile.h @@ -13,7 +13,10 @@ #include #include +#include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" + #include "ufo/filters/obsfunctions/DrawValueFromFile.h" #include "ufo/predictors/PredictorBase.h" @@ -30,10 +33,9 @@ class VariableCorrectionParameters : public oops::Parameters { DrawValueFromFileParametersWithoutGroup details{this}; }; -/// \brief Parameters recognized in the `options` section of the Configuration passed to -/// the InterpolateDataFromFile constructor. -class InterpolateDataFromFileParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(InterpolateDataFromFileParameters, Parameters) +/// Configuration parameters of the `interpolate_data_from_file` predictor. +class InterpolateDataFromFileParameters : public PredictorParametersBase { + OOPS_CONCRETE_PARAMETERS(InterpolateDataFromFileParameters, PredictorParametersBase); public: oops::Parameter> correctedVariables{ @@ -71,7 +73,11 @@ class InterpolateDataFromFileParameters : public oops::Parameters { /// DrawValueFromFile and DataExtractor. class InterpolateDataFromFile : public PredictorBase { public: - InterpolateDataFromFile(const eckit::Configuration &, const oops::Variables &); + /// The type of parameters accepted by the constructor of this predictor. + /// This typedef is used by the PredictorFactory. + typedef InterpolateDataFromFileParameters Parameters_; + + InterpolateDataFromFile(const Parameters_ &, const oops::Variables &); void compute(const ioda::ObsSpace &, const GeoVaLs &, const ObsDiagnostics &, ioda::ObsVector &) const override; diff --git a/src/ufo/predictors/LapseRate.cc b/src/ufo/predictors/LapseRate.cc index 7a725d0cc..20d64fb02 100644 --- a/src/ufo/predictors/LapseRate.cc +++ b/src/ufo/predictors/LapseRate.cc @@ -26,14 +26,12 @@ static PredictorMaker makerFuncLapseRate_("lapse_rate"); // ----------------------------------------------------------------------------- -LapseRate::LapseRate(const eckit::Configuration & conf, const oops::Variables & vars) - : PredictorBase(conf, vars), order_(1) +LapseRate::LapseRate(const Parameters_ & parameters, const oops::Variables & vars) + : PredictorBase(parameters, vars), + order_(parameters.order.value().value_or(1)) { - // get the order if it is provided in options - if (conf.has("options.order")) { - conf.get("options.order", order_); - - // override the predictor name for differentiable + if (parameters.order.value() != boost::none) { + // override the predictor name to distinguish between lapse_rate predictors of different orders name() = name() + "_order_" + std::to_string(order_); } @@ -50,29 +48,24 @@ LapseRate::LapseRate(const eckit::Configuration & conf, const oops::Variables & // This is a very preliminary method, please revisit // more flexibilites are needed - if (conf.has("options.tlapse")) { - const std::string tlapse_file = conf.getString("options.tlapse"); - std::ifstream infile(tlapse_file); - std::string nusis; // sensor/instrument/satellite - int nuchan; // channel number - float tlapse; - - if (infile.is_open()) { - while (!infile.eof()) { - infile >> nusis; - infile >> nuchan; - infile >> tlapse; - tlapmean_[nuchan] = tlapse; - } - infile.close(); - } else { - oops::Log::error() << "Unable to open file : " - << tlapse_file << std::endl; - ABORT("Unable to open tlap file "); + const std::string & tlapse_file = parameters.tlapse; + std::ifstream infile(tlapse_file); + std::string nusis; // sensor/instrument/satellite + int nuchan; // channel number + float tlapse; + + if (infile.is_open()) { + while (!infile.eof()) { + infile >> nusis; + infile >> nuchan; + infile >> tlapse; + tlapmean_[nuchan] = tlapse; } + infile.close(); } else { - oops::Log::error() << "tlapse file is not provided !" << std::endl; - ABORT("tlapse file is not provided !"); + oops::Log::error() << "Unable to open file : " + << tlapse_file << std::endl; + ABORT("Unable to open tlapse file "); } } diff --git a/src/ufo/predictors/LapseRate.h b/src/ufo/predictors/LapseRate.h index d6a8a3110..11b7b9ea9 100644 --- a/src/ufo/predictors/LapseRate.h +++ b/src/ufo/predictors/LapseRate.h @@ -9,12 +9,13 @@ #define UFO_PREDICTORS_LAPSERATE_H_ #include +#include -#include "ufo/predictors/PredictorBase.h" +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" -namespace eckit { - class Configuration; -} +#include "ufo/predictors/PredictorBase.h" namespace oops { class Variables; @@ -28,9 +29,30 @@ namespace ufo { // ----------------------------------------------------------------------------- +/// Configuration parameters of the LapseRate predictor. +class LapseRateParameters : public PredictorParametersBase { + OOPS_CONCRETE_PARAMETERS(LapseRateParameters, PredictorParametersBase); + + public: + /// Path to an input file. + oops::RequiredParameter tlapse{"tlapse", this}; + + /// Power to which to raise the lapse rate. By default, 1. + /// + /// \note If this option is set, a suffix containing its value (even if it's equal to 1) will be + /// appended to the predictor name. + oops::OptionalParameter order{"order", this}; +}; + +// ----------------------------------------------------------------------------- + class LapseRate : public PredictorBase { public: - LapseRate(const eckit::Configuration &, const oops::Variables &); + /// The type of parameters accepted by the constructor of this predictor. + /// This typedef is used by the PredictorFactory. + typedef LapseRateParameters Parameters_; + + LapseRate(const Parameters_ &, const oops::Variables &); void compute(const ioda::ObsSpace &, const GeoVaLs &, diff --git a/src/ufo/predictors/Legendre.cc b/src/ufo/predictors/Legendre.cc index 1a47be4da..184a8a705 100644 --- a/src/ufo/predictors/Legendre.cc +++ b/src/ufo/predictors/Legendre.cc @@ -17,16 +17,14 @@ static PredictorMaker makerFuncLegendre_("Legendre"); // ----------------------------------------------------------------------------- -Legendre::Legendre(const eckit::Configuration & conf, const oops::Variables & vars) - : PredictorBase(conf, vars), order_(1), nscan_(-99) { - // get the order if it is provided in options - if (conf.has("options.order")) { - conf.get("options.order", order_); - +Legendre::Legendre(const Parameters_ & parameters, const oops::Variables & vars) + : PredictorBase(parameters, vars), + order_(parameters.order.value().value_or(1)), + nscan_(parameters.numScanPositions) { + if (parameters.order.value() != boost::none) { // override the predictor name to distinguish between Legendre predictors of different orders name() = name() + "_" + std::to_string(order_); } - conf.get("number of scan positions", nscan_); } // ----------------------------------------------------------------------------- diff --git a/src/ufo/predictors/Legendre.h b/src/ufo/predictors/Legendre.h index a2907def5..9094ca49a 100644 --- a/src/ufo/predictors/Legendre.h +++ b/src/ufo/predictors/Legendre.h @@ -9,11 +9,13 @@ #define UFO_PREDICTORS_LEGENDRE_H_ #include -#include "ufo/predictors/PredictorBase.h" -namespace eckit { - class Configuration; -} +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" + +#include "ufo/predictors/PredictorBase.h" namespace ioda { class ObsSpace; @@ -21,6 +23,23 @@ namespace ioda { namespace ufo { +// ----------------------------------------------------------------------------- + +/// Configuration parameters of the Legendre predictor. +class LegendreParameters : public PredictorParametersBase { + OOPS_CONCRETE_PARAMETERS(LegendreParameters, PredictorParametersBase); + + public: + /// Number of scan positions. + oops::RequiredParameter numScanPositions{"number of scan positions", this}; + + /// Order of the Legendre polynomial. By default, 1. + /// + /// \note If this option is set, a suffix containing its value (even if it's equal to 1) will be + /// appended to the predictor name. + oops::OptionalParameter order{"order", this}; +}; + // ----------------------------------------------------------------------------- /** *This Legendre predictor is for fitting residual errors between the ends of the @@ -35,7 +54,11 @@ namespace ufo { class Legendre : public PredictorBase { public: - Legendre(const eckit::Configuration &, const oops::Variables &); + /// The type of parameters accepted by the constructor of this predictor. + /// This typedef is used by the PredictorFactory. + typedef LegendreParameters Parameters_; + + Legendre(const Parameters_ &, const oops::Variables &); ~Legendre() {} void compute(const ioda::ObsSpace &, diff --git a/src/ufo/predictors/OrbitalAngle.cc b/src/ufo/predictors/OrbitalAngle.cc index a5dd2acd7..be8e3e468 100644 --- a/src/ufo/predictors/OrbitalAngle.cc +++ b/src/ufo/predictors/OrbitalAngle.cc @@ -14,19 +14,22 @@ namespace ufo { +constexpr char FourierTermTypeParameterTraitsHelper::enumTypeName[]; +constexpr util::NamedEnumerator + FourierTermTypeParameterTraitsHelper::namedValues[]; + static PredictorMaker makerFuncOrbitalAngle_("orbital_angle"); // ----------------------------------------------------------------------------- -OrbitalAngle::OrbitalAngle(const eckit::Configuration & conf, const oops::Variables & vars) - : PredictorBase(conf, vars), order_(1) { - // get the order if it is provided in options - conf.get("options.order", order_); - conf.get("options.component", component_); - - // override the predictor name to distinguish between Orbital angle predictors of - // different orders as well as the two components, sine and cosine. - name() = name() + "_" + std::to_string(order_)+ "_" + component_; +OrbitalAngle::OrbitalAngle(const Parameters_ & parameters, const oops::Variables & vars) + : PredictorBase(parameters, vars), + order_(parameters.order), + component_(parameters.component) { + // override the predictor name to distinguish between Orbital angle predictors of + // different orders as well as the two components, sine and cosine. + name() = name() + "_" + std::to_string(order_)+ "_" + + (component_ == FourierTermType::SIN ? "sin" : "cos"); } // ----------------------------------------------------------------------------- @@ -42,10 +45,9 @@ void OrbitalAngle::compute(const ioda::ObsSpace & odb, std::vector orbital_angle(nlocs, 0.0); odb.get_db("MetaData", "satellite_orbital_angle", orbital_angle); - ASSERT(component_ == "cos" || component_ == "sin"); - switch (component_ == "cos") + switch (component_) { - case true: + case FourierTermType::COS: for (std::size_t jl = 0; jl < nlocs; ++jl) { double cos_oa{ std::cos(orbital_angle[jl]*order_*Constants::deg2rad)}; for (std::size_t jb = 0; jb < nvars; ++jb) { @@ -53,7 +55,7 @@ void OrbitalAngle::compute(const ioda::ObsSpace & odb, } } break; - case false: + case FourierTermType::SIN: for (std::size_t jl = 0; jl < nlocs; ++jl) { double sin_oa{ std::sin(orbital_angle[jl]*order_*Constants::deg2rad)}; for (std::size_t jb = 0; jb < nvars; ++jb) { diff --git a/src/ufo/predictors/OrbitalAngle.h b/src/ufo/predictors/OrbitalAngle.h index 1ddb8724d..aae899ec0 100644 --- a/src/ufo/predictors/OrbitalAngle.h +++ b/src/ufo/predictors/OrbitalAngle.h @@ -7,13 +7,15 @@ #ifndef UFO_PREDICTORS_ORBITALANGLE_H_ #define UFO_PREDICTORS_ORBITALANGLE_H_ + #include #include -#include "ufo/predictors/PredictorBase.h" -namespace eckit { - class Configuration; -} +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/ParameterTraits.h" +#include "oops/util/parameters/RequiredParameter.h" + +#include "ufo/predictors/PredictorBase.h" namespace ioda { class ObsSpace; @@ -21,6 +23,45 @@ namespace ioda { namespace ufo { +enum class FourierTermType { + SIN, COS +}; + +struct FourierTermTypeParameterTraitsHelper { + typedef FourierTermType EnumType; + static constexpr char enumTypeName[] = "FourierTermType"; + static constexpr util::NamedEnumerator namedValues[] = { + { FourierTermType::SIN, "sin" }, + { FourierTermType::COS, "cos" } + }; +}; + +} // namespace ufo + +namespace oops { + +template <> +struct ParameterTraits : + public EnumParameterTraits +{}; + +} // namespace oops + +namespace ufo { + +// ----------------------------------------------------------------------------- + +/// Configuration parameters of the OrbitalAngle predictor. +class OrbitalAngleParameters : public PredictorParametersBase { + OOPS_CONCRETE_PARAMETERS(OrbitalAngleParameters, PredictorParametersBase); + + public: + /// Order of the Fourier term. + oops::RequiredParameter order{"order", this}; + /// Type of the Fourier term (either `sin` or `cos`). + oops::RequiredParameter component{"component", this}; +}; + // ----------------------------------------------------------------------------- /** *This orbital angle predictor is used to fit residual errors as a function of satellite @@ -33,7 +74,11 @@ namespace ufo { class OrbitalAngle : public PredictorBase { public: - OrbitalAngle(const eckit::Configuration &, const oops::Variables &); + /// The type of parameters accepted by the constructor of this predictor. + /// This typedef is used by the PredictorFactory. + typedef OrbitalAngleParameters Parameters_; + + OrbitalAngle(const Parameters_ &, const oops::Variables &); void compute(const ioda::ObsSpace &, const GeoVaLs &, @@ -42,7 +87,7 @@ class OrbitalAngle : public PredictorBase { private: int order_; - std::string component_; // has two valid values: "cos" and "sin" + FourierTermType component_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/predictors/PredictorBase.cc b/src/ufo/predictors/PredictorBase.cc index 05dffb927..64c876509 100644 --- a/src/ufo/predictors/PredictorBase.cc +++ b/src/ufo/predictors/PredictorBase.cc @@ -18,8 +18,9 @@ namespace ufo { // ----------------------------------------------------------------------------- -PredictorBase::PredictorBase(const eckit::Configuration & conf, const oops::Variables & vars) - : func_name_(conf.getString("name")), +PredictorBase::PredictorBase(const PredictorParametersBase & parameters, + const oops::Variables & vars) + : func_name_(parameters.name), geovars_(), hdiags_(), vars_(vars) { } @@ -36,10 +37,10 @@ PredictorFactory::PredictorFactory(const std::string & name) { // ----------------------------------------------------------------------------- -PredictorBase * PredictorFactory::create(const eckit::Configuration & conf, - const oops::Variables & vars) { +std::unique_ptr PredictorFactory::create(const PredictorParametersBase & parameters, + const oops::Variables & vars) { oops::Log::trace() << "PredictorBase::create starting" << std::endl; - const std::string name = conf.getString("name"); + const std::string name = parameters.name; if (!predictorExists(name)) { oops::Log::error() << name << " does not exist in ufo::PredictorFactory." << std::endl; @@ -47,13 +48,25 @@ PredictorBase * PredictorFactory::create(const eckit::Configuration & conf, } typename std::map::iterator jloc = getMakers().find(name); - PredictorBase * ptr = jloc->second->make(conf, vars); + std::unique_ptr ptr = jloc->second->make(parameters, vars); oops::Log::trace() << "PredictorBase::create done" << std::endl; return ptr; } // ----------------------------------------------------------------------------- +std::unique_ptr PredictorFactory::createParameters( + const std::string &name) { + typename std::map::iterator it = + getMakers().find(name); + if (it == getMakers().end()) { + throw std::runtime_error(name + " does not exist in ufo::PredictorFactory"); + } + return it->second->makeParameters(); +} + +// ----------------------------------------------------------------------------- + bool PredictorFactory::predictorExists(const std::string & name) { return (getMakers().find(name) != getMakers().end()); } diff --git a/src/ufo/predictors/PredictorBase.h b/src/ufo/predictors/PredictorBase.h index a16d607f0..f1d77b1ae 100644 --- a/src/ufo/predictors/PredictorBase.h +++ b/src/ufo/predictors/PredictorBase.h @@ -13,17 +13,16 @@ #include #include +#include #include -#include "eckit/config/LocalConfiguration.h" - #include "ioda/ObsVector.h" #include "oops/base/Variables.h" - -namespace eckit { - class Configuration; -} +#include "oops/util/AssociativeContainers.h" +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" namespace ioda { class ObsSpace; @@ -34,11 +33,35 @@ namespace ufo { class ObsDiagnostics; // ----------------------------------------------------------------------------- -/// Base class for computing predictors +/// Base class for predictor parameters +class PredictorParametersBase : public oops::Parameters { + OOPS_ABSTRACT_PARAMETERS(PredictorParametersBase, Parameters) + + public: + /// \brief Predictor name. + oops::RequiredParameter name{"name", this}; +}; +// ----------------------------------------------------------------------------- +/// Concrete implementation of PredictorParametersBase with no new parameters. Useful for predictors +/// that don't take any options. +class EmptyPredictorParameters : public PredictorParametersBase { + OOPS_CONCRETE_PARAMETERS(EmptyPredictorParameters, PredictorParametersBase) + + // no other parameters needed +}; + +// ----------------------------------------------------------------------------- +/// Base class for computing predictors +/// +/// Note: each concrete implementation should typedef `Parameters_` to the name of a subclass of +/// PredictorParametersBase encapsulating its configuration options. It should also provide +/// a constructor with the following signature: +/// +/// PredictorBase(const Parameters_ &, const oops::Variables &); class PredictorBase : private boost::noncopyable { public: - explicit PredictorBase(const eckit::Configuration &, const oops::Variables &); + explicit PredictorBase(const PredictorParametersBase &, const oops::Variables &); virtual ~PredictorBase() = default; /// compute the predictor @@ -73,13 +96,38 @@ typedef std::vector> Predictors; /// Predictor Factory class PredictorFactory { public: - static PredictorBase * create(const eckit::Configuration &, const oops::Variables &); + /// \brief Create and return a new predictor. + /// + /// The predictor type is determined by the \c name attribute of \p parameters. + /// \p parameters must be an instance of the subclass of PredictorParametersBase + /// associated with that predictor type, otherwise an exception will be thrown. + static std::unique_ptr create(const PredictorParametersBase ¶meters, + const oops::Variables &vars); + + /// \brief Create and return an instance of the subclass of PredictorParametersBase + /// storing parameters of predictors of the specified type. + static std::unique_ptr createParameters(const std::string &name); + + /// \brief Return the names of all predictors that can be created by one of the registered makers. + static std::vector getMakerNames() { + return oops::keys(getMakers()); + } + + /// \brief Return true if a maker has been registered for a predictor of type \p name. + static bool predictorExists(const std::string &name); + virtual ~PredictorFactory() = default; - static bool predictorExists(const std::string &); + protected: - explicit PredictorFactory(const std::string &); + /// \brief Register a maker able to create predictors of type \p name. + explicit PredictorFactory(const std::string &name); + private: - virtual PredictorBase * make(const eckit::Configuration &, const oops::Variables &) = 0; + virtual std::unique_ptr make(const PredictorParametersBase &, + const oops::Variables &) = 0; + + virtual std::unique_ptr makeParameters() const = 0; + static std::map < std::string, PredictorFactory * > & getMakers() { static std::map < std::string, PredictorFactory * > makers_; return makers_; @@ -90,8 +138,18 @@ class PredictorFactory { template class PredictorMaker : public PredictorFactory { - virtual PredictorBase * make(const eckit::Configuration & conf, const oops::Variables & vars) - { return new T(conf, vars); } + typedef typename T::Parameters_ Parameters_; + + std::unique_ptr make(const PredictorParametersBase& parameters, + const oops::Variables & vars) override { + const auto &stronglyTypedParameters = dynamic_cast(parameters); + return boost::make_unique(stronglyTypedParameters, vars); + } + + std::unique_ptr makeParameters() const override { + return boost::make_unique(); + } + public: explicit PredictorMaker(const std::string & name) : PredictorFactory(name) {} diff --git a/src/ufo/predictors/ScanAngle.cc b/src/ufo/predictors/ScanAngle.cc index e794d667d..373bb797c 100644 --- a/src/ufo/predictors/ScanAngle.cc +++ b/src/ufo/predictors/ScanAngle.cc @@ -23,19 +23,14 @@ static PredictorMaker makerFuncScanAngle_("scan_angle"); // ----------------------------------------------------------------------------- -ScanAngle::ScanAngle(const eckit::Configuration & conf, const oops::Variables & vars) - : PredictorBase(conf, vars), order_(1) { - // get the order if it is provided in options - if (conf.has("options.order")) { - conf.get("options.order", order_); - - // override the predictor name for differentiable +ScanAngle::ScanAngle(const Parameters_ & parameters, const oops::Variables & vars) + : PredictorBase(parameters, vars), + order_(parameters.order.value().value_or(1)), + var_name_(parameters.varName) { + if (parameters.order.value() != boost::none) { + // override the predictor name to distinguish between scan_angle predictors of different orders name() = name() + "_order_" + std::to_string(order_); } - - if (conf.has("options.var_name")) { - conf.get("options.var_name", var_name_); - } } // ----------------------------------------------------------------------------- @@ -49,11 +44,7 @@ void ScanAngle::compute(const ioda::ObsSpace & odb, // retrieve the sensor view angle std::vector view_angle(nlocs, 0.0); - if ( var_name_.empty() ) { - odb.get_db("MetaData", "sensor_view_angle", view_angle); - } else { - odb.get_db("MetaData", var_name_, view_angle); - } + odb.get_db("MetaData", var_name_, view_angle); for (std::size_t jloc = 0; jloc < nlocs; ++jloc) { for (std::size_t jvar = 0; jvar < nvars; ++jvar) { diff --git a/src/ufo/predictors/ScanAngle.h b/src/ufo/predictors/ScanAngle.h index a35fa9e4e..794834f20 100644 --- a/src/ufo/predictors/ScanAngle.h +++ b/src/ufo/predictors/ScanAngle.h @@ -7,12 +7,14 @@ #ifndef UFO_PREDICTORS_SCANANGLE_H_ #define UFO_PREDICTORS_SCANANGLE_H_ + #include -#include "ufo/predictors/PredictorBase.h" -namespace eckit { - class Configuration; -} +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/Parameters.h" + +#include "ufo/predictors/PredictorBase.h" namespace oops { class Variables; @@ -26,9 +28,30 @@ namespace ufo { // ----------------------------------------------------------------------------- +/// Configuration parameters of the ScanAngle predictor. +class ScanAngleParameters : public PredictorParametersBase { + OOPS_CONCRETE_PARAMETERS(ScanAngleParameters, PredictorParametersBase); + + public: + /// Power to which to raise the scan angle. By default, 1. + /// + /// \note If this option is set, a suffix containing its value (even if it's equal to 1) will be + /// appended to the predictor name. + oops::OptionalParameter order{"order", this}; + + /// Name of the variable (from the MetaData group) containing the scan angle. + oops::Parameter varName{"var_name", "sensor_view_angle", this}; +}; + +// ----------------------------------------------------------------------------- + class ScanAngle : public PredictorBase { public: - ScanAngle(const eckit::Configuration &, const oops::Variables &); + /// The type of parameters accepted by the constructor of this predictor. + /// This typedef is used by the PredictorFactory. + typedef ScanAngleParameters Parameters_; + + ScanAngle(const Parameters_ &, const oops::Variables &); void compute(const ioda::ObsSpace &, const GeoVaLs &, diff --git a/src/ufo/predictors/SineOfLatitude.cc b/src/ufo/predictors/SineOfLatitude.cc index b2b6e7a55..4596eb42a 100644 --- a/src/ufo/predictors/SineOfLatitude.cc +++ b/src/ufo/predictors/SineOfLatitude.cc @@ -20,8 +20,8 @@ static PredictorMaker // ----------------------------------------------------------------------------- -SineOfLatitude::SineOfLatitude(const eckit::Configuration & conf, const oops::Variables & vars) - : PredictorBase(conf, vars) { +SineOfLatitude::SineOfLatitude(const Parameters_ & parameters, const oops::Variables & vars) + : PredictorBase(parameters, vars) { } // ----------------------------------------------------------------------------- diff --git a/src/ufo/predictors/SineOfLatitude.h b/src/ufo/predictors/SineOfLatitude.h index da925a3e6..b8c77ba72 100644 --- a/src/ufo/predictors/SineOfLatitude.h +++ b/src/ufo/predictors/SineOfLatitude.h @@ -10,10 +10,6 @@ #include "ufo/predictors/PredictorBase.h" -namespace eckit { - class Configuration; -} - namespace oops { class Variables; } @@ -28,7 +24,11 @@ namespace ufo { class SineOfLatitude : public PredictorBase { public: - SineOfLatitude(const eckit::Configuration &, const oops::Variables &); + /// The type of parameters accepted by the constructor of this predictor. + /// This typedef is used by the PredictorFactory. + typedef EmptyPredictorParameters Parameters_; + + SineOfLatitude(const Parameters_ &, const oops::Variables &); void compute(const ioda::ObsSpace &, const GeoVaLs &, diff --git a/src/ufo/predictors/Thickness.cc b/src/ufo/predictors/Thickness.cc index a93fcd9f3..77ba65076 100644 --- a/src/ufo/predictors/Thickness.cc +++ b/src/ufo/predictors/Thickness.cc @@ -17,14 +17,13 @@ static PredictorMaker makerFuncThickness_("thickness"); // ----------------------------------------------------------------------------- -Thickness::Thickness(const eckit::Configuration & conf, const oops::Variables & vars) - : PredictorBase(conf, vars) { +Thickness::Thickness(const Parameters_ & parameters, const oops::Variables & vars) + : PredictorBase(parameters, vars) { // required variables geovars_ += oops::Variables({"air_temperature", "air_pressure"}); // required options - eckit::LocalConfiguration bconf(conf, "options"); - parameters_.validateAndDeserialize(bconf); + parameters_ = parameters; // override the predictor name to distinguish between // thickness predictors at different pressure layers diff --git a/src/ufo/predictors/Thickness.h b/src/ufo/predictors/Thickness.h index 984d8ba7e..a38d3afe4 100644 --- a/src/ufo/predictors/Thickness.h +++ b/src/ufo/predictors/Thickness.h @@ -13,20 +13,17 @@ #include "oops/util/parameters/RequiredParameter.h" #include "ufo/predictors/PredictorBase.h" - -namespace eckit { - class Configuration; -} - namespace ioda { class ObsSpace; } namespace ufo { -/// Parameters controlling the thickness predictor. -class ThicknessParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(ThicknessParameters, Parameters) +// ----------------------------------------------------------------------------- + +/// Configuration parameters of the thickness predictor. +class ThicknessParameters : public PredictorParametersBase { + OOPS_CONCRETE_PARAMETERS(ThicknessParameters, PredictorParametersBase); public: /// Pressure value (Pa) at the top of the required thickness layer @@ -54,7 +51,11 @@ class ThicknessParameters : public oops::Parameters { class Thickness : public PredictorBase { public: - Thickness(const eckit::Configuration &, const oops::Variables &); + /// The type of parameters accepted by the constructor of this predictor. + /// This typedef is used by the PredictorFactory. + typedef ThicknessParameters Parameters_; + + Thickness(const Parameters_ &, const oops::Variables &); void compute(const ioda::ObsSpace &, const GeoVaLs &, @@ -62,7 +63,7 @@ class Thickness : public PredictorBase { ioda::ObsVector &) const override; private: - ThicknessParameters parameters_; + Parameters_ parameters_; }; // ----------------------------------------------------------------------------- diff --git a/test/testinput/airs_qc_filters.yaml b/test/testinput/airs_qc_filters.yaml index b1bc30520..3c09a83a0 100755 --- a/test/testinput/airs_qc_filters.yaml +++ b/test/testinput/airs_qc_filters.yaml @@ -51,22 +51,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &airs_aqua-atlap Data/ufo/testinput_tier_1/airs_aqua_tlapmean.txt + order: 2 + tlapse: &airs_aqua-atlap Data/ufo/testinput_tier_1/airs_aqua_tlapmean.txt - name: lapse_rate - options: - tlapse: *airs_aqua-atlap + tlapse: *airs_aqua-atlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/test/testinput/amsua_crtm_bc.yaml b/test/testinput/amsua_crtm_bc.yaml index 246a6b224..4c3afc527 100644 --- a/test/testinput/amsua_crtm_bc.yaml +++ b/test/testinput/amsua_crtm_bc.yaml @@ -32,26 +32,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua19tlap Data/ufo/testinput_tier_1/amsua_n19_tlapmean.txt + order: 2 + tlapse: &amsua19tlap Data/ufo/testinput_tier_1/amsua_n19_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua19tlap + tlapse: *amsua19tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle covariance: minimal required obs number: 20 diff --git a/test/testinput/amsua_qc_filters.yaml b/test/testinput/amsua_qc_filters.yaml index 579301676..5c38874d6 100755 --- a/test/testinput/amsua_qc_filters.yaml +++ b/test/testinput/amsua_qc_filters.yaml @@ -28,27 +28,21 @@ observations: predictors: - name: constant # - name: cloud_liquid_water - # options: - # clwret_ch238: 1 - # clwret_ch314: 2 - # clwret_types: [HofX] + # clwret_ch238: 1 + # clwret_ch314: 2 + # clwret_types: [HofX] - name: lapse_rate - options: - order: 2 - tlapse: &amsua19tlap Data/ufo/testinput_tier_1/amsua_n19_tlapmean.txt + order: 2 + tlapse: &amsua19tlap Data/ufo/testinput_tier_1/amsua_n19_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua19tlap + tlapse: *amsua19tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/test/testinput/atms_qc_filters.yaml b/test/testinput/atms_qc_filters.yaml index e0dcd7a81..5dec0084a 100755 --- a/test/testinput/atms_qc_filters.yaml +++ b/test/testinput/atms_qc_filters.yaml @@ -28,22 +28,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &atms_npp_tlap Data/ufo/testinput_tier_1/atms_npp_tlapmean.txt + order: 2 + tlapse: &atms_npp_tlap Data/ufo/testinput_tier_1/atms_npp_tlapmean.txt - name: lapse_rate - options: - tlapse: *atms_npp_tlap + tlapse: *atms_npp_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/test/testinput/bias_coeff.yaml b/test/testinput/bias_coeff.yaml index 23596314f..ea522e792 100644 --- a/test/testinput/bias_coeff.yaml +++ b/test/testinput/bias_coeff.yaml @@ -13,8 +13,6 @@ observations: variational bc: predictors: - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude obs bias test: norm: 0.0 @@ -78,26 +76,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua19tlap Data/ufo/testinput_tier_1/amsua_n19_tlapmean.txt + order: 2 + tlapse: &amsua19tlap Data/ufo/testinput_tier_1/amsua_n19_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua19tlap + tlapse: *amsua19tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs bias test: norm: 2.8888588267029465 diff --git a/test/testinput/bias_coeff_cov.yaml b/test/testinput/bias_coeff_cov.yaml index 8ea96b338..68b1589ca 100644 --- a/test/testinput/bias_coeff_cov.yaml +++ b/test/testinput/bias_coeff_cov.yaml @@ -14,26 +14,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua19tlap Data/ufo/testinput_tier_1/amsua_n19_tlapmean.txt + order: 2 + tlapse: &amsua19tlap Data/ufo/testinput_tier_1/amsua_n19_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua19tlap + tlapse: *amsua19tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle covariance: minimal required obs number: 20 diff --git a/test/testinput/cris_qc_filters.yaml b/test/testinput/cris_qc_filters.yaml index 33b164b74..e0ec08b26 100755 --- a/test/testinput/cris_qc_filters.yaml +++ b/test/testinput/cris_qc_filters.yaml @@ -61,22 +61,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &cris-fsr_npp-atlap Data/ufo/testinput_tier_1/cris-fsr_npp_tlapmean.txt + order: 2 + tlapse: &cris-fsr_npp-atlap Data/ufo/testinput_tier_1/cris-fsr_npp_tlapmean.txt - name: lapse_rate - options: - tlapse: *cris-fsr_npp-atlap + tlapse: *cris-fsr_npp-atlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/test/testinput/iasi_qc_filters.yaml b/test/testinput/iasi_qc_filters.yaml index 1e19c7000..90f159a69 100755 --- a/test/testinput/iasi_qc_filters.yaml +++ b/test/testinput/iasi_qc_filters.yaml @@ -93,22 +93,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &iasi_metop-atlap Data/ufo/testinput_tier_1/iasi_metop-a_tlapmean.txt + order: 2 + tlapse: &iasi_metop-atlap Data/ufo/testinput_tier_1/iasi_metop-a_tlapmean.txt - name: lapse_rate - options: - tlapse: *iasi_metop-atlap + tlapse: *iasi_metop-atlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/test/testinput/instrumentTests/airs/airs_aqua_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/airs/airs_aqua_gfs_HofX_bc.yaml index c67b72b8e..b69158c51 100644 --- a/test/testinput/instrumentTests/airs/airs_aqua_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/airs/airs_aqua_gfs_HofX_bc.yaml @@ -45,22 +45,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &airs_aqua_tlap Data/ufo/testinput_tier_1/instruments/radiance/airs_aqua_tlapmean.txt + order: 2 + tlapse: &airs_aqua_tlap Data/ufo/testinput_tier_1/instruments/radiance/airs_aqua_tlapmean.txt - name: lapse_rate - options: - tlapse: *airs_aqua_tlap + tlapse: *airs_aqua_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/airs/airs_aqua_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/airs/airs_aqua_gfs_HofX_qc.yaml index 8c97ff2f2..b0b31e226 100644 --- a/test/testinput/instrumentTests/airs/airs_aqua_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/airs/airs_aqua_gfs_HofX_qc.yaml @@ -45,22 +45,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &airs_aqua_tlap Data/ufo/testinput_tier_1/instruments/radiance/airs_aqua_tlapmean.txt + order: 2 + tlapse: &airs_aqua_tlap Data/ufo/testinput_tier_1/instruments/radiance/airs_aqua_tlapmean.txt - name: lapse_rate - options: - tlapse: *airs_aqua_tlap + tlapse: *airs_aqua_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_bc.yaml index 8157aa836..07f5904cd 100644 --- a/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_bc.yaml @@ -25,26 +25,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_aqua_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_aqua_tlapmean.txt + order: 2 + tlapse: &amsua_aqua_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_aqua_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_aqua_tlap + tlapse: *amsua_aqua_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_qc.yaml index 1ad30391d..4a2181461 100644 --- a/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_qc.yaml @@ -27,26 +27,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_aqua_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_aqua_tlapmean.txt + order: 2 + tlapse: &amsua_aqua_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_aqua_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_aqua_tlap + tlapse: *amsua_aqua_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_bc.yaml index 984b4e46c..92c4d0cd1 100644 --- a/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_bc.yaml @@ -25,26 +25,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-a_tlapmean.txt + order: 2 + tlapse: &amsua_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-a_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_metop-a_tlap + tlapse: *amsua_metop-a_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_qc.yaml index cf17e7507..27261643e 100644 --- a/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_qc.yaml @@ -27,26 +27,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-a_tlapmean.txt + order: 2 + tlapse: &amsua_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-a_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_metop-a_tlap + tlapse: *amsua_metop-a_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_bc.yaml index acfbba0c2..cb846aebc 100644 --- a/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_bc.yaml @@ -25,26 +25,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_metop-b_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-b_tlapmean.txt + order: 2 + tlapse: &amsua_metop-b_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-b_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_metop-b_tlap + tlapse: *amsua_metop-b_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_qc.yaml index 58c65d7dd..0e05bc985 100644 --- a/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_qc.yaml @@ -27,26 +27,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_metop-b_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-b_tlapmean.txt + order: 2 + tlapse: &amsua_metop-b_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-b_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_metop-b_tlap + tlapse: *amsua_metop-b_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_bc.yaml index c6e75f183..c4e0e8c1f 100644 --- a/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_bc.yaml @@ -25,26 +25,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_metop-c_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-c_tlapmean.txt + order: 2 + tlapse: &amsua_metop-c_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-c_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_metop-c_tlap + tlapse: *amsua_metop-c_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_qc.yaml index 1fddfcd08..7abad12b8 100644 --- a/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_qc.yaml @@ -27,26 +27,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_metop-c_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-c_tlapmean.txt + order: 2 + tlapse: &amsua_metop-c_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-c_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_metop-c_tlap + tlapse: *amsua_metop-c_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_bc.yaml index c133087ec..d6bac52d8 100644 --- a/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_bc.yaml @@ -25,26 +25,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_n15_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n15_tlapmean.txt + order: 2 + tlapse: &amsua_n15_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n15_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_n15_tlap + tlapse: *amsua_n15_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_qc.yaml index ee3643f40..ec03504c7 100644 --- a/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_qc.yaml @@ -27,26 +27,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_n15_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n15_tlapmean.txt + order: 2 + tlapse: &amsua_n15_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n15_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_n15_tlap + tlapse: *amsua_n15_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml index ec914e473..4bf2c0b08 100644 --- a/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml @@ -25,26 +25,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_n18_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n18_tlapmean.txt + order: 2 + tlapse: &amsua_n18_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n18_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_n18_tlap + tlapse: *amsua_n18_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_qc.yaml index 06194e396..0b58e080e 100644 --- a/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_qc.yaml @@ -27,26 +27,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_n18_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n18_tlapmean.txt + order: 2 + tlapse: &amsua_n18_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n18_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_n18_tlap + tlapse: *amsua_n18_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_bc.yaml index 6e10b4bd8..7190a25f4 100644 --- a/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_bc.yaml @@ -25,26 +25,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_n19_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n19_tlapmean.txt + order: 2 + tlapse: &amsua_n19_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n19_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_n19_tlap + tlapse: *amsua_n19_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_qc.yaml index 11b5353e6..0a72965ca 100644 --- a/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_qc.yaml @@ -27,26 +27,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &amsua_n19_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n19_tlapmean.txt + order: 2 + tlapse: &amsua_n19_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n19_tlapmean.txt - name: lapse_rate - options: - tlapse: *amsua_n19_tlap + tlapse: *amsua_n19_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_bc.yaml index d57020507..088c7d3f3 100644 --- a/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_bc.yaml @@ -25,26 +25,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &atms_n20_tlap Data/ufo/testinput_tier_1/instruments/radiance/atms_n20_tlapmean.txt + order: 2 + tlapse: &atms_n20_tlap Data/ufo/testinput_tier_1/instruments/radiance/atms_n20_tlapmean.txt - name: lapse_rate - options: - tlapse: *atms_n20_tlap + tlapse: *atms_n20_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_qc.yaml index a2d181c9a..64ab3e7b6 100644 --- a/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_qc.yaml @@ -27,26 +27,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &atms_n20_tlap Data/ufo/testinput_tier_1/instruments/radiance/atms_n20_tlapmean.txt + order: 2 + tlapse: &atms_n20_tlap Data/ufo/testinput_tier_1/instruments/radiance/atms_n20_tlapmean.txt - name: lapse_rate - options: - tlapse: *atms_n20_tlap + tlapse: *atms_n20_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_bc.yaml index d9baa9015..f5415e8b5 100644 --- a/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_bc.yaml @@ -25,26 +25,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &atms_npp_tlap Data/ufo/testinput_tier_1/instruments/radiance/atms_npp_tlapmean.txt + order: 2 + tlapse: &atms_npp_tlap Data/ufo/testinput_tier_1/instruments/radiance/atms_npp_tlapmean.txt - name: lapse_rate - options: - tlapse: *atms_npp_tlap + tlapse: *atms_npp_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_qc.yaml index f308ed0a9..68525b839 100644 --- a/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_qc.yaml @@ -27,26 +27,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &atms_npp_tlap Data/ufo/testinput_tier_1/instruments/radiance/atms_npp_tlapmean.txt + order: 2 + tlapse: &atms_npp_tlap Data/ufo/testinput_tier_1/instruments/radiance/atms_npp_tlapmean.txt - name: lapse_rate - options: - tlapse: *atms_npp_tlap + tlapse: *atms_npp_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: - filter: BlackList diff --git a/test/testinput/instrumentTests/avhrr/avhrr_metop-a_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/avhrr/avhrr_metop-a_gfs_HofX_bc.yaml index 44b3b0a6a..ea0da8fdd 100644 --- a/test/testinput/instrumentTests/avhrr/avhrr_metop-a_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/avhrr/avhrr_metop-a_gfs_HofX_bc.yaml @@ -23,22 +23,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &avhrr_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/avhrr_metop-a_tlapmean.txt + order: 2 + tlapse: &avhrr_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/avhrr_metop-a_tlapmean.txt - name: lapse_rate - options: - tlapse: *avhrr_metop-a_tlap + tlapse: *avhrr_metop-a_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-6 diff --git a/test/testinput/instrumentTests/avhrr/avhrr_metop-a_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/avhrr/avhrr_metop-a_gfs_HofX_qc.yaml index bf2cfbb46..18f84d131 100644 --- a/test/testinput/instrumentTests/avhrr/avhrr_metop-a_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/avhrr/avhrr_metop-a_gfs_HofX_qc.yaml @@ -25,22 +25,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &avhrr_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/avhrr_metop-a_tlapmean.txt + order: 2 + tlapse: &avhrr_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/avhrr_metop-a_tlapmean.txt - name: lapse_rate - options: - tlapse: *avhrr_metop-a_tlap + tlapse: *avhrr_metop-a_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/test/testinput/instrumentTests/avhrr/avhrr_n18_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/avhrr/avhrr_n18_gfs_HofX_bc.yaml index 3d88da8b1..aeaa77f07 100644 --- a/test/testinput/instrumentTests/avhrr/avhrr_n18_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/avhrr/avhrr_n18_gfs_HofX_bc.yaml @@ -23,22 +23,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &avhrr_n18_tlap Data/ufo/testinput_tier_1/instruments/radiance/avhrr_n18_tlapmean.txt + order: 2 + tlapse: &avhrr_n18_tlap Data/ufo/testinput_tier_1/instruments/radiance/avhrr_n18_tlapmean.txt - name: lapse_rate - options: - tlapse: *avhrr_n18_tlap + tlapse: *avhrr_n18_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-6 diff --git a/test/testinput/instrumentTests/avhrr/avhrr_n18_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/avhrr/avhrr_n18_gfs_HofX_qc.yaml index 744606766..b270d16c8 100644 --- a/test/testinput/instrumentTests/avhrr/avhrr_n18_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/avhrr/avhrr_n18_gfs_HofX_qc.yaml @@ -25,22 +25,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &avhrr_n18_tlap Data/ufo/testinput_tier_1/instruments/radiance/avhrr_n18_tlapmean.txt + order: 2 + tlapse: &avhrr_n18_tlap Data/ufo/testinput_tier_1/instruments/radiance/avhrr_n18_tlapmean.txt - name: lapse_rate - options: - tlapse: *avhrr_n18_tlap + tlapse: *avhrr_n18_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/test/testinput/instrumentTests/cris/cris-fsr_n20_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/cris/cris-fsr_n20_gfs_HofX_bc.yaml index e6b60fa71..3fa96e312 100644 --- a/test/testinput/instrumentTests/cris/cris-fsr_n20_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/cris/cris-fsr_n20_gfs_HofX_bc.yaml @@ -55,22 +55,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &cris-fsr_n20_tlap Data/ufo/testinput_tier_1/instruments/radiance/cris-fsr_n20_tlapmean.txt + order: 2 + tlapse: &cris-fsr_n20_tlap Data/ufo/testinput_tier_1/instruments/radiance/cris-fsr_n20_tlapmean.txt - name: lapse_rate - options: - tlapse: *cris-fsr_n20_tlap + tlapse: *cris-fsr_n20_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/cris/cris-fsr_n20_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/cris/cris-fsr_n20_gfs_HofX_qc.yaml index c34653560..0885bd128 100644 --- a/test/testinput/instrumentTests/cris/cris-fsr_n20_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/cris/cris-fsr_n20_gfs_HofX_qc.yaml @@ -57,22 +57,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &cris-fsr_n20_tlap Data/ufo/testinput_tier_1/instruments/radiance/cris-fsr_n20_tlapmean.txt + order: 2 + tlapse: &cris-fsr_n20_tlap Data/ufo/testinput_tier_1/instruments/radiance/cris-fsr_n20_tlapmean.txt - name: lapse_rate - options: - tlapse: *cris-fsr_n20_tlap + tlapse: *cris-fsr_n20_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/test/testinput/instrumentTests/cris/cris-fsr_npp_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/cris/cris-fsr_npp_gfs_HofX_bc.yaml index 3660f2c72..34aecbc54 100644 --- a/test/testinput/instrumentTests/cris/cris-fsr_npp_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/cris/cris-fsr_npp_gfs_HofX_bc.yaml @@ -55,22 +55,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &cris-fsr_npp_tlap Data/ufo/testinput_tier_1/instruments/radiance/cris-fsr_npp_tlapmean.txt + order: 2 + tlapse: &cris-fsr_npp_tlap Data/ufo/testinput_tier_1/instruments/radiance/cris-fsr_npp_tlapmean.txt - name: lapse_rate - options: - tlapse: *cris-fsr_npp_tlap + tlapse: *cris-fsr_npp_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/cris/cris-fsr_npp_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/cris/cris-fsr_npp_gfs_HofX_qc.yaml index e3c230ab2..babdc5e4a 100644 --- a/test/testinput/instrumentTests/cris/cris-fsr_npp_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/cris/cris-fsr_npp_gfs_HofX_qc.yaml @@ -57,22 +57,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &cris-fsr_npp_tlap Data/ufo/testinput_tier_1/instruments/radiance/cris-fsr_npp_tlapmean.txt + order: 2 + tlapse: &cris-fsr_npp_tlap Data/ufo/testinput_tier_1/instruments/radiance/cris-fsr_npp_tlapmean.txt - name: lapse_rate - options: - tlapse: *cris-fsr_npp_tlap + tlapse: *cris-fsr_npp_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/test/testinput/instrumentTests/iasi/iasi_metop-a_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/iasi/iasi_metop-a_gfs_HofX_bc.yaml index 83acd1fb5..916319cf1 100644 --- a/test/testinput/instrumentTests/iasi/iasi_metop-a_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/iasi/iasi_metop-a_gfs_HofX_bc.yaml @@ -71,22 +71,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &iasi_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/iasi_metop-a_tlapmean.txt + order: 2 + tlapse: &iasi_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/iasi_metop-a_tlapmean.txt - name: lapse_rate - options: - tlapse: *iasi_metop-a_tlap + tlapse: *iasi_metop-a_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/iasi/iasi_metop-a_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/iasi/iasi_metop-a_gfs_HofX_qc.yaml index fd4297d06..de474ed12 100644 --- a/test/testinput/instrumentTests/iasi/iasi_metop-a_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/iasi/iasi_metop-a_gfs_HofX_qc.yaml @@ -73,22 +73,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &iasi_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/iasi_metop-a_tlapmean.txt + order: 2 + tlapse: &iasi_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/iasi_metop-a_tlapmean.txt - name: lapse_rate - options: - tlapse: *iasi_metop-a_tlap + tlapse: *iasi_metop-a_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/test/testinput/instrumentTests/iasi/iasi_metop-b_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/iasi/iasi_metop-b_gfs_HofX_bc.yaml index 24885b056..21fba5b12 100644 --- a/test/testinput/instrumentTests/iasi/iasi_metop-b_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/iasi/iasi_metop-b_gfs_HofX_bc.yaml @@ -71,22 +71,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &iasi_metop-b_tlap Data/ufo/testinput_tier_1/instruments/radiance/iasi_metop-b_tlapmean.txt + order: 2 + tlapse: &iasi_metop-b_tlap Data/ufo/testinput_tier_1/instruments/radiance/iasi_metop-b_tlapmean.txt - name: lapse_rate - options: - tlapse: *iasi_metop-b_tlap + tlapse: *iasi_metop-b_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/iasi/iasi_metop-b_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/iasi/iasi_metop-b_gfs_HofX_qc.yaml index e80d9b2fa..71c1840c9 100644 --- a/test/testinput/instrumentTests/iasi/iasi_metop-b_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/iasi/iasi_metop-b_gfs_HofX_qc.yaml @@ -73,22 +73,17 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &iasi_metop-b_tlap Data/ufo/testinput_tier_1/instruments/radiance/iasi_metop-b_tlapmean.txt + order: 2 + tlapse: &iasi_metop-b_tlap Data/ufo/testinput_tier_1/instruments/radiance/iasi_metop-b_tlapmean.txt - name: lapse_rate - options: - tlapse: *iasi_metop-b_tlap + tlapse: *iasi_metop-b_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Wavenumber Check diff --git a/test/testinput/instrumentTests/mhs/mhs_metop-b_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/mhs/mhs_metop-b_gfs_HofX_bc.yaml index 3efbb4225..06d2204f9 100644 --- a/test/testinput/instrumentTests/mhs/mhs_metop-b_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/mhs/mhs_metop-b_gfs_HofX_bc.yaml @@ -23,26 +23,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &mhs_metop-b_tlap Data/ufo/testinput_tier_1/instruments/radiance/mhs_metop-b_tlapmean.txt + order: 2 + tlapse: &mhs_metop-b_tlap Data/ufo/testinput_tier_1/instruments/radiance/mhs_metop-b_tlapmean.txt - name: lapse_rate - options: - tlapse: *mhs_metop-b_tlap + tlapse: *mhs_metop-b_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/mhs/mhs_metop-c_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/mhs/mhs_metop-c_gfs_HofX_bc.yaml index a40bcaa97..65146263c 100644 --- a/test/testinput/instrumentTests/mhs/mhs_metop-c_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/mhs/mhs_metop-c_gfs_HofX_bc.yaml @@ -23,26 +23,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &mhs_metop-c_tlap Data/ufo/testinput_tier_1/instruments/radiance/mhs_metop-c_tlapmean.txt + order: 2 + tlapse: &mhs_metop-c_tlap Data/ufo/testinput_tier_1/instruments/radiance/mhs_metop-c_tlapmean.txt - name: lapse_rate - options: - tlapse: *mhs_metop-c_tlap + tlapse: *mhs_metop-c_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/mhs/mhs_n19_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/mhs/mhs_n19_gfs_HofX_bc.yaml index 82dfe389c..52ce8b273 100644 --- a/test/testinput/instrumentTests/mhs/mhs_n19_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/mhs/mhs_n19_gfs_HofX_bc.yaml @@ -23,26 +23,19 @@ observations: predictors: - name: constant - name: cosine_of_latitude_times_orbit_node - options: - preconditioner: 0.01 - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &mhs_n19_tlap Data/ufo/testinput_tier_1/instruments/radiance/mhs_n19_tlapmean.txt + order: 2 + tlapse: &mhs_n19_tlap Data/ufo/testinput_tier_1/instruments/radiance/mhs_n19_tlapmean.txt - name: lapse_rate - options: - tlapse: *mhs_n19_tlap + tlapse: *mhs_n19_tlap - name: emissivity - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-7 diff --git a/test/testinput/instrumentTests/seviri/seviri_m11_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/seviri/seviri_m11_gfs_HofX_bc.yaml index 589947de3..2d72fb02a 100644 --- a/test/testinput/instrumentTests/seviri/seviri_m11_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/seviri/seviri_m11_gfs_HofX_bc.yaml @@ -23,23 +23,18 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &seviri_m11_tlap Data/ufo/testinput_tier_1/instruments/radiance/seviri_m11_tlapmean.txt + order: 2 + tlapse: &seviri_m11_tlap Data/ufo/testinput_tier_1/instruments/radiance/seviri_m11_tlapmean.txt - name: lapse_rate - options: - tlapse: *seviri_m11_tlap + tlapse: *seviri_m11_tlap - name: emissivity # The scan angle here actually is used as for scan positions in the input ioda file to replicate gsi - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle vector ref: GsiHofXBc tolerance: 1.e-6 diff --git a/test/testinput/instrumentTests/seviri/seviri_m11_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/seviri/seviri_m11_gfs_HofX_qc.yaml index 69fc9eccc..ac0a3e01a 100644 --- a/test/testinput/instrumentTests/seviri/seviri_m11_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/seviri/seviri_m11_gfs_HofX_qc.yaml @@ -23,23 +23,18 @@ observations: predictors: - name: constant - name: lapse_rate - options: - order: 2 - tlapse: &seviri_m11_tlap Data/ufo/testinput_tier_1/instruments/radiance/seviri_m11_tlapmean.txt + order: 2 + tlapse: &seviri_m11_tlap Data/ufo/testinput_tier_1/instruments/radiance/seviri_m11_tlapmean.txt - name: lapse_rate - options: - tlapse: *seviri_m11_tlap + tlapse: *seviri_m11_tlap - name: emissivity # The scan angle here actually is used as for scan positions in the input ioda file to replicate gsi - name: scan_angle - options: - order: 4 + order: 4 - name: scan_angle - options: - order: 3 + order: 3 - name: scan_angle - options: - order: 2 + order: 2 - name: scan_angle obs filters: # Observation Range Sanity Check diff --git a/test/testinput/instrumentTests/ssmis/ssmis_f17_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/ssmis/ssmis_f17_gfs_HofX_bc.yaml index 8c24f9bc6..f39b7cdba 100644 --- a/test/testinput/instrumentTests/ssmis/ssmis_f17_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/ssmis/ssmis_f17_gfs_HofX_bc.yaml @@ -25,39 +25,32 @@ observations: predictors: - name: constant - name: cloud_liquid_water - options: - satellite: SSMIS - ch19h: 12 - ch19v: 13 - ch22v: 14 - ch37h: 15 - ch37v: 16 - ch91v: 17 - ch91h: 18 + satellite: SSMIS + ch19h: 12 + ch19v: 13 + ch22v: 14 + ch37h: 15 + ch37v: 16 + ch91v: 17 + ch91h: 18 - name: cosine_of_latitude_times_orbit_node - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &ssmis_f17_tlap Data/ufo/testinput_tier_1/instruments/radiance/ssmis_f17_tlapmean.txt + order: 2 + tlapse: &ssmis_f17_tlap Data/ufo/testinput_tier_1/instruments/radiance/ssmis_f17_tlapmean.txt - name: lapse_rate - options: - tlapse: *ssmis_f17_tlap + tlapse: *ssmis_f17_tlap - name: emissivity - name: scan_angle - options: - var_name: scan_position - order: 4 + var_name: scan_position + order: 4 - name: scan_angle - options: - var_name: scan_position - order: 3 + var_name: scan_position + order: 3 - name: scan_angle - options: - var_name: scan_position - order: 2 + var_name: scan_position + order: 2 - name: scan_angle - options: - var_name: scan_position + var_name: scan_position vector ref: GsiHofXBc tolerance: 1.e-5 diff --git a/test/testinput/instrumentTests/ssmis/ssmis_f17_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/ssmis/ssmis_f17_gfs_HofX_qc.yaml index 0b71ce802..54d8d0f91 100644 --- a/test/testinput/instrumentTests/ssmis/ssmis_f17_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/ssmis/ssmis_f17_gfs_HofX_qc.yaml @@ -25,40 +25,33 @@ observations: predictors: - name: constant - name: cloud_liquid_water - options: - satellite: SSMIS - ch19h: 12 - ch19v: 13 - ch22v: 14 - ch37h: 15 - ch37v: 16 - ch91v: 17 - ch91h: 18 + satellite: SSMIS + ch19h: 12 + ch19v: 13 + ch22v: 14 + ch37h: 15 + ch37v: 16 + ch91v: 17 + ch91h: 18 - name: cosine_of_latitude_times_orbit_node - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &ssmis_f17_tlap Data/ufo/testinput_tier_1/instruments/radiance/ssmis_f17_tlapmean.txt + order: 2 + tlapse: &ssmis_f17_tlap Data/ufo/testinput_tier_1/instruments/radiance/ssmis_f17_tlapmean.txt - name: lapse_rate - options: - tlapse: *ssmis_f17_tlap + tlapse: *ssmis_f17_tlap - name: emissivity - name: scan_angle - options: - var_name: scan_position - order: 4 + var_name: scan_position + order: 4 - name: scan_angle - options: - var_name: scan_position - order: 3 + var_name: scan_position + order: 3 - name: scan_angle - options: - var_name: scan_position - order: 2 + var_name: scan_position + order: 2 - name: scan_angle - options: - var_name: scan_position + var_name: scan_position obs filters: #step1: Gross check (setuprad) - filter: Background Check diff --git a/test/testinput/interpolate_data_from_file_predictor.yaml b/test/testinput/interpolate_data_from_file_predictor.yaml index 660f58cd7..db5356bce 100644 --- a/test/testinput/interpolate_data_from_file_predictor.yaml +++ b/test/testinput/interpolate_data_from_file_predictor.yaml @@ -11,18 +11,17 @@ observations: static bc: predictors: - name: interpolate_data_from_file - options: - corrected variables: - - name: air_temperature - file: Data/ufo/testinput_tier_1/bias_interpolation_air_temperature.nc4 - interpolation: - - name: station_id@MetaData - method: exact - - name: specific_humidity - file: Data/ufo/testinput_tier_1/bias_interpolation_specific_humidity.nc4 - interpolation: - - name: station_id@MetaData - method: exact + corrected variables: + - name: air_temperature + file: Data/ufo/testinput_tier_1/bias_interpolation_air_temperature.nc4 + interpolation: + - name: station_id@MetaData + method: exact + - name: specific_humidity + file: Data/ufo/testinput_tier_1/bias_interpolation_specific_humidity.nc4 + interpolation: + - name: station_id@MetaData + method: exact tolerance: 1.0e-16 - obs space: name: Satellite @@ -34,15 +33,14 @@ observations: static bc: predictors: - name: interpolate_data_from_file - options: - corrected variables: - - name: brightness_temperature - channels: *channels - file: Data/ufo/testinput_tier_1/bias_interpolation_brightness_temperature.nc4 - interpolation: - - name: scan_position@MetaData - method: nearest # In practice, 'exact' would probably be used. 'nearest' is used - # here to reduce the size of the NetCDF file used by this test. - # The predictor's output will depend on the scan position and also (because the - # 'channels' option is set) on the channel number + corrected variables: + - name: brightness_temperature + channels: *channels + file: Data/ufo/testinput_tier_1/bias_interpolation_brightness_temperature.nc4 + interpolation: + - name: scan_position@MetaData + method: nearest # In practice, 'exact' would probably be used. 'nearest' is used + # here to reduce the size of the NetCDF file used by this test. + # The predictor's output will depend on the scan position and also (because the + # 'channels' option is set) on the channel number tolerance: 1.0e-16 diff --git a/test/testinput/legendre_predictor.yaml b/test/testinput/legendre_predictor.yaml index 719cc54df..0eacd8ad4 100644 --- a/test/testinput/legendre_predictor.yaml +++ b/test/testinput/legendre_predictor.yaml @@ -17,18 +17,14 @@ observations: predictors: - name: Legendre number of scan positions: 32 - options: - order: 1 + order: 1 - name: Legendre number of scan positions: 32 - options: - order: 2 + order: 2 - name: Legendre number of scan positions: 32 - options: - order: 3 + order: 3 - name: Legendre number of scan positions: 32 - options: - order: 4 + order: 4 tolerance: 1.0e-6 diff --git a/test/testinput/orbital_angle_predictor.yaml b/test/testinput/orbital_angle_predictor.yaml index 11a4c0c42..715e6a51e 100644 --- a/test/testinput/orbital_angle_predictor.yaml +++ b/test/testinput/orbital_angle_predictor.yaml @@ -15,51 +15,39 @@ observations: variational bc: predictors: - name: orbital_angle - options: - order: 1 - component: "cos" + order: 1 + component: "cos" - name: orbital_angle - options: - order: 1 - component: "sin" + order: 1 + component: "sin" - name: orbital_angle - options: - order: 2 - component: "cos" + order: 2 + component: "cos" - name: orbital_angle - options: - order: 2 - component: "sin" + order: 2 + component: "sin" - name: orbital_angle - options: - order: 3 - component: "cos" + order: 3 + component: "cos" - name: orbital_angle - options: - order: 3 - component: "sin" + order: 3 + component: "sin" - name: orbital_angle - options: - order: 4 - component: "cos" + order: 4 + component: "cos" - name: orbital_angle - options: - order: 4 - component: "sin" + order: 4 + component: "sin" - name: orbital_angle - options: - order: 5 - component: "cos" + order: 5 + component: "cos" - name: orbital_angle - options: - order: 5 - component: "sin" + order: 5 + component: "sin" - name: orbital_angle - options: - order: 6 - component: "cos" + order: 6 + component: "cos" - name: orbital_angle - options: - order: 6 - component: "sin" + order: 6 + component: "sin" tolerance: 1.e-5 diff --git a/test/testinput/ssmis_f17_gfs_backgroundcheck_bc.yaml b/test/testinput/ssmis_f17_gfs_backgroundcheck_bc.yaml index 89374812a..0ba13bbdb 100644 --- a/test/testinput/ssmis_f17_gfs_backgroundcheck_bc.yaml +++ b/test/testinput/ssmis_f17_gfs_backgroundcheck_bc.yaml @@ -25,40 +25,33 @@ observations: predictors: - name: constant - name: cloud_liquid_water - options: - satellite: SSMIS - ch19h: 12 - ch19v: 13 - ch22v: 14 - ch37h: 15 - ch37v: 16 - ch91v: 17 - ch91h: 18 + satellite: SSMIS + ch19h: 12 + ch19v: 13 + ch22v: 14 + ch37h: 15 + ch37v: 16 + ch91v: 17 + ch91h: 18 - name: cosine_of_latitude_times_orbit_node - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &ssmis_f17_tlap Data/ufo/testinput_tier_1/instruments/radiance/ssmis_f17_tlapmean.txt + order: 2 + tlapse: &ssmis_f17_tlap Data/ufo/testinput_tier_1/instruments/radiance/ssmis_f17_tlapmean.txt - name: lapse_rate - options: - tlapse: *ssmis_f17_tlap + tlapse: *ssmis_f17_tlap - name: emissivity - name: scan_angle - options: - var_name: scan_position - order: 4 + var_name: scan_position + order: 4 - name: scan_angle - options: - var_name: scan_position - order: 3 + var_name: scan_position + order: 3 - name: scan_angle - options: - var_name: scan_position - order: 2 + var_name: scan_position + order: 2 - name: scan_angle - options: - var_name: scan_position + var_name: scan_position obs filters: #step1: Gross check using abs(O-B-bias) - filter: Background Check diff --git a/test/testinput/ssmis_f17_gfs_backgroundcheck_nbc.yaml b/test/testinput/ssmis_f17_gfs_backgroundcheck_nbc.yaml index 25e18a6ee..07796045b 100644 --- a/test/testinput/ssmis_f17_gfs_backgroundcheck_nbc.yaml +++ b/test/testinput/ssmis_f17_gfs_backgroundcheck_nbc.yaml @@ -25,40 +25,33 @@ observations: predictors: - name: constant - name: cloud_liquid_water - options: - satellite: SSMIS - ch19h: 12 - ch19v: 13 - ch22v: 14 - ch37h: 15 - ch37v: 16 - ch91v: 17 - ch91h: 18 + satellite: SSMIS + ch19h: 12 + ch19v: 13 + ch22v: 14 + ch37h: 15 + ch37v: 16 + ch91v: 17 + ch91h: 18 - name: cosine_of_latitude_times_orbit_node - name: sine_of_latitude - name: lapse_rate - options: - order: 2 - tlapse: &ssmis_f17_tlap Data/ufo/testinput_tier_1/instruments/radiance/ssmis_f17_tlapmean.txt + order: 2 + tlapse: &ssmis_f17_tlap Data/ufo/testinput_tier_1/instruments/radiance/ssmis_f17_tlapmean.txt - name: lapse_rate - options: - tlapse: *ssmis_f17_tlap + tlapse: *ssmis_f17_tlap - name: emissivity - name: scan_angle - options: - var_name: scan_position - order: 4 + var_name: scan_position + order: 4 - name: scan_angle - options: - var_name: scan_position - order: 3 + var_name: scan_position + order: 3 - name: scan_angle - options: - var_name: scan_position - order: 2 + var_name: scan_position + order: 2 - name: scan_angle - options: - var_name: scan_position + var_name: scan_position obs filters: #step1: Gross check using abs(O-B) - filter: Background Check diff --git a/test/testinput/thickness_predictor.yaml b/test/testinput/thickness_predictor.yaml index 521e0fd41..b4d65e8b7 100644 --- a/test/testinput/thickness_predictor.yaml +++ b/test/testinput/thickness_predictor.yaml @@ -14,11 +14,10 @@ observations: variational bc: predictors: - name: thickness - options: - layer top: 300000 - layer base: 850000 - mean: 7.6 - standard deviation: 0.4 + layer top: 300000 + layer base: 850000 + mean: 7.6 + standard deviation: 0.4 expectExceptionWithMessage: layer top is greater than largest model pressure level - obs space: @@ -33,21 +32,18 @@ observations: variational bc: predictors: - name: thickness - options: - layer top: 30000 - layer base: 85000 - mean: 7.6 - standard deviation: 0.4 + layer top: 30000 + layer base: 85000 + mean: 7.6 + standard deviation: 0.4 - name: thickness - options: - layer top: 5000 - layer base: 20000 - mean: 8.6 - standard deviation: 0.4 + layer top: 5000 + layer base: 20000 + mean: 8.6 + standard deviation: 0.4 - name: thickness - options: - layer top: 85000 - layer base: 100000 - mean: 1.2 - standard deviation: 0.4 + layer top: 85000 + layer base: 100000 + mean: 1.2 + standard deviation: 0.4 tolerance: 5.0e-5 From def9813ce0208797a6da06db10553aa9a1fcfd9f Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Fri, 25 Jun 2021 20:38:47 +0100 Subject: [PATCH 19/98] Convert Identity operator to C++ (#1263) * C++ Identity operator * todo note * move vector declaration outside loop * IdentityCpp -> Identity * improve comments * initialise adjoint with zeros; deal with missing values * fix adjoint code; linopr test * change position of yaml in list * remove some includes * deal with operator variable indices correctly * fix coding norms * remove ufo_get_ioda_test_data * correct treatment of adjoint * remove zeroing from TL; test with two variables * just get first GeoVaL level * get/put at level * Removed ObsBias from function sigs * fix geoval indexing * remove allocation and zeroing of GeoVaLs Co-authored-by: Ryan Honeyager --- src/ufo/identity/CMakeLists.txt | 6 - src/ufo/identity/ObsIdentity.cc | 45 +++--- src/ufo/identity/ObsIdentity.h | 47 ++++-- src/ufo/identity/ObsIdentity.interface.F90 | 100 ------------ src/ufo/identity/ObsIdentity.interface.h | 48 ------ src/ufo/identity/ObsIdentityTLAD.cc | 65 +++++--- src/ufo/identity/ObsIdentityTLAD.h | 26 ++-- .../identity/ObsIdentityTLAD.interface.F90 | 134 ---------------- src/ufo/identity/ObsIdentityTLAD.interface.h | 52 ------- src/ufo/identity/ufo_identity_mod.F90 | 77 ---------- src/ufo/identity/ufo_identity_tlad_mod.F90 | 145 ------------------ test/CMakeLists.txt | 15 ++ test/testinput/identity.yaml | 40 +++++ 13 files changed, 172 insertions(+), 628 deletions(-) delete mode 100644 src/ufo/identity/ObsIdentity.interface.F90 delete mode 100644 src/ufo/identity/ObsIdentity.interface.h delete mode 100644 src/ufo/identity/ObsIdentityTLAD.interface.F90 delete mode 100644 src/ufo/identity/ObsIdentityTLAD.interface.h delete mode 100644 src/ufo/identity/ufo_identity_mod.F90 delete mode 100644 src/ufo/identity/ufo_identity_tlad_mod.F90 create mode 100644 test/testinput/identity.yaml diff --git a/src/ufo/identity/CMakeLists.txt b/src/ufo/identity/CMakeLists.txt index 6ecb5b6f5..8ffdc3c83 100644 --- a/src/ufo/identity/CMakeLists.txt +++ b/src/ufo/identity/CMakeLists.txt @@ -8,12 +8,6 @@ set ( identity_files ObsIdentity.cc ObsIdentityTLAD.h ObsIdentityTLAD.cc - ObsIdentity.interface.F90 - ObsIdentity.interface.h - ObsIdentityTLAD.interface.F90 - ObsIdentityTLAD.interface.h - ufo_identity_mod.F90 - ufo_identity_tlad_mod.F90 ) PREPEND( _p_identity_files "identity" ${identity_files} ) diff --git a/src/ufo/identity/ObsIdentity.cc b/src/ufo/identity/ObsIdentity.cc index 11c0c29e3..f39d56013 100644 --- a/src/ufo/identity/ObsIdentity.cc +++ b/src/ufo/identity/ObsIdentity.cc @@ -1,8 +1,8 @@ /* - * (C) Copyright 2017-2018 UCAR - * + * (C) Copyright 2021 UK Met Office + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ #include "ufo/identity/ObsIdentity.h" @@ -12,7 +12,6 @@ #include "ioda/ObsVector.h" -#include "oops/base/Variables.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" @@ -22,43 +21,51 @@ namespace ufo { // ----------------------------------------------------------------------------- -static ObsOperatorMaker makerIdentity_("Identity"); +static ObsOperatorMaker obsIdentityMaker_("Identity"); // ----------------------------------------------------------------------------- ObsIdentity::ObsIdentity(const ioda::ObsSpace & odb, const eckit::Configuration & config) - : ObsOperatorBase(odb, config), keyOperObsIdentity_(0), odb_(odb), varin_() + : ObsOperatorBase(odb, config) { - std::vector operatorVarIndices; - getOperatorVariables(config, odb.obsvariables(), operatorVars_, operatorVarIndices); + oops::Log::trace() << "ObsIdentity constructor starting" << std::endl; - ufo_identity_setup_f90(keyOperObsIdentity_, config, - operatorVars_, operatorVarIndices.data(), operatorVarIndices.size(), - varin_); + getOperatorVariables(config, odb.obsvariables(), operatorVars_, operatorVarIndices_); + requiredVars_ += operatorVars_; - oops::Log::trace() << "ObsIdentity created." << std::endl; + oops::Log::trace() << "ObsIdentity constructor finished" << std::endl; } // ----------------------------------------------------------------------------- ObsIdentity::~ObsIdentity() { - ufo_identity_delete_f90(keyOperObsIdentity_); oops::Log::trace() << "ObsIdentity destructed" << std::endl; } // ----------------------------------------------------------------------------- -void ObsIdentity::simulateObs(const GeoVaLs & gom, ioda::ObsVector & ovec, - ObsDiagnostics &) const { - ufo_identity_simobs_f90(keyOperObsIdentity_, gom.toFortran(), odb_, - ovec.nvars(), ovec.nlocs(), ovec.toFortran()); - oops::Log::trace() << "ObsIdentity: observation operator run" << std::endl; +void ObsIdentity::simulateObs(const GeoVaLs & gv, ioda::ObsVector & ovec, + ObsDiagnostics & ydiags) const { + oops::Log::trace() << "ObsIdentity: simulateObs starting" << std::endl; + + std::vector vec(ovec.nlocs()); + for (int jvar : operatorVarIndices_) { + const std::string& varname = ovec.varnames().variables()[jvar]; + // Get GeoVaL at the lowest level. + gv.getAtLevel(vec, varname, 0); + for (size_t jloc = 0; jloc < ovec.nlocs(); ++jloc) { + const size_t idx = jloc * ovec.nvars() + jvar; + ovec[idx] = vec[jloc]; + } + } + + oops::Log::trace() << "ObsIdentity: simulateObs finished" << std::endl; } // ----------------------------------------------------------------------------- void ObsIdentity::print(std::ostream & os) const { - os << "ObsIdentity::print not implemented"; + os << "ObsIdentity operator" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/ufo/identity/ObsIdentity.h b/src/ufo/identity/ObsIdentity.h index 586862b43..4b3368b01 100644 --- a/src/ufo/identity/ObsIdentity.h +++ b/src/ufo/identity/ObsIdentity.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2017-2018 UCAR + * (C) Copyright 2021 UK Met Office * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -10,10 +10,11 @@ #include #include +#include #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" -#include "ufo/identity/ObsIdentity.interface.h" + #include "ufo/ObsOperatorBase.h" /// Forward declarations @@ -30,33 +31,49 @@ namespace ufo { class GeoVaLs; class ObsDiagnostics; -// ----------------------------------------------------------------------------- -/// Generic identity observation operator class +/// \brief Identity observation operator. +/// +/// This observation operator transfers model values directly to the H(x) vector, after horizontal +/// interpolation has been performed, with no further processing. +/// For GeoVaLs with more than one vertical level, only the first entry in the GeoVaL is processed +/// in this way. +/// +/// An example yaml configuration is: +/// +/// obs operator: +/// name: Identity +/// +/// This operator also accepts an optional `variables` parameter, which controls which ObsSpace +/// variables will be simulated. This option should only be set if this operator is used as a +/// component of the Composite operator. If `variables` is not set, the operator will simulate +/// all ObsSpace variables. Please see the documentation of the Composite operator for further +/// details. class ObsIdentity : public ObsOperatorBase, - private util::ObjectCounter { + private util::ObjectCounter { public: static const std::string classname() {return "ufo::ObsIdentity";} ObsIdentity(const ioda::ObsSpace &, const eckit::Configuration &); - virtual ~ObsIdentity(); + ~ObsIdentity() override; -// Obs Operator void simulateObs(const GeoVaLs &, ioda::ObsVector &, ObsDiagnostics &) const override; -// Other - const oops::Variables & requiredVars() const override {return varin_;} + const oops::Variables & requiredVars() const override { return requiredVars_; } oops::Variables simulatedVars() const override { return operatorVars_; } - int & toFortran() {return keyOperObsIdentity_;} - const int & toFortran() const {return keyOperObsIdentity_;} - private: void print(std::ostream &) const override; - F90hop keyOperObsIdentity_; - const ioda::ObsSpace& odb_; - oops::Variables varin_; + + private: + /// Required variables. + oops::Variables requiredVars_; + + /// Operator variables. oops::Variables operatorVars_; + + /// Indices of operator variables. + std::vector operatorVarIndices_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/identity/ObsIdentity.interface.F90 b/src/ufo/identity/ObsIdentity.interface.F90 deleted file mode 100644 index 60d85668e..000000000 --- a/src/ufo/identity/ObsIdentity.interface.F90 +++ /dev/null @@ -1,100 +0,0 @@ -! (C) Copyright 2017-2018 UCAR -! -! This software is licensed under the terms of the Apache Licence Version 2.0 -! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -!> Fortran identity module for functions on the interface between C++ and Fortran -! to handle observation operators - -module ufo_identity_mod_c - - use iso_c_binding - use ufo_identity_mod - use ufo_geovals_mod_c, only: ufo_geovals_registry - use ufo_geovals_mod, only: ufo_geovals - implicit none - - private - -#define LISTED_TYPE ufo_identity - - !> Linked list interface - defines registry_t type -#include "oops/util/linkedList_i.f" - - !> Global registry - type(registry_t) :: ufo_identity_registry - - ! ------------------------------------------------------------------------------ - -contains - - ! ------------------------------------------------------------------------------ - !> Linked list implementation -#include "oops/util/linkedList_c.f" - -! ------------------------------------------------------------------------------ - -subroutine ufo_identity_setup_c(c_key_self, c_conf, c_obsvars, c_obsvarindices, c_nobsvars, & - c_geovars) bind(c,name='ufo_identity_setup_f90') -use fckit_configuration_module, only: fckit_configuration -use oops_variables_mod -implicit none -integer(c_int), intent(inout) :: c_key_self -type(c_ptr), value, intent(in) :: c_conf -! variables to be simulated -type(c_ptr), intent(in), value :: c_obsvars ! variables to be simulated... -integer(c_int), intent(in), value :: c_nobsvars -integer(c_int), intent(in) :: c_obsvarindices(c_nobsvars) ! ... and their global indices -type(c_ptr), intent(in), value :: c_geovars ! variables requested from the model - -type(ufo_identity), pointer :: self - -call ufo_identity_registry%setup(c_key_self, self) - -self%obsvars = oops_variables(c_obsvars) -allocate(self%obsvarindices(self%obsvars%nvars())) -self%obsvarindices(:) = c_obsvarindices(:) + 1 ! Convert from C to Fortran indexing - -self%geovars = oops_variables(c_geovars) -call self%setup() - -end subroutine ufo_identity_setup_c - -! ------------------------------------------------------------------------------ - -subroutine ufo_identity_delete_c(c_key_self) bind(c,name='ufo_identity_delete_f90') -implicit none -integer(c_int), intent(inout) :: c_key_self - -type(ufo_identity), pointer :: self - -call ufo_identity_registry%get(c_key_self, self) - -call ufo_identity_registry%remove(c_key_self) - -end subroutine ufo_identity_delete_c - -! ------------------------------------------------------------------------------ - -subroutine ufo_identity_simobs_c(c_key_self, c_key_geovals, c_obsspace, c_nvars, c_nlocs, & - c_hofx) bind(c,name='ufo_identity_simobs_f90') -implicit none -integer(c_int), intent(in) :: c_key_self -integer(c_int), intent(in) :: c_key_geovals -type(c_ptr), value, intent(in) :: c_obsspace -integer(c_int), intent(in) :: c_nvars, c_nlocs -real(c_double), intent(inout) :: c_hofx(c_nvars, c_nlocs) - -type(ufo_identity), pointer :: self -type(ufo_geovals), pointer :: geovals - -call ufo_identity_registry%get(c_key_self, self) -call ufo_geovals_registry%get(c_key_geovals, geovals) - -call self%simobs(geovals, c_obsspace, c_nvars, c_nlocs, c_hofx) - -end subroutine ufo_identity_simobs_c - -! ------------------------------------------------------------------------------ - -end module ufo_identity_mod_c diff --git a/src/ufo/identity/ObsIdentity.interface.h b/src/ufo/identity/ObsIdentity.interface.h deleted file mode 100644 index d545ed8c1..000000000 --- a/src/ufo/identity/ObsIdentity.interface.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * (C) Copyright 2017-2018 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef UFO_IDENTITY_OBSIDENTITY_INTERFACE_H_ -#define UFO_IDENTITY_OBSIDENTITY_INTERFACE_H_ - -#include "ioda/ObsSpace.h" -#include "oops/base/Variables.h" -#include "ufo/Fortran.h" - -namespace ufo { - -extern "C" { - -// ----------------------------------------------------------------------------- - - /// \param operatorVars - /// Variables to be simulated by this operator. - /// \param operatorVarIndices - /// Indices of the variables from \p operatorVar in the list of all simulated - /// variables in the ObsSpace. - /// \param numOperatorVarIndices - /// Size of the \p operatorVarIndices array (must be the same as the number of variables in - /// \p operatorVars). - /// \param[out] requiredVars - /// GeoVaLs required for the simulation of the variables \p operatorVars. - /// - /// Example: if the list of simulated variables in the ObsSpace is - /// [air_temperature, northward_wind, eastward_wind] and \p operatorVars is - /// [northward_wind, eastward_wind], then \p operatorVarIndices should be set to [1, 2]. - void ufo_identity_setup_f90(F90hop &, const eckit::Configuration &, - const oops::Variables &operatorVars, - const int *operatorVarIndices, const int numOperatorVarIndices, - oops::Variables &requiredVars); - void ufo_identity_delete_f90(F90hop &); - void ufo_identity_simobs_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &, const int &, double &); - -// ----------------------------------------------------------------------------- - -} // extern C - -} // namespace ufo -#endif // UFO_IDENTITY_OBSIDENTITY_INTERFACE_H_ diff --git a/src/ufo/identity/ObsIdentityTLAD.cc b/src/ufo/identity/ObsIdentityTLAD.cc index b3bfdecb8..71c69707a 100644 --- a/src/ufo/identity/ObsIdentityTLAD.cc +++ b/src/ufo/identity/ObsIdentityTLAD.cc @@ -1,5 +1,5 @@ /* - * (C) Copyright 2017-2021 UCAR + * (C) Copyright 2021 UK Met Office * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -15,6 +15,7 @@ #include "oops/base/Variables.h" #include "oops/util/Logger.h" +#include "oops/util/missingValues.h" #include "ufo/GeoVaLs.h" #include "ufo/utils/OperatorUtils.h" // for getOperatorVariables @@ -27,53 +28,77 @@ static LinearObsOperatorMaker makerIdentityTL_("Identity"); ObsIdentityTLAD::ObsIdentityTLAD(const ioda::ObsSpace & odb, const eckit::Configuration & config) - : LinearObsOperatorBase(odb), keyOperObsIdentity_(0), varin_() + : LinearObsOperatorBase(odb) { - std::vector operatorVarIndices; - getOperatorVariables(config, odb.obsvariables(), operatorVars_, operatorVarIndices); + oops::Log::trace() << "ObsIdentityTLAD constructor starting" << std::endl; - ufo_identity_tlad_setup_f90(keyOperObsIdentity_, config, - operatorVars_, - operatorVarIndices.data(), operatorVarIndices.size(), - varin_); + getOperatorVariables(config, odb.obsvariables(), operatorVars_, operatorVarIndices_); + requiredVars_ += operatorVars_; - oops::Log::trace() << "ObsIdentityTLAD created." << std::endl; + oops::Log::trace() << "ObsIdentityTLAD constructor finished" << std::endl; } // ----------------------------------------------------------------------------- ObsIdentityTLAD::~ObsIdentityTLAD() { - ufo_identity_tlad_delete_f90(keyOperObsIdentity_); oops::Log::trace() << "ObsIdentityTLAD destructed" << std::endl; } // ----------------------------------------------------------------------------- void ObsIdentityTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { - ufo_identity_tlad_settraj_f90(keyOperObsIdentity_, geovals.toFortran(), obsspace()); + // The trajectory is not needed because the observation operator is linear. oops::Log::trace() << "ObsIdentityTLAD: trajectory set" << std::endl; } // ----------------------------------------------------------------------------- -void ObsIdentityTLAD::simulateObsTL(const GeoVaLs & geovals, ioda::ObsVector & ovec) const { - ufo_identity_simobs_tl_f90(keyOperObsIdentity_, geovals.toFortran(), obsspace(), - ovec.nvars(), ovec.nlocs(), ovec.toFortran()); - oops::Log::trace() << "ObsIdentityTLAD: TL observation operator run" << std::endl; +void ObsIdentityTLAD::simulateObsTL(const GeoVaLs & dx, ioda::ObsVector & dy) const { + oops::Log::trace() << "ObsIdentityTLAD: TL observation operator starting" << std::endl; + + std::vector vec(dy.nlocs()); + for (int jvar : operatorVarIndices_) { + const std::string& varname = dy.varnames().variables()[jvar]; + // Fill dy with dx at the lowest level. + dx.getAtLevel(vec, varname, 0); + for (size_t jloc = 0; jloc < dy.nlocs(); ++jloc) { + const size_t idx = jloc * dy.nvars() + jvar; + dy[idx] = vec[jloc]; + } + } + + oops::Log::trace() << "ObsIdentityTLAD: TL observation operator finished" << std::endl; } // ----------------------------------------------------------------------------- -void ObsIdentityTLAD::simulateObsAD(GeoVaLs & geovals, const ioda::ObsVector & ovec) const { - ufo_identity_simobs_ad_f90(keyOperObsIdentity_, geovals.toFortran(), obsspace(), - ovec.nvars(), ovec.nlocs(), ovec.toFortran()); - oops::Log::trace() << "ObsIdentityTLAD: adjoint observation operator run" << std::endl; +void ObsIdentityTLAD::simulateObsAD(GeoVaLs & dx, const ioda::ObsVector & dy) const { + oops::Log::trace() << "ObsIdentityTLAD: adjoint observation operator starting" << std::endl; + + const double missing = util::missingValue(missing); + + std::vector vec(dy.nlocs()); + for (int jvar : operatorVarIndices_) { + const std::string& varname = dy.varnames().variables()[jvar]; + // Get current value of dx at the lowest level. + dx.getAtLevel(vec, varname, 0); + // Increment dx with non-missing values of dy. + for (size_t jloc = 0; jloc < dy.nlocs(); ++jloc) { + const size_t idx = jloc * dy.nvars() + jvar; + if (dy[idx] != missing) + vec[jloc] += dy[idx]; + } + // Store new value of dx. + dx.putAtLevel(vec, varname, 0); + } + + oops::Log::trace() << "ObsIdentityTLAD: adjoint observation operator finished" << std::endl; } // ----------------------------------------------------------------------------- void ObsIdentityTLAD::print(std::ostream & os) const { - os << "ObsIdentityTLAD::print not implemented" << std::endl; + os << "ObsIdentityTLAD operator" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/ufo/identity/ObsIdentityTLAD.h b/src/ufo/identity/ObsIdentityTLAD.h index 163e0db04..f5b376f6e 100644 --- a/src/ufo/identity/ObsIdentityTLAD.h +++ b/src/ufo/identity/ObsIdentityTLAD.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2017-2018 UCAR + * (C) Copyright 2021 UK Met Office * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -10,11 +10,11 @@ #include #include +#include #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" -#include "ufo/identity/ObsIdentityTLAD.interface.h" #include "ufo/LinearObsOperatorBase.h" // Forward declarations @@ -32,33 +32,35 @@ namespace ufo { class ObsDiagnostics; // ----------------------------------------------------------------------------- -/// Identity TL/AD observation operator class +/// TL/AD code for the Identity observation operator. class ObsIdentityTLAD : public LinearObsOperatorBase, - private util::ObjectCounter { + private util::ObjectCounter { public: static const std::string classname() {return "ufo::ObsIdentityTLAD";} ObsIdentityTLAD(const ioda::ObsSpace &, const eckit::Configuration &); virtual ~ObsIdentityTLAD(); - // Obs Operators void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; - // Other - const oops::Variables & requiredVars() const override {return varin_;} + const oops::Variables & requiredVars() const override {return requiredVars_;} oops::Variables simulatedVars() const override {return operatorVars_;} - int & toFortran() {return keyOperObsIdentity_;} - const int & toFortran() const {return keyOperObsIdentity_;} - private: void print(std::ostream &) const override; - F90hop keyOperObsIdentity_; - oops::Variables varin_; + + private: + /// Required variables. + oops::Variables requiredVars_; + + /// Operator variables. oops::Variables operatorVars_; + + /// Indices of operator variables. + std::vector operatorVarIndices_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/identity/ObsIdentityTLAD.interface.F90 b/src/ufo/identity/ObsIdentityTLAD.interface.F90 deleted file mode 100644 index 9123160e4..000000000 --- a/src/ufo/identity/ObsIdentityTLAD.interface.F90 +++ /dev/null @@ -1,134 +0,0 @@ -! (C) Copyright 2017-2018 UCAR -! -! This software is licensed under the terms of the Apache Licence Version 2.0 -! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -!> Fortran identity module for functions on the interface between C++ and Fortran -! to handle tl/ad observation operators - -module ufo_identity_tlad_mod_c - - use iso_c_binding - use ufo_identity_tlad_mod - use ufo_geovals_mod_c, only: ufo_geovals_registry - use ufo_geovals_mod, only: ufo_geovals - implicit none - - private - -#define LISTED_TYPE ufo_identity_tlad - - !> Linked list interface - defines registry_t type -#include "oops/util/linkedList_i.f" - - !> Global registry - type(registry_t) :: ufo_identity_tlad_registry - -contains - - ! ------------------------------------------------------------------------------ - !> Linked list implementation -#include "oops/util/linkedList_c.f" - -! ------------------------------------------------------------------------------ - -subroutine ufo_identity_tlad_setup_c(c_key_self, c_conf, c_obsvars, c_obsvarindices, c_nobsvars, & - c_geovars) bind(c,name='ufo_identity_tlad_setup_f90') -use oops_variables_mod -implicit none -integer(c_int), intent(inout) :: c_key_self -type(c_ptr), value, intent(in) :: c_conf -type(c_ptr), intent(in), value :: c_obsvars ! variables to be simulated... -integer(c_int), intent(in), value :: c_nobsvars -integer(c_int), intent(in) :: c_obsvarindices(c_nobsvars) ! ... and their global indices -type(c_ptr), value, intent(in) :: c_geovars ! variables requested from the model - -type(ufo_identity_tlad), pointer :: self - -call ufo_identity_tlad_registry%setup(c_key_self, self) - -self%obsvars = oops_variables(c_obsvars) -allocate(self%obsvarindices(self%obsvars%nvars())) -self%obsvarindices(:) = c_obsvarindices(:) + 1 ! Convert from C to Fortran indexing -self%geovars = oops_variables(c_geovars) - -call self%setup() - -end subroutine ufo_identity_tlad_setup_c - -! ------------------------------------------------------------------------------ - -subroutine ufo_identity_tlad_delete_c(c_key_self) bind(c,name='ufo_identity_tlad_delete_f90') -implicit none -integer(c_int), intent(inout) :: c_key_self - -type(ufo_identity_tlad), pointer :: self - -call ufo_identity_tlad_registry%delete(c_key_self, self) - - -end subroutine ufo_identity_tlad_delete_c - -! ------------------------------------------------------------------------------ - -subroutine ufo_identity_tlad_settraj_c(c_key_self, c_key_geovals, c_obsspace) bind(c,name='ufo_identity_tlad_settraj_f90') - -implicit none -integer(c_int), intent(in) :: c_key_self -integer(c_int), intent(in) :: c_key_geovals -type(c_ptr), value, intent(in) :: c_obsspace - -type(ufo_identity_tlad), pointer :: self -type(ufo_geovals), pointer :: geovals - -call ufo_identity_tlad_registry%get(c_key_self, self) -call ufo_geovals_registry%get(c_key_geovals, geovals) -call self%settraj(geovals, c_obsspace) - -end subroutine ufo_identity_tlad_settraj_c - -! ------------------------------------------------------------------------------ - -subroutine ufo_identity_simobs_tl_c(c_key_self, c_key_geovals, c_obsspace, c_nvars, c_nlocs, c_hofx) bind(c,name='ufo_identity_simobs_tl_f90') - -implicit none -integer(c_int), intent(in) :: c_key_self -integer(c_int), intent(in) :: c_key_geovals -type(c_ptr), value, intent(in) :: c_obsspace -integer(c_int), intent(in) :: c_nvars, c_nlocs -real(c_double), intent(inout) :: c_hofx(c_nvars, c_nlocs) - -type(ufo_identity_tlad), pointer :: self -type(ufo_geovals), pointer :: geovals - -call ufo_identity_tlad_registry%get(c_key_self, self) -call ufo_geovals_registry%get(c_key_geovals, geovals) - -call self%simobs_tl(geovals, c_obsspace, c_nvars, c_nlocs, c_hofx) - -end subroutine ufo_identity_simobs_tl_c - -! ------------------------------------------------------------------------------ - -subroutine ufo_identity_simobs_ad_c(c_key_self, c_key_geovals, c_obsspace, c_nvars, c_nlocs, c_hofx) bind(c,name='ufo_identity_simobs_ad_f90') - -implicit none -integer(c_int), intent(in) :: c_key_self -integer(c_int), intent(in) :: c_key_geovals -type(c_ptr), value, intent(in) :: c_obsspace -integer(c_int), intent(in) :: c_nvars, c_nlocs -real(c_double), intent(in) :: c_hofx(c_nvars, c_nlocs) - -type(ufo_identity_tlad), pointer :: self -type(ufo_geovals), pointer :: geovals - -call ufo_identity_tlad_registry%get(c_key_self, self) -call ufo_geovals_registry%get(c_key_geovals, geovals) -call self%simobs_ad(geovals, c_obsspace, c_nvars, c_nlocs, c_hofx) - -end subroutine ufo_identity_simobs_ad_c - -! ------------------------------------------------------------------------------ - - -end module ufo_identity_tlad_mod_c diff --git a/src/ufo/identity/ObsIdentityTLAD.interface.h b/src/ufo/identity/ObsIdentityTLAD.interface.h deleted file mode 100644 index 70a89dcf5..000000000 --- a/src/ufo/identity/ObsIdentityTLAD.interface.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * (C) Copyright 2017-2018 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef UFO_IDENTITY_OBSIDENTITYTLAD_INTERFACE_H_ -#define UFO_IDENTITY_OBSIDENTITYTLAD_INTERFACE_H_ - -#include "ioda/ObsSpace.h" -#include "oops/base/Variables.h" -#include "ufo/Fortran.h" - -namespace ufo { - -/// Interface to Fortran UFO generic/identity routines - -extern "C" { - -// ----------------------------------------------------------------------------- - - /// \param operatorVars - /// Variables to be simulated by this operator. - /// \param operatorVarIndices - /// Indices of the variables from \p operatorVar in the list of all simulated - /// variables in the ObsSpace. - /// \param numOperatorVarIndices - /// Size of the \p operatorVarIndices array (must be the same as the number of variables in - /// \p operatorVars). - /// \param[out] requiredVars - /// GeoVaLs required for the simulation of the variables \p operatorVars. - /// - /// Example: if the list of simulated variables in the ObsSpace is - /// [air_temperature, northward_wind, eastward_wind] and \p operatorVars is - /// [northward_wind, eastward_wind], then \p operatorVarIndices should be set to [1, 2]. - void ufo_identity_tlad_setup_f90(F90hop &, const eckit::Configuration &, - const oops::Variables &operatorVars, - const int *operatorVarIndices, const int numOperatorVarIndices, - oops::Variables &requiredVars); - void ufo_identity_tlad_delete_f90(F90hop &); - void ufo_identity_tlad_settraj_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &); - void ufo_identity_simobs_tl_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &nvars, const int &nlocs, double &hofx); - void ufo_identity_simobs_ad_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &nvars, const int &nlocs, const double &hofx); -// ----------------------------------------------------------------------------- - -} // extern C - -} // namespace ufo -#endif // UFO_IDENTITY_OBSIDENTITYTLAD_INTERFACE_H_ diff --git a/src/ufo/identity/ufo_identity_mod.F90 b/src/ufo/identity/ufo_identity_mod.F90 deleted file mode 100644 index 496ca3317..000000000 --- a/src/ufo/identity/ufo_identity_mod.F90 +++ /dev/null @@ -1,77 +0,0 @@ -! (C) Copyright 2017-2018 UCAR -! -! This software is licensed under the terms of the Apache Licence Version 2.0 -! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -! Fortran module for identity observation operator -!--------------------------------------------------------------------------------------------------- -module ufo_identity_mod - - use oops_variables_mod - use ufo_vars_mod - -! Fortran derived type for the observation type -!--------------------------------------------------------------------------------------------------- - type, public :: ufo_identity - type(oops_variables), public :: obsvars ! Variables to be simulated - integer, allocatable, public :: obsvarindices(:) ! Indices of obsvars in the list of all - ! simulated variables in the ObsSpace - type(oops_variables), public :: geovars - contains - procedure :: setup => identity_setup_ - procedure :: simobs => identity_simobs_ - end type ufo_identity - -contains - -! ------------------------------------------------------------------------------ -subroutine identity_setup_(self) -implicit none -class(ufo_identity), intent(inout) :: self - -integer :: nvars, ivar - -! set variables requested from the model (same as simulated): -nvars = self%obsvars%nvars() -do ivar = 1, nvars - call self%geovars%push_back(self%obsvars%variable(ivar)) -enddo - -end subroutine identity_setup_ - - -! ------------------------------------------------------------------------------ -subroutine identity_simobs_(self, geovals, obss, nvars, nlocs, hofx) - use ufo_geovals_mod - use obsspace_mod - use iso_c_binding - implicit none - class(ufo_identity), intent(in) :: self - type(ufo_geovals), intent(in) :: geovals - integer, intent(in) :: nvars, nlocs - real(c_double), intent(inout) :: hofx(nvars, nlocs) - type(c_ptr), value, intent(in) :: obss - - integer :: iobs, ivar, iobsvar - type(ufo_geoval), pointer :: point - character(len=MAXVARLEN) :: geovar - - do iobsvar = 1, size(self%obsvarindices) - ! Get the index of the row of hofx to fill - ivar = self%obsvarindices(iobsvar) - - ! Get the name of input variable in geovals - geovar = self%geovars%variable(iobsvar) - - !> Get profile for this variable from geovals - call ufo_geovals_get_var(geovals, geovar, point) - - !> Here we just apply a identity hofx - do iobs = 1, nlocs - hofx(ivar,iobs) = point%vals(1,iobs) - enddo - enddo - -end subroutine identity_simobs_ - -end module ufo_identity_mod diff --git a/src/ufo/identity/ufo_identity_tlad_mod.F90 b/src/ufo/identity/ufo_identity_tlad_mod.F90 deleted file mode 100644 index 852ed92a0..000000000 --- a/src/ufo/identity/ufo_identity_tlad_mod.F90 +++ /dev/null @@ -1,145 +0,0 @@ -! (C) Copyright 2017-2018 UCAR -! -! This software is licensed under the terms of the Apache Licence Version 2.0 -! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -!> Fortran module for identity tl/ad observation operator - -module ufo_identity_tlad_mod - - use oops_variables_mod - use ufo_vars_mod - implicit none - - ! ------------------------------------------------------------------------------ - - !> Fortran derived type for the tl/ad observation operator - type, public :: ufo_identity_tlad - private - type(oops_variables), public :: obsvars ! Variables to be simulated - integer, allocatable, public :: obsvarindices(:) ! Indices of obsvars in the list of all - ! simulated variables in the ObsSpace - type(oops_variables), public :: geovars - contains - procedure :: setup => identity_tlad_setup_ - procedure :: settraj => identity_tlad_settraj_ - procedure :: simobs_tl => identity_simobs_tl_ - procedure :: simobs_ad => identity_simobs_ad_ - end type ufo_identity_tlad - -contains - -! ------------------------------------------------------------------------------ -subroutine identity_tlad_setup_(self) - implicit none - class(ufo_identity_tlad), intent(inout) :: self - - integer :: ivar, nvars - - !> copy simulated variables to variables requested from the model - nvars = self%obsvars%nvars() - do ivar = 1, nvars - call self%geovars%push_back(self%obsvars%variable(ivar)) - enddo - -end subroutine identity_tlad_setup_ - -!------------------------------------------------------------------------------ -subroutine identity_tlad_settraj_(self, geovals, obss) - use iso_c_binding - use ufo_geovals_mod, only: ufo_geovals - implicit none - class(ufo_identity_tlad), intent(inout) :: self - type(ufo_geovals), intent(in) :: geovals - type(c_ptr), value, intent(in) :: obss - -! since observation operator is linear, don't care about trajectory itself - -end subroutine identity_tlad_settraj_ - -! ------------------------------------------------------------------------------ -subroutine identity_simobs_tl_(self, geovals, obss, nvars, nlocs, hofx) - use iso_c_binding - use kinds - use ufo_geovals_mod, only: & - ufo_geovals, & - ufo_geoval, & - ufo_geovals_get_var - use obsspace_mod - implicit none - class(ufo_identity_tlad), intent(in) :: self - type(ufo_geovals), intent(in) :: geovals - type(c_ptr), value, intent(in) :: obss - integer, intent(in) :: nvars, nlocs - real(c_double), intent(inout) :: hofx(nvars, nlocs) - - integer :: iobs, iobsvar, ivar - type(ufo_geoval), pointer :: point - character(len=MAXVARLEN) :: geovar - - do iobsvar = 1, size(self%obsvarindices) - ! Get the index of the row of hofx to fill - ivar = self%obsvarindices(iobsvar) - - !> Get the name of input variable in geovals - geovar = self%geovars%variable(iobsvar) - - !> Get profile for this variable from geovals - call ufo_geovals_get_var(geovals, geovar, point) - - !> Here we just apply a identity hofx - do iobs = 1, nlocs - hofx(ivar, iobs) = point%vals(1, iobs) - enddo - enddo - -end subroutine identity_simobs_tl_ - -! ------------------------------------------------------------------------------ -subroutine identity_simobs_ad_(self, geovals, obss, nvars, nlocs, hofx) - use iso_c_binding - use kinds - use ufo_geovals_mod, only: & - ufo_geovals, & - ufo_geoval, & - ufo_geovals_get_var - use obsspace_mod - use missing_values_mod - implicit none - class(ufo_identity_tlad), intent(in) :: self - type(ufo_geovals), intent(inout) :: geovals - type(c_ptr), value, intent(in) :: obss - integer, intent(in) :: nvars, nlocs - real(c_double), intent(in) :: hofx(nvars, nlocs) - - integer :: iobs, iobsvar, ivar - type(ufo_geoval), pointer :: point - character(len=MAXVARLEN) :: geovar - real(c_double) :: missing - - !> Set missing value - missing = missing_value(missing) - - if (.not. geovals%linit ) geovals%linit=.true. - - do iobsvar = 1, size(self%obsvarindices) - ! Get the index of the row of hofx to fill - ivar = self%obsvarindices(iobsvar) - - !> Get the name of input variable in geovals - geovar = self%geovars%variable(iobsvar) - - !> Get profile for this variable from geovals - call ufo_geovals_get_var(geovals, geovar, point) - - ! backward obs operator - do iobs = 1, nlocs - if (hofx(ivar, iobs) /= missing) then - point%vals(1, iobs) = hofx(ivar, iobs) - endif - enddo - enddo - -end subroutine identity_simobs_ad_ - -end module ufo_identity_tlad_mod diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 11b3a59d6..da0257805 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -129,6 +129,7 @@ list( APPEND ufo_test_input testinput/iasi_crtm.yaml testinput/iasi_qc.yaml testinput/iasi_qc_filters.yaml + testinput/identity.yaml testinput/interpolate_data_from_file_predictor.yaml testinput/legendre_predictor.yaml testinput/locations.yaml @@ -1576,6 +1577,20 @@ ecbuild_add_test( TARGET test_ufo_opr_profile_average DEPENDS test_ObsOperator.x TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_opr_identity + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x + ARGS "testinput/identity.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsOperator.x + TEST_DEPENDS ufo_get_ufo_test_data ) + +ecbuild_add_test( TARGET test_ufo_linopr_identity + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperatorTLAD.x + ARGS "testinput/identity.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsOperatorTLAD.x + TEST_DEPENDS ufo_get_ufo_test_data ) + # Test UFO ObsFilters (generic) ecbuild_add_test( TARGET test_ufo_qc_gen_processwhere diff --git a/test/testinput/identity.yaml b/test/testinput/identity.yaml new file mode 100644 index 000000000..d4e2bea8e --- /dev/null +++ b/test/testinput/identity.yaml @@ -0,0 +1,40 @@ +window begin: 2018-04-14T21:00:00Z +window end: 2018-04-15T03:00:00Z + +observations: + +# One simulated variable. +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/sondes_obs_2018041500_s.nc4 + simulated variables: [air_temperature] + obs operator: + name: Identity + geovals: + filename: Data/ufo/testinput_tier_1/sondes_geoval_2018041500_s.nc4 + linear obs operator test: + coef TL: 0.1 + tolerance TL: 1.0e-11 + tolerance AD: 1.0e-13 + rms ref: 294.79799709699643 + tolerance: 1.0e-06 + +# Two simulated variables. +# The expected rms is calculated as follows: +# sqrt((rms(air_temperature)^2 + rms(eastward_wind)^2) / 2). +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/sondes_obs_2018041500_s.nc4 + simulated variables: [air_temperature, eastward_wind] + obs operator: + name: Identity + geovals: + filename: Data/ufo/testinput_tier_1/sondes_geoval_2018041500_s.nc4 + linear obs operator test: + coef TL: 0.1 + tolerance TL: 1.0e-11 + tolerance AD: 1.0e-13 + rms ref: 208.52332007792242 + tolerance: 1.0e-06 From 232c7c5023e56a3d4bcfc9371df8837218bc8d34 Mon Sep 17 00:00:00 2001 From: Stuart Newman <50145691+smnewman@users.noreply.github.com> Date: Fri, 25 Jun 2021 21:31:47 +0100 Subject: [PATCH 20/98] Commit bugfix changes (#1283) Co-authored-by: Ryan Honeyager --- .../filters/obsfunctions/CloudCostFunction.cc | 19 ++++++++++++++++--- test/testinput/function_cloudcost.yaml | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/ufo/filters/obsfunctions/CloudCostFunction.cc b/src/ufo/filters/obsfunctions/CloudCostFunction.cc index d64ef96fe..474f43d98 100755 --- a/src/ufo/filters/obsfunctions/CloudCostFunction.cc +++ b/src/ufo/filters/obsfunctions/CloudCostFunction.cc @@ -117,7 +117,20 @@ void CloudCostFunction::compute(const ObsFilterData & in, // Ensure specific humidity is within limits gv_qgas[iloc] = std::max(gv_qgas[iloc], options_.min_q.value()); gv_qgas[iloc] = std::min(gv_qgas[iloc], qsaturated[iloc]); - humidity_total[iloc] = gv_qgas[iloc] + gv_clw[iloc] + gv_ciw[iloc]; + humidity_total[iloc] = gv_qgas[iloc] + gv_clw[iloc]; // ice is neglected for partitioning + } + int qsplit_partition_mode = 1; // partition total water into vapour, liquid and ice + std::vector qsplit_gas(nlocs), qsplit_clw(nlocs), qsplit_ciw(nlocs); + ufo_ops_satrad_qsplit_f90(qsplit_partition_mode, static_cast(nlocs), gv_pres.data(), + gv_temp.data(), humidity_total.data(), qsplit_gas.data(), + qsplit_clw.data(), qsplit_ciw.data(), split_rain); + for (size_t iloc = 0; iloc < nlocs; ++iloc) { + // For scattering use sum of geovals for qtotal, otherwise use partitioned quantities + if (options_.scattering_switch.value()) { + humidity_total[iloc] += gv_ciw[iloc]; + } else { + humidity_total[iloc] = qsplit_gas[iloc] + qsplit_clw[iloc] + qsplit_ciw[iloc]; + } } } @@ -131,8 +144,8 @@ void CloudCostFunction::compute(const ObsFilterData & in, in.get(Variable("brightness_temperature_jacobian_"+ciw_name+"@ObsDiag", channels_)[ichan], level_jac, jac_ciw); std::vector dq_dqtotal(nlocs), dql_dqtotal(nlocs), dqi_dqtotal(nlocs); - int qsplit_mode = 2; // compute derivatives - ufo_ops_satrad_qsplit_f90(qsplit_mode, static_cast(nlocs), gv_pres.data(), + int qsplit_derivative_mode = 2; // compute derivatives + ufo_ops_satrad_qsplit_f90(qsplit_derivative_mode, static_cast(nlocs), gv_pres.data(), gv_temp.data(), humidity_total.data(), dq_dqtotal.data(), dql_dqtotal.data(), dqi_dqtotal.data(), split_rain); // Jacobian dy/dx for observation y, humdity x in units kg/kg diff --git a/test/testinput/function_cloudcost.yaml b/test/testinput/function_cloudcost.yaml index 8b73ed426..220ab45ab 100644 --- a/test/testinput/function_cloudcost.yaml +++ b/test/testinput/function_cloudcost.yaml @@ -57,4 +57,4 @@ observations: name: cloud_cost@TestReference test: name: cloud_cost@MetaData - absTol: 1.0e-4 + absTol: 1.0e-5 From 0643a151f2e304ddeb320ab54ce9e1cd7015b1d1 Mon Sep 17 00:00:00 2001 From: JoanneAWaller <52206323+JoanneAWaller@users.noreply.github.com> Date: Fri, 25 Jun 2021 22:35:42 +0100 Subject: [PATCH 21/98] Add obs function to calculate adjusted surface temperatures (#1287) * Adding function to adjust observed temperature to be valid at model surface height. * Adhere to coding norms and tidy code. * Restored test/CMakeLists.text * Fixed comments. * Addressed Chris' comments * Address Wojciech's comments. Co-authored-by: Ryan Honeyager --- src/ufo/filters/CMakeLists.txt | 2 + .../ModelHeightAdjustedAirTemperature.cc | 74 +++++++++++++++++++ .../ModelHeightAdjustedAirTemperature.h | 55 ++++++++++++++ test/CMakeLists.txt | 10 ++- .../function_ModelSurfaceHeightAdjust.yaml | 22 ++++++ 5 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.cc create mode 100644 src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.h create mode 100755 test/testinput/unit_tests/function_ModelSurfaceHeightAdjust.yaml diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index 9483d0b5e..014e431f7 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -181,6 +181,8 @@ set ( filters_files obsfunctions/CloudDetectMinResidualAVHRR.h obsfunctions/CloudDetectMinResidualIR.cc obsfunctions/CloudDetectMinResidualIR.h + obsfunctions/ModelHeightAdjustedAirTemperature.cc + obsfunctions/ModelHeightAdjustedAirTemperature.h obsfunctions/ObsErrorBoundIR.cc obsfunctions/ObsErrorBoundIR.h obsfunctions/ObsErrorBoundMW.cc diff --git a/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.cc b/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.cc new file mode 100644 index 000000000..e93eafe17 --- /dev/null +++ b/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.cc @@ -0,0 +1,74 @@ +/* ----------------------------------------------------------------------------- + * (C) British Crown Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * ----------------------------------------------------------------------------- + */ + +#include + +#include "ioda/ObsDataVector.h" +#include "oops/util/missingValues.h" +#include "ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.h" +#include "ufo/filters/Variable.h" +#include "ufo/utils/Constants.h" + +namespace ufo { + +static ObsFunctionMaker + makerModelHeightAdjustedAirTemperature_("ModelHeightAdjustedAirTemperature"); + +// ----------------------------------------------------------------------------- + +ModelHeightAdjustedAirTemperature::ModelHeightAdjustedAirTemperature( + const eckit::LocalConfiguration & conf): invars_() { + // Required observation data + invars_ += Variable("air_temperature_at_2m@ObsValue"); + // Required model surface altitude + invars_ += Variable("surface_altitude@GeoVaLs"); + + // Required observation station height + parameters_.validateAndDeserialize(conf); + const Variable elevation = parameters_.elevation.value(); + invars_ += elevation; +} + +// ----------------------------------------------------------------------------- + +ModelHeightAdjustedAirTemperature::~ModelHeightAdjustedAirTemperature() {} + +// ----------------------------------------------------------------------------- + +void ModelHeightAdjustedAirTemperature::compute(const ObsFilterData & in, + ioda::ObsDataVector & out) const { + const size_t nlocs = in.nlocs(); + std::vector t2(nlocs); + std::vector ModelHeight(nlocs); + std::vector StationHeight(nlocs); + + in.get(Variable("air_temperature_at_2m@ObsValue"), t2); + in.get(Variable("surface_altitude@GeoVaLs"), ModelHeight); + in.get(parameters_.elevation.value(), StationHeight); + + const float missing = util::missingValue(missing); + + // compute temperature correction and adjusted temperature. + for (size_t jj = 0; jj < nlocs; ++jj) { + if (StationHeight[jj] == missing || ModelHeight[jj] == missing || t2[jj] == missing) { + out[0][jj] = missing; + } else { + out[0][jj] = t2[jj] + Constants::Lclr*(StationHeight[jj] - ModelHeight[jj]); + } + } +} + +// ----------------------------------------------------------------------------- + +const ufo::Variables & ModelHeightAdjustedAirTemperature::requiredVariables() const { + return invars_; +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.h b/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.h new file mode 100644 index 000000000..84ccd6bc2 --- /dev/null +++ b/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.h @@ -0,0 +1,55 @@ +/* ----------------------------------------------------------------------------- + * (C) British Crown Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * ----------------------------------------------------------------------------- + */ + +#ifndef UFO_FILTERS_OBSFUNCTIONS_MODELHEIGHTADJUSTEDAIRTEMPERATURE_H_ +#define UFO_FILTERS_OBSFUNCTIONS_MODELHEIGHTADJUSTEDAIRTEMPERATURE_H_ + +#include + +#include "ufo/filters/ObsFilterData.h" +#include "ufo/filters/obsfunctions/ObsFunctionBase.h" +#include "ufo/filters/Variables.h" + +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" +#include "ufo/utils/parameters/ParameterTraitsVariable.h" + +namespace ufo { + +/// \brief Options controlling ModelHeightAdjustedAirTemperature ObsFunction +class ModelHeightAdjustedAirTemperatureParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ModelHeightAdjustedAirTemperatureParameters, Parameters) + + public: + /// Input observation station height to be used + oops::RequiredParameter elevation{"elevation", this}; +}; + +/// \brief Function to calculate surface temperature observation value +/// adjusted from station height to model surface height. Outputs a derived +/// 2m air temperature adjusted to the model surface height. The correction +/// applied to the temperature is calculated using a standard lapse rate +/// (Constants::Lclr). + +class ModelHeightAdjustedAirTemperature : public ObsFunctionBase { + public: + explicit ModelHeightAdjustedAirTemperature(const eckit::LocalConfiguration & + = eckit::LocalConfiguration()); + ~ModelHeightAdjustedAirTemperature(); + + void compute(const ObsFilterData &, + ioda::ObsDataVector &) const; + const ufo::Variables & requiredVariables() const; + + private: + ModelHeightAdjustedAirTemperatureParameters parameters_; + ufo::Variables invars_; +}; +} // namespace ufo + +#endif // UFO_FILTERS_OBSFUNCTIONS_MODELHEIGHTADJUSTEDAIRTEMPERATURE_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index da0257805..8a093e345 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -278,6 +278,7 @@ list( APPEND ufo_test_input testinput/unit_tests/function_tropo_estimate.yaml testinput/unit_tests/function_lam_domaincheck.yaml testinput/unit_tests/function_lam_domaincheck_circle.yaml + testinput/unit_tests/function_ModelSurfaceHeightAdjust.yaml testinput/unit_tests/function_obserrorfactorquotient.yaml testinput/unit_tests/function_satwindsLNVDcheck.yaml testinput/unit_tests/function_satwindsSPDBcheck.yaml @@ -2489,7 +2490,7 @@ ecbuild_add_test( TARGET test_ufo_variabletransforms_humidityrelative_part2 ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ioda_test_data ) - + ecbuild_add_test( TARGET test_ufo_variabletransforms_humidityspecific COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x ARGS "testinput/unit_tests/variabletransforms_shumidity.yaml" @@ -2781,6 +2782,13 @@ ecbuild_add_test( TARGET test_ufo_function_setsurfacetype TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_function_modelsurfaceheightadjust + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x + ARGS "testinput/unit_tests/function_ModelSurfaceHeightAdjust.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFunction.x + TEST_DEPENDS ufo_get_ufo_test_data) + # Test Diagnostics ecbuild_add_test( TARGET test_ufo_obsdiag_background_error_vert_interp_air_pressure diff --git a/test/testinput/unit_tests/function_ModelSurfaceHeightAdjust.yaml b/test/testinput/unit_tests/function_ModelSurfaceHeightAdjust.yaml new file mode 100755 index 000000000..0931248e0 --- /dev/null +++ b/test/testinput/unit_tests/function_ModelSurfaceHeightAdjust.yaml @@ -0,0 +1,22 @@ +window begin: 2021-05-21T00:00:00Z +window end: 2021-05-21T23:00:00Z + +observations: +- obs space: + name: Surface + obsdatain: + obsfile: Data/ufo/testinput_tier_1/surface_obs_20210521T1200Z.nc4 + obsdataout: + obsfile: Data/surface_obs_20210521T1200Z_out.nc + simulated variables: [air_temperature_at_2m] + geovals: + filename: Data/ufo/testinput_tier_1/surface_geovals_20210521T1200Z.nc4 + + obs function: + name: ModelHeightAdjustedAirTemperature@ObsFunction + options: + elevation: + name: station_elevation@MetaData + variables: [air_temperature_at_2m] + tolerance: 1.0e-6 + From b51e001975721fdafa40b66cccb1a12af97040d6 Mon Sep 17 00:00:00 2001 From: Travis Sluka Date: Wed, 30 Jun 2021 16:53:45 -0600 Subject: [PATCH 22/98] obs files for soca ewok hofx (#1311) * starting soca obs files * add more ocean obs --- ewok/jedi-godas/adt_3a.yaml | 21 +++++++++++++++++++++ ewok/jedi-godas/adt_3b.yaml | 21 +++++++++++++++++++++ ewok/jedi-godas/adt_c2.yaml | 21 +++++++++++++++++++++ ewok/jedi-godas/adt_j3.yaml | 21 +++++++++++++++++++++ ewok/jedi-godas/adt_sa.yaml | 21 +++++++++++++++++++++ ewok/jedi-godas/icec_ssmi.yaml | 25 +++++++++++++++++++++++++ ewok/jedi-godas/icec_ssmis_285_286.yaml | 25 +++++++++++++++++++++++++ ewok/jedi-godas/insitu_prof_s.yaml | 25 +++++++++++++++++++++++++ ewok/jedi-godas/insitu_prof_t.yaml | 25 +++++++++++++++++++++++++ ewok/jedi-godas/sss_smap.yaml | 25 +++++++++++++++++++++++++ ewok/jedi-godas/sst_amsr2.yaml | 21 +++++++++++++++++++++ ewok/jedi-godas/sst_gmi.yaml | 21 +++++++++++++++++++++ ewok/jedi-godas/sst_viirs.yaml | 21 +++++++++++++++++++++ ewok/jedi-godas/sst_windsat.yaml | 21 +++++++++++++++++++++ 14 files changed, 314 insertions(+) create mode 100644 ewok/jedi-godas/adt_3a.yaml create mode 100644 ewok/jedi-godas/adt_3b.yaml create mode 100644 ewok/jedi-godas/adt_c2.yaml create mode 100644 ewok/jedi-godas/adt_j3.yaml create mode 100644 ewok/jedi-godas/adt_sa.yaml create mode 100644 ewok/jedi-godas/icec_ssmi.yaml create mode 100644 ewok/jedi-godas/icec_ssmis_285_286.yaml create mode 100644 ewok/jedi-godas/insitu_prof_s.yaml create mode 100644 ewok/jedi-godas/insitu_prof_t.yaml create mode 100644 ewok/jedi-godas/sss_smap.yaml create mode 100644 ewok/jedi-godas/sst_amsr2.yaml create mode 100644 ewok/jedi-godas/sst_gmi.yaml create mode 100644 ewok/jedi-godas/sst_viirs.yaml create mode 100644 ewok/jedi-godas/sst_windsat.yaml diff --git a/ewok/jedi-godas/adt_3a.yaml b/ewok/jedi-godas/adt_3a.yaml new file mode 100644 index 000000000..a2c023494 --- /dev/null +++ b/ewok/jedi-godas/adt_3a.yaml @@ -0,0 +1,21 @@ +obs space: + name: adt_3a + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/adt_3a.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).adt_3a.{{window_begin}}.nc4 + simulated variables: [obs_absolute_dynamic_topography] +obs operator: + name: ADT +obs error: + covariance model: diagonal +obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 5.0 \ No newline at end of file diff --git a/ewok/jedi-godas/adt_3b.yaml b/ewok/jedi-godas/adt_3b.yaml new file mode 100644 index 000000000..669892a5d --- /dev/null +++ b/ewok/jedi-godas/adt_3b.yaml @@ -0,0 +1,21 @@ +obs space: + name: adt_3b + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/adt_3b.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).adt_3b.{{window_begin}}.nc4 + simulated variables: [obs_absolute_dynamic_topography] +obs operator: + name: ADT +obs error: + covariance model: diagonal +obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 5.0 \ No newline at end of file diff --git a/ewok/jedi-godas/adt_c2.yaml b/ewok/jedi-godas/adt_c2.yaml new file mode 100644 index 000000000..dacfaf5e5 --- /dev/null +++ b/ewok/jedi-godas/adt_c2.yaml @@ -0,0 +1,21 @@ +obs space: + name: adt_c2 + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/adt_c2.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).adt_c2.{{window_begin}}.nc4 + simulated variables: [obs_absolute_dynamic_topography] +obs operator: + name: ADT +obs error: + covariance model: diagonal +obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 5.0 \ No newline at end of file diff --git a/ewok/jedi-godas/adt_j3.yaml b/ewok/jedi-godas/adt_j3.yaml new file mode 100644 index 000000000..09d4fed5c --- /dev/null +++ b/ewok/jedi-godas/adt_j3.yaml @@ -0,0 +1,21 @@ +obs space: + name: adt_j3 + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/adt_j3.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).adt_j3.{{window_begin}}.nc4 + simulated variables: [obs_absolute_dynamic_topography] +obs operator: + name: ADT +obs error: + covariance model: diagonal +obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 5.0 \ No newline at end of file diff --git a/ewok/jedi-godas/adt_sa.yaml b/ewok/jedi-godas/adt_sa.yaml new file mode 100644 index 000000000..30a546fb8 --- /dev/null +++ b/ewok/jedi-godas/adt_sa.yaml @@ -0,0 +1,21 @@ +obs space: + name: adt_sa + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/adt_sa.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).adt_sa.{{window_begin}}.nc4 + simulated variables: [obs_absolute_dynamic_topography] +obs operator: + name: ADT +obs error: + covariance model: diagonal +obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 5.0 \ No newline at end of file diff --git a/ewok/jedi-godas/icec_ssmi.yaml b/ewok/jedi-godas/icec_ssmi.yaml new file mode 100644 index 000000000..bd3e406e0 --- /dev/null +++ b/ewok/jedi-godas/icec_ssmi.yaml @@ -0,0 +1,25 @@ +obs space: + name: icec_ssmi + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/icec_ssmi.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).icec_ssmi.{{window_begin}}.nc4 + simulated variables: [sea_ice_area_fraction] +obs operator: + name: Identity +obs error: + covariance model: diagonal +obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Bounds Check + minvalue: 0.0 + maxvalue: 1.0 + - filter: Background Check + threshold: 5.0 + - filter: Domain Check + where: + - variable: { name: sea_surface_temperature@GeoVaLs} + maxvalue: 0.9 \ No newline at end of file diff --git a/ewok/jedi-godas/icec_ssmis_285_286.yaml b/ewok/jedi-godas/icec_ssmis_285_286.yaml new file mode 100644 index 000000000..642515464 --- /dev/null +++ b/ewok/jedi-godas/icec_ssmis_285_286.yaml @@ -0,0 +1,25 @@ +obs space: + name: icec_ssmis_285_286 + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/icec_ssmis_285_286.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).icec_ssmis_285_286.{{window_begin}}.nc4 + simulated variables: [sea_ice_area_fraction] +obs operator: + name: Identity +obs error: + covariance model: diagonal +obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Bounds Check + minvalue: 0.0 + maxvalue: 1.0 + - filter: Background Check + threshold: 5.0 + - filter: Domain Check + where: + - variable: { name: sea_surface_temperature@GeoVaLs} + maxvalue: 0.9 \ No newline at end of file diff --git a/ewok/jedi-godas/insitu_prof_s.yaml b/ewok/jedi-godas/insitu_prof_s.yaml new file mode 100644 index 000000000..d3129b9c1 --- /dev/null +++ b/ewok/jedi-godas/insitu_prof_s.yaml @@ -0,0 +1,25 @@ +obs space: + name: insitu_prof + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/insitu_prof.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).insitu_prof_s.{{window_begin}}.nc4 + simulated variables: [sea_water_salinity] +obs operator: + name: MarineVertInterp +obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Domain Check + where: + - variable: {name: z@ObsError} + minvalue: 0.0001 + - filter: Bounds Check + minvalue: 1.0 + maxvalue: 40.0 + - filter: Background Check + threshold: 5.0 diff --git a/ewok/jedi-godas/insitu_prof_t.yaml b/ewok/jedi-godas/insitu_prof_t.yaml new file mode 100644 index 000000000..5efcafa8a --- /dev/null +++ b/ewok/jedi-godas/insitu_prof_t.yaml @@ -0,0 +1,25 @@ +obs space: + name: insitu_prof + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/insitu_prof.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).insitu_prof_t.{{window_begin}}.nc4 + simulated variables: [sea_water_temperature] +obs operator: + name: InsituTemperature +obs error: + covariance model: diagonal +obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Domain Check + where: + - variable: {name: sea_water_temperature@ObsError} + minvalue: 0.001 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 5.0 \ No newline at end of file diff --git a/ewok/jedi-godas/sss_smap.yaml b/ewok/jedi-godas/sss_smap.yaml new file mode 100644 index 000000000..cd6a313e2 --- /dev/null +++ b/ewok/jedi-godas/sss_smap.yaml @@ -0,0 +1,25 @@ +obs space: + name: sss_smap + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/sss_smap.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).sss_smap.{{window_begin}}.nc4 + simulated variables: [sea_surface_salinity] +obs operator: + name: Identity +obs error: + covariance model: diagonal +obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Bounds Check + minvalue: 0.1 + maxvalue: 40.0 + - filter: Background Check + threshold: 5.0 + - filter: Domain Check + where: + - variable: {name: sea_surface_temperature@GeoVaLs} + minvalue: 10.0 \ No newline at end of file diff --git a/ewok/jedi-godas/sst_amsr2.yaml b/ewok/jedi-godas/sst_amsr2.yaml new file mode 100644 index 000000000..92fc6c61e --- /dev/null +++ b/ewok/jedi-godas/sst_amsr2.yaml @@ -0,0 +1,21 @@ +obs space: + name: sst_amsr2 + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/sst_amsr2.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).sst_amsr2.{{window_begin}}.nc4 + simulated variables: [sea_surface_temperature] +obs operator: + name: Identity +obs error: + covariance model: diagonal +obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 5.0 \ No newline at end of file diff --git a/ewok/jedi-godas/sst_gmi.yaml b/ewok/jedi-godas/sst_gmi.yaml new file mode 100644 index 000000000..583eee43e --- /dev/null +++ b/ewok/jedi-godas/sst_gmi.yaml @@ -0,0 +1,21 @@ +obs space: + name: sst_gmi + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/sst_gmi.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).sst_gmi.{{window_begin}}.nc4 + simulated variables: [sea_surface_temperature] +obs operator: + name: Identity +obs error: + covariance model: diagonal +obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 5.0 \ No newline at end of file diff --git a/ewok/jedi-godas/sst_viirs.yaml b/ewok/jedi-godas/sst_viirs.yaml new file mode 100644 index 000000000..48c7f3831 --- /dev/null +++ b/ewok/jedi-godas/sst_viirs.yaml @@ -0,0 +1,21 @@ +obs space: + name: sst_viirs + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/sst_viirs.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).sst_viirs.{{window_begin}}.nc4 + simulated variables: [sea_surface_temperature] +obs operator: + name: Identity +obs error: + covariance model: diagonal +obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 5.0 \ No newline at end of file diff --git a/ewok/jedi-godas/sst_windsat.yaml b/ewok/jedi-godas/sst_windsat.yaml new file mode 100644 index 000000000..c34968b93 --- /dev/null +++ b/ewok/jedi-godas/sst_windsat.yaml @@ -0,0 +1,21 @@ +obs space: + name: sst_windsat + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/sst_windsat.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).sst_windsat.{{window_begin}}.nc4 + simulated variables: [sea_surface_temperature] +obs operator: + name: Identity +obs error: + covariance model: diagonal +obs filters: + - filter: Domain Check + where: + - variable: {name: sea_area_fraction@GeoVaLs} + minvalue: 0.9 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 5.0 \ No newline at end of file From 21620694b660a2b1cd6fea188485b7cdf4d80086 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Tue, 6 Jul 2021 08:14:24 -0600 Subject: [PATCH 23/98] add ObsBias::write implementation and a test (#1316) * add ObsBias::write implementation and a test * construct predictors instead of copying --- src/ufo/ObsBias.cc | 77 +++++++++++++++++++++++-- src/ufo/ObsBias.h | 3 + src/ufo/ObsBiasParameters.h | 3 + test/CMakeLists.txt | 7 +++ test/mains/TestUfoObsBias.cc | 15 +++++ test/testinput/bias_coeff.yaml | 35 +++++++++-- test/ufo/ObsBias.h | 102 +++++++++++++++++++++++++++++++++ 7 files changed, 232 insertions(+), 10 deletions(-) create mode 100644 test/mains/TestUfoObsBias.cc create mode 100644 test/ufo/ObsBias.h diff --git a/src/ufo/ObsBias.cc b/src/ufo/ObsBias.cc index 5c0aa215d..f96db49f9 100644 --- a/src/ufo/ObsBias.cc +++ b/src/ufo/ObsBias.cc @@ -8,11 +8,13 @@ #include "ufo/ObsBias.h" #include +#include #include #include #include #include "ioda/Engines/Factory.h" +#include "ioda/Engines/HH.h" #include "ioda/Layout.h" #include "ioda/ObsGroup.h" #include "ioda/ObsVector.h" @@ -147,10 +149,70 @@ void ObsBias::read(const Parameters_ & params) { oops::Log::trace() << "ObsBias::read and initilization done " << std::endl; } +// ----------------------------------------------------------------------------- +/// Create ObsGroup with dimensions npredictors = size(predictors) and +/// nchannels = size(channels), variables predictors, channels and +/// bias_cooefficients (npredictors x nchannels) +ioda::ObsGroup saveBiasCoeffsWithChannels(ioda::Group & parent, + const std::vector & predictors, + const std::vector & channels, + const Eigen::MatrixXd & coeffs) { + // dimensions + ioda::NewDimensionScales_t dims { + ioda::NewDimensionScale("npredictors", predictors.size()), + ioda::NewDimensionScale("nchannels", channels.size()) + }; + // new ObsGroup + ioda::ObsGroup ogrp = ioda::ObsGroup::generate(parent, dims); + + // save the predictors + ioda::Variable predsVar = ogrp.vars.createWithScales( + "predictors", {ogrp.vars["npredictors"]}); + predsVar.write(predictors); + // and the variables + ioda::Variable chansVar = ogrp.vars.createWithScales("channels", {ogrp.vars["nchannels"]}); + chansVar.write(channels); + + // Set up the creation parameters for the bias coefficients variable + ioda::VariableCreationParameters float_params; + float_params.chunk = true; // allow chunking + float_params.compressWithGZIP(); // compress using gzip + float missing_value = util::missingValue(missing_value); + float_params.setFillValue(missing_value); + + // Create a variable for bias coefficients, save bias coeffs to the variable + ioda::Variable biasVar = ogrp.vars.createWithScales("bias_coefficients", + {ogrp.vars["npredictors"], ogrp.vars["nchannels"]}, float_params); + biasVar.writeWithEigenRegular(coeffs); + return ogrp; +} + // ----------------------------------------------------------------------------- void ObsBias::write(const Parameters_ & params) const { - oops::Log::trace() << "ObsBias::write to file not implemented" << std::endl; + if (params.outputFile.value() != boost::none) { + // FIXME: only implemented for channels currently + if (vars_.channels().size() == 0) { + throw eckit::NotImplemented("ObsBias::write not implemented for variables without channels", + Here()); + } + // Create a file, overwrite if exists + const std::string output_filename = *params.outputFile.value(); + ioda::Group group = ioda::Engines::HH::createFile(output_filename, + ioda::Engines::BackendCreateModes::Truncate_If_Exists); + + // put only variable bias predictors into the predictors vector + std::vector predictors(prednames_.begin() + numStaticPredictors_, + prednames_.end()); + // map coefficients to 2D for saving + Eigen::Map + coeffs(biascoeffs_.data(), numVariablePredictors_, vars_.size()); + + saveBiasCoeffsWithChannels(group, predictors, vars_.channels(), coeffs); + } else { + throw eckit::UserError("obs bias.output file section missing from the configuration", + Here()); + } } // ----------------------------------------------------------------------------- @@ -161,7 +223,7 @@ double ObsBias::norm() const { // Static predictors const int numUnitCoeffs = numStaticPredictors_ * vars_.size(); - zz += numUnitCoeffs * numUnitCoeffs; + zz += numUnitCoeffs; // Variable predictors zz += biascoeffs_.squaredNorm(); @@ -176,6 +238,12 @@ double ObsBias::norm() const { // ----------------------------------------------------------------------------- +void ObsBias::zero() { + biascoeffs_ = Eigen::VectorXd::Zero(numVariablePredictors_ * vars_.size()); +} + +// ----------------------------------------------------------------------------- + std::vector> ObsBias::variablePredictors() const { return std::vector>( predictors_.begin() + numStaticPredictors_, predictors_.end()); @@ -188,7 +256,7 @@ void ObsBias::print(std::ostream & os) const { // map bias coeffs to eigen matrix Eigen::Map coeffs(biascoeffs_.data(), numVariablePredictors_, vars_.size()); - os << "ObsBias::print " << std::endl; + os << "Obs bias coefficients: " << std::endl; os << "---------------------------------------------------------------" << std::endl; for (std::size_t p = 0; p < numStaticPredictors_; ++p) { os << std::fixed << std::setw(20) << prednames_[p] @@ -210,8 +278,7 @@ void ObsBias::print(std::ostream & os) const { << coeffs.row(p).norm() << std::endl; } - os << "---------------------------------------------------------------" << std::endl; - os << "ObsBias::print done" << std::endl; + os << "---------------------------------------------------------------"; } } diff --git a/src/ufo/ObsBias.h b/src/ufo/ObsBias.h index 6cd82af27..be4c8c476 100644 --- a/src/ufo/ObsBias.h +++ b/src/ufo/ObsBias.h @@ -84,6 +84,9 @@ class ObsBias : public util::Printable, /// Return the list of bias-corrected variables. const oops::Variables & correctedVars() const {return vars_;} + /// Set all variable predictors coefficients to zero (used in the test) + void zero(); + // Operator operator bool() const { return (numStaticPredictors_ > 0 || numVariablePredictors_ > 0) && vars_.size() > 0; diff --git a/src/ufo/ObsBiasParameters.h b/src/ufo/ObsBiasParameters.h index cec94e455..8ae16de8e 100644 --- a/src/ufo/ObsBiasParameters.h +++ b/src/ufo/ObsBiasParameters.h @@ -95,6 +95,9 @@ class ObsBiasParameters : public oops::Parameters { /// Path to a NetCDF file containing initial values of the coefficients of predictors used /// in VarBC. oops::OptionalParameter inputFile{"input file", this}; + /// Path to a NetCDF file containing final values of the coefficients of predictors used + /// in VarBC. + oops::OptionalParameter outputFile{"output file", this}; /// Options controlling the covariance matrix. oops::OptionalParameter covariance{"covariance", this}; }; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8a093e345..7bade7ca0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3113,6 +3113,13 @@ ecbuild_add_test( TARGET test_ufo_bias_coeffs DEPENDS test_ObsBias.x TEST_DEPENDS ufo_get_ufo_test_data) +ecbuild_add_test( TARGET test_ufo_bias_coeffs_readwrite + SOURCES mains/TestUfoObsBias.cc + ARGS "testinput/bias_coeff.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + LIBS ufo + TEST_DEPENDS ufo_get_ufo_test_data) + ecbuild_add_test( TARGET test_ufo_bias_coeff_increment SOURCES mains/TestObsBiasIncrement.cc ARGS "testinput/bias_coeff_cov.yaml" diff --git a/test/mains/TestUfoObsBias.cc b/test/mains/TestUfoObsBias.cc new file mode 100644 index 000000000..a2f3f512e --- /dev/null +++ b/test/mains/TestUfoObsBias.cc @@ -0,0 +1,15 @@ +/* + * (C) Copyright 2021- UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "../ufo/ObsBias.h" +#include "oops/runs/Run.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + ufo::test::ObsBias tests; + return run.execute(tests); +} diff --git a/test/testinput/bias_coeff.yaml b/test/testinput/bias_coeff.yaml index ea522e792..cd570fecf 100644 --- a/test/testinput/bias_coeff.yaml +++ b/test/testinput/bias_coeff.yaml @@ -7,9 +7,10 @@ observations: obsdatain: obsfile: Data/ufo/testinput_tier_1/amsua_n19_obs_2018041500_m_qc.nc4 simulated variables: [brightness_temperature] - channels: &channels 1-15 + channels: 1-15 obs bias: # test reading coefficients for 2 predictors; coeffs are zero for all channels input file: Data/ufo/testinput_tier_1/satbias_amsua_n19.nc4 + output file: Data/satbias_amsua_n19_test1.nc4 variational bc: predictors: - name: cosine_of_latitude_times_orbit_node @@ -23,9 +24,10 @@ observations: obsdatain: obsfile: Data/ufo/testinput_tier_1/amsua_n19_obs_2018041500_m_qc.nc4 simulated variables: [brightness_temperature] - channels: &channels 1-15 + channels: 1-15 obs bias: # test reading coefficients for 1 predictor (all channels) input file: Data/ufo/testinput_tier_1/satbias_amsua_n19.nc4 + output file: Data/satbias_amsua_n19_test2.nc4 variational bc: predictors: - name: emissivity @@ -38,9 +40,10 @@ observations: obsdatain: obsfile: Data/ufo/testinput_tier_1/amsua_n19_obs_2018041500_m_qc.nc4 simulated variables: [brightness_temperature] - channels: &channels 7-9, 12 + channels: 7-9, 12 obs bias: # test reading coefficients for 1 predictor (subset of channels) input file: Data/ufo/testinput_tier_1/satbias_amsua_n19.nc4 + output file: Data/satbias_amsua_n19_test3.nc4 variational bc: predictors: - name: emissivity @@ -53,9 +56,10 @@ observations: obsdatain: obsfile: Data/ufo/testinput_tier_1/amsua_n19_obs_2018041500_m_qc.nc4 simulated variables: [brightness_temperature] - channels: &channels 7-9, 12 + channels: 7-9, 12 obs bias: # test reading coefficients for 2 predictors and subset of channels input file: Data/ufo/testinput_tier_1/satbias_amsua_n19.nc4 + output file: Data/satbias_amsua_n19_test4.nc4 variational bc: predictors: - name: emissivity @@ -69,9 +73,30 @@ observations: obsdatain: obsfile: Data/ufo/testinput_tier_1/amsua_n19_obs_2018041500_m_qc.nc4 simulated variables: [brightness_temperature] - channels: &channels 1-15 + channels: 7-9, 12 + obs bias: # test reading coefficients for 1 predictor and subset of channels + # and static (==1) coefficients for another predictor + input file: Data/ufo/testinput_tier_1/satbias_amsua_n19.nc4 + output file: Data/satbias_amsua_n19_test5.nc4 + variational bc: + predictors: + - name: emissivity + static bc: + predictors: + - name: constant + obs bias test: + norm: 0.99229224635 + relative tolerance: 1.e-7 + +- obs space: + name: amsua_n19 + obsdatain: + obsfile: Data/ufo/testinput_tier_1/amsua_n19_obs_2018041500_m_qc.nc4 + simulated variables: [brightness_temperature] + channels: 1-15 obs bias: # test reading all channels and most predictors input file: Data/ufo/testinput_tier_1/satbias_amsua_n19.nc4 + output file: Data/satbias_amsua_n19_test6.nc4 variational bc: predictors: - name: constant diff --git a/test/ufo/ObsBias.h b/test/ufo/ObsBias.h new file mode 100644 index 000000000..f41ba878f --- /dev/null +++ b/test/ufo/ObsBias.h @@ -0,0 +1,102 @@ +/* + * (C) Copyright 2021- UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef TEST_UFO_OBSBIAS_H_ +#define TEST_UFO_OBSBIAS_H_ + +#include +#include + +#define ECKIT_TESTING_SELF_REGISTER_CASES 0 + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/testing/Test.h" +#include "ioda/ObsSpace.h" +#include "oops/mpi/mpi.h" +#include "oops/runs/Test.h" +#include "oops/util/DateTime.h" +#include "oops/util/FloatCompare.h" +#include "oops/util/Logger.h" +#include "test/TestEnvironment.h" +#include "ufo/ObsBias.h" + +namespace ufo { +namespace test { +// ----------------------------------------------------------------------------- + +/// \brief Tests ObsBias::read and ObsBias::write methods +/// \details Test that if we write and then read or initialize ObsBias the coefficients +/// are correct for the read/initialized ObsBias. +void testObsBiasReadWrite() { + eckit::LocalConfiguration conf(::test::TestEnvironment::config()); + + util::DateTime bgn(conf.getString("window begin")); + util::DateTime end(conf.getString("window end")); + std::vector obsconfs + = conf.getSubConfigurations("observations"); + + for (auto & oconf : obsconfs) { + ioda::ObsSpace odb(oconf.getSubConfiguration("obs space"), oops::mpi::world(), + bgn, end, oops::mpi::myself()); + + // setup ObsBias parameters + eckit::LocalConfiguration biasconf = oconf.getSubConfiguration("obs bias"); + ObsBiasParameters biasparams; + biasparams.validateAndDeserialize(biasconf); + + // setup ObsBias parameters with input file == output file from the original yaml + eckit::LocalConfiguration biasconf_forread = biasconf; + biasconf_forread.set("input file", biasconf.getString("output file")); + ObsBiasParameters biasparams_forread; + biasparams_forread.validateAndDeserialize(biasconf_forread); + + // init ObsBias + ObsBias ybias(odb, biasparams); + oops::Log::test() << "Initialized obs bias: " << ybias << std::endl; + // save original coefficients for comparison later + const Eigen::VectorXd original_coeffs = ybias.data(); + // write out; assign zero; read back in; compare with original coefficients + ybias.write(biasparams); + ybias.zero(); + EXPECT_EQUAL(ybias.data().norm(), 0.0); + ybias.read(biasparams_forread); + oops::Log::test() << "Obs bias after write out; assign zero; read back in: " + << ybias << std::endl; + EXPECT_EQUAL((original_coeffs - ybias.data()).norm(), 0.0); + // create ObsBias from output coefficients; compare with original coefficients + ObsBias ybias2(odb, biasparams_forread); + oops::Log::test() << "Obs bias initialized from the written out file: " << ybias2 << std::endl; + EXPECT_EQUAL((original_coeffs - ybias2.data()).norm(), 0.0); + } +} + +// ----------------------------------------------------------------------------- + +class ObsBias : public oops::Test { + public: + ObsBias() = default; + virtual ~ObsBias() = default; + + private: + std::string testid() const override {return "ufo::test::ObsBias";} + + void register_tests() const override { + std::vector& ts = eckit::testing::specification(); + + ts.emplace_back(CASE("ufo/ObsBias/testObsBiasReadWrite") + { testObsBiasReadWrite(); }); + } + + void clear() const override {} +}; + +// ----------------------------------------------------------------------------- + +} // namespace test +} // namespace ufo + +#endif // TEST_UFO_OBSBIAS_H_ From 4a88688bbbbbf27faf43efaa22a3ccc6fad2bb88 Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan <2984440+yaswant@users.noreply.github.com> Date: Tue, 6 Jul 2021 16:10:06 +0100 Subject: [PATCH 24/98] conditional addition of crtm test data download (#1315) Co-authored-by: Ryan Honeyager --- test/CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7bade7ca0..a011baf6c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -434,15 +434,17 @@ endfunction() # add tests to download ufo & crtm data ADD_DOWNLOAD_TEST( "ufo" ${ufo_test_data} UFO_UFO_TESTFILES_PATH ) -ADD_DOWNLOAD_TEST( "crtm" ${crtm_test_data} UFO_CRTM_TESTFILES_PATH ) execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ${UFO_UFO_TESTFILES_PATH} ${CMAKE_CURRENT_BINARY_DIR}/Data/ufo) -execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink - ${UFO_CRTM_TESTFILES_PATH} - ${CMAKE_CURRENT_BINARY_DIR}/Data/crtm) +if( crtm_FOUND ) + ADD_DOWNLOAD_TEST( "crtm" ${crtm_test_data} UFO_CRTM_TESTFILES_PATH ) + execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink + ${UFO_CRTM_TESTFILES_PATH} + ${CMAKE_CURRENT_BINARY_DIR}/Data/crtm) +endif( crtm_FOUND ) ##################################################################### # Install headers used in tests of interfaces that may be implemented by clients. From b4642801abeb849286e92e7143bee9b77c1ebdb1 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Tue, 6 Jul 2021 14:26:49 -0600 Subject: [PATCH 25/98] don't throw exception on missing obsbias.outputfile (#1322) --- src/ufo/ObsBias.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ufo/ObsBias.cc b/src/ufo/ObsBias.cc index f96db49f9..863461f44 100644 --- a/src/ufo/ObsBias.cc +++ b/src/ufo/ObsBias.cc @@ -210,8 +210,10 @@ void ObsBias::write(const Parameters_ & params) const { saveBiasCoeffsWithChannels(group, predictors, vars_.channels(), coeffs); } else { - throw eckit::UserError("obs bias.output file section missing from the configuration", - Here()); + if (numVariablePredictors_ > 0) { + oops::Log::warning() << "obs bias.output file is NOT available, bias coefficients " + << "will not be saved." << std::endl; + } } } From 4b0e6fe45b3951dd1cc9cc5049366ad32f322335 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Thu, 8 Jul 2021 18:13:53 +0100 Subject: [PATCH 26/98] Assimilation of derived variables (#1304) * Make PreQC a subclass of FilterBase. Do work in applyFilter() rather than the constructor. * Added a FinalCheck filter. * Apply the FinalCheck filter at the end of the ObsFilters test. * QCmanager: don't reject observations with missing obs errors. Initially reject all derived obs. * Allow copying variables using the Variable Assignment filter. * FinalCheck: Inherit from ObsProcessorBase. * VariableAssignment: Extra documentation. * VariableAssignment: do int comparisons before float comparisons. * QCmanager: Add documentation. remove unused variable. * QCmanager: Look both at ObsValue and DerivedObsValue. * Rewritten an error message in FinalCheck. * Removed duplicate anchors. * Replace DerivedValue by DerivedObsValue in an example. * Added tests verifying correct handling of derived ObsValues and derived simulated variables. * QCmanager: If the ObsValue array of a derived simulated variable already exists, treat it like a non-derived variable. * Removed inaccurate comments. * Test the "source variable" option of Variable Assignment. * PreQC: Fixed handling of the where statement. Added a unit test. * Skip missing value conversion when copying a variable to another. At present a variable can be copied only to another variable of the same type. Co-authored-by: Stephen Herbener <32968781+srherbener@users.noreply.github.com> Co-authored-by: Anna Shlyaeva --- src/ufo/filters/CMakeLists.txt | 2 + src/ufo/filters/FinalCheck.cc | 73 +++++++++++ src/ufo/filters/FinalCheck.h | 54 ++++++++ src/ufo/filters/PreQC.cc | 71 +++++------ src/ufo/filters/PreQC.h | 50 +++++--- src/ufo/filters/QCmanager.cc | 70 ++++++++--- src/ufo/filters/QCmanager.h | 8 +- src/ufo/filters/VariableAssignment.cc | 108 +++++++++++++--- src/ufo/filters/VariableAssignment.h | 15 ++- .../filters/obsfunctions/DrawValueFromFile.h | 4 +- src/ufo/instantiateObsFilterFactory.h | 3 + test/CMakeLists.txt | 8 ++ test/testinput/qc_finalcheck.yaml | 119 ++++++++++++++++++ test/testinput/qc_preqc.yaml | 15 +++ test/testinput/qc_variableassignment.yaml | 84 +++++++++++-- test/ufo/ObsFilters.h | 22 ++++ 16 files changed, 588 insertions(+), 118 deletions(-) create mode 100644 src/ufo/filters/FinalCheck.cc create mode 100644 src/ufo/filters/FinalCheck.h create mode 100644 test/testinput/qc_finalcheck.yaml diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index 014e431f7..2aad855c7 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -17,6 +17,8 @@ set ( filters_files FilterBase.cc FilterBase.h FilterParametersBase.h + FinalCheck.cc + FinalCheck.h GenericFilterParameters.h getScalarOrFilterData.cc getScalarOrFilterData.h diff --git a/src/ufo/filters/FinalCheck.cc b/src/ufo/filters/FinalCheck.cc new file mode 100644 index 000000000..2c883c26f --- /dev/null +++ b/src/ufo/filters/FinalCheck.cc @@ -0,0 +1,73 @@ +/* + * (C) Copyright 2018-2019 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/filters/FinalCheck.h" + +#include +#include +#include + +#include "ioda/ObsSpace.h" +#include "oops/base/Variables.h" +#include "oops/util/Logger.h" +#include "oops/util/missingValues.h" +#include "ufo/filters/QCflags.h" + +namespace ufo { + +// ----------------------------------------------------------------------------- + +FinalCheck::FinalCheck(ioda::ObsSpace & obsdb, const Parameters_ &, + std::shared_ptr> qcflags, + std::shared_ptr> obserr) + : ObsProcessorBase(obsdb, true /*deferToPost?*/, std::move(qcflags), std::move(obserr)) +{ + oops::Log::trace() << "FinalCheck constructed" << std::endl; +} + +// ----------------------------------------------------------------------------- + +FinalCheck::~FinalCheck() { + oops::Log::trace() << "FinalCheck destructed" << std::endl; +} + +// ----------------------------------------------------------------------------- + +void FinalCheck::doFilter() const { + oops::Log::trace() << "FinalCheck doFilter starts" << std::endl; + + const oops::Variables &derived = obsdb_.derived_obsvariables(); + for (size_t jv = 0; jv < derived.size(); ++jv) { + if (!obsdb_.has("ObsValue", derived[jv])) + throw eckit::UnexpectedState( + "All filters have been run, but the derived simulated variable " + derived[jv] + + " can't be found either in the ObsValue or the DerivedObsValue group", Here()); + } + + // Set the QC flag to missing for any observations that haven't been rejected yet, + // but have missing error estimates. + const float missing = util::missingValue(missing); + for (size_t jv = 0; jv < obsdb_.obsvariables().size(); ++jv) { + for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { + if ((*flags_)[jv][jobs] == QCflags::pass && (*obserr_)[jv][jobs] == missing) { + (*flags_)[jv][jobs] = QCflags::missing; + } + } + } + + oops::Log::trace() << "FinalCheck doFilter done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +void FinalCheck::print(std::ostream & os) const { + os << "Final Check" << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/filters/FinalCheck.h b/src/ufo/filters/FinalCheck.h new file mode 100644 index 000000000..945c5ef1b --- /dev/null +++ b/src/ufo/filters/FinalCheck.h @@ -0,0 +1,54 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_FILTERS_FINALCHECK_H_ +#define UFO_FILTERS_FINALCHECK_H_ + +#include +#include +#include + +#include "oops/base/ObsFilterParametersBase.h" +#include "oops/util/ObjectCounter.h" +#include "ufo/filters/ObsProcessorBase.h" + +namespace ufo { + +class FinalCheckParameters : public oops::ObsFilterParametersBase { + OOPS_CONCRETE_PARAMETERS(FinalCheckParameters, ObsFilterParametersBase) + // No extra parameters needed +}; + +/// \brief A filter run automatically at the end of the whole sequence of filters. +/// +/// It does two things: +/// - verifies that all derived simulated variables have been created and if not, throws an +/// exception +/// - sets the QC flag of all observations with missing error estimates to `missing`. +class FinalCheck : public ObsProcessorBase, + private util::ObjectCounter { + public: + /// The type of parameters accepted by the constructor of this filter. + /// This typedef is used by the FilterFactory. + typedef FinalCheckParameters Parameters_; + + static const std::string classname() {return "ufo::FinalCheck";} + + FinalCheck(ioda::ObsSpace & obsdb, const Parameters_ & params, + std::shared_ptr> qcflags, + std::shared_ptr> obserr); + ~FinalCheck() override; + + void doFilter() const override; + + private: + void print(std::ostream &) const override; +}; + +} // namespace ufo + +#endif // UFO_FILTERS_FINALCHECK_H_ diff --git a/src/ufo/filters/PreQC.cc b/src/ufo/filters/PreQC.cc index 6ca324601..09b3d8ae5 100644 --- a/src/ufo/filters/PreQC.cc +++ b/src/ufo/filters/PreQC.cc @@ -8,70 +8,61 @@ #include "ufo/filters/PreQC.h" #include - -#include "eckit/config/Configuration.h" +#include #include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" -#include "oops/base/Variables.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/Logger.h" -#include "oops/util/missingValues.h" -#include "ufo/filters/QCflags.h" namespace ufo { -// Presets for QC filters could be performed in a function outside of any class. -// We keep them as a filter for now. The main reason for this is to be able to use -// the factory for models not in UFO/IODA. - -// ----------------------------------------------------------------------------- - -PreQC::PreQC(ioda::ObsSpace & obsdb, const eckit::Configuration & config, - std::shared_ptr > qcflags, +PreQC::PreQC(ioda::ObsSpace & obsdb, const Parameters_ & parameters, + std::shared_ptr > flags, std::shared_ptr > obserr) - : nogeovals_() + : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { - oops::Log::trace() << "PreQC::PreQC starting " << config << std::endl; - const int missing = util::missingValue(missing); + oops::Log::debug() << "PreQC: config = " << parameters_ << std::endl; +} -// Basic arguments checks - ASSERT(qcflags); - ASSERT(obserr); +// ----------------------------------------------------------------------------- - const oops::Variables observed = obsdb.obsvariables(); +void PreQC::applyFilter(const std::vector & apply, + const Variables & filtervars, + std::vector> & flagged) const { + oops::Log::trace() << "PreQC applyFilter starting " << std::endl; - ASSERT(qcflags->nvars() == observed.size()); - ASSERT(qcflags->nlocs() == obsdb.nlocs()); - ASSERT(obserr->nvars() == observed.size()); - ASSERT(obserr->nlocs() == obsdb.nlocs()); + const int missing = util::missingValue(missing); -// Read QC flags from pre-processing - const std::string qcin = config.getString("inputQC", "PreQC"); - ioda::ObsDataVector preqc(obsdb, observed, qcin); + // Read QC flags from pre-processing + ioda::ObsDataVector preqc(obsdb_, + filtervars.toOopsVariables(), + parameters_.inputQC); oops::Log::debug() << "PreQC::PreQC preqc: " << preqc; -// Get min and max values and reject outside range - const int qcmin = config.getInt("minvalue", 0); - const int qcmax = config.getInt("maxvalue", 0); - - for (size_t jv = 0; jv < observed.size(); ++jv) { - for (size_t jobs = 0; jobs < obsdb.nlocs(); ++jobs) { - if (preqc[jv][jobs] == missing || - preqc[jv][jobs] > qcmax || - preqc[jv][jobs] < qcmin) { - (*qcflags)[jv][jobs] = QCflags::preQC; + // Get min and max values and reject outside range + const int qcmin = parameters_.minvalue; + const int qcmax = parameters_.maxvalue; + + for (size_t jv = 0; jv < filtervars.nvars(); ++jv) { + const ioda::ObsDataRow ¤tPreQC = preqc[jv]; + std::vector ¤tFlagged = flagged[jv]; + for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { + if (apply[jobs] && + (currentPreQC[jobs] == missing || + currentPreQC[jobs] > qcmax || + currentPreQC[jobs] < qcmin)) { + currentFlagged[jobs] = true; } } } - oops::Log::trace() << "PreQC::PreQC done" << std::endl; + oops::Log::trace() << "PreQC applyFilter done" << std::endl; } // ----------------------------------------------------------------------------- void PreQC::print(std::ostream & os) const { - os << "PreQC"; + os << "PreQC: config = " << parameters_ << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/PreQC.h b/src/ufo/filters/PreQC.h index 0dda1120b..27a116a02 100644 --- a/src/ufo/filters/PreQC.h +++ b/src/ufo/filters/PreQC.h @@ -10,41 +10,51 @@ #include #include +#include +#include -#include "eckit/config/LocalConfiguration.h" -#include "ioda/ObsDataVector.h" -#include "oops/base/Variables.h" -#include "oops/util/Printable.h" +#include "oops/util/ObjectCounter.h" +#include "ufo/filters/FilterBase.h" +#include "ufo/filters/QCflags.h" namespace ioda { template class ObsDataVector; class ObsSpace; - class ObsVector; } namespace ufo { -class GeoVaLs; -class ObsDiagnostics; -class PreQC : public util::Printable { +class PreQCParameters : public FilterParametersBase { + OOPS_CONCRETE_PARAMETERS(PreQCParameters, FilterParametersBase) + public: - PreQC(ioda::ObsSpace &, const eckit::Configuration &, - std::shared_ptr >, - std::shared_ptr >); - ~PreQC() {} + /// The ObsSpace group holding PreQC flags. By default, 'PreQC'. + oops::Parameter inputQC{"inputQC", "PreQC", this}; + /// Minimum PreQC flag denoting "pass". By default, zero. + oops::Parameter minvalue{"minvalue", 0, this}; + /// Maximum PreQC flag denoting "pass". By default, zero. + oops::Parameter maxvalue{"maxvalue", 0, this}; +}; - void preProcess() const {} - void priorFilter(const GeoVaLs &) const {} - void postFilter(const ioda::ObsVector &, const ObsDiagnostics &) const {} +class PreQC : public FilterBase, private util::ObjectCounter { + public: + /// The type of parameters accepted by the constructor of this filter. + /// This typedef is used by the FilterFactory. + typedef PreQCParameters Parameters_; + + static const std::string classname() {return "ufo::PreQC";} - const oops::Variables & requiredVars() const {return nogeovals_;} - const oops::Variables & requiredHdiagnostics() const {return nodiagvars_;} + PreQC(ioda::ObsSpace &, const Parameters_ &, + std::shared_ptr >, + std::shared_ptr >); private: - void print(std::ostream &) const; + void print(std::ostream &) const override; + void applyFilter(const std::vector &, const Variables &, + std::vector> &) const override; + int qcFlag() const override {return QCflags::preQC;} - const oops::Variables nogeovals_; - const oops::Variables nodiagvars_; + Parameters_ parameters_; }; } // namespace ufo diff --git a/src/ufo/filters/QCmanager.cc b/src/ufo/filters/QCmanager.cc index 0e72412c8..c9e46059e 100644 --- a/src/ufo/filters/QCmanager.cc +++ b/src/ufo/filters/QCmanager.cc @@ -30,34 +30,63 @@ namespace ufo { // We keep them as a filter for now. The main reason for this is to be able to use // the factory for models not in UFO/IODA. +namespace { + +/// At each location, set the QC flag to QCflags::missing if the current QC flag is invalid +/// or if the ObsValue is missing. +void updateQCFlags(const std::vector *obsValues, std::vector& qcflags) { + const float rmiss = util::missingValue(rmiss); + const int imiss = util::missingValue(imiss); + for (size_t jobs = 0; jobs < qcflags.size(); ++jobs) { + if (qcflags[jobs] == imiss || !obsValues || (*obsValues)[jobs] == rmiss) { + qcflags[jobs] = QCflags::missing; + } + } +} + +} // namespace + // ----------------------------------------------------------------------------- -QCmanager::QCmanager(ioda::ObsSpace & obsdb, const eckit::Configuration & config, +QCmanager::QCmanager(ioda::ObsSpace & obsdb, const eckit::Configuration & /*config*/, std::shared_ptr > qcflags, - std::shared_ptr > obserr) - : obsdb_(obsdb), config_(config), nogeovals_(), nodiags_(), flags_(qcflags), - observed_(obsdb.obsvariables()) + std::shared_ptr > /*obserr*/) + : obsdb_(obsdb), nogeovals_(), nodiags_(), flags_(qcflags) { - oops::Log::trace() << "QCmanager::QCmanager starting " << config_ << std::endl; + oops::Log::trace() << "QCmanager::QCmanager starting" << std::endl; ASSERT(qcflags); - ASSERT(obserr); - ASSERT(flags_->nvars() == observed_.size()); + const oops::Variables &allSimulatedVars = obsdb.obsvariables(); + const oops::Variables &initialSimulatedVars = obsdb.initial_obsvariables(); + const oops::Variables &derivedSimulatedVars = obsdb.derived_obsvariables(); + + ASSERT(allSimulatedVars.size() == initialSimulatedVars.size() + derivedSimulatedVars.size()); + ASSERT(flags_->nvars() == allSimulatedVars.size()); ASSERT(flags_->nlocs() == obsdb_.nlocs()); - ASSERT(obserr->nvars() == observed_.size()); - ASSERT(obserr->nlocs() == obsdb_.nlocs()); const float rmiss = util::missingValue(rmiss); const int imiss = util::missingValue(imiss); - const ioda::ObsDataVector obs(obsdb, observed_, "ObsValue"); + const ioda::ObsDataVector obs(obsdb, initialSimulatedVars, "ObsValue"); - for (size_t jv = 0; jv < observed_.size(); ++jv) { - for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - if ((*flags_)[jv][jobs] == imiss || obs[jv][jobs] == rmiss || (*obserr)[jv][jobs] == rmiss) { - (*flags_)[jv][jobs] = QCflags::missing; - } + // Iterate over initial simulated variables + for (size_t jv = 0; jv < initialSimulatedVars.size(); ++jv) { + const ioda::ObsDataRow ¤tObsValues = obs[jv]; + ioda::ObsDataRow ¤tQCFlags = (*qcflags)[obs.varnames()[jv]]; + updateQCFlags(¤tObsValues, currentQCFlags); + } + + // Iterate over derived simulated variables and if they don't exist yet, set their QC flags to + // 'missing'. + for (size_t jv = 0; jv < derivedSimulatedVars.size(); ++jv) { + ioda::ObsDataRow ¤tQCFlags = (*qcflags)[derivedSimulatedVars[jv]]; + if (!obsdb.has("ObsValue", derivedSimulatedVars[jv])) { + updateQCFlags(nullptr, currentQCFlags); + } else { + std::vector currentObsValues(obsdb_.nlocs()); + obsdb_.get_db("ObsValue", derivedSimulatedVars[jv], currentObsValues); + updateQCFlags(¤tObsValues, currentQCFlags); } } @@ -70,10 +99,11 @@ void QCmanager::postFilter(const ioda::ObsVector & hofx, const ObsDiagnostics &) oops::Log::trace() << "QCmanager postFilter" << std::endl; const double missing = util::missingValue(missing); + const oops::Variables &allSimulatedVars = obsdb_.obsvariables(); - for (size_t jv = 0; jv < observed_.size(); ++jv) { + for (size_t jv = 0; jv < allSimulatedVars.size(); ++jv) { for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - size_t iobs = observed_.size() * jobs + jv; + size_t iobs = allSimulatedVars.size() * jobs + jv; if ((*flags_)[jv][jobs] == 0 && hofx[iobs] == missing) { (*flags_)[jv][jobs] = QCflags::Hfailed; } @@ -124,7 +154,9 @@ void QCmanager::print(std::ostream & os) const { const size_t nlocs = obsdb_.nlocs(); const size_t gnlocs = obsdb_.globalNumLocs(); - for (size_t jvar = 0; jvar < observed_.size(); ++jvar) { + const oops::Variables &allSimulatedVars = obsdb_.obsvariables(); + + for (size_t jvar = 0; jvar < allSimulatedVars.size(); ++jvar) { std::unique_ptr>> accumulator = obsdb_.distribution()->createAccumulator(cases.size()); @@ -137,7 +169,7 @@ void QCmanager::print(std::ostream & os) const { const std::vector counts = accumulator->computeResult(); if (obsdb_.comm().rank() == 0) { - const std::string info = "QC " + flags_->obstype() + " " + observed_[jvar] + ": "; + const std::string info = "QC " + flags_->obstype() + " " + allSimulatedVars[jvar] + ": "; // Normal cases for (size_t i = numSpecialCases; i < counts.size(); ++i) diff --git a/src/ufo/filters/QCmanager.h b/src/ufo/filters/QCmanager.h index eb73011f7..b6f6852a5 100644 --- a/src/ufo/filters/QCmanager.h +++ b/src/ufo/filters/QCmanager.h @@ -26,6 +26,11 @@ namespace ufo { class GeoVaLs; class ObsDiagnostics; +/// \brief Always the first filter to be run. +/// +/// The constructor sets the QC flag to `missing` at all locations with missing obs values of QC +/// flags. The postFilter() function sets the QC flag to `Hfailed` if it was previously set to +/// `pass`, but the obs operator failed to produce a valid value. class QCmanager : public util::Printable { public: QCmanager(ioda::ObsSpace &, const eckit::Configuration &, @@ -44,12 +49,9 @@ class QCmanager : public util::Printable { void print(std::ostream &) const; ioda::ObsSpace & obsdb_; - const eckit::LocalConfiguration config_; const oops::Variables nogeovals_; const oops::Variables nodiags_; std::shared_ptr> flags_; - std::shared_ptr> obserr_; - const oops::Variables & observed_; }; } // namespace ufo diff --git a/src/ufo/filters/VariableAssignment.cc b/src/ufo/filters/VariableAssignment.cc index 1dbf7790c..b167014b0 100644 --- a/src/ufo/filters/VariableAssignment.cc +++ b/src/ufo/filters/VariableAssignment.cc @@ -22,6 +22,7 @@ #include "oops/util/missingValues.h" #include "ufo/filters/processWhere.h" +#include "ufo/filters/QCflags.h" #include "ufo/utils/StringUtils.h" namespace ufo { @@ -47,6 +48,26 @@ void assignValue(const std::string &valueAsString, } } +/// Retrieve the variable \p variable and assign its components (channels) to successive +/// vectors in \p values (only at locations selected by the `where` statement). +template +void assignVariable(const ufo::Variable &variable, + const std::vector &apply, + const ObsFilterData &data, + ioda::ObsDataVector &values) { + ioda::ObsDataVector newValues(data.obsspace(), variable.toOopsVariables()); + data.get(variable, newValues); + + for (size_t ival = 0; ival < values.nvars(); ++ival) { + std::vector ¤tValues = values[ival]; + const ioda::ObsDataRow ¤tNewValues = newValues[ival]; + for (size_t iloc = 0; iloc < apply.size(); ++iloc) { + if (apply[iloc]) + currentValues[iloc] = currentNewValues[iloc]; + } + } +} + /// Evaluate the ObsFunction \p function and assign the vectors it produced to successive vectors /// in \p values (only at locations selected by the `where` statement). template @@ -81,6 +102,8 @@ void assignNumericValues(const AssignmentParameters ¶ms, ioda::ObsDataVector &values) { if (params.value_.value() != boost::none) { assignValue(*params.value_.value(), apply, values); + } else if (params.sourceVariable.value() != boost::none) { + assignVariable(*params.sourceVariable.value(), apply, data, values); } else { ASSERT(params.function.value() != boost::none); assignFunction(*params.function.value(), variable, apply, data, values); @@ -91,9 +114,11 @@ void assignNumericValues(const AssignmentParameters ¶ms, template void assignNonnumericValues(const AssignmentParameters ¶ms, const std::vector &apply, + const ObsFilterData &data, ioda::ObsDataVector &values) { if (params.value_.value() != boost::none) { assignValue(*params.value_.value(), apply, values); + // TODO(wsmigaj): Handle sourceVariable once PR #1280 is merged in. } else { ASSERT(params.function.value() != boost::none); throw eckit::BadValue("ObsFunction values cannot be assigned to non-numeric variables", Here()); @@ -129,20 +154,55 @@ void saveValues(const ufo::Variable &variable, obsdb.put_db(variable.group(), variable.variable(ich), values[ich]); } -/// Retrieve the current values of a numeric variable \p variable from \p obsdb (or if it doesn't -/// already exist, fill it with missing values), assign new values to elements selected by the -/// `where` clause and save the results to \p obsdb. -template -void assignToNumericVariable(const ufo::Variable &variable, - const AssignmentParameters ¶ms, - const std::vector &apply, - const ObsFilterData &data, - ioda::ObsSpace &obsdb) { - ioda::ObsDataVector values = getCurrentValues(variable, obsdb); +/// Change the QC flag from `miss` to `pass` if the obs value is no longer missing or from `pass` to +/// `miss` if the obs value is now missing. +void updateQCFlags(const ioda::ObsDataVector &obsvalues, ioda::ObsDataVector &qcflags) { + const float missing = util::missingValue(float()); + + for (size_t ivar = 0; ivar < obsvalues.nvars(); ++ivar) { + if (qcflags.varnames().has(obsvalues.varnames()[ivar])) { + const ioda::ObsDataRow ¤tValues = obsvalues[ivar]; + ioda::ObsDataRow ¤tFlags = qcflags[obsvalues.varnames()[ivar]]; + for (size_t iloc = 0; iloc < obsvalues.nlocs(); ++iloc) { + if (currentFlags[iloc] == QCflags::missing && currentValues[iloc] != missing) + currentFlags[iloc] = QCflags::pass; + else if (currentFlags[iloc] == QCflags::pass && currentValues[iloc] == missing) + currentFlags[iloc] = QCflags::missing; + } + } + } +} + +/// Retrieve the current values of an int-valued variable \p variable from \p obsdb (or if it +/// doesn't already exist, fill it with missing values), assign new values to elements selected by +/// the `where` clause and save the results to \p obsdb. +void assignToIntVariable(const ufo::Variable &variable, + const AssignmentParameters ¶ms, + const std::vector &apply, + const ObsFilterData &data, + ioda::ObsSpace &obsdb) { + ioda::ObsDataVector values = getCurrentValues(variable, obsdb); assignNumericValues(params, variable, apply, data, values); saveValues(variable, values, obsdb); } +/// Works like `assignToIntVariable()`, but in addition if \p variable belongs to the `ObsValue` or +/// `DerivedObsValue` group and is one of the simulated variables, it updates `qcflags` by (a) +/// changing `miss` to `pass` if the obs value is no longer missing and (b) changing `pass` to +/// `miss` if the obs value is now missing. +void assignToFloatVariable(const ufo::Variable &variable, + const AssignmentParameters ¶ms, + const std::vector &apply, + const ObsFilterData &data, + ioda::ObsSpace &obsdb, + ioda::ObsDataVector &qcflags) { + ioda::ObsDataVector values = getCurrentValues(variable, obsdb); + assignNumericValues(params, variable, apply, data, values); + saveValues(variable, values, obsdb); + if (variable.group() == "ObsValue" || variable.group() == "DerivedObsValue") + updateQCFlags(values, qcflags); +} + /// Retrieve the current values of a non-numeric variable \p variable from \p obsdb (or if it /// doesn't already exist, fill it with missing values), assign new values to elements selected by /// the `where` clause and save the results to \p obsdb. @@ -150,9 +210,10 @@ template void assignToNonnumericVariable(const ufo::Variable &variable, const AssignmentParameters ¶ms, const std::vector &apply, + const ObsFilterData &data, ioda::ObsSpace &obsdb) { ioda::ObsDataVector values = getCurrentValues(variable, obsdb); - assignNonnumericValues(params, apply, values); + assignNonnumericValues(params, apply, data, values); saveValues(variable, values, obsdb); } @@ -163,19 +224,20 @@ void assignToVariable(const ufo::Variable &variable, const AssignmentParameters ¶ms, const std::vector &apply, const ObsFilterData &data, - ioda::ObsSpace &obsdb) { + ioda::ObsSpace &obsdb, + ioda::ObsDataVector &qcflags) { switch (dtype) { case ioda::ObsDtype::Float: - assignToNumericVariable(variable, params, apply, data, obsdb); + assignToFloatVariable(variable, params, apply, data, obsdb, qcflags); break; case ioda::ObsDtype::Integer: - assignToNumericVariable(variable, params, apply, data, obsdb); + assignToIntVariable(variable, params, apply, data, obsdb); break; case ioda::ObsDtype::String: - assignToNonnumericVariable(variable, params, apply, obsdb); + assignToNonnumericVariable(variable, params, apply, data, obsdb); break; case ioda::ObsDtype::DateTime: - assignToNonnumericVariable(variable, params, apply, obsdb); + assignToNonnumericVariable(variable, params, apply, data, obsdb); break; default: ASSERT_MSG(false, "Unrecognized data type"); @@ -224,10 +286,13 @@ void AssignmentParameters::deserialize(util::CompositePath &path, // These checks should really be done at the validation stage (using JSON Schema), // but this isn't supported yet, so this is better than nothing. - if ((value_.value() == boost::none && function.value() == boost::none) || - (value_.value() != boost::none && function.value() != boost::none)) + const int numOptionsSet = static_cast(value_.value() != boost::none) + + static_cast(sourceVariable.value() != boost::none) + + static_cast(function.value() != boost::none); + if (numOptionsSet != 1) throw eckit::UserError(path.path() + - ": Exactly one of the 'value' and 'function' options must be present"); + ": Exactly one of the 'value', 'source variable' and 'function' options " + "must be present"); } @@ -241,6 +306,9 @@ VariableAssignment::VariableAssignment(ioda::ObsSpace & obsdb, const Parameters_ allvars_ += getAllWhereVariables(parameters.where); for (const AssignmentParameters &assignment : parameters.assignments.value()) { + if (assignment.sourceVariable.value() != boost::none) { + allvars_ += *assignment.sourceVariable.value(); + } if (assignment.function.value() != boost::none) { allvars_ += *assignment.function.value(); } @@ -257,7 +325,7 @@ void VariableAssignment::doFilter() const { for (const AssignmentParameters &assignment : parameters_.assignments.value()) { const ufo::Variable variable = getVariable(assignment); const ioda::ObsDtype dtype = getDataType(assignment.type, variable, obsdb_); - assignToVariable(variable, dtype, assignment, apply, data_, obsdb_); + assignToVariable(variable, dtype, assignment, apply, data_, obsdb_, *flags_); } oops::Log::trace() << "VariableAssignment doFilter end" << std::endl; diff --git a/src/ufo/filters/VariableAssignment.h b/src/ufo/filters/VariableAssignment.h index 4d87f462b..badeb2ec4 100644 --- a/src/ufo/filters/VariableAssignment.h +++ b/src/ufo/filters/VariableAssignment.h @@ -44,10 +44,16 @@ class AssignmentParameters : public oops::Parameters { /// Exactly one of the `value` and `function` options must be given. oops::OptionalParameter value_{"value", this}; - /// Variable (typically an ObsFunction) that should be evaluated and assigned to the specified + /// Variable that should be copied into the destination variable specified using the `name` option + /// (at all locations selected be the `where` statement, if present). + /// + /// Exactly one of the `value`, `source variable` and `function` options must be given. + oops::OptionalParameter sourceVariable{"source variable", this}; + + /// An ObsFunction that should be evaluated and assigned to the specified /// variable (at all locations selected be the `where` statement, if present). /// - /// Exactly one of the `value` and `function` options must be given. + /// Exactly one of the `value`, `source variable` and `function` options must be given. oops::OptionalParameter function{"function", this}; /// Type (int, float, string or datetime) of the variable to which new values should be assigned. @@ -86,6 +92,11 @@ class VariableAssignmentParameters : public oops::ObsFilterParametersBase { /// yet, they will be created; in this case elements not selected by the where clause will be /// initialized with the missing value markers. /// +/// If the modified variable belongs to the `DerivedObsValue` group and is a simulated variable, QC +/// flags previously set to `missing` are reset to `pass` at locations where a valid obs value has +/// been assigned. Conversely, QC flags previously set to `pass` are reset to `missing` at +/// locations where the obs value has been set to missing. +/// /// Example 1: Create new variables `air_temperature@GrossErrorProbability` and /// `relative_humidity@GrossErrorProbability` and set them to 0.1 at all locations. /// diff --git a/src/ufo/filters/obsfunctions/DrawValueFromFile.h b/src/ufo/filters/obsfunctions/DrawValueFromFile.h index f682546c1..1c16a47b5 100644 --- a/src/ufo/filters/obsfunctions/DrawValueFromFile.h +++ b/src/ufo/filters/obsfunctions/DrawValueFromFile.h @@ -118,14 +118,14 @@ class DrawValueFromFileParameters : public DrawValueFromFileParametersWithoutGro /// \code{.yaml} /// - filter: Variable Assignment /// assignments: -/// - name: interpolated_value@DerivedValue +/// - name: interpolated_value@DerivedObsValue /// function: /// name: DrawValueFromFile@ObsFunction /// channels: 1-3 /// options: /// file: /// channels: 1-3 -/// group: DerivedValue +/// group: DerivedObsValue /// interpolation: /// - name: satellite_id@MetaData /// method: exact diff --git a/src/ufo/instantiateObsFilterFactory.h b/src/ufo/instantiateObsFilterFactory.h index a60f085cf..574e4c554 100644 --- a/src/ufo/instantiateObsFilterFactory.h +++ b/src/ufo/instantiateObsFilterFactory.h @@ -16,6 +16,7 @@ #include "ufo/filters/BayesianBackgroundQCFlags.h" #include "ufo/filters/BlackList.h" #include "ufo/filters/DifferenceCheck.h" +#include "ufo/filters/FinalCheck.h" #include "ufo/filters/Gaussian_Thinning.h" #include "ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.h" #include "ufo/filters/HistoryCheck.h" @@ -57,6 +58,8 @@ template void instantiateObsFilterFactory() { oops::instantiateObsFilterFactory(); static oops::FilterMaker > qcManagerMaker("QCmanager"); + static oops::FilterMaker > + finalCheckMaker("Final Check"); static oops::FilterMaker > preQCMaker("PreQC"); static oops::FilterMaker > diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a011baf6c..9ec4edfcf 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -178,6 +178,7 @@ list( APPEND ufo_test_input testinput/qc_derivative_dpdt.yaml testinput/qc_derivative_dxdt.yaml testinput/qc_differencecheck.yaml + testinput/qc_finalcheck.yaml testinput/qc_gauss_thinning.yaml testinput/qc_gauss_thinning_unittests.yaml testinput/qc_grosserrorwholereport.yaml @@ -1722,6 +1723,13 @@ ecbuild_add_test( TARGET test_ufo_qc_derivative_speed DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_qc_finalcheck + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x + ARGS "testinput/qc_finalcheck.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFilters.x + TEST_DEPENDS ufo_get_ufo_test_data ) + ecbuild_add_test( TARGET test_ufo_qc_gen_defer_to_post COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x ARGS "testinput/qc_defer_to_post.yaml" diff --git a/test/testinput/qc_finalcheck.yaml b/test/testinput/qc_finalcheck.yaml new file mode 100644 index 000000000..1e8b0ea55 --- /dev/null +++ b/test/testinput/qc_finalcheck.yaml @@ -0,0 +1,119 @@ +window begin: 2000-01-01T00:00:00Z +window end: 2030-01-01T00:00:00Z + +observations: + +# The following two tests verify the Variable Assignment filter makes expected updates to QC flags +# when derived obs values change from missing to non-missing and vice versa + +- obs space: + name: Assign missing values to some observations + obsdatain: + obsfile: Data/ufo/testinput_tier_1/filters_testdata.nc4 + simulated variables: [variable1] + obs filters: + # Blacklist locations 2 and 3 + - filter: BlackList + where: + - variable: + name: latitude@MetaData # [1, 2, ..., 10] + minvalue: 3 + maxvalue: 4 + # This filter will put missing values in the first 3 elements of variable1@DerivedObsValue. + # It should also set the QC flag to "missing" at locations 0 and 1 (location 2 has already been + # rejected by the BlackList, so its QC flag should not change) + - filter: Variable Assignment + where: + - variable: + name: latitude@MetaData # [1, 2, ..., 10] + minvalue: 4 # Select only the last 7 locations + assignments: + - name: variable1@DerivedObsValue + type: float + value: 1 + benchmarkFlag: 10 # missing + flaggedObservationsBenchmark: [0, 1] + failedObservationsBenchmark: [0, 1, 2, 3] + +- obs space: + name: Remove missing values from some observations + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + # The first two elements of ObsValue and the last two of ObsError are missing + simulated variables: [surface_pressure] + obs filters: + # Blacklist location 2 + - filter: BlackList + where: + - variable: + name: latitude@MetaData + minvalue: 0 + maxvalue: 0 + # Leave only the first element of surface_pressure@DerivedObsValue set to a missing value + - filter: Variable Assignment + where: + - variable: + name: latitude@MetaData + minvalue: -2 # Select all locations except the first + assignments: + - name: surface_pressure@DerivedObsValue + type: float + value: 100000 + benchmarkFlag: 10 # missing + flaggedObservationsBenchmark: [0, 3, 4] # loc 0 has a missing ob, locs 3 and 4 -- a missing error + failedObservationsBenchmark: [0, 2, 3, 4] # in addition, loc 2 was blacklisted + +# The following tests check if derived simulated variables are handled appropriately + +- obs space: + name: Derived simulated variable not assigned + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + derived simulated variables: [eastward_wind] + expectExceptionWithMessage: All filters have been run, but the derived simulated variable eastward_wind can't be found either in the ObsValue or the DerivedObsValue group + +- obs space: + name: Derived simulated variable assigned values, but not error estimates + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + derived simulated variables: [eastward_wind] + obs filters: + - filter: Variable Assignment + assignments: + - name: eastward_wind@DerivedObsValue + type: float + value: 10 + passedBenchmark: 5 # air_temperature + failedBenchmark: 5 # eastward_wind -- its error estimates are missing + +- obs space: + name: Derived simulated variable assigned both values and error estimates + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + derived simulated variables: [eastward_wind] + obs filters: + - filter: Variable Assignment + assignments: + - name: eastward_wind@DerivedObsValue + type: float + value: 10 + - filter: Perform Action + action: + name: assign error + error parameter: 1 + passedBenchmark: 10 + +- obs space: + name: Derived simulated variable already exists + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [] + # Already exists. But the first two elements of ObsValue and the last two of ObsError are missing + derived simulated variables: [surface_pressure] + obs filters: + # Add a dummy filter to force the QCmanager and Final Check to be run. + - filter: Domain Check + failedObservationsBenchmark: [0, 1, 3, 4] diff --git a/test/testinput/qc_preqc.yaml b/test/testinput/qc_preqc.yaml index 0fa3a57d4..171455d1d 100644 --- a/test/testinput/qc_preqc.yaml +++ b/test/testinput/qc_preqc.yaml @@ -27,3 +27,18 @@ observations: minvalue: -2 maxvalue: 8 passedBenchmark: 377 +- obs space: + name: amsua_n19, with where statement + obsdatain: + obsfile: Data/ufo/testinput_tier_1/amsua_n19_obs_2018041500_m.nc4 + simulated variables: [brightness_temperature] + channels: 4-8 + obs filters: + - filter: PreQC + where: # will select 4 locations + - variable: + name: latitude@MetaData + maxvalue: -70 + minvalue: -2 + maxvalue: 8 + failedBenchmark: 9 diff --git a/test/testinput/qc_variableassignment.yaml b/test/testinput/qc_variableassignment.yaml index 9e126fe0b..4c4afb3dc 100644 --- a/test/testinput/qc_variableassignment.yaml +++ b/test/testinput/qc_variableassignment.yaml @@ -265,6 +265,72 @@ observations: reference: name: assigned_int_variable_4@TestReference +- obs space: + <<: *ObsSpace + name: Copy a variable to another (pre-existing) float variable + obs filters: + - filter: Variable Assignment + where: *Where + assignments: + - name: float_variable_1@MetaData + source variable: + name: assigned_float_variable_3@TestReference + compareVariables: + - test: + name: float_variable_1@MetaData + reference: + name: assigned_float_variable_3@TestReference + +- obs space: + <<: *ObsSpace + name: Copy a variable to a new float variable + obs filters: + - filter: Variable Assignment + where: *Where + assignments: + - name: new_variable@MetaData + type: float + source variable: + name: assigned_float_variable_3@TestReference + compareVariables: + - test: + name: new_variable@MetaData + reference: + name: assigned_float_variable_4@TestReference + +- obs space: + <<: *ObsSpace + name: Copy a variable to another (pre-existing) int variable + obs filters: + - filter: Variable Assignment + where: *Where + assignments: + - name: int_variable_1@MetaData + source variable: + name: assigned_int_variable_3@TestReference + compareVariables: + - test: + name: int_variable_1@MetaData + reference: + name: assigned_int_variable_3@TestReference + +- obs space: + <<: *ObsSpace + name: Copy a variable to new int variable + obs filters: + - filter: Variable Assignment + where: *Where + assignments: + - name: new_variable@MetaData + type: int + source variable: + name: assigned_int_variable_3@TestReference + compareVariables: + - test: + name: new_variable@MetaData + reference: + name: assigned_int_variable_4@TestReference + ## Test Conditional ObsFunction "add on" - Example 1 ## Float variable assignment including multiple where in a case - obs space: @@ -561,11 +627,9 @@ observations: err1: [5] expectExceptionWithMessage: ObsFunction values cannot be assigned to non-numeric variables -- obs space: &ObsSpace +- obs space: + <<: *ObsSpace name: Try to assign values to an existing variable from the ObsValue group - obsdatain: - obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc - simulated variables: [air_temperature] obs filters: - filter: Variable Assignment assignments: @@ -573,11 +637,9 @@ observations: value: 75.5 expectExceptionWithMessage: Assignment to variables from the ObsValue group is not allowed -- obs space: &ObsSpace +- obs space: + <<: *ObsSpace name: Try to assign values to a new variable from the ObsValue group - obsdatain: - obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc - simulated variables: [air_temperature] obs filters: - filter: Variable Assignment assignments: @@ -586,11 +648,9 @@ observations: type: float expectExceptionWithMessage: Assignment to variables from the ObsValue group is not allowed -- obs space: &ObsSpace +- obs space: + <<: *ObsSpace name: Try to assign values to a new variable without specifying its type - obsdatain: - obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc - simulated variables: [air_temperature] obs filters: - filter: Variable Assignment assignments: diff --git a/test/ufo/ObsFilters.h b/test/ufo/ObsFilters.h index 6ef5a99e1..51c1b57d2 100644 --- a/test/ufo/ObsFilters.h +++ b/test/ufo/ObsFilters.h @@ -35,6 +35,7 @@ #include "oops/util/parameters/RequiredParameter.h" #include "test/interface/ObsTestsFixture.h" #include "test/TestEnvironment.h" +#include "ufo/filters/FinalCheck.h" #include "ufo/filters/QCflags.h" #include "ufo/filters/Variable.h" #include "ufo/ObsBiasParameters.h" @@ -176,6 +177,23 @@ class ObsFiltersParameters : public oops::Parameters { // ----------------------------------------------------------------------------- +//! +//! \brief Run the FinalCheck filter. +//! +//! This needs to be done manually if post-filters aren't run because the HofX vector +//! is not available. +//! +void runFinalCheck(oops::ObsSpace &obsspace, + oops::ObsDataVector &qcflags, + oops::ObsVector &obserr) { + FinalCheck finalCheck(obsspace.obsspace(), FinalCheckParameters(), + qcflags.obsdatavectorptr(), + std::make_shared>(obserr.obsvector())); + finalCheck.doFilter(); +} + +// ----------------------------------------------------------------------------- + //! //! \brief Convert indices of observations held by this process to global observation indices. //! @@ -445,10 +463,14 @@ void testFilters(size_t obsSpaceIndex, oops::ObsSpace &obspace, filters.priorFilter(gval); oops::Log::info() << "HofX or ObsOperator sections not provided for filters, " << "postFilter not called" << std::endl; +/// apply the FinalCheck filter (which should always be run after all other filters). + runFinalCheck(obspace, *qcflags, obserr); } else { /// no need to run priorFilter or postFilter oops::Log::info() << "GeoVaLs not required, HofX or ObsOperator sections not " << "provided for filters, only preProcess was called" << std::endl; +/// apply the FinalCheck filter (which should always be run after all other filters). + runFinalCheck(obspace, *qcflags, obserr); } qcflags->save("EffectiveQC"); From 049ee3f2db86aafab78a77ba782e25b2f36c8246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Thu, 8 Jul 2021 20:03:28 +0100 Subject: [PATCH 27/98] Allow ObsFunctions to produce values of types different than float (#1280) * Removed an unused include. * Added a Variable::fullName() function. * Convert ObsFunction to templates. They can now produce values of types other than float. * Make the ObsFunction test compatible with functions returning values other than floats. * PrimitiveVariables: handle all ObsFunction types in the same way. * Variables: handle all ObsFunction types in the same way. * ObsFilterData: handle all ObsFunction types in the same way. Reduce code duplication. * Make the Variable Assignment filter compatible with ObsFunctions producing values other than floats. * Make existing ObsFunctions inherit from ObsFunctionBase rather than plain ObsFunctionBase. * Define variants of the Conditional obs function producing ints, strings and datetimes. * Add an extra test verifying error handling in the Variable Assignment filter. * Enable tests of the VariableAssignment filter producing datetime variables. They didn't work with ioda v1, but do with ioda v2. * Remove unused function declaration. * Work around a floating-point exception generated by code built with Clang. * Test Variables::allFromGroup() and hasGroup() for all ObsFunction types. * Corrected various comments. * Added a template parameter to ObsFunctionBase in some ObsFunctions added recently on develop. * Added a comment explaining the purpose of a fragment of a YAML file. * Allow DrawValueFromFile to produce values of types other than float (#1286) * Replaced static member functions in DrawValueFromFile with standalone functions in an anonymous namespace. * Removed an unused member function of DataExtractor. * Replaced the public member function template DataExtractor::extract() with a public method overloaded over all the supported types and a private template member function. * Moved definitions of template member functions of DataExtractor from the header file to the cc file. * Make it possible to verify that certain ObsFunction options cause its compute() method to throw an exception. * Make DrawValueFromFile and DataExtractor templates parametrised by the type of the extracted values. * Avoid specialising linearInterpolation() three times. * Instantiate the DrawValueFromFile template explicitly. This seems to be required by Clang. Not sure why -- I'd have thought the template would be instantiated anyway in the declarations of ObsFunctionMaker> objects. * Corrected comments. * Removed a superfluous function. * Removed unnecessary namespace qualifications. * Removed unused static functions. * Corrected exception messages. * Added extra tests to increase code coverage. Co-authored-by: Ryan Honeyager --- src/ufo/filters/ObsBoundsCheck.cc | 1 - src/ufo/filters/ObsFilterData.cc | 195 +++---- src/ufo/filters/ObsFilterData.h | 88 ++- src/ufo/filters/Variable.cc | 11 + src/ufo/filters/Variable.h | 4 + src/ufo/filters/VariableAssignment.cc | 46 +- src/ufo/filters/Variables.cc | 36 +- .../filters/obsfunctions/BennartzScatIndex.h | 2 +- .../obsfunctions/BgdDepartureAnomaly.h | 2 +- .../filters/obsfunctions/CLWMatchIndexMW.h | 2 +- src/ufo/filters/obsfunctions/CLWRetMW.h | 2 +- src/ufo/filters/obsfunctions/CLWRetMW_SSMIS.h | 2 +- .../filters/obsfunctions/CLWRetSymmetricMW.h | 2 +- .../obsfunctions/ChannelUseflagCheckRad.h | 2 +- .../filters/obsfunctions/CloudCostFunction.h | 2 +- .../CloudDetectMinResidualAVHRR.h | 2 +- .../obsfunctions/CloudDetectMinResidualIR.h | 2 +- src/ufo/filters/obsfunctions/Conditional.cc | 23 +- src/ufo/filters/obsfunctions/Conditional.h | 93 ++- .../obsfunctions/DrawObsErrorFromFile.h | 6 +- .../filters/obsfunctions/DrawValueFromFile.cc | 78 ++- .../filters/obsfunctions/DrawValueFromFile.h | 48 +- .../obsfunctions/HydrometeorCheckAMSUA.h | 2 +- .../obsfunctions/HydrometeorCheckATMS.h | 2 +- src/ufo/filters/obsfunctions/ImpactHeight.h | 2 +- .../InterChannelConsistencyCheck.h | 2 +- .../LAMDomainCheck/LAMDomainCheck.h | 2 +- .../ModelHeightAdjustedAirTemperature.h | 2 +- .../filters/obsfunctions/NearSSTRetCheckIR.h | 2 +- .../filters/obsfunctions/ObsErrorBoundIR.h | 2 +- .../filters/obsfunctions/ObsErrorBoundMW.h | 2 +- .../obsfunctions/ObsErrorFactorConventional.h | 2 +- .../obsfunctions/ObsErrorFactorLatRad.h | 2 +- .../obsfunctions/ObsErrorFactorQuotient.h | 2 +- .../obsfunctions/ObsErrorFactorSfcPressure.h | 2 +- .../obsfunctions/ObsErrorFactorSituDependMW.h | 2 +- .../ObsErrorFactorSurfJacobianRad.h | 2 +- .../obsfunctions/ObsErrorFactorTopoRad.h | 2 +- .../ObsErrorFactorTransmitTopRad.h | 2 +- .../obsfunctions/ObsErrorFactorWavenumIR.h | 2 +- .../filters/obsfunctions/ObsErrorModelQuad.h | 2 +- .../filters/obsfunctions/ObsErrorModelRamp.h | 2 +- .../ObsErrorModelStepwiseLinear.h | 2 +- src/ufo/filters/obsfunctions/ObsFunction.cc | 25 +- src/ufo/filters/obsfunctions/ObsFunction.h | 11 +- .../filters/obsfunctions/ObsFunctionBase.cc | 45 +- .../filters/obsfunctions/ObsFunctionBase.h | 67 ++- .../ObsFunctionLinearCombination.h | 2 +- .../obsfunctions/ObsFunctionScattering.h | 2 +- .../obsfunctions/ObsFunctionVelocity.h | 2 +- src/ufo/filters/obsfunctions/SCATRetMW.h | 2 +- src/ufo/filters/obsfunctions/SIRetMW.h | 2 +- .../filters/obsfunctions/SIRetSymmetricMW.h | 2 +- .../filters/obsfunctions/SatWindsLNVDCheck.h | 2 +- .../filters/obsfunctions/SatWindsSPDBCheck.h | 2 +- .../filters/obsfunctions/SatwindIndivErrors.h | 2 +- src/ufo/filters/obsfunctions/SetSurfaceType.h | 2 +- src/ufo/filters/obsfunctions/SunGlintAngle.h | 2 +- .../filters/obsfunctions/SymmCldImpactIR.h | 2 +- .../obsfunctions/TotalColumnVaporGuess.h | 2 +- .../filters/obsfunctions/TropopauseEstimate.h | 2 +- .../filters/obsfunctions/WindDirAngleDiff.h | 2 +- src/ufo/predictors/InterpolateDataFromFile.cc | 6 +- src/ufo/predictors/InterpolateDataFromFile.h | 2 +- src/ufo/utils/CMakeLists.txt | 1 + src/ufo/utils/PrimitiveVariables.cc | 3 +- .../utils/dataextractor/ConstrainedRange.h | 55 ++ src/ufo/utils/dataextractor/DataExtractor.cc | 532 ++++++++++++++++-- src/ufo/utils/dataextractor/DataExtractor.h | 364 ++---------- .../dataextractor/DataExtractorBackend.h | 8 +- .../dataextractor/DataExtractorCSVBackend.cc | 83 ++- .../dataextractor/DataExtractorCSVBackend.h | 10 +- .../utils/dataextractor/DataExtractorInput.h | 30 +- .../DataExtractorNetCDFBackend.cc | 76 ++- .../DataExtractorNetCDFBackend.h | 8 +- test/CMakeLists.txt | 16 + test/testinput/obserror_assign_unittests.yaml | 145 ++++- test/testinput/qc_variableassignment.yaml | 87 +-- .../unit_tests/function_conditional.yaml | 229 ++++++++ .../function_drawvaluefromfile.yaml | 368 ++++++++++++ test/testinput/variables.yaml | 36 ++ test/ufo/ObsFunction.h | 183 ++++-- test/ufo/Variables.h | 64 ++- 83 files changed, 2323 insertions(+), 852 deletions(-) create mode 100644 src/ufo/utils/dataextractor/ConstrainedRange.h create mode 100644 test/testinput/unit_tests/function_conditional.yaml create mode 100644 test/testinput/unit_tests/function_drawvaluefromfile.yaml diff --git a/src/ufo/filters/ObsBoundsCheck.cc b/src/ufo/filters/ObsBoundsCheck.cc index c281cb610..e2fff8fc4 100644 --- a/src/ufo/filters/ObsBoundsCheck.cc +++ b/src/ufo/filters/ObsBoundsCheck.cc @@ -19,7 +19,6 @@ #include "oops/util/IntSetParser.h" #include "oops/util/Logger.h" #include "oops/util/missingValues.h" -#include "ufo/filters/obsfunctions/ObsFunction.h" #include "ufo/filters/QCflags.h" #include "ufo/utils/PrimitiveVariables.h" #include "ufo/utils/StringUtils.h" diff --git a/src/ufo/filters/ObsFilterData.cc b/src/ufo/filters/ObsFilterData.cc index 3c041aec3..22a224899 100644 --- a/src/ufo/filters/ObsFilterData.cc +++ b/src/ufo/filters/ObsFilterData.cc @@ -10,6 +10,7 @@ #include #include +#include "eckit/utils/StringTools.h" #include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" @@ -78,8 +79,14 @@ bool ObsFilterData::has(const Variable & varname) const { const std::string grp = varname.group(); if (grp == "GeoVaLs") { return (gvals_ && gvals_->has(var)); - } else if (grp == "ObsFunction") { - return ObsFunctionFactory::functionExists(var); + } else if (grp == ObsFunctionTraits::groupName) { + return ObsFunctionFactory::functionExists(var); + } else if (grp == ObsFunctionTraits::groupName) { + return ObsFunctionFactory::functionExists(var); + } else if (grp == ObsFunctionTraits::groupName) { + return ObsFunctionFactory::functionExists(var); + } else if (grp == ObsFunctionTraits::groupName) { + return ObsFunctionFactory::functionExists(var); } else if (grp == "ObsDiag" || grp == "ObsBiasTerm") { return (diags_ && diags_->has(var)); } else { @@ -122,118 +129,48 @@ bool ObsFilterData::hasDataVectorInt(const std::string & grp, const std::string } // ----------------------------------------------------------------------------- -/*! Gets requested data from ObsFilterData - * \param[in] varname is a name of a variable requested - * \param[out] values on output is data from varname (undefined on input) - * \warning if data are unavailable, assertions would fail and method abort - */ -void ObsFilterData::get(const Variable & varname, std::vector & values) const { - const std::string var = varname.variable(0); - const std::string grp = varname.group(); +// Overloads of get() taking a std::vector. - if (grp == "VarMetaData") { - values.resize(obsdb_.nvars()); - obsdb_.get_db(grp, var, values); - } else { - ioda::ObsDataVector vec(obsdb_, varname.toOopsVariables(), grp, false); - this->get(varname, vec); - values.resize(obsdb_.nlocs()); - for (size_t jj = 0; jj < obsdb_.nlocs(); ++jj) { - values[jj] = vec[var][jj]; - } - } +// ----------------------------------------------------------------------------- +void ObsFilterData::get(const Variable & varname, std::vector & values) const { + getVector(varname, values); } +// ----------------------------------------------------------------------------- +void ObsFilterData::get(const Variable & varname, std::vector & values) const { + getVector(varname, values); +} // ----------------------------------------------------------------------------- -/*! Gets requested data from ObsFilterData - * \param[in] varname is a name of a variable requested - * \param[out] values on output is data from varname (undefined on input) - * \warning if data are unavailable, assertions would fail and method abort - */ void ObsFilterData::get(const Variable & varname, std::vector & values) const { - const std::string var = varname.variable(); - const std::string grp = varname.group(); - - if (grp == "GeoVaLs" || grp == "HofX" || grp == "ObsDiag" || - grp == "ObsBiasTerm" || grp == "ObsFunction") { - oops::Log::error() << "ObsFilterData::get std::string and int values only " - << "supported for ObsSpace" - << std::endl; - ABORT("ObsFilterData::get std::string, int and util::DateTime values only supported for " - "ObsSpace"); - } else { - values.resize(obsdb_.nlocs()); - obsdb_.get_db(grp, var, values); - } + getVector(varname, values); } - // ----------------------------------------------------------------------------- -/*! Gets requested data from ObsFilterData - * \param[in] varname is a name of a variable requested - * \param[out] values on output is data from varname (undefined on input) - * \warning if data are unavailable, assertions would fail and method abort - */ void ObsFilterData::get(const Variable & varname, std::vector & values) const { - const std::string var = varname.variable(); - const std::string grp = varname.group(); - - if (grp == "GeoVaLs" || grp == "HofX" || grp == "ObsDiag" || - grp == "ObsBiasTerm" || grp == "ObsFunction") { - oops::Log::error() << "ObsFilterData::get std::string and int values only " - << "supported for ObsSpace" - << std::endl; - ABORT("ObsFilterData::get std::string, int and util::DateTime values only supported for " - "ObsSpace"); - } else { - values.resize(obsdb_.nlocs()); - obsdb_.get_db(grp, var, values); - } + getVector(varname, values); } - // ----------------------------------------------------------------------------- -/*! Gets requested integer data from ObsFilterData - * \param[in] varname is a name of a variable requested - * \param[out] values on output is data from varname (undefined on input) - * \warning if data are unavailable, assertions would fail and method abort - * only ObsSpace int data are supported currently - */ -void ObsFilterData::get(const Variable & varname, std::vector & values) const { - const std::string var = varname.variable(); +template +void ObsFilterData::getVector(const Variable & varname, std::vector & values) const { + const std::string var = varname.variable(0); const std::string grp = varname.group(); if (grp == "VarMetaData") { values.resize(obsdb_.nvars()); obsdb_.get_db(grp, var, values); } else { - if (grp == "GeoVaLs" || grp == "HofX" || grp == "ObsDiag" || - grp == "ObsBiasTerm" || grp == "ObsFunction") { - oops::Log::error() << "ObsFilterData::get std::string and int values only " - << "supported for ObsSpace" - << std::endl; - ABORT("ObsFilterData::get std::string and int values only supported for ObsSpace"); - } else { - ioda::ObsDataVector vec(obsdb_, varname.toOopsVariables(), grp, false); - this->get(varname, vec); - values.resize(obsdb_.nlocs()); - for (size_t jj = 0; jj < obsdb_.nlocs(); ++jj) { - values[jj] = vec[var][jj]; - } - } + ioda::ObsDataVector vec(obsdb_, varname.toOopsVariables(), grp, false); + this->get(varname, vec); + values = vec[var]; } } +// ----------------------------------------------------------------------------- +// Overload of get() taking an std::vector and a level index. // ----------------------------------------------------------------------------- -/*! Gets requested data at requested level from ObsFilterData - * \param[in] varname is a name of a variable requested - * group must be either GeoVaLs or ObsDiag - * \param[in] level is a level variable is requested at - * \param[out] values on output is data from varname (undefined on input) - * \warning if data are unavailable, assertions would fail and method abort - */ void ObsFilterData::get(const Variable & varname, const int level, std::vector & values) const { const std::string var = varname.variable(); @@ -253,12 +190,9 @@ void ObsFilterData::get(const Variable & varname, const int level, } // ----------------------------------------------------------------------------- -/*! Gets requested data from ObsFilterData into ObsDataVector - * \param[in] varname is a name of a variable requested - * \param[out] values on output is data from varname (should be allocated on input) - * \warning only works for ObsFunction; - * if data are unavailable, assertions would fail and method abort - */ +// Overloads of get() taking an ioda::ObsDataVector. + +// ----------------------------------------------------------------------------- void ObsFilterData::get(const Variable & varname, ioda::ObsDataVector & values) const { const std::string var = varname.variable(0); const std::string grp = varname.group(); @@ -269,8 +203,8 @@ void ObsFilterData::get(const Variable & varname, ioda::ObsDataVector & v gvals_->get(vec, var); values[var] = vec; /// For Function call compute - } else if (grp == "ObsFunction") { - ObsFunction obsfunc(varname); + } else if (grp == ObsFunctionTraits::groupName) { + ObsFunction obsfunc(varname); obsfunc.compute(*this, values); /// For HofX get from ObsVector H(x) (should be available) } else if (this->hasVector(grp, var)) { @@ -294,33 +228,78 @@ void ObsFilterData::get(const Variable & varname, ioda::ObsDataVector & v std::map *>::const_iterator jv = dvecsf_.find(grp); values = *jv->second; + /// Produce a clear error message for incompatible ObsFunction types + } else if (eckit::StringTools::endsWith(grp, "ObsFunction")) { + throw eckit::BadParameter("ObsFilterData::get(): " + varname.fullName() + + " is not a function producing values of type " + + ObsFunctionTraits::valueTypeName, Here()); } else { values.read(grp); } } // ----------------------------------------------------------------------------- -/*! Gets requested data from ObsFilterData into ObsDataVector - * \param[in] varname is a name of a variable requested - * \param[out] values on output is data from varname (should be allocated on input) - * \return data associated with varname, in ioda::ObsDataVector -*/ void ObsFilterData::get(const Variable & varname, ioda::ObsDataVector & values) const { const std::string var = varname.variable(0); const std::string grp = varname.group(); - if (this->hasDataVectorInt(grp, var)) { + /// For Function call compute + if (grp == ObsFunctionTraits::groupName) { + ObsFunction obsfunc(varname); + obsfunc.compute(*this, values); + /// For ObsDataVector + } else if (this->hasDataVectorInt(grp, var)) { std::map *>::const_iterator jv = dvecsi_.find(grp); values = *jv->second; + /// Produce a clear error message for incompatible ObsFunction types + } else if (eckit::StringTools::endsWith(grp, "ObsFunction")) { + throw eckit::BadParameter("ObsFilterData::get(): " + varname.fullName() + + " is not a function producing values of type " + + ObsFunctionTraits::valueTypeName, Here()); + /// Produce a clear error message for other incompatible groups + } else if (grp == "GeoVaLs" || grp == "HofX" || grp == "ObsDiag" || grp == "ObsBiasTerm") { + throw eckit::BadParameter("ObsFilterData::get(): variables from the group " + grp + + " are of type float", Here()); } else { values.read(grp); } } // ----------------------------------------------------------------------------- -/*! Returns number of levels in 3D geovals and obsdiags or - * one if not 3D geovals or obsdiag - * - */ +void ObsFilterData::get(const Variable & varname, ioda::ObsDataVector & values) const { + getNonNumeric(varname, values); +} + +// ----------------------------------------------------------------------------- +void ObsFilterData::get(const Variable & varname, + ioda::ObsDataVector & values) const { + getNonNumeric(varname, values); +} + +// ----------------------------------------------------------------------------- +template +void ObsFilterData::getNonNumeric(const Variable & varname, ioda::ObsDataVector & values) const { + const std::string &grp = varname.group(); + /// For Function call compute + if (grp == ObsFunctionTraits::groupName) { + ObsFunction obsfunc(varname); + obsfunc.compute(*this, values); + } else if (eckit::StringTools::endsWith(grp, "ObsFunction")) { + throw eckit::BadParameter("ObsFilterData::get(): " + varname.fullName() + + " is not a function producing values of type " + + ObsFunctionTraits::valueTypeName, Here()); + /// Produce a clear error message for other incompatible groups + } else if (grp == "GeoVaLs" || grp == "HofX" || grp == "ObsDiag" || grp == "ObsBiasTerm") { + throw eckit::BadParameter("ObsFilterData::get(): variables from the group " + grp + + " are of type float", Here()); + } else { + values.read(grp); + } +} + +// ----------------------------------------------------------------------------- +// End of overloads of get(). + +// ----------------------------------------------------------------------------- size_t ObsFilterData::nlevs(const Variable & varname) const { const std::string var = varname.variable(); const std::string grp = varname.group(); diff --git a/src/ufo/filters/ObsFilterData.h b/src/ufo/filters/ObsFilterData.h index 08e618960..e2ed18b8f 100644 --- a/src/ufo/filters/ObsFilterData.h +++ b/src/ufo/filters/ObsFilterData.h @@ -30,18 +30,17 @@ namespace ufo { // ----------------------------------------------------------------------------- /*! \brief ObsFilterData provides access to all data related to an ObsFilter * - * \details ObsFilterData can always provide access to all data from ObsSpace - * and optionally to data from H(x) ObsVector, GeoVaLs and ObsDiagnostics. - * The latter three can be associated with ObsFilterData by using associate() + * \details ObsFilterData can always provide access to all data from ObsSpace, values computed on + * the fly by ObsFunctions, and optionally to data from the H(x) ObsVector, GeoVaLs and + * ObsDiagnostics. The latter three can be associated with ObsFilterData by using associate() * method. - * */ class ObsFilterData : public util::Printable, private util::ObjectCounter { public: static const std::string classname() {return "ufo::ObsFilterData";} - //! Constructs ObsFilterData and associates ObsSpace with it + //! Constructs ObsFilterData and associates an ObsSpace with it. explicit ObsFilterData(ioda::ObsSpace &); ~ObsFilterData(); @@ -56,29 +55,66 @@ class ObsFilterData : public util::Printable, //! Associates ObsDataVector with this ObsFilterData (int) void associate(const ioda::ObsDataVector &, const std::string &); - //! Gets requested data from ObsFilterData - void get(const Variable &, std::vector &) const; - //! Gets requested data at requested level from ObsFilterData - void get(const Variable &, const int, std::vector &) const; - //! Gets requested data from ObsFilterData - void get(const Variable &, std::vector &) const; - //! Gets requested data from ObsFilterData - void get(const Variable &, std::vector &) const; - //! Gets requested data from ObsFilterData - void get(const Variable &, std::vector &) const; - //! Gets requested data from ObsFilterData (ObsDataVector has to be allocated) - void get(const Variable &, ioda::ObsDataVector &) const; - //! Gets requested data from ObsFilterData (ObsDataVector has to be allocated) - void get(const Variable &, ioda::ObsDataVector &) const; - //! Checks if requested data exists in ObsFilterData - bool has(const Variable &) const; + //! \brief Fills a `std::vector` with values of the specified variable. + //! + //! \param varname + //! The requested variable. + //! \param[out] values + //! Vector to be filled with values of the requested variable. + //! + //! An exception is thrown if the requested variable does not exist or is not of the correct type. + void get(const Variable &varname, std::vector &values) const; + //! \overload + void get(const Variable &varname, std::vector &values) const; + //! \overload + void get(const Variable &varname, std::vector &values) const; + //! \overload + void get(const Variable &varname, std::vector &values) const; + + //! \brief Fills a `std::vector` with values of the specified variable at a single level. + //! + //! \param varname + //! The requested variable, which must belong to one of the following groups: GeoVaLs, ObsDiag + //! and ObsBiasTerm. + //! \param level + //! Requested level. + //! \param[out] + //! Vector to be filled with values of the requested variable. + //! + //! An exception is thrown if the requested variable does not exist or if it is not in one of + //! the groups listed above. + void get(const Variable & varname, const int level, + std::vector & values) const; + + //! brief Fills a `ioda::ObsDataVector` with values of the specified variable. + //! + //! \param varname + //! The requested variable. + //! \param[out] values + //! An `ObsDataVector` to be filled with values of the requested variable. Must be + //! pre-populated, i.e. contain a row corresponding to each requested channel of that variable. + //! + //! An exception is thrown if the requested variable does not exist or is not of the correct type. + void get(const Variable &varname, ioda::ObsDataVector &values) const; + //! \overload + void get(const Variable &varname, ioda::ObsDataVector &values) const; + //! \overload + void get(const Variable &varname, ioda::ObsDataVector &values) const; + //! \overload + void get(const Variable &varname, ioda::ObsDataVector &values) const; + + //! Returns true if variable `varname` is known to ObsFilterData, false otherwise. + bool has(const Variable &varname) const; //! Determines dtype of the provided variable ioda::ObsDtype dtype(const Variable &) const; - //! Returns number of locations + //! \brief Returns the number of locations in the associated ObsSpace. size_t nlocs() const; - //! Returns number of levels for specified variable if 3D GeoVaLs or ObsDiags + //! \brief Returns the number of levels in the specified variable. + //! + //! This is useful primarily for variables belonging to the GeoVaLs, ObsDiag and ObsBiasTerm + //! groups. For all other variables this function returns 1. size_t nlevs(const Variable &) const; //! Returns reference to ObsSpace associated with ObsFilterData ioda::ObsSpace & obsspace() const {return obsdb_;} @@ -92,6 +128,12 @@ class ObsFilterData : public util::Printable, bool hasDataVector(const std::string &, const std::string &) const; bool hasDataVectorInt(const std::string &, const std::string &) const; + template + void getVector(const Variable &varname, std::vector &values) const; + /// Called by the overloads of get() taking an ioda::ObsDataVector of strings or datetimes. + template + void getNonNumeric(const Variable &varname, ioda::ObsDataVector &values) const; + ioda::ObsSpace & obsdb_; //!< ObsSpace associated with this object const GeoVaLs mutable * gvals_; //!< pointer to GeoVaLs associated with this object std::map ovecs_; //!< Associated ObsVectors diff --git a/src/ufo/filters/Variable.cc b/src/ufo/filters/Variable.cc index 11c7597d1..3ffdfa59f 100644 --- a/src/ufo/filters/Variable.cc +++ b/src/ufo/filters/Variable.cc @@ -125,6 +125,17 @@ const std::vector & Variable::channels() const { // ----------------------------------------------------------------------------- +std::string Variable::fullName() const { + std::string result; + result.reserve(varname_.size() + 1 + grpname_.size()); + result = varname_; + result += '@'; + result += grpname_; + return result; +} + +// ----------------------------------------------------------------------------- + oops::Variables Variable::toOopsVariables() const { oops::Variables vars; for (size_t jj = 0; jj < this->size(); ++jj) { diff --git a/src/ufo/filters/Variable.h b/src/ufo/filters/Variable.h index 03795f153..e8c071464 100644 --- a/src/ufo/filters/Variable.h +++ b/src/ufo/filters/Variable.h @@ -38,6 +38,10 @@ class Variable: public util::Printable { const std::string & group() const; const std::vector & channels() const; + /// Return the full variable name including the group name (but no channel number), e.g. + /// `air_pressure@MetaData`. + std::string fullName() const; + oops::Variables toOopsVariables() const; const eckit::LocalConfiguration & options() const {return options_;} diff --git a/src/ufo/filters/VariableAssignment.cc b/src/ufo/filters/VariableAssignment.cc index b167014b0..7afc88682 100644 --- a/src/ufo/filters/VariableAssignment.cc +++ b/src/ufo/filters/VariableAssignment.cc @@ -7,6 +7,8 @@ #include "ufo/filters/VariableAssignment.h" +#include +#include #include #include #include @@ -21,6 +23,7 @@ #include "oops/util/Logger.h" #include "oops/util/missingValues.h" +#include "ufo/filters/obsfunctions/ObsFunction.h" #include "ufo/filters/processWhere.h" #include "ufo/filters/QCflags.h" #include "ufo/utils/StringUtils.h" @@ -70,25 +73,36 @@ void assignVariable(const ufo::Variable &variable, /// Evaluate the ObsFunction \p function and assign the vectors it produced to successive vectors /// in \p values (only at locations selected by the `where` statement). -template +template void assignFunction(const ufo::Variable &function, const ufo::Variable &variable, const std::vector &apply, const ObsFilterData &data, ioda::ObsDataVector &values) { - ioda::ObsDataVector newValues(data.obsspace(), variable.toOopsVariables()); + ioda::ObsDataVector newValues(data.obsspace(), variable.toOopsVariables()); data.get(function, newValues); - const VariableType missing = util::missingValue(VariableType()); - const float missingfloat = util::missingValue(float()); + const VariableType missingVariableValue = util::missingValue(VariableType()); + const FunctionValueType missingFunctionValue = util::missingValue(FunctionValueType()); + const FunctionValueType maxFunctionValue = std::numeric_limits::max(); for (size_t ival = 0; ival < values.nvars(); ++ival) { std::vector ¤tValues = values[ival]; - const ioda::ObsDataRow ¤tNewValues = newValues[ival]; + const ioda::ObsDataRow ¤tNewValues = newValues[ival]; for (size_t iloc = 0; iloc < apply.size(); ++iloc) { - if (apply[iloc] && currentNewValues[iloc] != missingfloat) - currentValues[iloc] = static_cast(currentNewValues[iloc]); - if (apply[iloc] && currentNewValues[iloc] == missingfloat) - currentValues[iloc] = missing; + if (apply[iloc]) { + if (currentNewValues[iloc] != missingFunctionValue) + // The call to min() below is a hack preventing clang from generating optimized code + // that speculatively converts currentNewValues[iloc] to VariableType regardless of + // whether it's equal to missingFunctionValue or not. This causes a floating exception + // and thus breaks the test suite. After migration to a more recent version of clang + // we may be able to remove the call to min() and use the -ffp-exception-behavior=maytrap + // flag instead to instruct the compiler to avoid transformations that may raise + // floating-point exceptions that would not have been raised by the original code. + currentValues[iloc] = + static_cast(std::min(currentNewValues[iloc], maxFunctionValue)); + else + currentValues[iloc] = missingVariableValue; + } } } } @@ -106,13 +120,20 @@ void assignNumericValues(const AssignmentParameters ¶ms, assignVariable(*params.sourceVariable.value(), apply, data, values); } else { ASSERT(params.function.value() != boost::none); - assignFunction(*params.function.value(), variable, apply, data, values); + if (params.function.value()->group() == ObsFunctionTraits::groupName) + assignFunction(*params.function.value(), variable, apply, data, values); + else if (params.function.value()->group() == ObsFunctionTraits::groupName) + assignFunction(*params.function.value(), variable, apply, data, values); + else + throw eckit::BadParameter(params.function.value()->fullName() + + " is not a function producing numeric values", Here()); } } /// Assign values to a non-numeric variable (of type string or DateTime). template void assignNonnumericValues(const AssignmentParameters ¶ms, + const ufo::Variable &variable, const std::vector &apply, const ObsFilterData &data, ioda::ObsDataVector &values) { @@ -120,8 +141,7 @@ void assignNonnumericValues(const AssignmentParameters ¶ms, assignValue(*params.value_.value(), apply, values); // TODO(wsmigaj): Handle sourceVariable once PR #1280 is merged in. } else { - ASSERT(params.function.value() != boost::none); - throw eckit::BadValue("ObsFunction values cannot be assigned to non-numeric variables", Here()); + assignFunction(*params.function.value(), variable, apply, data, values); } } @@ -213,7 +233,7 @@ void assignToNonnumericVariable(const ufo::Variable &variable, const ObsFilterData &data, ioda::ObsSpace &obsdb) { ioda::ObsDataVector values = getCurrentValues(variable, obsdb); - assignNonnumericValues(params, apply, data, values); + assignNonnumericValues(params, variable, apply, data, values); saveValues(variable, values, obsdb); } diff --git a/src/ufo/filters/Variables.cc b/src/ufo/filters/Variables.cc index d1fad63c1..dda57f7df 100644 --- a/src/ufo/filters/Variables.cc +++ b/src/ufo/filters/Variables.cc @@ -130,8 +130,20 @@ Variables Variables::allFromGroup(const std::string & group) const { for (size_t ivar = 0; ivar < vars_.size(); ++ivar) { if (vars_[ivar].group() == group) { vars += vars_[ivar]; - } else if (vars_[ivar].group() == "ObsFunction") { - ObsFunction obsfunc(vars_[ivar]); + } else if (vars_[ivar].group() == ObsFunctionTraits::groupName) { + ObsFunction obsfunc(vars_[ivar]); + ufo::Variables funcvars = obsfunc.requiredVariables(); + vars += funcvars.allFromGroup(group); + } else if (vars_[ivar].group() == ObsFunctionTraits::groupName) { + ObsFunction obsfunc(vars_[ivar]); + ufo::Variables funcvars = obsfunc.requiredVariables(); + vars += funcvars.allFromGroup(group); + } else if (vars_[ivar].group() == ObsFunctionTraits::groupName) { + ObsFunction obsfunc(vars_[ivar]); + ufo::Variables funcvars = obsfunc.requiredVariables(); + vars += funcvars.allFromGroup(group); + } else if (vars_[ivar].group() == ObsFunctionTraits::groupName) { + ObsFunction obsfunc(vars_[ivar]); ufo::Variables funcvars = obsfunc.requiredVariables(); vars += funcvars.allFromGroup(group); } @@ -156,9 +168,23 @@ oops::Variables Variables::toOopsVariables() const { bool Variables::hasGroup(const std::string & group) const { bool found = false; for (size_t jj = 0; jj < vars_.size(); ++jj) { - if (vars_[jj].group() == group) found = true; - if (vars_[jj].group() == "ObsFunction") { - ObsFunction obsfunc(vars_[jj]); + if (vars_[jj].group() == group) + found = true; + + if (vars_[jj].group() == ObsFunctionTraits::groupName) { + ObsFunction obsfunc(vars_[jj]); + ufo::Variables funcvars = obsfunc.requiredVariables(); + found = found || funcvars.hasGroup(group); + } else if (vars_[jj].group() == ObsFunctionTraits::groupName) { + ObsFunction obsfunc(vars_[jj]); + ufo::Variables funcvars = obsfunc.requiredVariables(); + found = found || funcvars.hasGroup(group); + } else if (vars_[jj].group() == ObsFunctionTraits::groupName) { + ObsFunction obsfunc(vars_[jj]); + ufo::Variables funcvars = obsfunc.requiredVariables(); + found = found || funcvars.hasGroup(group); + } else if (vars_[jj].group() == ObsFunctionTraits::groupName) { + ObsFunction obsfunc(vars_[jj]); ufo::Variables funcvars = obsfunc.requiredVariables(); found = found || funcvars.hasGroup(group); } diff --git a/src/ufo/filters/obsfunctions/BennartzScatIndex.h b/src/ufo/filters/obsfunctions/BennartzScatIndex.h index 68701d696..5b05a642c 100755 --- a/src/ufo/filters/obsfunctions/BennartzScatIndex.h +++ b/src/ufo/filters/obsfunctions/BennartzScatIndex.h @@ -75,7 +75,7 @@ class BennartzScatIndexParameters : public oops::Parameters { /// support of nowcasting applications /// Meteorol. Appl. 9, 177-189 (2002) DOI:10.1017/S1350482702002037 -class BennartzScatIndex : public ObsFunctionBase { +class BennartzScatIndex : public ObsFunctionBase { public: explicit BennartzScatIndex(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/BgdDepartureAnomaly.h b/src/ufo/filters/obsfunctions/BgdDepartureAnomaly.h index dd74038c4..63895f065 100755 --- a/src/ufo/filters/obsfunctions/BgdDepartureAnomaly.h +++ b/src/ufo/filters/obsfunctions/BgdDepartureAnomaly.h @@ -69,7 +69,7 @@ class BgdDepartureAnomalyParameters : public oops::Parameters { /// is used to diagnose cloudy scenes. /// -class BgdDepartureAnomaly : public ObsFunctionBase { +class BgdDepartureAnomaly : public ObsFunctionBase { public: explicit BgdDepartureAnomaly(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); void compute(const ObsFilterData &, ioda::ObsDataVector &) const; diff --git a/src/ufo/filters/obsfunctions/CLWMatchIndexMW.h b/src/ufo/filters/obsfunctions/CLWMatchIndexMW.h index 9a29c2eb0..cd5627b2a 100755 --- a/src/ufo/filters/obsfunctions/CLWMatchIndexMW.h +++ b/src/ufo/filters/obsfunctions/CLWMatchIndexMW.h @@ -52,7 +52,7 @@ class CLWMatchIndexMWParameters : public oops::Parameters { /// 1: both background and observation cont contain clouds /// 0: either background has cloud or observation has cloud detected /// -class CLWMatchIndexMW : public ObsFunctionBase { +class CLWMatchIndexMW : public ObsFunctionBase { public: explicit CLWMatchIndexMW(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/CLWRetMW.h b/src/ufo/filters/obsfunctions/CLWRetMW.h index 3428a1f16..0e2e76b92 100755 --- a/src/ufo/filters/obsfunctions/CLWRetMW.h +++ b/src/ufo/filters/obsfunctions/CLWRetMW.h @@ -85,7 +85,7 @@ class CLWRetMWParameters : public oops::Parameters { /// the NOAA 15 advanced microwave sounding unit /// Journal of Geophysical Research (Vol. 106, No. D3, Pages 2943-2953) /// -class CLWRetMW : public ObsFunctionBase { +class CLWRetMW : public ObsFunctionBase { public: explicit CLWRetMW(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/CLWRetMW_SSMIS.h b/src/ufo/filters/obsfunctions/CLWRetMW_SSMIS.h index 9630c726b..d8814ef7b 100644 --- a/src/ufo/filters/obsfunctions/CLWRetMW_SSMIS.h +++ b/src/ufo/filters/obsfunctions/CLWRetMW_SSMIS.h @@ -51,7 +51,7 @@ class CLWRetMW_SSMISParameters : public oops::Parameters { /// Sounder (SSMIS) and Special Sensor Microwave Imager (SSM/I)', TGARS Special /// Issue on the DMSP SSMIS, 46, 984-995. /// -class CLWRetMW_SSMIS : public ObsFunctionBase { +class CLWRetMW_SSMIS : public ObsFunctionBase { public: explicit CLWRetMW_SSMIS(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/CLWRetSymmetricMW.h b/src/ufo/filters/obsfunctions/CLWRetSymmetricMW.h index 007357fef..b09d1e59c 100755 --- a/src/ufo/filters/obsfunctions/CLWRetSymmetricMW.h +++ b/src/ufo/filters/obsfunctions/CLWRetSymmetricMW.h @@ -31,7 +31,7 @@ typedef CLWRetMWParameters CLWRetSymmetricMWParameters; /// \brief Calculate symmetric (mean) cloud amount from the cloud amount retrieved /// from the observed and simulated measurements /// -class CLWRetSymmetricMW : public ObsFunctionBase { +class CLWRetSymmetricMW : public ObsFunctionBase { public: explicit CLWRetSymmetricMW(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/ChannelUseflagCheckRad.h b/src/ufo/filters/obsfunctions/ChannelUseflagCheckRad.h index 6f382a9f9..7a4068170 100755 --- a/src/ufo/filters/obsfunctions/ChannelUseflagCheckRad.h +++ b/src/ufo/filters/obsfunctions/ChannelUseflagCheckRad.h @@ -38,7 +38,7 @@ class ChannelUseflagCheckRadParameters : public oops::Parameters { /// /// \brief Channel useflag check: remove channel if useflag is less than one /// -class ChannelUseflagCheckRad : public ObsFunctionBase { +class ChannelUseflagCheckRad : public ObsFunctionBase { public: explicit ChannelUseflagCheckRad(const eckit::LocalConfiguration &); ~ChannelUseflagCheckRad(); diff --git a/src/ufo/filters/obsfunctions/CloudCostFunction.h b/src/ufo/filters/obsfunctions/CloudCostFunction.h index 1dd061ebf..f7728a4cc 100755 --- a/src/ufo/filters/obsfunctions/CloudCostFunction.h +++ b/src/ufo/filters/obsfunctions/CloudCostFunction.h @@ -109,7 +109,7 @@ class CloudCostFunctionParameters : public oops::Parameters { /// Quart. J. Royal Meterol. Soc., Vol. 125, pp. 2359-2378 (1999). /// https://doi.org/10.1002/qj.49712555902 -class CloudCostFunction : public ObsFunctionBase { +class CloudCostFunction : public ObsFunctionBase { public: explicit CloudCostFunction(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.h b/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.h index 42ae13471..5bea6c56f 100755 --- a/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.h +++ b/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.h @@ -63,7 +63,7 @@ class CloudDetectMinResidualAVHRRParameters : public oops::Parameters { /// 1 = channel is affected by clouds (cloudy channel) /// 2 = channel is not affected by clouds but too sensitive to surface condition /// -class CloudDetectMinResidualAVHRR : public ObsFunctionBase { +class CloudDetectMinResidualAVHRR : public ObsFunctionBase { public: explicit CloudDetectMinResidualAVHRR(const eckit::LocalConfiguration &); ~CloudDetectMinResidualAVHRR(); diff --git a/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.h b/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.h index 4b85b4270..1a50f8916 100755 --- a/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.h +++ b/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.h @@ -60,7 +60,7 @@ class CloudDetectMinResidualIRParameters : public oops::Parameters { /// 1 = channel is affected by clouds (cloudy channel) /// 2 = channel is not affected by clouds but too sensitive to surface condition /// -class CloudDetectMinResidualIR : public ObsFunctionBase { +class CloudDetectMinResidualIR : public ObsFunctionBase { public: explicit CloudDetectMinResidualIR(const eckit::LocalConfiguration &); ~CloudDetectMinResidualIR(); diff --git a/src/ufo/filters/obsfunctions/Conditional.cc b/src/ufo/filters/obsfunctions/Conditional.cc index 2b0fb519e..116314a01 100755 --- a/src/ufo/filters/obsfunctions/Conditional.cc +++ b/src/ufo/filters/obsfunctions/Conditional.cc @@ -13,23 +13,29 @@ namespace ufo { -static ObsFunctionMaker makerConditional_("Conditional"); +static ObsFunctionMaker> floatMaker("Conditional"); +static ObsFunctionMaker> intMaker("Conditional"); +static ObsFunctionMaker> stringMaker("Conditional"); +static ObsFunctionMaker> dateTimeMaker("Conditional"); -Conditional::Conditional(const eckit::LocalConfiguration & conf) +// ----------------------------------------------------------------------------- +template +Conditional::Conditional(const eckit::LocalConfiguration & conf) : invars_() { // Initialize options options_.validateAndDeserialize(conf); // Populate invars_ - for (const LocalConditionalParameters &lcp : options_.cases.value()) + for (const LocalConditionalParameters &lcp : options_.cases.value()) invars_ += getAllWhereVariables(lcp.where); } // ----------------------------------------------------------------------------- -void Conditional::compute(const ObsFilterData & in, - ioda::ObsDataVector & out) const { +template +void Conditional::compute(const ObsFilterData & in, + ioda::ObsDataVector & out) const { // Assign default value to array - const float missing = util::missingValue(float()); + const FunctionValue missing = util::missingValue(FunctionValue()); for (size_t ivar = 0; ivar < out.nvars(); ++ivar) { std::fill(out[ivar].begin(), out[ivar].end(), options_.defaultvalue.value().value_or(missing)); } // ivar @@ -38,7 +44,7 @@ void Conditional::compute(const ObsFilterData & in, // if firstmatchingcase is true, the first case that is true assigns the value. // if firstmatchingcase is false, the last matching case will assign the value. std::vector applied(out.nlocs(), false); - for (const LocalConditionalParameters &lcp : options_.cases.value()) { + for (const LocalConditionalParameters &lcp : options_.cases.value()) { std::vector apply = processWhere(lcp.where, in); for (size_t iloc = 0; iloc < out.nlocs(); ++iloc) { if (apply[iloc] && applied[iloc] == false) { @@ -51,7 +57,8 @@ void Conditional::compute(const ObsFilterData & in, } // compute // ----------------------------------------------------------------------------- -const ufo::Variables & Conditional::requiredVariables() const { +template +const ufo::Variables & Conditional::requiredVariables() const { return invars_; } diff --git a/src/ufo/filters/obsfunctions/Conditional.h b/src/ufo/filters/obsfunctions/Conditional.h index 0d89e3a33..9ab654a03 100755 --- a/src/ufo/filters/obsfunctions/Conditional.h +++ b/src/ufo/filters/obsfunctions/Conditional.h @@ -34,6 +34,7 @@ namespace ufo { /// minvalue: 0 /// value: 0.5 /// +template class LocalConditionalParameters : public oops::Parameters { OOPS_CONCRETE_PARAMETERS(LocalConditionalParameters, Parameters) @@ -43,21 +44,23 @@ class LocalConditionalParameters : public oops::Parameters { oops::RequiredParameter> where{"where", this}; /// \brief Value to be assigned when this particular where clause is true. - oops::RequiredParameter value{"value", this}; + oops::RequiredParameter value{"value", this}; }; /// Parameters controlling the Conditional obs function. +template class ConditionalParameters : public oops::Parameters { OOPS_CONCRETE_PARAMETERS(ConditionalParameters, Parameters) public: /// List of cases for assignment. See LocalConditionalParameters /// for what each where clause requires. - oops::RequiredParameter> cases{"cases", this}; + oops::RequiredParameter>> cases{ + "cases", this}; /// Default value for the array to be assigned. Missing value is used /// if this is not present in the yaml. - oops::OptionalParameter defaultvalue{"defaultvalue", this}; + oops::OptionalParameter defaultvalue{"defaultvalue", this}; /// When this flag is true a value is assigned for the first matching where case /// for a location. This matches with the python case logic. @@ -71,43 +74,73 @@ class ConditionalParameters : public oops::Parameters { /// /// The obs function has been designed primarily to work with the Variable assignment filter /// to simplify the assignment of more complicated variables. Any functionality in the -/// processWhere class can be used with this obs function. This can only be used to assign -/// float and int variables because the ObsFunction only computes floating point arrays. +/// processWhere class can be used with this obs function. /// -/// Example : Create a new integer variable `emissivity@ObsDerived` and assign values based +/// This template is used to define four ObsFunctions, each producing values of a different type: +/// * `Conditional@ObsFunction` produces floats +/// * `Conditional@IntObsFunction` produces ints +/// * `Conditional@StringObsFunction` produces strings +/// * `Conditional@DateTimeObsFunction` produces datetimes. +/// +/// Example 1: Create a new floating-point variable `emissivity@ObsDerived` and assign values based /// on the surface type. /// -/// - filter: Variable Assignment -/// assignments: -/// - name: emissivity@ObsDerived -/// type: int -/// function: -/// name: Conditional@ObsFunction -/// options: -/// defaultvalue: 0.0 # default value - rttov to calculate. -/// cases: -/// - where: -/// - variable: -/// name: surface_type@MetaData -/// is_in: 1 -/// # if necessary, further conditions could be specified in extra items -/// # in the 'where' list -/// value: 0.3 -/// - where: -/// - variable: -/// name: surface_type@MetaData -/// is_in: 2 -/// value: 0.5 +/// - filter: Variable Assignment +/// assignments: +/// - name: emissivity@ObsDerived +/// type: float +/// function: +/// name: Conditional@ObsFunction +/// options: +/// defaultvalue: 0.0 # default value - rttov to calculate. +/// cases: +/// - where: +/// - variable: +/// name: surface_type@MetaData +/// is_in: 1 +/// # if necessary, further conditions could be specified in extra items +/// # in the 'where' list +/// value: 0.3 +/// - where: +/// - variable: +/// name: surface_type@MetaData +/// is_in: 2 +/// value: 0.5 +/// +/// Example 2: Create a new string variable `surface_description@MetaData` and set it to `land`, +/// `sea` or `unknown` depending on the value of the `surface_type@MetaData` variable. +/// +/// - filter: Variable Assignment +/// assignments: +/// - name: surface_description@MetaData +/// type: string +/// function: +/// name: Conditional@StringObsFunction +/// options: +/// defaultvalue: unknown +/// cases: +/// - where: +/// - variable: +/// name: surface_type@MetaData +/// is_in: 1 +/// value: land +/// - where: +/// - variable: +/// name: surface_type@MetaData +/// is_in: 2 +/// value: sea + /// -class Conditional : public ObsFunctionBase { +template +class Conditional : public ObsFunctionBase { public: explicit Conditional(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); void compute(const ObsFilterData &, - ioda::ObsDataVector &) const; + ioda::ObsDataVector &) const; const ufo::Variables & requiredVariables() const; private: ufo::Variables invars_; - ConditionalParameters options_; + ConditionalParameters options_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/obsfunctions/DrawObsErrorFromFile.h b/src/ufo/filters/obsfunctions/DrawObsErrorFromFile.h index cd5b29d18..6a05cdcd6 100644 --- a/src/ufo/filters/obsfunctions/DrawObsErrorFromFile.h +++ b/src/ufo/filters/obsfunctions/DrawObsErrorFromFile.h @@ -45,17 +45,15 @@ namespace ufo { /// /// Note that channel number extraction is implicit, using the channels selected and performed as /// an exact match before any user defined interpolation takes place. -class DrawObsErrorFromFile : public ObsFunctionBase { +class DrawObsErrorFromFile : public ObsFunctionBase { public: - static const std::string classname() {return "DrawObsErrorFromFile";} - explicit DrawObsErrorFromFile(const eckit::LocalConfiguration &); void compute(const ObsFilterData &, ioda::ObsDataVector &) const; const ufo::Variables & requiredVariables() const; private: - DrawValueFromFile drawValueFromFile_; + DrawValueFromFile drawValueFromFile_; }; } // namespace ufo diff --git a/src/ufo/filters/obsfunctions/DrawValueFromFile.cc b/src/ufo/filters/obsfunctions/DrawValueFromFile.cc index c93e33928..2f7f01c06 100644 --- a/src/ufo/filters/obsfunctions/DrawValueFromFile.cc +++ b/src/ufo/filters/obsfunctions/DrawValueFromFile.cc @@ -5,22 +5,61 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ #include "eckit/exception/Exceptions.h" -#include "oops/util/IntSetParser.h" #include "ufo/filters/obsfunctions/DrawValueFromFile.h" namespace ufo { +namespace { + +// ----------------------------------------------------------------------------- +typedef std::list, + std::vector, + std::vector> + >> ObData; + + +// ----------------------------------------------------------------------------- +/// \brief This is a convenience function for updating our container for useful observation data +template +void updateObData(const ObsFilterData &in, const Variable &var, ObData &obData) { + std::vector dat; + in.get(var, dat); + obData.emplace_back(var.fullName(), std::move(dat)); +} + +// ----------------------------------------------------------------------------- +/// \brief Add datetime observation information data to our container. +/// \details We simply convert the datetimes to strings as our implementation does not discriminate +/// between the two types. +void updateObDataDateTime(const ObsFilterData &in, const Variable &var, ObData &obData) { + std::vector dat; + std::vector datConv; + in.get(var, dat); + datConv.resize(dat.size()); + + // Convert the vec. of datetime. to strings + std::transform(dat.begin(), dat.end(), datConv.begin(), + [](util::DateTime dt){return dt.toString();}); + obData.emplace_back(var.fullName(), std::move(datConv)); +} + +} // anonymous namespace + +// ----------------------------------------------------------------------------- constexpr char InterpMethodParameterTraitsHelper::enumTypeName[]; constexpr util::NamedEnumerator InterpMethodParameterTraitsHelper::namedValues[]; -static ObsFunctionMaker - makerNetCDF_("DrawValueFromFile"); - +// ----------------------------------------------------------------------------- +static ObsFunctionMaker> floatMaker("DrawValueFromFile"); +static ObsFunctionMaker> intMaker("DrawValueFromFile"); +static ObsFunctionMaker> stringMaker("DrawValueFromFile"); // ----------------------------------------------------------------------------- -DrawValueFromFile::DrawValueFromFile(const eckit::LocalConfiguration &config) +// Could be moved to a non-templated base class +template +DrawValueFromFile::DrawValueFromFile(const eckit::LocalConfiguration &config) : allvars_() { // Initialize options options_.deserialize(config); @@ -50,12 +89,10 @@ DrawValueFromFile::DrawValueFromFile(const eckit::LocalConfiguration &config) // ----------------------------------------------------------------------------- -DrawValueFromFile::~DrawValueFromFile() {} - - +template class ExtractVisitor : public boost::static_visitor { public: - ExtractVisitor(DataExtractor &interpolator, + ExtractVisitor(DataExtractor &interpolator, const size_t &iloc) : interpolator(interpolator), iloc(iloc) {} @@ -65,17 +102,16 @@ class ExtractVisitor : public boost::static_visitor { interpolator.extract(obVal); } - DataExtractor &interpolator; + DataExtractor &interpolator; const size_t &iloc; }; // ----------------------------------------------------------------------------- -void DrawValueFromFile::compute(const ObsFilterData & in, - ioda::ObsDataVector & out) const { - const float missing = util::missingValue(missing); - - DataExtractor interpolator{fpath_, options_.group}; +template +void DrawValueFromFile::compute(const ObsFilterData & in, + ioda::ObsDataVector & out) const { + DataExtractor interpolator{fpath_, options_.group}; // Channel number handling if (options_.chlist.value() != boost::none) @@ -86,7 +122,7 @@ void DrawValueFromFile::compute(const ObsFilterData & in, oops::Log::debug() << "Extracting " << allvars_[ind].variable() << " from the obsSpace" << std::endl; - const std::string &varName = get_full_name(allvars_[ind]); + const std::string varName = allvars_[ind].fullName(); const InterpMethod &interpolationMethod = interpMethod_.at(varName); interpolator.scheduleSort(varName, interpolationMethod); switch (in.dtype(allvars_[ind])) { @@ -116,7 +152,7 @@ void DrawValueFromFile::compute(const ObsFilterData & in, // Perform any extraction methods (exact, nearest and linear interp.) for (auto &od : obData) { - ExtractVisitor visitor(interpolator, iloc); + ExtractVisitor visitor(interpolator, iloc); boost::apply_visitor(visitor, od.second); } out[jvar][iloc] = interpolator.getResult(); @@ -125,11 +161,15 @@ void DrawValueFromFile::compute(const ObsFilterData & in, } // ----------------------------------------------------------------------------- -const ufo::Variables & DrawValueFromFile::requiredVariables() const { +template +const ufo::Variables & DrawValueFromFile::requiredVariables() const { return allvars_; } - // ----------------------------------------------------------------------------- +// Explicit instantiations +template class DrawValueFromFile; +template class DrawValueFromFile; +template class DrawValueFromFile; } // namespace ufo diff --git a/src/ufo/filters/obsfunctions/DrawValueFromFile.h b/src/ufo/filters/obsfunctions/DrawValueFromFile.h index 1c16a47b5..9010a61ae 100644 --- a/src/ufo/filters/obsfunctions/DrawValueFromFile.h +++ b/src/ufo/filters/obsfunctions/DrawValueFromFile.h @@ -111,6 +111,9 @@ class DrawValueFromFileParameters : public DrawValueFromFileParametersWithoutGro /// \brief Produce values by interpolating an array loaded from a file, indexed by /// coordinates whose names correspond to ObsSpace variables. /// +/// \tparam T +/// Type of values produced by this ObsFunction. Must be `float`, `int` or `std::string`. +/// /// \details See DataExtractor for details on the format of this file. /// /// ### example configurations: ### @@ -137,60 +140,21 @@ class DrawValueFromFileParameters : public DrawValueFromFileParametersWithoutGro /// /// Note that channel number extraction is implicit, using the channels selected and performed as /// an exact match before any user defined interpolation takes place. -class DrawValueFromFile : public ObsFunctionBase { +template +class DrawValueFromFile : public ObsFunctionBase { public: - static const std::string classname() {return "DrawValueFromFile";} - explicit DrawValueFromFile(const eckit::LocalConfiguration &); - ~DrawValueFromFile(); void compute(const ObsFilterData &, - ioda::ObsDataVector &) const; + ioda::ObsDataVector &) const; const ufo::Variables & requiredVariables() const; private: - typedef std::list, - std::vector, - std::vector> - >> ObData; - Variables allvars_; std::unordered_map interpMethod_; std::string fpath_; DrawValueFromFileParameters options_; std::vector channels_; - - static std::string get_full_name(const ufo::Variable &variable) { - std::string name = variable.variable(); - if (variable.group().size() > 0) name += ("@"+variable.group()); - return name; - } - - /// \brief This is a convenience function for updating our container for useful observation data - template - static void updateObData(const ObsFilterData &in, const ufo::Variable &var, ObData &obData) { - std::vector dat; - in.get(var, dat); - std::string name = get_full_name(var); - obData.emplace_back(name, std::move(dat)); - } - - /// \brief Add datetime observation information data to our container. - /// \details We simply convert the datetimes to strings as our implementation is not discriminate - /// between the two types. - static void updateObDataDateTime(const ObsFilterData &in, const ufo::Variable &var, - ObData &obData) { - std::vector dat; - std::vector datConv; - in.get(var, dat); - datConv.resize(dat.size()); - - // Convert the vec. of datetime. to strings - std::transform(dat.begin(), dat.end(), datConv.begin(), - [](util::DateTime dt){return dt.toString();}); - std::string name = get_full_name(var); - obData.emplace_back(name, std::move(datConv)); - } }; } // namespace ufo diff --git a/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.h b/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.h index ac8b07cd0..7ae58a97d 100755 --- a/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.h +++ b/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.h @@ -77,7 +77,7 @@ class HydrometeorCheckAMSUAParameters : public oops::Parameters { /// 0 = channel is not affected by thick clouds and precipitation /// 1 = channel is affected by thick clouds and precipitataion /// -class HydrometeorCheckAMSUA : public ObsFunctionBase { +class HydrometeorCheckAMSUA : public ObsFunctionBase { public: explicit HydrometeorCheckAMSUA(const eckit::LocalConfiguration &); ~HydrometeorCheckAMSUA(); diff --git a/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.h b/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.h index fdd78a3ed..041f29f73 100755 --- a/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.h +++ b/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.h @@ -77,7 +77,7 @@ class HydrometeorCheckATMSParameters : public oops::Parameters { /// 0 = channel is not affected by thick clouds and precipitation /// 1 = channel is affected by thick clouds and precipitataion /// -class HydrometeorCheckATMS : public ObsFunctionBase { +class HydrometeorCheckATMS : public ObsFunctionBase { public: explicit HydrometeorCheckATMS(const eckit::LocalConfiguration &); ~HydrometeorCheckATMS(); diff --git a/src/ufo/filters/obsfunctions/ImpactHeight.h b/src/ufo/filters/obsfunctions/ImpactHeight.h index 890916521..73c191548 100644 --- a/src/ufo/filters/obsfunctions/ImpactHeight.h +++ b/src/ufo/filters/obsfunctions/ImpactHeight.h @@ -20,7 +20,7 @@ namespace ufo { /// \brief Function calculates the GNSS-RO impact height as the difference /// between the impact parameter and earth's radius of curvature. /// -class ImpactHeight : public ObsFunctionBase { +class ImpactHeight : public ObsFunctionBase { public: explicit ImpactHeight(const eckit::LocalConfiguration &); ~ImpactHeight(); diff --git a/src/ufo/filters/obsfunctions/InterChannelConsistencyCheck.h b/src/ufo/filters/obsfunctions/InterChannelConsistencyCheck.h index fbf383f8d..03556eb24 100755 --- a/src/ufo/filters/obsfunctions/InterChannelConsistencyCheck.h +++ b/src/ufo/filters/obsfunctions/InterChannelConsistencyCheck.h @@ -49,7 +49,7 @@ class InterChannelConsistencyCheckParameters : public oops::Parameters { /// /// \brief Inter-channel consistency check /// -class InterChannelConsistencyCheck : public ObsFunctionBase { +class InterChannelConsistencyCheck : public ObsFunctionBase { public: explicit InterChannelConsistencyCheck(const eckit::LocalConfiguration &); ~InterChannelConsistencyCheck(); diff --git a/src/ufo/filters/obsfunctions/LAMDomainCheck/LAMDomainCheck.h b/src/ufo/filters/obsfunctions/LAMDomainCheck/LAMDomainCheck.h index 46b87ced4..4d73fff70 100644 --- a/src/ufo/filters/obsfunctions/LAMDomainCheck/LAMDomainCheck.h +++ b/src/ufo/filters/obsfunctions/LAMDomainCheck/LAMDomainCheck.h @@ -49,7 +49,7 @@ class LAMDomainCheckParameters : public oops::Parameters { // ----------------------------------------------------------------------------- -class LAMDomainCheck : public ObsFunctionBase { +class LAMDomainCheck : public ObsFunctionBase { public: static const std::string classname() {return "LAMDomainCheck";} diff --git a/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.h b/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.h index 84ccd6bc2..5df56edf7 100644 --- a/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.h +++ b/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.h @@ -36,7 +36,7 @@ class ModelHeightAdjustedAirTemperatureParameters : public oops::Parameters { /// applied to the temperature is calculated using a standard lapse rate /// (Constants::Lclr). -class ModelHeightAdjustedAirTemperature : public ObsFunctionBase { +class ModelHeightAdjustedAirTemperature : public ObsFunctionBase { public: explicit ModelHeightAdjustedAirTemperature(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/NearSSTRetCheckIR.h b/src/ufo/filters/obsfunctions/NearSSTRetCheckIR.h index 85e034f6f..10e344496 100755 --- a/src/ufo/filters/obsfunctions/NearSSTRetCheckIR.h +++ b/src/ufo/filters/obsfunctions/NearSSTRetCheckIR.h @@ -57,7 +57,7 @@ class NearSSTRetCheckIRParameters : public oops::Parameters { /// 0 = channel is retained for assimilation /// 1 = channel is not retained for assimilation /// -class NearSSTRetCheckIR : public ObsFunctionBase { +class NearSSTRetCheckIR : public ObsFunctionBase { public: explicit NearSSTRetCheckIR(const eckit::LocalConfiguration &); ~NearSSTRetCheckIR(); diff --git a/src/ufo/filters/obsfunctions/ObsErrorBoundIR.h b/src/ufo/filters/obsfunctions/ObsErrorBoundIR.h index 7f6290868..4eb93f038 100755 --- a/src/ufo/filters/obsfunctions/ObsErrorBoundIR.h +++ b/src/ufo/filters/obsfunctions/ObsErrorBoundIR.h @@ -60,7 +60,7 @@ class ObsErrorBoundIRParameters : public oops::Parameters { /// Residual Threshold = MIN( (3.0 * ( 1 / Errflat )^2 * (1 / Errftaotop )^2), ErrobsMax ) /// Filter out data if |obs-h(x)| > Residual Threshold /// -class ObsErrorBoundIR : public ObsFunctionBase { +class ObsErrorBoundIR : public ObsFunctionBase { public: explicit ObsErrorBoundIR(const eckit::LocalConfiguration &); ~ObsErrorBoundIR(); diff --git a/src/ufo/filters/obsfunctions/ObsErrorBoundMW.h b/src/ufo/filters/obsfunctions/ObsErrorBoundMW.h index 72c5b5547..80cb530ae 100755 --- a/src/ufo/filters/obsfunctions/ObsErrorBoundMW.h +++ b/src/ufo/filters/obsfunctions/ObsErrorBoundMW.h @@ -68,7 +68,7 @@ class ObsErrorBoundMWParameters : public oops::Parameters { /// Residual Threshold = MIN( (3.0 * ( 1 / Errflat )^2 * (1 / Errftaotop )^2), ErrobsMax ) /// Filter out data if |obs-h(x)| > Residual Threshold /// -class ObsErrorBoundMW : public ObsFunctionBase { +class ObsErrorBoundMW : public ObsFunctionBase { public: explicit ObsErrorBoundMW(const eckit::LocalConfiguration &); ~ObsErrorBoundMW(); diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorConventional.h b/src/ufo/filters/obsfunctions/ObsErrorFactorConventional.h index 92a98fdf4..603fdfad3 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorConventional.h +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorConventional.h @@ -97,7 +97,7 @@ class ObsErrorFactorConventionalParameters : public oops::Parameters { /// sort variable: "air_pressure" /// sort order: "descending" /// -class ObsErrorFactorConventional : public ObsFunctionBase { +class ObsErrorFactorConventional : public ObsFunctionBase { public: static const std::string classname() {return "ObsErrorFactorConventional";} diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorLatRad.h b/src/ufo/filters/obsfunctions/ObsErrorFactorLatRad.h index 756a903cc..0d7cb961e 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorLatRad.h +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorLatRad.h @@ -41,7 +41,7 @@ class ObsErrorFactorLatRadParameters : public oops::Parameters { /// The function gives the maximum error bound reduction at equator and decreasing /// towards higher latitudes. /// -class ObsErrorFactorLatRad : public ObsFunctionBase { +class ObsErrorFactorLatRad : public ObsFunctionBase { public: explicit ObsErrorFactorLatRad(const eckit::LocalConfiguration &); ~ObsErrorFactorLatRad(); diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorQuotient.h b/src/ufo/filters/obsfunctions/ObsErrorFactorQuotient.h index 28767e8e4..2b0d9cfb1 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorQuotient.h +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorQuotient.h @@ -62,7 +62,7 @@ class ObsErrorFactorQuotientParameters : public oops::Parameters { /// name: air_temperature@ObsError /// defer to post: true # Likely necessary for order of filters /// -class ObsErrorFactorQuotient : public ObsFunctionBase { +class ObsErrorFactorQuotient : public ObsFunctionBase { public: static const std::string classname() {return "ObsErrorFactorQuotient";} diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.h b/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.h index b6ac851db..d13cbe9d7 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.h +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.h @@ -69,7 +69,7 @@ class ObsErrorFactorSfcPressureParameters : public oops::Parameters { /// error_max: 300 # 3 mb /// geovar_sfc_geomz: surface_geopotential_height # default is surface_altitude /// -class ObsErrorFactorSfcPressure : public ObsFunctionBase { +class ObsErrorFactorSfcPressure : public ObsFunctionBase { public: static const std::string classname() {return "ObsErrorFactorSfcPressure";} diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorSituDependMW.h b/src/ufo/filters/obsfunctions/ObsErrorFactorSituDependMW.h index fa721d25d..ca774e617 100755 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorSituDependMW.h +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorSituDependMW.h @@ -67,7 +67,7 @@ class ObsErrorFactorSituDependMWParameters : public oops::Parameters { /// retrieved cloud liquid water from background and observation, scattering index, /// surface wind speed, and cloud information match index over the ocean /// -class ObsErrorFactorSituDependMW : public ObsFunctionBase { +class ObsErrorFactorSituDependMW : public ObsFunctionBase { public: explicit ObsErrorFactorSituDependMW(const eckit::LocalConfiguration &); ~ObsErrorFactorSituDependMW(); diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorSurfJacobianRad.h b/src/ufo/filters/obsfunctions/ObsErrorFactorSurfJacobianRad.h index 3e4f10cdc..95373346b 100755 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorSurfJacobianRad.h +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorSurfJacobianRad.h @@ -58,7 +58,7 @@ class ObsErrorFactorSurfJacobianRadParameters : public oops::Parameters { /// Errinv = inverse of effective observation error variance /// EIF = SQRT [ 1 / ( 1 / (1 + Errinv * Beta) ] /// -class ObsErrorFactorSurfJacobianRad : public ObsFunctionBase { +class ObsErrorFactorSurfJacobianRad : public ObsFunctionBase { public: explicit ObsErrorFactorSurfJacobianRad(const eckit::LocalConfiguration &); ~ObsErrorFactorSurfJacobianRad(); diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.h b/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.h index 38fb2da78..7ed9d14ad 100755 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.h +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.h @@ -53,7 +53,7 @@ class ObsErrorFactorTopoRadParameters : public oops::Parameters { /// EIF = SQRT [ 1 / ( 2000 / H ) ] for 2000 < H < 4000 and Channels 1-6,15 /// EIF = SQRT [ 1 / ( 4000 / H ) ] for H > 4000 and Channel 7 /// -class ObsErrorFactorTopoRad : public ObsFunctionBase { +class ObsErrorFactorTopoRad : public ObsFunctionBase { public: explicit ObsErrorFactorTopoRad(const eckit::LocalConfiguration &); ~ObsErrorFactorTopoRad(); diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorTransmitTopRad.h b/src/ufo/filters/obsfunctions/ObsErrorFactorTransmitTopRad.h index b9cd4fe1c..c432e1d42 100755 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorTransmitTopRad.h +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorTransmitTopRad.h @@ -39,7 +39,7 @@ class ObsErrorFactorTransmitTopRadParameters : public oops::Parameters { /// tao = model top-to-space transmittance /// EIF = SQRT ( 1.0 / tao ) /// -class ObsErrorFactorTransmitTopRad : public ObsFunctionBase { +class ObsErrorFactorTransmitTopRad : public ObsFunctionBase { public: explicit ObsErrorFactorTransmitTopRad(const eckit::LocalConfiguration &); ~ObsErrorFactorTransmitTopRad(); diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorWavenumIR.h b/src/ufo/filters/obsfunctions/ObsErrorFactorWavenumIR.h index 7b9cfc5cd..231a719fb 100755 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorWavenumIR.h +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorWavenumIR.h @@ -42,7 +42,7 @@ class ObsErrorFactorWavenumIRParameters : public oops::Parameters { /// z = solar zenith angle [radian] /// EIF = SQRT[ 1 / ( 1 - (x - 2000)) * y * MAX(0, COS(z)) / 4000 ] /// -class ObsErrorFactorWavenumIR : public ObsFunctionBase { +class ObsErrorFactorWavenumIR : public ObsFunctionBase { public: explicit ObsErrorFactorWavenumIR(const eckit::LocalConfiguration &); ~ObsErrorFactorWavenumIR(); diff --git a/src/ufo/filters/obsfunctions/ObsErrorModelQuad.h b/src/ufo/filters/obsfunctions/ObsErrorModelQuad.h index 096b1fa44..2b330ab9e 100755 --- a/src/ufo/filters/obsfunctions/ObsErrorModelQuad.h +++ b/src/ufo/filters/obsfunctions/ObsErrorModelQuad.h @@ -113,7 +113,7 @@ class ObsErrorModelQuadParameters : public oops::Parameters { /// err1: [17.0, 20.5, 21.1] /// {save: true} /// -class ObsErrorModelQuad : public ObsFunctionBase { +class ObsErrorModelQuad : public ObsFunctionBase { public: static const std::string classname() {return "ObsErrorModelQuad";} diff --git a/src/ufo/filters/obsfunctions/ObsErrorModelRamp.h b/src/ufo/filters/obsfunctions/ObsErrorModelRamp.h index 94e15b177..a16c9953d 100755 --- a/src/ufo/filters/obsfunctions/ObsErrorModelRamp.h +++ b/src/ufo/filters/obsfunctions/ObsErrorModelRamp.h @@ -175,7 +175,7 @@ class ObsErrorModelRampParameters : public oops::Parameters { /// err0: [{ERR0}] /// err1: [{ERR1}] /// -class ObsErrorModelRamp : public ObsFunctionBase { +class ObsErrorModelRamp : public ObsFunctionBase { public: static const std::string classname() {return "ObsErrorModelRamp";} diff --git a/src/ufo/filters/obsfunctions/ObsErrorModelStepwiseLinear.h b/src/ufo/filters/obsfunctions/ObsErrorModelStepwiseLinear.h index cfaf0ead3..fcafea498 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorModelStepwiseLinear.h +++ b/src/ufo/filters/obsfunctions/ObsErrorModelStepwiseLinear.h @@ -91,7 +91,7 @@ class ObsErrorModelStepwiseLinearParameters : public oops::Parameters { /// xvals: [110000, 85000, 50000, 25000, 10000, 1] #Pressure (Pa) /// errors: [1.1, 1.3, 1.8, 2.4, 4.0, 4.5] /// -class ObsErrorModelStepwiseLinear : public ObsFunctionBase { +class ObsErrorModelStepwiseLinear : public ObsFunctionBase { public: static const std::string classname() {return "ObsErrorModelStepwiseLinear";} diff --git a/src/ufo/filters/obsfunctions/ObsFunction.cc b/src/ufo/filters/obsfunctions/ObsFunction.cc index 89e7bd523..7ecd3ea40 100644 --- a/src/ufo/filters/obsfunctions/ObsFunction.cc +++ b/src/ufo/filters/obsfunctions/ObsFunction.cc @@ -8,33 +8,46 @@ #include "ufo/filters/obsfunctions/ObsFunction.h" #include "ioda/ObsDataVector.h" +#include "oops/util/DateTime.h" #include "ufo/filters/Variables.h" namespace ufo { // ----------------------------------------------------------------------------- -ObsFunction::ObsFunction(const Variable & var) - : obsfct_(ObsFunctionFactory::create(var)) +template +ObsFunction::ObsFunction(const Variable & var) + : obsfct_(ObsFunctionFactory::create(var)) {} // ----------------------------------------------------------------------------- -ObsFunction::~ObsFunction() {} +template +ObsFunction::~ObsFunction() {} // ----------------------------------------------------------------------------- -void ObsFunction::compute(const ObsFilterData & in, - ioda::ObsDataVector & out) const { +template +void ObsFunction::compute(const ObsFilterData & in, + ioda::ObsDataVector & out) const { obsfct_->compute(in, out); } // ----------------------------------------------------------------------------- -const ufo::Variables & ObsFunction::requiredVariables() const { +template +const ufo::Variables & ObsFunction::requiredVariables() const { return obsfct_->requiredVariables(); } // ----------------------------------------------------------------------------- +// Explicit instantiations for the supported value types +template class ObsFunction; +template class ObsFunction; +template class ObsFunction; +template class ObsFunction; + +// ----------------------------------------------------------------------------- + } // namespace ufo diff --git a/src/ufo/filters/obsfunctions/ObsFunction.h b/src/ufo/filters/obsfunctions/ObsFunction.h index 29e4b9fdd..b22413db8 100644 --- a/src/ufo/filters/obsfunctions/ObsFunction.h +++ b/src/ufo/filters/obsfunctions/ObsFunction.h @@ -23,6 +23,12 @@ namespace ufo { // ----------------------------------------------------------------------------- +/// \brief A function of observation data. +/// +/// \tparam FunctionValue +/// Type of the values produced by the function. Must be `float`, `int`, `std::string` +/// or `util::DateTime`. +template class ObsFunction : private boost::noncopyable { public: /// constructor takes function name (for factory) on input @@ -31,11 +37,12 @@ class ObsFunction : private boost::noncopyable { /// compute(metadata, obs values, output) void compute(const ObsFilterData &, - ioda::ObsDataVector &) const; + ioda::ObsDataVector &) const; /// required variables const ufo::Variables & requiredVariables() const; + private: - std::unique_ptr obsfct_; + std::unique_ptr> obsfct_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/obsfunctions/ObsFunctionBase.cc b/src/ufo/filters/obsfunctions/ObsFunctionBase.cc index 7c69ed980..b5b044287 100644 --- a/src/ufo/filters/obsfunctions/ObsFunctionBase.cc +++ b/src/ufo/filters/obsfunctions/ObsFunctionBase.cc @@ -10,43 +10,66 @@ #include #include -#include "oops/util/abor1_cpp.h" +#include "eckit/exception/Exceptions.h" +#include "oops/util/DateTime.h" #include "oops/util/Logger.h" namespace ufo { +const char *ObsFunctionTraits::valueTypeName = "float"; +const char *ObsFunctionTraits::valueTypeName = "int"; +const char *ObsFunctionTraits::valueTypeName = "std::string"; +const char *ObsFunctionTraits::valueTypeName = "util::DateTime"; + +const char *ObsFunctionTraits::groupName = "ObsFunction"; +const char *ObsFunctionTraits::groupName = "IntObsFunction"; +const char *ObsFunctionTraits::groupName = "StringObsFunction"; +const char *ObsFunctionTraits::groupName = "DateTimeObsFunction"; + // ----------------------------------------------------------------------------- -ObsFunctionFactory::ObsFunctionFactory(const std::string & name) { +template +ObsFunctionFactory::ObsFunctionFactory(const std::string & name) { if (getMakers().find(name) != getMakers().end()) { - oops::Log::error() << name << " already registered in ufo::ObsFunctionFactory." << std::endl; - ABORT("Element already registered in ufo::ObsFunctionFactory."); + throw eckit::UserError(name + " already registered in ufo::ObsFunctionFactory<" + + std::string(ObsFunctionTraits::valueTypeName) + ">", + Here()); } getMakers()[name] = this; } // ----------------------------------------------------------------------------- -ObsFunctionBase * ObsFunctionFactory::create(const Variable & var) { +template +ObsFunctionBase * ObsFunctionFactory::create(const Variable & var) { oops::Log::trace() << "ObsFunctionBase::create starting" << std::endl; typename std::map::iterator jloc = getMakers().find(var.variable()); if (jloc == getMakers().end()) { - oops::Log::error() << var.variable() << " does not exist in ufo::ObsFunctionFactory." - << std::endl; - ABORT("Element does not exist in ufo::ObsFunctionFactory."); + throw eckit::UserError(var.variable() + " does not exist in ufo::ObsFunctionFactory<" + + std::string(ObsFunctionTraits::valueTypeName) + ">", + Here()); } - ObsFunctionBase * ptr = jloc->second->make(var.options()); - oops::Log::trace() << "ObsFunctionBase::create done" << std::endl; + ObsFunctionBase * ptr = jloc->second->make(var.options()); + oops::Log::trace() << "ObsFunctionFactory::create done" << std::endl; return ptr; } // ----------------------------------------------------------------------------- -bool ObsFunctionFactory::functionExists(const std::string & name) { +template +bool ObsFunctionFactory::functionExists(const std::string & name) { return (getMakers().find(name) != getMakers().end()); } // ----------------------------------------------------------------------------- +// Explicit instantiations for the supported value types +template class ObsFunctionFactory; +template class ObsFunctionFactory; +template class ObsFunctionFactory; +template class ObsFunctionFactory; + +// ----------------------------------------------------------------------------- + } // namespace ufo diff --git a/src/ufo/filters/obsfunctions/ObsFunctionBase.h b/src/ufo/filters/obsfunctions/ObsFunctionBase.h index c4a205b75..9f8667957 100644 --- a/src/ufo/filters/obsfunctions/ObsFunctionBase.h +++ b/src/ufo/filters/obsfunctions/ObsFunctionBase.h @@ -22,15 +22,22 @@ class Variables; // ----------------------------------------------------------------------------- /// Base class for computing functions on observation data - +/// +/// \tparam FunctionValue +/// Type of the values produced by the function. Must be `float`, `int`, `std::string` +/// or `util::DateTime`. +template class ObsFunctionBase : private boost::noncopyable { public: + /// Type of the values produced by the function. + typedef FunctionValue Value_; + explicit ObsFunctionBase(const eckit::LocalConfiguration conf = eckit::LocalConfiguration()) {} virtual ~ObsFunctionBase() {} /// compute the result of the function virtual void compute(const ObsFilterData &, - ioda::ObsDataVector &) const = 0; + ioda::ObsDataVector &) const = 0; /// geovals required to compute the function virtual const ufo::Variables & requiredVariables() const = 0; @@ -38,16 +45,55 @@ class ObsFunctionBase : private boost::noncopyable { // ----------------------------------------------------------------------------- -/// Obs Function Factory +/// \brief Common properties of ObsFunctions producing values of type `FunctionValue`. +template +struct ObsFunctionTraits; + +template <> +struct ObsFunctionTraits{ + /// Name of the type of values produced by subclasses of ObsFunctionBase. + static const char *valueTypeName; + /// Name of the group identifying ObsFunctions producing floats. + static const char *groupName; +}; + +template <> +struct ObsFunctionTraits{ + /// Name of the type of values produced by subclasses of ObsFunctionBase. + static const char *valueTypeName; + /// Name of the group identifying ObsFunctions producing ints. + static const char *groupName; +}; + +template <> +struct ObsFunctionTraits{ + /// Name of the type of values produced by subclasses of ObsFunctionBase. + static const char *valueTypeName; + /// Name of the group identifying ObsFunctions producing strings. + static const char *groupName; +}; + +template <> +struct ObsFunctionTraits{ + /// Name of the type of values produced by subclasses of ObsFunctionBase. + static const char *valueTypeName; + /// Name of the group identifying ObsFunctions producing datetimes. + static const char *groupName; +}; + +// ----------------------------------------------------------------------------- + +/// Factory of ObsFunctions producing values of type `FunctionValue`. +template class ObsFunctionFactory { public: - static ObsFunctionBase * create(const Variable &); + static ObsFunctionBase * create(const Variable &); virtual ~ObsFunctionFactory() = default; static bool functionExists(const std::string &); protected: explicit ObsFunctionFactory(const std::string &); private: - virtual ObsFunctionBase * make(const eckit::LocalConfiguration conf) = 0; + virtual ObsFunctionBase * make(const eckit::LocalConfiguration conf) = 0; static std::map < std::string, ObsFunctionFactory * > & getMakers() { static std::map < std::string, ObsFunctionFactory * > makers_; return makers_; @@ -56,13 +102,16 @@ class ObsFunctionFactory { // ----------------------------------------------------------------------------- -template -class ObsFunctionMaker : public ObsFunctionFactory { - virtual ObsFunctionBase * make(const eckit::LocalConfiguration conf) +template +class ObsFunctionMaker : public ObsFunctionFactory { + typedef typename T::Value_ Value_; + typedef ObsFunctionFactory Factory_; + + virtual ObsFunctionBase * make(const eckit::LocalConfiguration conf) { return new T(conf); } public: explicit ObsFunctionMaker(const std::string & name) - : ObsFunctionFactory(name) {} + : Factory_(name) {} }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/obsfunctions/ObsFunctionLinearCombination.h b/src/ufo/filters/obsfunctions/ObsFunctionLinearCombination.h index c68726442..b9d279b92 100644 --- a/src/ufo/filters/obsfunctions/ObsFunctionLinearCombination.h +++ b/src/ufo/filters/obsfunctions/ObsFunctionLinearCombination.h @@ -54,7 +54,7 @@ class LinearCombinationParameters : public oops::Parameters { /// -class LinearCombination : public ObsFunctionBase { +class LinearCombination : public ObsFunctionBase { public: explicit LinearCombination(const eckit::LocalConfiguration &); ~LinearCombination(); diff --git a/src/ufo/filters/obsfunctions/ObsFunctionScattering.h b/src/ufo/filters/obsfunctions/ObsFunctionScattering.h index 37600e078..91abae36b 100644 --- a/src/ufo/filters/obsfunctions/ObsFunctionScattering.h +++ b/src/ufo/filters/obsfunctions/ObsFunctionScattering.h @@ -16,7 +16,7 @@ namespace ufo { // ----------------------------------------------------------------------------- -class ObsFunctionScattering : public ObsFunctionBase { +class ObsFunctionScattering : public ObsFunctionBase { public: explicit ObsFunctionScattering(const eckit::LocalConfiguration conf = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/ObsFunctionVelocity.h b/src/ufo/filters/obsfunctions/ObsFunctionVelocity.h index 238db156a..a1d2e77f4 100644 --- a/src/ufo/filters/obsfunctions/ObsFunctionVelocity.h +++ b/src/ufo/filters/obsfunctions/ObsFunctionVelocity.h @@ -18,7 +18,7 @@ namespace ufo { // ----------------------------------------------------------------------------- -class ObsFunctionVelocity : public ObsFunctionBase { +class ObsFunctionVelocity : public ObsFunctionBase { public: explicit ObsFunctionVelocity(const eckit::LocalConfiguration); ~ObsFunctionVelocity(); diff --git a/src/ufo/filters/obsfunctions/SCATRetMW.h b/src/ufo/filters/obsfunctions/SCATRetMW.h index ad47391a7..09541c700 100755 --- a/src/ufo/filters/obsfunctions/SCATRetMW.h +++ b/src/ufo/filters/obsfunctions/SCATRetMW.h @@ -77,7 +77,7 @@ class SCATRetMWParameters : public oops::Parameters { /// Application of AMSU for obtaining hydrological parameters /// Microw. Radiomet. Remote Sens. Eatch's Surf. Atmosphere, pp. 339-351 /// -class SCATRetMW : public ObsFunctionBase { +class SCATRetMW : public ObsFunctionBase { public: explicit SCATRetMW(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/SIRetMW.h b/src/ufo/filters/obsfunctions/SIRetMW.h index e73e1a455..54735436a 100755 --- a/src/ufo/filters/obsfunctions/SIRetMW.h +++ b/src/ufo/filters/obsfunctions/SIRetMW.h @@ -69,7 +69,7 @@ class SIRetMWParameters : public oops::Parameters { /// Geer, A. J., Fabrizio, B., Bormann, N., & English, S. (2014). All-sky assimilation of /// microwave humidity sounders. European Centre for Medium-Range Weather Forecasts. /// -class SIRetMW : public ObsFunctionBase { +class SIRetMW : public ObsFunctionBase { public: explicit SIRetMW(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/SIRetSymmetricMW.h b/src/ufo/filters/obsfunctions/SIRetSymmetricMW.h index 95231c430..ad07c6971 100755 --- a/src/ufo/filters/obsfunctions/SIRetSymmetricMW.h +++ b/src/ufo/filters/obsfunctions/SIRetSymmetricMW.h @@ -31,7 +31,7 @@ typedef SIRetMWParameters SIRetSymmetricMWParameters; /// \brief Calculate symmetric (mean) cloud amount from the cloud amount retrieved /// from the observed and simulated measurements /// -class SIRetSymmetricMW : public ObsFunctionBase { +class SIRetSymmetricMW : public ObsFunctionBase { public: explicit SIRetSymmetricMW(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/SatWindsLNVDCheck.h b/src/ufo/filters/obsfunctions/SatWindsLNVDCheck.h index 6258d24c8..1f327c059 100644 --- a/src/ufo/filters/obsfunctions/SatWindsLNVDCheck.h +++ b/src/ufo/filters/obsfunctions/SatWindsLNVDCheck.h @@ -49,7 +49,7 @@ class SatWindsLNVDCheckParameters : public oops::Parameters { /// test_hofx: GsiHofX /// maxvalue: 3 /// -class SatWindsLNVDCheck : public ObsFunctionBase { +class SatWindsLNVDCheck : public ObsFunctionBase { public: static const std::string classname() {return "SatWindsLNVDCheck";} diff --git a/src/ufo/filters/obsfunctions/SatWindsSPDBCheck.h b/src/ufo/filters/obsfunctions/SatWindsSPDBCheck.h index b058b2579..e3017a499 100644 --- a/src/ufo/filters/obsfunctions/SatWindsSPDBCheck.h +++ b/src/ufo/filters/obsfunctions/SatWindsSPDBCheck.h @@ -64,7 +64,7 @@ class SatWindsSPDBCheckParameters : public oops::Parameters { /// error_max: 20.0 /// maxvalue: 1.75 # gross error * 0.7 /// -class SatWindsSPDBCheck : public ObsFunctionBase { +class SatWindsSPDBCheck : public ObsFunctionBase { public: static const std::string classname() {return "SatWindsSPDBCheck";} diff --git a/src/ufo/filters/obsfunctions/SatwindIndivErrors.h b/src/ufo/filters/obsfunctions/SatwindIndivErrors.h index b9642ce85..63efe73fb 100644 --- a/src/ufo/filters/obsfunctions/SatwindIndivErrors.h +++ b/src/ufo/filters/obsfunctions/SatwindIndivErrors.h @@ -52,7 +52,7 @@ class SatwindIndivErrorsParameters : public oops::Parameters { /// dependent on an input height (pressure) error estimate and the wind shear. /// -class SatwindIndivErrors : public ObsFunctionBase { +class SatwindIndivErrors : public ObsFunctionBase { public: explicit SatwindIndivErrors(const eckit::LocalConfiguration &); ~SatwindIndivErrors(); diff --git a/src/ufo/filters/obsfunctions/SetSurfaceType.h b/src/ufo/filters/obsfunctions/SetSurfaceType.h index 4f2596a2d..92041e61e 100644 --- a/src/ufo/filters/obsfunctions/SetSurfaceType.h +++ b/src/ufo/filters/obsfunctions/SetSurfaceType.h @@ -79,7 +79,7 @@ class SetSurfaceTypeParameters : public oops::Parameters { /// /// \brief Set observation operator surface type based on model and observation data /// -class SetSurfaceType : public ObsFunctionBase { +class SetSurfaceType : public ObsFunctionBase { public: explicit SetSurfaceType(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/SunGlintAngle.h b/src/ufo/filters/obsfunctions/SunGlintAngle.h index f4c29d357..1c9b6aecd 100644 --- a/src/ufo/filters/obsfunctions/SunGlintAngle.h +++ b/src/ufo/filters/obsfunctions/SunGlintAngle.h @@ -20,7 +20,7 @@ namespace ufo { /// /// \brief Calculate Sun glint angles at observation locations. /// -class SunGlintAngle : public ObsFunctionBase { +class SunGlintAngle : public ObsFunctionBase { public: explicit SunGlintAngle(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/SymmCldImpactIR.h b/src/ufo/filters/obsfunctions/SymmCldImpactIR.h index 8e318c3b1..451d06e23 100755 --- a/src/ufo/filters/obsfunctions/SymmCldImpactIR.h +++ b/src/ufo/filters/obsfunctions/SymmCldImpactIR.h @@ -37,7 +37,7 @@ class SymmCldImpactIRParameters : public oops::Parameters { /// Okamoto, K., McNally, A.P. and Bell, W. (2014), Progress towards the /// assimilation of all‐sky infrared radiances: an evaluation of cloud /// effects. Q.J.R. Meteorol. Soc., 140: 1603-1614. doi:10.1002/qj.2242 -class SymmCldImpactIR : public ObsFunctionBase { +class SymmCldImpactIR : public ObsFunctionBase { public: explicit SymmCldImpactIR(const eckit::LocalConfiguration); ~SymmCldImpactIR(); diff --git a/src/ufo/filters/obsfunctions/TotalColumnVaporGuess.h b/src/ufo/filters/obsfunctions/TotalColumnVaporGuess.h index 74f6a9927..5c5e73151 100644 --- a/src/ufo/filters/obsfunctions/TotalColumnVaporGuess.h +++ b/src/ufo/filters/obsfunctions/TotalColumnVaporGuess.h @@ -24,7 +24,7 @@ namespace ufo { /// /// \brief Calculate column water vapor mass in guess at observation locations. /// -class TotalColumnVaporGuess : public ObsFunctionBase { +class TotalColumnVaporGuess : public ObsFunctionBase { public: explicit TotalColumnVaporGuess(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); diff --git a/src/ufo/filters/obsfunctions/TropopauseEstimate.h b/src/ufo/filters/obsfunctions/TropopauseEstimate.h index 0108d7e27..2e8679247 100644 --- a/src/ufo/filters/obsfunctions/TropopauseEstimate.h +++ b/src/ufo/filters/obsfunctions/TropopauseEstimate.h @@ -68,7 +68,7 @@ class TropopauseEstimateParameters : public oops::Parameters { /// value: air_pressure@MetaData /// minvalue: -5000 # 50 hPa above tropopause level, negative p-diff /// -class TropopauseEstimate : public ObsFunctionBase { +class TropopauseEstimate : public ObsFunctionBase { public: static const std::string classname() {return "TropopauseEstimate";} diff --git a/src/ufo/filters/obsfunctions/WindDirAngleDiff.h b/src/ufo/filters/obsfunctions/WindDirAngleDiff.h index 84780a82a..aad9bc4cc 100644 --- a/src/ufo/filters/obsfunctions/WindDirAngleDiff.h +++ b/src/ufo/filters/obsfunctions/WindDirAngleDiff.h @@ -51,7 +51,7 @@ class WindDirAngleDiffParameters : public oops::Parameters { /// test_hofx: GsiHofX /// maxvalue: 50 /// -class WindDirAngleDiff : public ObsFunctionBase { +class WindDirAngleDiff : public ObsFunctionBase { public: static const std::string classname() {return "WindDirAngleDiff";} diff --git a/src/ufo/predictors/InterpolateDataFromFile.cc b/src/ufo/predictors/InterpolateDataFromFile.cc index aadcd5bea..995ed1fe3 100644 --- a/src/ufo/predictors/InterpolateDataFromFile.cc +++ b/src/ufo/predictors/InterpolateDataFromFile.cc @@ -67,11 +67,11 @@ InterpolateDataFromFile::InterpolateDataFromFile(const Parameters_ & parameters, "' is not in the list of bias-corrected variables", Here()); eckit::LocalConfiguration varConfig = varParams.details.toConfiguration(); varConfig.set("group", "ObsBias"); - obsFunctions_[varParams.name] = boost::make_unique(varConfig); + obsFunctions_[varParams.name] = boost::make_unique>(varConfig); } for (const auto &varAndObsFunction : obsFunctions_) { - const DrawValueFromFile &obsFunction = *varAndObsFunction.second; + const DrawValueFromFile &obsFunction = *varAndObsFunction.second; const ufo::Variables &requiredVariables = obsFunction.requiredVariables(); geovars_ += requiredVariables.allFromGroup("GeoVaLs").toOopsVariables(); hdiags_ += requiredVariables.allFromGroup("ObsDiag").toOopsVariables(); @@ -90,7 +90,7 @@ void InterpolateDataFromFile::compute(const ioda::ObsSpace & /*odb*/, for (const auto &varAndObsFunction : obsFunctions_) { const std::string &varName = varAndObsFunction.first; - const DrawValueFromFile &obsFunction = *varAndObsFunction.second; + const DrawValueFromFile &obsFunction = *varAndObsFunction.second; oops::Variables currentVars({varName}, vars_.channels()); ioda::ObsDataVector obsFunctionResult(out.space(), currentVars); diff --git a/src/ufo/predictors/InterpolateDataFromFile.h b/src/ufo/predictors/InterpolateDataFromFile.h index 3b7608ca3..9903eeffb 100644 --- a/src/ufo/predictors/InterpolateDataFromFile.h +++ b/src/ufo/predictors/InterpolateDataFromFile.h @@ -90,7 +90,7 @@ class InterpolateDataFromFile : public PredictorBase { // (http://www.open-std.org/jtc1/sc22/wg21/docs/lwg-defects.html#2397), later resolved by // amending the C++11 standard as described in N4387 // (http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4387.html). - std::map> obsFunctions_; + std::map>> obsFunctions_; }; } // namespace ufo diff --git a/src/ufo/utils/CMakeLists.txt b/src/ufo/utils/CMakeLists.txt index 288abc310..6980e7b13 100644 --- a/src/ufo/utils/CMakeLists.txt +++ b/src/ufo/utils/CMakeLists.txt @@ -6,6 +6,7 @@ set ( utils_files ArrowProxy.h Constants.h + dataextractor/ConstrainedRange.h dataextractor/DataExtractor.h dataextractor/DataExtractor.cc dataextractor/DataExtractorBackend.h diff --git a/src/ufo/utils/PrimitiveVariables.cc b/src/ufo/utils/PrimitiveVariables.cc index 8262607a1..ad41f0fc9 100644 --- a/src/ufo/utils/PrimitiveVariables.cc +++ b/src/ufo/utils/PrimitiveVariables.cc @@ -5,6 +5,7 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ +#include "eckit/utils/StringTools.h" #include "ufo/filters/ObsFilterData.h" #include "ufo/utils/PrimitiveVariables.h" @@ -14,7 +15,7 @@ void PrimitiveVariablesIterator::loadCurrentVariable() { if (variableIndex_ < variables_.size()) { const Variable &variable = variables_[variableIndex_]; vector_.reset(new ioda::ObsDataVector(data_.obsspace(), variable.toOopsVariables())); - if (variable.group() == "ObsFunction") { + if (eckit::StringTools::endsWith(variable.group(), "ObsFunction")) { data_.get(variable, *vector_); } else { for (size_t i = 0; i < variable.size(); ++i) { diff --git a/src/ufo/utils/dataextractor/ConstrainedRange.h b/src/ufo/utils/dataextractor/ConstrainedRange.h new file mode 100644 index 000000000..5b0b506a5 --- /dev/null +++ b/src/ufo/utils/dataextractor/ConstrainedRange.h @@ -0,0 +1,55 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_UTILS_DATAEXTRACTOR_CONSTRAINEDRANGE_H_ +#define UFO_UTILS_DATAEXTRACTOR_CONSTRAINEDRANGE_H_ + +#include + +namespace ufo { + +/// \brief A range of indices. +class ConstrainedRange { + public: + /// \brief Create an unconstrained range of size `size`. + explicit ConstrainedRange(int size = 0) : size_(size) { + reset(); + } + + /// \brief Return the index of the first element in the range. + int begin() const { return begin_; } + /// \brief Return the index of the element past the end of the range. + int end() const { return end_; } + + /// \brief Return the range length. + int size() const { return end_ - begin_; } + /// \brief Return true if the range is empty, false otherwise. + bool empty() const { return end_ == begin_; } + + /// \brief Constrain the range. + void constrain(int newBegin, int newEnd) { + assert(newBegin >= begin_); + assert(newEnd <= end_); + begin_ = newBegin; + end_ = newEnd; + } + + /// \brief Remove any constraints, resetting the range to its original size. + void reset() { + begin_ = 0; + end_ = size_; + } + + private: + int size_; + int begin_; + int end_; +}; + +} // namespace ufo + +#endif // UFO_UTILS_DATAEXTRACTOR_CONSTRAINEDRANGE_H_ diff --git a/src/ufo/utils/dataextractor/DataExtractor.cc b/src/ufo/utils/dataextractor/DataExtractor.cc index 174b08500..2243aef83 100644 --- a/src/ufo/utils/dataextractor/DataExtractor.cc +++ b/src/ufo/utils/dataextractor/DataExtractor.cc @@ -5,17 +5,32 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ +#include // sort +#include // greater +#include // std::numeric_limits +#include // list +#include // stringstream +#include // pair + #include +#include "eckit/exception/Exceptions.h" #include "eckit/utils/StringTools.h" #include "ioda/Misc/StringFuncs.h" +#include "oops/util/Logger.h" +#include "oops/util/missingValues.h" + #include "ufo/utils/dataextractor/DataExtractor.h" #include "ufo/utils/dataextractor/DataExtractorCSVBackend.h" #include "ufo/utils/dataextractor/DataExtractorInput.h" #include "ufo/utils/dataextractor/DataExtractorNetCDFBackend.h" +namespace ufo { + +namespace { + /// \brief Boost visitor which allows us to sort a vector. class SortUpdateVisitor : public boost::static_visitor { public: @@ -59,45 +74,401 @@ class SortVisitor : public boost::static_visitor { }; -namespace ufo { +/// \brief Update our extract constraint based on an exact match against the specified coordinate +/// indexing a dimension of the payload array. +/// +/// \param[in] varName +/// Name of the coordinate to match against. +/// \param[in] varValues +/// Vector of values of that coordinate. +/// \param[in] obVal +/// Value to match. +/// \param[inout] range +/// On input, the range of slices of the payload array along the dimension indexed by +/// `varValues` that matches all constraints considered so far. On output, the subrange of +/// slices matching also the current constraint. +template +void exactMatch(const std::string &varName, + const std::vector &varValues, + const T &obVal, + ConstrainedRange &range) { + // Find the first and last matching index + auto bounds = std::equal_range(varValues.begin() + range.begin(), + varValues.begin() + range.end(), + obVal); + if (bounds.first == bounds.second) { + // No matching coordinate found. If the coordinate contains a 'missing value' entry, + // use it as a fallback. (If it doesn't, the 'bounds' range will stay empty, so an error will + // be reported). + bounds = std::equal_range(varValues.begin() + range.begin(), + varValues.begin() + range.end(), + util::missingValue(obVal)); + } + + range.constrain(static_cast(bounds.first - varValues.begin()), + static_cast(bounds.second - varValues.begin())); + + if (range.begin() == range.end()) { + std::stringstream msg; + msg << "No match found for exact match extraction of value '" << obVal + << "' of the variable '" << varName << "'"; + throw eckit::Exception(msg.str(), Here()); + } + oops::Log::debug() << "Exact match; name: " << varName << " range: " << + range.begin() << "," << range.end() << std::endl; +} + + +/// \brief Update our extract constraint based on a nearest match against the specified +/// coordinate indexing a dimension of the payload array. +/// \details +/// +/// Method: +/// - Find **first** discovered nearest value in our loop. +/// - Determine which indices match this nearest value. +/// (more than one index could have this one value). +/// +/// [1, 1, 2, 3, 4, 5] +/// +/// Nearest neighbour extraction of “1”, has more than one neighbour. +/// That is, more than one index with the same value have the same distance: +/// +/// [1, 1] i.e. range=(0, 2) +/// +/// - Note that an alternative implementation could consider equidistant +/// values, though it was decided this was not desirable behaviour: +/// +/// [1, 1, 2, 3, 4, 5] +/// +/// Nearest neighbour extraction of “1.5” could be then considered to have 3 +/// equidistant neighbours (1, 1, 2). That is, two different values with the +/// same distance. +/// +/// [1, 1, 2] i.e. range=(0, 3) +/// +/// \param[in] varName +/// Name of the coordinate to match against. +/// \param[in] varValues +/// Vector of values of that coordinate. +/// \param[in] obVal +/// Value to match. +/// \param[inout] range +/// On input, the range of slices of the payload array along the dimension indexed by +/// `varValues` that matches all constraints considered so far. On output, the subrange of +/// slices matching also the current constraint. +template +void nearestMatch(const std::string &varName, + const std::vector &varValues, + const T &obVal, + ConstrainedRange &range) { + // Find first index of varValues >= obVal + int nnIndex = std::lower_bound(varValues.begin() + range.begin(), + varValues.begin() + range.end(), + obVal) - varValues.begin(); + if (nnIndex >= range.end()) { + nnIndex = range.end() - 1; + } + + // Now fetch the nearest neighbour index (lower index prioritised for different values with + // same distance) + T dist = std::abs(varValues[nnIndex] - obVal); + if ((varValues[nnIndex] > obVal) && (nnIndex > range.begin()) && + (std::abs(varValues[nnIndex - 1] - obVal) <= dist)) + nnIndex--; + + // Now find **same value** equidistant neighbours + auto bounds = std::equal_range(varValues.begin() + range.begin(), + varValues.begin() + range.end(), + varValues[nnIndex]); + range.constrain(static_cast(bounds.first - varValues.begin()), + static_cast(bounds.second - varValues.begin())); + oops::Log::debug() << "Nearest match; name: " << varName << " range: " << + range.begin() << "," << range.end() << std::endl; +} + + +void nearestMatch(const std::string &varName, + const std::vector &varValues, + const std::string &obVal, + ConstrainedRange &range) { + throw eckit::UserError("The 'nearest' method cannot be used for string variables.", Here()); +} + + +/// \brief Update our extract constraint based on a least-upper-bound match against the specified +/// coordinate indexing a dimension of the payload array. +/// +/// \param[in] varName +/// Name of the coordinate to match against. +/// \param[in] varValues +/// Vector of values of that coordinate. +/// \param[in] obVal +/// Value to match. +/// \param[inout] range +/// On input, the range of slices of the payload array along the dimension indexed by +/// `varValues` that matches all constraints considered so far. On output, the subrange of +/// slices matching also the current constraint. +template +void leastUpperBoundMatch(const std::string &varName, + const std::vector &varValues, + const T &obVal, + ConstrainedRange &range) { + // Find index of the first varValues >= obVal + typedef typename std::vector::const_iterator It; + const It rangeBegin(varValues.begin() + range.begin()); + const It rangeEnd(varValues.begin() + range.end()); + + const It leastUpperBoundIt = std::lower_bound(rangeBegin, rangeEnd, obVal); + if (leastUpperBoundIt == rangeEnd) { + std::stringstream msg; + msg << "No match found for 'least upper bound' extraction of value '" << obVal + << "' of the variable '" << varName << "'"; + throw eckit::Exception(msg.str(), Here()); + } + + // Find the range of items with the same value of this coordinate + const auto bounds = std::equal_range(rangeBegin, rangeEnd, *leastUpperBoundIt); + range.constrain(static_cast(bounds.first - varValues.begin()), + static_cast(bounds.second - varValues.begin())); + oops::Log::debug() << "Least upper bound match; name: " << varName << " range: " + << range.begin() << "," << range.end() << std::endl; +} + +void leastUpperBoundMatch(const std::string &varName, + const std::vector &varValues, + const std::string &obVal, + ConstrainedRange &range) { + throw eckit::UserError("The 'least upper bound' method cannot be used for string variables.", + Here()); +} + +/// \brief Update our extract constraint based on a greatest-lower-bound match against the +/// specified coordinate indexing a dimension of the payload array. +/// +/// \param[in] varName +/// Name of the coordinate to match against. +/// \param[in] varValues +/// Vector of values of that coordinate. +/// \param[in] obVal +/// Value to match. +/// \param[inout] range +/// On input, the range of slices of the payload array along the dimension indexed by +/// `varValues` that matches all constraints considered so far. On output, the subrange of +/// slices matching also the current constraint. +template +void greatestLowerBoundMatch(const std::string &varName, + const std::vector &varValues, + const T &obVal, + ConstrainedRange &range) { + // Find index of the last varValues <= obVal + typedef typename std::vector::const_reverse_iterator ReverseIt; + typedef std::greater Compare; + const ReverseIt reverseRangeBegin(varValues.begin() + range.end()); + const ReverseIt reverseRangeEnd(varValues.begin() + range.begin()); + + const ReverseIt greatestLowerBoundIt = + std::lower_bound(reverseRangeBegin, reverseRangeEnd, obVal, Compare()); + if (greatestLowerBoundIt == reverseRangeEnd) { + std::stringstream msg; + msg << "No match found for 'greatest lower bound' extraction of value '" << obVal + << "' of the variable '" << varName << "'"; + throw eckit::Exception(msg.str(), Here()); + } + + // Find the range of items with the same value of this coordinate + const auto bounds = std::equal_range(varValues.begin() + range.begin(), + varValues.begin() + range.end(), + *greatestLowerBoundIt); + range.constrain(static_cast(bounds.first - varValues.begin()), + static_cast(bounds.second - varValues.begin())); + oops::Log::debug() << "Greatest lower bound match; name: " << varName << " range: " + << range.begin() << "," << range.end() << std::endl; +} + +void greatestLowerBoundMatch(const std::string &varName, + const std::vector &varValues, + const std::string &obVal, + ConstrainedRange &range) { + throw eckit::UserError("The 'greatest lower bound' method cannot be used for string variables.", + Here()); +} + + +/// \brief Restrict `range` to the subrange of `varValues` matching `obVal` according to the +/// criteria of `method`. +/// +/// \param[in] method +/// Matching method. +/// \param[in] varName +/// Name of the coordinate to match against. +/// \param[in] varValues +/// Vector of values of that coordinate. +/// \param[in] obVal +/// Value to match. +/// \param[inout] range +/// On input, the range of slices of the payload array along the dimension indexed by +/// `varValues` that matches all constraints considered so far. On output, the subrange of +/// slices matching also the current constraint. +template +void match(InterpMethod method, + const std::string &varName, + const std::vector &varValues, + const T &obVal, + ConstrainedRange &range) { + switch (method) { + case InterpMethod::EXACT: + exactMatch(varName, varValues, obVal, range); + break; + case InterpMethod::NEAREST: + nearestMatch(varName, varValues, obVal, range); + break; + case InterpMethod::LEAST_UPPER_BOUND: + leastUpperBoundMatch(varName, varValues, obVal, range); + break; + case InterpMethod::GREATEST_LOWER_BOUND: + greatestLowerBoundMatch(varName, varValues, obVal, range); + break; + default: + throw eckit::BadParameter("Unrecognized interpolation method", Here()); + } +} + + +/// \brief Perform piecewise linear interpolation along axis `dimIndex` at the 'location' `obVal`. +/// +/// \details It is assumed that previous calls to extract() have extracted a single 1D slice of +/// the interpolated array parallel to the axis `varName`. This function returns the value +/// produced by piecewise linear interpolation of this slice at the point `obVal`. +/// +/// \param[in] dimIndex +/// Axis of `interpolatedArray` along which to interpolate. +/// \param[in] ranges +/// Ranges of slices of `interpolatedArray` matching all constraints considered so far. +/// \param[in] varName +/// Name of the coordinate along which to interpolate. +/// \param[in] varValues +/// Vector of values of that coordinate. +/// \param[in] obVal +/// Interpolation location. +/// \param[in] interpolatedArray +/// Interpolated array. +template +float linearInterpolation(size_t dimIndex, + const std::array &ranges, + const std::string &varName, + const std::vector &varValues, + const CoordinateValue &obVal, + const boost::multi_array &interpolatedArray) { + // Sanity check constraint + int sizeDim0 = ranges[0].size(); + int sizeDim1 = ranges[1].size(); + if ((dimIndex == 1 && !(sizeDim1 > 1 && sizeDim0 == 1)) || + (dimIndex == 0 && !(sizeDim0 > 1 && sizeDim1 == 1))) { + throw eckit::Exception("Linear interpolation failed - data must be 1D.", Here()); + } + + // Constrain our index range in the relevant dimension. + const ConstrainedRange &range = ranges[static_cast(dimIndex)]; + + if ((obVal > varValues[range.end() - 1]) || (obVal < varValues[range.begin()])) { + throw eckit::Exception("Linear interpolation failed, value is beyond grid extent." + "No extrapolation supported.", + Here()); + } + // Find first index of varValues >= obVal + int nnIndex = std::lower_bound(varValues.begin() + range.begin(), + varValues.begin() + range.end(), + obVal) - varValues.begin(); + + // Determine upper or lower indices from this + if (varValues[nnIndex] == obVal) { + // No interpolation required (is equal) + float res; + if (dimIndex == 1) { + res = interpolatedArray[ranges[0].begin()][nnIndex]; + } else { + res = interpolatedArray[nnIndex][ranges[1].begin()]; + } + return res; + } + // Linearly interpolate between these two indices. + float zUpper; + float zLower; + if (dimIndex == 1) { + zUpper = interpolatedArray[ranges[0].begin()][nnIndex]; + zLower = interpolatedArray[ranges[0].begin()][nnIndex-1]; + } else { + zUpper = interpolatedArray[nnIndex][ranges[1].begin()]; + zLower = interpolatedArray[nnIndex-1][ranges[1].begin()]; + } + float res = ((static_cast(obVal - varValues[nnIndex-1]) / + static_cast(varValues[nnIndex] - varValues[nnIndex-1])) * + (zUpper - zLower)) + zLower; + return res; +} -DataExtractor::DataExtractor(const std::string &filepath, const std::string &group) { + +float linearInterpolation(size_t dimIndex, + const std::array &ranges, + const std::string &varName, + const std::vector &varValues, + const std::string &obVal, + const boost::multi_array &interpolatedArray) { + throw eckit::UserError("Linear interpolation cannot be performed along coordinate axes indexed " + "by string variables such as " + varName + ".", Here()); +} + +} // namespace + + +template +DataExtractor::DataExtractor(const std::string &filepath, + const std::string &group) { // Read the data from the file load(filepath, group); // Start by constraining to the full range of our data resetExtract(); // Initialise splitter for both dimensions - splitter_.emplace_back(ufo::RecursiveSplitter(static_cast(interpolatedArray2D_.rows()))); - splitter_.emplace_back(ufo::RecursiveSplitter(static_cast(interpolatedArray2D_.cols()))); + splitter_.emplace_back(ufo::RecursiveSplitter(interpolatedArray2D_.shape()[0])); + splitter_.emplace_back(ufo::RecursiveSplitter(interpolatedArray2D_.shape()[1])); } -void DataExtractor::load(const std::string &filepath, - const std::string &interpolatedArrayGroup) { - std::unique_ptr backend = createBackendFor(filepath); - DataExtractorInput input = backend->loadData(interpolatedArrayGroup); +template +void DataExtractor::load(const std::string &filepath, + const std::string &interpolatedArrayGroup) { + std::unique_ptr> backend = createBackendFor(filepath); + DataExtractorInput input = backend->loadData(interpolatedArrayGroup); coord2DimMapping_ = std::move(input.coord2DimMapping); dim2CoordMapping_ = std::move(input.dim2CoordMapping); coordsVals_ = std::move(input.coordsVals); + interpolatedArray2D_.resize(boost::extents[input.payloadArray.shape()[0]] + [input.payloadArray.shape()[1]]); interpolatedArray2D_ = std::move(input.payloadArray); + // Set the unconstrained size of matching ranges along both axes of the payload array. + for (size_t i = 0; i < constrainedRanges_.size(); ++i) + constrainedRanges_[i] = ConstrainedRange(input.payloadArray.shape()[i]); } -std::unique_ptr DataExtractor::createBackendFor( - const std::string &filepath) { + +template +std::unique_ptr> +DataExtractor::createBackendFor(const std::string &filepath) { const std::string lowercasePath = eckit::StringTools::lower(filepath); if (eckit::StringTools::endsWith(lowercasePath, ".nc") || eckit::StringTools::endsWith(lowercasePath, ".nc4")) - return boost::make_unique(filepath); + return boost::make_unique>(filepath); else if (eckit::StringTools::endsWith(lowercasePath, ".csv")) - return boost::make_unique(filepath); + return boost::make_unique>(filepath); else throw eckit::BadValue("File '" + filepath + "' has an unrecognized extension. " "The supported extensions are .nc, .nc4 and .csv", Here()); } -void DataExtractor::sort() { - Eigen::ArrayXXf sortedArray = interpolatedArray2D_; +template +void DataExtractor::sort() { + boost::multi_array sortedArray = interpolatedArray2D_; nextCoordToExtractBy_ = coordsToExtractBy_.begin(); for (size_t dim = 0; dim < dim2CoordMapping_.size(); ++dim) { @@ -115,8 +486,8 @@ void DataExtractor::sort() { ind++; oops::Log::debug() << "Sort index dim0; index-from: " << ind << " index-to: " << index << std::endl; - for (Eigen::Index j = 0; j < interpolatedArray2D_.cols(); j++) { - sortedArray(ind, j) = interpolatedArray2D_(static_cast(index), j); + for (size_t j = 0; j < interpolatedArray2D_.shape()[1]; j++) { + sortedArray[ind][j] = interpolatedArray2D_[index][j]; } } } @@ -129,8 +500,8 @@ void DataExtractor::sort() { ind++; oops::Log::debug() << "Sort index dim1; index-from: " << ind << " index-to: " << index << std::endl; - for (Eigen::Index i = 0; i < interpolatedArray2D_.rows(); i++) { - sortedArray(i, ind) = interpolatedArray2D_(i, static_cast(index)); + for (size_t i = 0; i < interpolatedArray2D_.shape()[0]; i++) { + sortedArray[i][ind] = interpolatedArray2D_[i][index]; } } } @@ -144,7 +515,13 @@ void DataExtractor::sort() { } -void DataExtractor::scheduleSort(const std::string &varName, const InterpMethod &method) { +template +void DataExtractor::scheduleSort(const std::string &varName, + const InterpMethod &method) { + if (method == InterpMethod::LINEAR && !std::is_floating_point::value) + throw eckit::BadParameter("Linear interpolation can be used when extracting floating-point " + "values, but not integers or strings.", Here()); + // Map any names of the form var@Group to Group/var const std::string canonicalVarName = ioda::convertV1PathToV2Path(varName); @@ -159,17 +536,83 @@ void DataExtractor::scheduleSort(const std::string &varName, const InterpMethod } -void DataExtractor::resetExtract() { - constrainedRanges_[0].begin = 0; - constrainedRanges_[0].end = static_cast(interpolatedArray2D_.rows()); - constrainedRanges_[1].begin = 0; - constrainedRanges_[1].end = static_cast(interpolatedArray2D_.cols()); - resultSet_ = false; - nextCoordToExtractBy_ = coordsToExtractBy_.begin(); +template +void DataExtractor::extract(float obVal) { + extractImpl(obVal); +} + + +template +void DataExtractor::extract(int obVal) { + extractImpl(obVal); +} + + +template +void DataExtractor::extract(const std::string &obVal) { + extractImpl(obVal); +} + + +template +template +void DataExtractor::extractImpl(const T &obVal) { + if (nextCoordToExtractBy_ == coordsToExtractBy_.cend()) { + throw eckit::UserError("Too many extract() calls made for the expected number of variables.", + Here()); + } + + // Perform the extraction using the selected method + if (nextCoordToExtractBy_->method == InterpMethod::LINEAR) + maybeExtractByLinearInterpolation(obVal); + else + match(nextCoordToExtractBy_->method, + nextCoordToExtractBy_->name, + boost::get>(nextCoordToExtractBy_->values), + obVal, + constrainedRanges_[nextCoordToExtractBy_->payloadDim]); + + ++nextCoordToExtractBy_; +} + + +// Primary template, used for all ExtractedValue types except float. +template +template +void DataExtractor::maybeExtractByLinearInterpolation(const T &obVal) { + // Should never be called -- this error should be detected earlier. + throw eckit::BadParameter("Linear interpolation can be used when extracting floating-point " + "values, but not integers or strings.", Here()); +} + + +// Specialization for ExtractedValue = float. +template <> +template +void DataExtractor::maybeExtractByLinearInterpolation(const T &obVal) { + result_ = linearInterpolation(nextCoordToExtractBy_->payloadDim, + constrainedRanges_, + nextCoordToExtractBy_->name, + boost::get>(nextCoordToExtractBy_->values), + obVal, + interpolatedArray2D_); + resultSet_ = true; +} + + +// Primary template, used for all ExtractedValue types except float. +template +ExtractedValue DataExtractor::getResult() { + // Fetch the result + ExtractedValue res = getUniqueMatch(); + resetExtract(); + return res; } -float DataExtractor::getResult() { +// Specialization adding support for linear interpolation. +template <> +float DataExtractor::getResult() { // Fetch the result if (resultSet_) { // This was derived from linear interpolation so return it. @@ -177,16 +620,41 @@ float DataExtractor::getResult() { return result_; } - int sizeDim0 = constrainedRanges_[0].end - constrainedRanges_[0].begin; - int sizeDim1 = constrainedRanges_[1].end - constrainedRanges_[1].begin; + float res = getUniqueMatch(); + resetExtract(); + return res; +} + + +template +ExtractedValue DataExtractor::getUniqueMatch() const { + // This function should be called only if linear interpolation is not used within the + // extraction process. + ASSERT(!resultSet_); + + int sizeDim0 = constrainedRanges_[0].size(); + int sizeDim1 = constrainedRanges_[1].size(); if (sizeDim0 != 1 || sizeDim1 != 1) { throw eckit::Exception("Previous calls to extract() have failed to identify " "a single value to return.", Here()); } - float res = interpolatedArray2D_(constrainedRanges_[0].begin, constrainedRanges_[1].begin); - resetExtract(); - return res; + return interpolatedArray2D_[constrainedRanges_[0].begin()] + [constrainedRanges_[1].begin()]; } +template +void DataExtractor::resetExtract() { + for (ConstrainedRange &range : constrainedRanges_) + range.reset(); + resultSet_ = false; + nextCoordToExtractBy_ = coordsToExtractBy_.begin(); +} + + +// Explicit instantiations +template class DataExtractor; +template class DataExtractor; +template class DataExtractor; + } // namespace ufo diff --git a/src/ufo/utils/dataextractor/DataExtractor.h b/src/ufo/utils/dataextractor/DataExtractor.h index 4dc40d668..4c2e9836c 100644 --- a/src/ufo/utils/dataextractor/DataExtractor.h +++ b/src/ufo/utils/dataextractor/DataExtractor.h @@ -8,24 +8,15 @@ #ifndef UFO_UTILS_DATAEXTRACTOR_DATAEXTRACTOR_H_ #define UFO_UTILS_DATAEXTRACTOR_DATAEXTRACTOR_H_ -#include // sort -#include // greater -#include // std::numeric_limits -#include // list #include // unique_ptr -#include // stringstream #include #include -#include // pair #include -#include "boost/variant.hpp" -#include "Eigen/Dense" // Eigen Arrays and Matrices +#include +#include -#include "eckit/exception/Exceptions.h" -#include "oops/util/Logger.h" -#include "oops/util/missingValues.h" -#include "ufo/filters/Variables.h" +#include "ufo/utils/dataextractor/ConstrainedRange.h" #include "ufo/utils/RecursiveSplitter.h" namespace ioda { @@ -36,8 +27,10 @@ struct Named_Variable; namespace ufo { +template class DataExtractorBackend; + /// \brief Method used by the DataExtractor to map the value of an ObsSpace variable to /// a range of slices of the interpolated array along the dimension indexed by that variable. enum class InterpMethod { @@ -70,13 +63,17 @@ enum class InterpMethod { /// \brief Perform a piecewise linear interpolation along the dimension indexed by the ObsSpace /// variable. /// - /// This method can only be used for the last indexing variable. + /// This method can only be used for the last indexing variable. It is supported only when the + /// DataExtractor produces values of type `float`, but not `int` or `std::string`. LINEAR }; /// \brief This class makes it possible to extract and interpolate data loaded from a file. /// +/// \tparam ExtractedValue +/// Type of the values to extract. Must be `float`, `int` or `std::string`. +/// /// Currently the following file formats are supported: NetCDF and CSV. See /// DataExtractorNetCDFBackend and DataExtractorCSVBackend for more information about these /// formats. @@ -111,6 +108,7 @@ enum class InterpMethod { /// /// Both 1 and 2 are equidistant, but we take the first found equidistant neighbour (1), then /// return all indices matching this neighbour (indices 0 and 1 in this case). +template class DataExtractor { public: @@ -151,310 +149,31 @@ class DataExtractor /// method). The extract then utilises the observation value ('obVal') to perform this /// extraction. /// \param[in] obVal is the observation value used for the extract operation. - template - void extract(const T &obVal) { - if (nextCoordToExtractBy_ == coordsToExtractBy_.cend()) { - throw eckit::UserError("Too many extract() calls made for the expected number of variables.", - Here()); - } - - // Fetch coordinate values - const std::vector &coordValues = boost::get>(nextCoordToExtractBy_->values); - // Perform the extraction using the selected method - switch (nextCoordToExtractBy_->method) { - case InterpMethod::EXACT: - exactMatch(nextCoordToExtractBy_->name, coordValues, - nextCoordToExtractBy_->payloadDim, obVal); - break; - case InterpMethod::LINEAR: - result_ = getResult(nextCoordToExtractBy_->name, coordValues, - nextCoordToExtractBy_->payloadDim, obVal); - resultSet_ = true; - break; - case InterpMethod::NEAREST: - nearestMatch(nextCoordToExtractBy_->name, coordValues, - nextCoordToExtractBy_->payloadDim, obVal); - break; - case InterpMethod::LEAST_UPPER_BOUND: - leastUpperBoundMatch(nextCoordToExtractBy_->name, coordValues, - nextCoordToExtractBy_->payloadDim, obVal); - break; - case InterpMethod::GREATEST_LOWER_BOUND: - greatestLowerBoundMatch(nextCoordToExtractBy_->name, coordValues, - nextCoordToExtractBy_->payloadDim, obVal); - break; - default: - throw eckit::UserError("Only 'linear', 'nearest', 'exact', 'least upper bound' and " - "'greatest lower bound' interpolation methods supported.", - Here()); - } - ++nextCoordToExtractBy_; - } + void extract(float obVal); + void extract(int obVal); + void extract(const std::string &obVal); /// \brief Fetch the final interpolated value. /// \details This will only be succesful if previous calls to extract() have produced a single /// value to return. - float getResult(); + ExtractedValue getResult(); private: - /// \brief Fetch the final interpolated value at the 'location' `obVal`. - /// - /// \details It is assumed that previous calls to extract() have extracted a single 1D slice of - /// the interpolated array parallel to the axis `varName`. This function returns the value - /// produced by piecewise linear interpolation of this slice at the point `obVal`. - /// - /// \param[in] varName is the name of the coordinate along which to interpolate. - /// \param[in] varValues is the vector of values of that coordinate. - /// \param[in] dimIndex is the dimension of the payload array indexed by the coordinate. - /// \param[in] obVal is the interpolation location. - template - float getResult(const std::string &varName, const std::vector &varValues, - int dimIndex, const T &obVal) { - // Sanity check constraint - int sizeDim0 = constrainedRanges_[0].end - constrainedRanges_[0].begin; - int sizeDim1 = constrainedRanges_[1].end - constrainedRanges_[1].begin; - if ((dimIndex == 1 && !(sizeDim1 > 1 && sizeDim0 == 1)) || - (dimIndex == 0 && !(sizeDim0 > 1 && sizeDim1 == 1))) { - throw eckit::Exception("Linear interpolation failed - data must be 1D.", Here()); - } - - // Constrain our index range in the relevant dimension. - const Range &range = constrainedRanges_[static_cast(dimIndex)]; - - if ((obVal > varValues[range.end - 1]) || (obVal < varValues[range.begin])) { - throw eckit::Exception("Linear interpolation failed, value is beyond grid extent." - "No extrapolation supported.", - Here()); - } - // Find first index of varValues >= obVal - int nnIndex = std::lower_bound(varValues.begin() + range.begin, - varValues.begin() + range.end, - obVal) - varValues.begin(); - - // Determine upper or lower indices from this - if (varValues[nnIndex] == obVal) { - // No interpolation required (is equal) - float res; - if (dimIndex == 1) { - res = static_cast(interpolatedArray2D_(constrainedRanges_[0].begin, nnIndex)); - } else { - res = static_cast(interpolatedArray2D_(nnIndex, constrainedRanges_[1].begin)); - } - return res; - } - // Linearly interpolate between these two indices. - auto zUpper = *(interpolatedArray2D_.data()); - auto zLower = *(interpolatedArray2D_.data()); - if (dimIndex == 1) { - zUpper = interpolatedArray2D_(constrainedRanges_[0].begin, nnIndex); - zLower = interpolatedArray2D_(constrainedRanges_[0].begin, nnIndex-1); - } else { - zUpper = interpolatedArray2D_(nnIndex, constrainedRanges_[1].begin); - zLower = interpolatedArray2D_(nnIndex-1, constrainedRanges_[1].begin); - } - float res = ((static_cast(obVal - varValues[nnIndex-1]) / - static_cast(varValues[nnIndex] - varValues[nnIndex-1])) * - (zUpper - zLower)) + zLower; - return res; - } - - float getResult(const std::string &varName, const std::vector &varValues, - int dimIndex, const std::string &obVal) { - throw eckit::UserError("VarName: " + varName + - " - linear interpolation not compatible with string type.", Here()); - } - - /// \brief Update our extract constraint based on a nearest match against the specified - /// coordinate indexing a dimension of the payload array. - /// \details - /// - /// Method: - /// - Find **first** discovered nearest value in our loop. - /// - Determine which indices match this nearest value. - /// (more than one index could have this one value). - /// - /// [1, 1, 2, 3, 4, 5] - /// - /// Nearest neighbour extraction of “1”, has more than one neighbour. - /// That is, more than one index with the same value have the same distance: - /// - /// [1, 1] i.e. range=(0, 2) - /// - /// - Note that an alternative implementation could consider equidistant - /// values, though it was decided this was not desirable behaviour: - /// - /// [1, 1, 2, 3, 4, 5] - /// - /// Nearest neighbour extraction of “1.5” could be then considered to have 3 - /// equidistant neighbours (1, 1, 2). That is, two different values with the - /// same distance. - /// - /// [1, 1, 2] i.e. range=(0, 3) - /// - /// \param[in] varName is the name of the coordinate to match against. - /// \param[in] varValues is the vector of values of that coordinate. - /// \param[in] dimIndex is the dimension of the payload array indexed by the coordinate. - /// \param[in] obVal is the value to match. - template - void nearestMatch(const std::string &varName, const std::vector &varValues, - int dimIndex, const T &obVal) { - // Constrain our index range in the relevant dimension. - Range &range = constrainedRanges_[static_cast(dimIndex)]; - - // Find first index of varValues >= obVal - int nnIndex = std::lower_bound(varValues.begin() + range.begin, - varValues.begin() + range.end, - obVal) - varValues.begin(); - if (nnIndex >= range.end) { - nnIndex = range.end - 1; - } - - // Now fetch the nearest neighbour index (lower index prioritised for different values with - // same distance) - T dist = std::abs(varValues[nnIndex] - obVal); - if ((varValues[nnIndex] > obVal) && (nnIndex > range.begin) && - (std::abs(varValues[nnIndex - 1] - obVal) <= dist)) - nnIndex--; - - // Now find **same value** equidistant neighbours - auto bounds = std::equal_range(varValues.begin() + range.begin, - varValues.begin() + range.end, - varValues[nnIndex]); - range = {static_cast(bounds.first - varValues.begin()), - static_cast(bounds.second - varValues.begin())}; - oops::Log::debug() << "Nearest match; name: " << varName << " range: " << - range.begin << "," << range.end << std::endl; - } - - void nearestMatch(const std::string &varName, const std::vector &varValues, - int dimIndex, const std::string &obVal) { - throw eckit::UserError("Nearest match not compatible with string type.", Here()); - } - - /// \brief Update our extract constraint based on an exact match against the specified coordinate - /// indexing a dimension of the payload array. - /// - /// \param[in] varName is the name of the coordinate to match against. - /// \param[in] varValues is the vector of values of that coordinate. - /// \param[in] dimIndex is the dimension of the payload array indexed by the coordinate. - /// \param[in] obVal is the value to match. - template - void exactMatch(const std::string &varName, const std::vector &varValues, - int dimIndex, const T &obVal) { - // Constrain our index range in the relevant dimension. - Range &range = constrainedRanges_[static_cast(dimIndex)]; - - // Find the first and last matching index - auto bounds = std::equal_range(varValues.begin() + range.begin, - varValues.begin() + range.end, - obVal); - if (bounds.first == bounds.second) { - // No matching coordinate found. If the coordinate contains a 'missing value' entry, - // use it as a fallback. (If it doesn't, the 'bounds' range will stay empty, so an error will - // be reported). - bounds = std::equal_range(varValues.begin() + range.begin, - varValues.begin() + range.end, - util::missingValue(obVal)); - } - - range = {static_cast(bounds.first - varValues.begin()), - static_cast(bounds.second - varValues.begin())}; - - if (range.begin == range.end) { - std::stringstream msg; - msg << "No match found for exact match extraction of value '" << obVal - << "' of the variable '" << varName << "'"; - throw eckit::Exception(msg.str(), Here()); - } - oops::Log::debug() << "Exact match; name: " << varName << " range: " << - range.begin << "," << range.end << std::endl; - } - - /// \brief Update our extract constraint based on a least-upper-bound match against the specified - /// coordinate indexing a dimension of the payload array. - /// - /// \param[in] varName is the name of the coordinate to match against. - /// \param[in] varValues is the vector of values of that coordinate. - /// \param[in] dimIndex is the dimension of the payload array indexed by the coordinate. - /// \param[in] obVal is the value to match. - template - void leastUpperBoundMatch(const std::string &varName, const std::vector &varValues, - int dimIndex, const T &obVal) { - // Constrain our index range in the relevant dimension. - Range &range = constrainedRanges_[static_cast(dimIndex)]; - - // Find index of the first varValues >= obVal - typedef typename std::vector::const_iterator It; - const It rangeBegin(varValues.begin() + range.begin); - const It rangeEnd(varValues.begin() + range.end); - - const It leastUpperBoundIt = std::lower_bound(rangeBegin, rangeEnd, obVal); - if (leastUpperBoundIt == rangeEnd) { - std::stringstream msg; - msg << "No match found for 'least upper bound' extraction of value '" << obVal - << "' of the variable '" << varName << "'"; - throw eckit::Exception(msg.str(), Here()); - } - - // Find the range of items with the same value of this coordinate - const auto bounds = std::equal_range(rangeBegin, rangeEnd, *leastUpperBoundIt); - range = {static_cast(bounds.first - varValues.begin()), - static_cast(bounds.second - varValues.begin())}; - oops::Log::debug() << "Least upper bound match; name: " << varName << " range: " - << range.begin << "," << range.end << std::endl; - } - - void leastUpperBoundMatch(const std::string &varName, const std::vector &varValues, - int dimIndex, const std::string &obVal) { - throw eckit::UserError("The 'least upper bound' method cannot be used for string variables.", - Here()); - } - - /// \brief Update our extract constraint based on a greatest-lower-bound match against the - /// specified coordinate indexing a dimension of the payload array. + /// \brief Common implementation of the overloaded public function extract(). + template + void extractImpl(const T &obVal); + + /// \brief Perform extraction using piecewise linear interpolation, if it's compatible with the + /// ExtractedValue type in use; otherwise throw an exception. + template + void maybeExtractByLinearInterpolation(const T &obVal); + + /// \brief Fetch the result produced by previous calls to extract(), none of which may have + /// used linear interpolation. /// - /// \param[in] varName is the name of the coordinate to match against. - /// \param[in] varValues is the vector of values of that coordinate. - /// \param[in] dimIndex is the dimension of the payload array indexed by the coordinate. - /// \param[in] obVal is the value to match, against the NetCDF coordinate of name 'varName'. - template - void greatestLowerBoundMatch(const std::string &varName, const std::vector &varValues, - int dimIndex, const T &obVal) { - // Constrain our index range in the relevant dimension. - Range &range = constrainedRanges_[static_cast(dimIndex)]; - - // Find index of the last varValues <= obVal - - typedef typename std::vector::const_reverse_iterator ReverseIt; - typedef std::greater Compare; - const ReverseIt reverseRangeBegin(varValues.begin() + range.end); - const ReverseIt reverseRangeEnd(varValues.begin() + range.begin); - - const ReverseIt greatestLowerBoundIt = - std::lower_bound(reverseRangeBegin, reverseRangeEnd, obVal, Compare()); - if (greatestLowerBoundIt == reverseRangeEnd) { - std::stringstream msg; - msg << "No match found for 'greatest lower bound' extraction of value '" << obVal - << "' of the variable '" << varName << "'"; - throw eckit::Exception(msg.str(), Here()); - } - - // Find the range of items with the same value of this coordinate - const auto bounds = std::equal_range(varValues.begin() + range.begin, - varValues.begin() + range.end, - *greatestLowerBoundIt); - range = {static_cast(bounds.first - varValues.begin()), - static_cast(bounds.second - varValues.begin())}; - oops::Log::debug() << "Greatest lower bound match; name: " << varName << " range: " - << range.begin << "," << range.end << std::endl; - } - - void greatestLowerBoundMatch(const std::string &varName, - const std::vector &varValues, - int dimIndex, const std::string &obVal) { - throw eckit::UserError("The 'greatest lower bound' method cannot be used for string variables.", - Here()); - } + /// An exception is thrown if these calls haven't produced a unique match of the extraction + /// criteria. + ExtractedValue getUniqueMatch() const; /// \brief Reset the extraction range for this object. /// \details Each time an exactMatch, nearestMatch, leastUpperBoundMatch or @@ -462,30 +181,19 @@ class DataExtractor /// the extraction range is further constrained to match our updated match conditions. After /// the final 'extract' is made (i.e. an interpolated value is derived) it is desirable to reset /// the extraction range by calling this method. - /// \internal This is called by the getObsErrorValue member functions just before returning the + /// \internal This is called by the getResult member function just before returning the /// interpolated value. void resetExtract(); - /// \brief Fetch the coordinate data of specified name. - /// \details We cache this data so as not to require performing multiple loads from disk. - template - T& get(const std::string &key) { - try { - return boost::get (coordsVals_[key]); - } catch (boost::bad_get) { - throw eckit::BadParameter("Unable to find coordinate with this type", Here()); - } - } - /// \brief Load all data from the input file. void load(const std::string &filepath, const std::string &interpolatedArrayGroup); /// \brief Create a backend able to read file \p filepath. - static std::unique_ptr createBackendFor(const std::string &filepath); + static std::unique_ptr> createBackendFor( + const std::string &filepath); // Object represent the extraction range in both dimensions. - struct Range {int begin, end;}; - std::array constrainedRanges_; + std::array constrainedRanges_; // Container holding coordinate arrays (of all supported types) loaded from the input file. typedef boost::variant, @@ -493,8 +201,10 @@ class DataExtractor std::vector> CoordinateValues; std::unordered_map coordsVals_; // The array to be interpolated (the payload array). - Eigen::ArrayXXf interpolatedArray2D_; + boost::multi_array interpolatedArray2D_; + // Linear interpolation result. Only used when ExtractedValue = float. interpolation. float result_; + // Set to true if result_ is a valid value. bool resultSet_; // Container for re-ordering our data std::vector splitter_; @@ -518,7 +228,7 @@ class DataExtractor /// Coordinates to use in successive calls to extract(). std::vector coordsToExtractBy_; - std::vector::const_iterator nextCoordToExtractBy_; + typename std::vector::const_iterator nextCoordToExtractBy_; }; } // namespace ufo diff --git a/src/ufo/utils/dataextractor/DataExtractorBackend.h b/src/ufo/utils/dataextractor/DataExtractorBackend.h index c7cd2531a..e11e1c0b2 100644 --- a/src/ufo/utils/dataextractor/DataExtractorBackend.h +++ b/src/ufo/utils/dataextractor/DataExtractorBackend.h @@ -13,9 +13,15 @@ namespace ufo { +template struct DataExtractorInput; /// \brief Provides data to the DataExtractor. +/// +/// \tparam ExtractedValue +/// Type of values extracted by the DataExtractor. Must be `float`, `int` or `std::string`. +/// +template class DataExtractorBackend { public: virtual ~DataExtractorBackend() = default; @@ -29,7 +35,7 @@ class DataExtractorBackend { /// /// \returns An object encapsulating the payload variable, all coordinates indexing it /// and the mapping between dimensions of the payload array and coordinates. - virtual DataExtractorInput loadData(const std::string &payloadGroup) const = 0; + virtual DataExtractorInput loadData(const std::string &payloadGroup) const = 0; }; } // namespace ufo diff --git a/src/ufo/utils/dataextractor/DataExtractorCSVBackend.cc b/src/ufo/utils/dataextractor/DataExtractorCSVBackend.cc index 45490a18d..5706f9484 100644 --- a/src/ufo/utils/dataextractor/DataExtractorCSVBackend.cc +++ b/src/ufo/utils/dataextractor/DataExtractorCSVBackend.cc @@ -10,8 +10,8 @@ #include // for move #include +#include #include -#include "Eigen/Core" #include "eckit/exception/Exceptions.h" #include "eckit/parser/CSVParser.h" @@ -74,27 +74,46 @@ class AppendValueVisitor : public boost::static_visitor { const eckit::Value &value_; }; -/// Visitor that converts a (numeric) std::vector to an Eigen column vector. -class ConvertToEigenArrayVisitor : public boost::static_visitor { +template +void convertVectorToColumnArray(const std::vector &source, + boost::multi_array &destination) { + const Source missingSource = util::missingValue(Source()); + const Destination missingDestination = util::missingValue(Destination()); + destination.resize(boost::extents[source.size()][1]); + for (size_t i = 0; i < source.size(); ++i) + if (source[i] != missingSource) + destination[i][0] = static_cast(source[i]); + else + destination[i][0] = missingDestination; +} + +/// Visitor that converts an std::vector to a boost::multi_array with one column. +template +class ConvertToBoostMultiArrayVisitor : public boost::static_visitor { public: - explicit ConvertToEigenArrayVisitor(Eigen::ArrayXXf &output) : + explicit ConvertToBoostMultiArrayVisitor(boost::multi_array &output) : output_(output) {} - template + template ::value, bool>::type + = true> void operator()(const std::vector &values) { - output_.resize(values.size(), 1); + output_.resize(boost::extents[values.size()][1]); for (size_t i = 0; i < values.size(); ++i) - output_(i, 0) = values[i]; + output_[i][0] = values[i]; } - void operator()(const std::vector &) { + template ::value, bool>::type + = true> + void operator()(const std::vector &) { // Should never be called throw eckit::NotImplemented(Here()); } private: - Eigen::ArrayXXf &output_; + boost::multi_array &output_; }; /// \brief Find the index of the column whose name ends with `@` followed by `payloadGroup` @@ -128,15 +147,40 @@ std::vector createColumn(size_t numValues) { return values; } +/// \brief Throw an exception if contents of columns of type `type` can't be converted to values +/// of type `ExtractedValue`. +template +void checkPayloadColumnType(const std::string &type); + +template <> +void checkPayloadColumnType(const std::string &type) { + if (type != "float" && type != "int") + throw eckit::UserError("The payload column must contain numeric data", Here()); +} + +template <> +void checkPayloadColumnType(const std::string &type) { + if (type != "float" && type != "int") + throw eckit::UserError("The payload column must contain numeric data", Here()); +} + +template <> +void checkPayloadColumnType(const std::string &type) { + if (type != "string" && type != "datetime") + throw eckit::UserError("The payload column must contain strings or datetimes", Here()); +} + } // namespace -DataExtractorCSVBackend::DataExtractorCSVBackend(const std::string &filepath) +template +DataExtractorCSVBackend::DataExtractorCSVBackend(const std::string &filepath) : filepath_(filepath) {} -DataExtractorInput DataExtractorCSVBackend::loadData( +template +DataExtractorInput DataExtractorCSVBackend::loadData( const std::string &interpolatedArrayGroup) const { - DataExtractorInput result; + DataExtractorInput result; const eckit::Value contents = eckit::CSVParser::decodeFile(filepath_, false /* hasHeader? */); const size_t numRows = contents.size(); @@ -169,12 +213,12 @@ DataExtractorInput DataExtractorCSVBackend::loadData( throw eckit::UserError("The number of columns in line 2 differs from that in line 1", Here()); // Allocate vectors for values to be loaded from subsequent lines - std::vector columns(numColumns); + std::vector columns(numColumns); for (size_t column = 0; column < numColumns; ++column) { const std::string type = typeHeader[column]; + if (column == payloadColumnIndex) + checkPayloadColumnType(type); if (type == "string" || type == "datetime") { - if (column == payloadColumnIndex) - throw eckit::UserError("The payload column must contain numeric data", Here()); columns[column] = createColumn(numValues); } else if (type == "int" || type == "integer") { columns[column] = createColumn(numValues); @@ -202,7 +246,7 @@ DataExtractorInput DataExtractorCSVBackend::loadData( result.dim2CoordMapping.resize(1); for (size_t column = 0; column < numColumns; ++column) { if (column == payloadColumnIndex) { - ConvertToEigenArrayVisitor visitor(result.payloadArray); + ConvertToBoostMultiArrayVisitor visitor(result.payloadArray); boost::apply_visitor(visitor, columns[column]); } else { result.coordsVals[columnNames[column]] = std::move(columns[column]); @@ -211,10 +255,15 @@ DataExtractorInput DataExtractorCSVBackend::loadData( } } - if (result.payloadArray.rows() == 0) + if (result.payloadArray.shape()[0] == 0) throw eckit::UserError("No data could be loaded from the file '" + filepath_ + "'", Here()); return result; } +// Explicit instantiations +template class DataExtractorCSVBackend; +template class DataExtractorCSVBackend; +template class DataExtractorCSVBackend; + } // namespace ufo diff --git a/src/ufo/utils/dataextractor/DataExtractorCSVBackend.h b/src/ufo/utils/dataextractor/DataExtractorCSVBackend.h index 1885b7395..cb6369515 100644 --- a/src/ufo/utils/dataextractor/DataExtractorCSVBackend.h +++ b/src/ufo/utils/dataextractor/DataExtractorCSVBackend.h @@ -43,8 +43,9 @@ namespace ufo /// observation. The details of this comparison (e.g. whether an exact match is required, the /// nearest match is used, or piecewise linear interpolation is performed) depend on how the class /// using the extracted data (e.g. the DrawValueFromFile ObsFunction) is configured. The data type -/// of each column must match the data type of the corresponding ObsSpace variable. The payload -/// column must be of type `float` or `int`. The column order does not matter. +/// of each column must match the data type of the corresponding ObsSpace variable. The type of the +/// payload column should match the template parameter `ExtractedValue`, which must be set to +/// either `float`, `int` or `std::string`. The column order does not matter. /// /// Notes: /// @@ -80,14 +81,15 @@ namespace ufo /// /// Refer to the documentation of the DrawValueFromFile ObsFunction for more information about the /// available extraction methods. -class DataExtractorCSVBackend : public DataExtractorBackend { +template +class DataExtractorCSVBackend : public DataExtractorBackend { public: /// \brief Create a new instance. /// /// \param filepath Path to the CSV file that will be read by loadData(). explicit DataExtractorCSVBackend(const std::string &filepath); - DataExtractorInput loadData(const std::string &payloadGroup) const override; + DataExtractorInput loadData(const std::string &payloadGroup) const override; private: std::string filepath_; diff --git a/src/ufo/utils/dataextractor/DataExtractorInput.h b/src/ufo/utils/dataextractor/DataExtractorInput.h index 241b01018..19f71c9d2 100644 --- a/src/ufo/utils/dataextractor/DataExtractorInput.h +++ b/src/ufo/utils/dataextractor/DataExtractorInput.h @@ -12,27 +12,28 @@ #include #include +#include #include -// cpplint misclassifies this file as a c system include -#include // NOLINT(build/include_order) namespace ufo { -/// \brief Input data for the DataExtractor. +/// \brief Parts of the input data for the DataExtractor that don't depend on the type of the +/// extracted values. /// /// Note: the names of all coordinates are expected to be of the form `Group/var` (ioda-v2 style) /// rather than `var@Group` (ioda-v1 style). -struct DataExtractorInput { - /// Array from which values will be extracted - Eigen::ArrayXXf payloadArray; - +struct DataExtractorInputBase { + /// \brief A coordinate indexing a dimension of the payload array, i.e. the array from which + /// a DataExtractor will extract data. typedef boost::variant, std::vector, std::vector > Coordinate; + /// \brief A collection of named coordinate vectors. typedef std::unordered_map Coordinates; - /// Coordinates indexing payloadArray + + /// Coordinates indexing the payload array Coordinates coordsVals; /// Maps coordinate names to dimensions (0 or 1) of the payload array @@ -41,6 +42,19 @@ struct DataExtractorInput { std::vector> dim2CoordMapping; }; +/// \brief Input data for the DataExtractor. +/// +/// \tparam ExtractedValue +/// Type of the values to be extracted. Must be `float`, `int` or `std::string`. +/// +/// Note: the names of all coordinates are expected to be of the form `Group/var` (ioda-v2 style) +/// rather than `var@Group` (ioda-v1 style). +template +struct DataExtractorInput : public DataExtractorInputBase { + /// Array from which values will be extracted + boost::multi_array payloadArray; +}; + } // namespace ufo #endif // UFO_UTILS_DATAEXTRACTOR_DATAEXTRACTORINPUT_H_ diff --git a/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.cc b/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.cc index 1e4289525..f03f85731 100644 --- a/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.cc +++ b/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.cc @@ -11,9 +11,8 @@ #include // pair #include +#include #include -#include "Eigen/Core" -#include "unsupported/Eigen/CXX11/Tensor" #include "eckit/exception/Exceptions.h" #include "eckit/utils/StringTools.h" @@ -60,7 +59,7 @@ std::vector fetchDimNameMapping( /// \brief Add variable `var` of type `T` to `coordsVals` under key `key`. template void updateVariable(const std::string &key, ioda::Variable var, - DataExtractorInput::Coordinates &coordsVals) { + DataExtractorInputBase::Coordinates &coordsVals) { // Read the variable from the input file std::vector values = var.readAsVector(); @@ -81,7 +80,7 @@ void updateVariable(const std::string &key, ioda::Variable var, /// \brief Add variable `var` to `coordsVals` under key `key`. void update(const std::string &key, ioda::Variable var, - DataExtractorInput::Coordinates &coordsVals) { + DataExtractorInputBase::Coordinates &coordsVals) { if (var.isA()) { updateVariable(key, var, coordsVals); } else if (var.isA()) { @@ -133,25 +132,28 @@ const std::string &findPayloadVariable(const std::vector &varNames, }; auto payloadVarIt = std::find_if(varNames.begin(), varNames.end(), isInPayloadGroup); if (payloadVarIt == varNames.end()) - throw eckit::UserError("No payload column found: no column name begins with '" + prefix + + throw eckit::UserError("No payload variable found: no variable name begins with '" + prefix + "' or ends with '" + suffix + "'", Here()); if (std::any_of(payloadVarIt + 1, varNames.end(), isInPayloadGroup)) throw eckit::UserError("Multiple payload candidates found: " - "more than one column name begins with '" + prefix + + "more than one variable name begins with '" + prefix + "' or ends with '" + suffix + "'", Here()); return *payloadVarIt; } } // namespace -DataExtractorNetCDFBackend::DataExtractorNetCDFBackend(const std::string &filepath) +template +DataExtractorNetCDFBackend::DataExtractorNetCDFBackend( + const std::string &filepath) : filepath_(filepath) {} -DataExtractorInput DataExtractorNetCDFBackend::loadData( +template +DataExtractorInput DataExtractorNetCDFBackend::loadData( const std::string &interpolatedArrayGroup) const { - DataExtractorInput result; + DataExtractorInput result; // Open the input file const ioda::Group group = ioda::Engines::HH::openFile( @@ -206,7 +208,7 @@ DataExtractorInput DataExtractorNetCDFBackend::loadData( // Maps dimensions of the payload array (0 or 1) to coordinate names std::unordered_map> dim2CoordMapping; // Coordinate values - DataExtractorInput::Coordinates coordsVals; + DataExtractorInputBase::Coordinates coordsVals; for (const std::string& varName : vars) { const ioda::Variable &variable = coords[varName]; @@ -237,25 +239,30 @@ DataExtractorInput DataExtractorNetCDFBackend::loadData( ASSERT(dimdim.dimsCur[0] == dimdim.dimsCur[1]); // Sanity check the matrix is square. if (dimdim.dimensionality == 3) { - // ioda::Variable requires us to define the size of the tensor beforehand - // We use row major ordering so that it resemblence handling the NetCDF data structure. - Eigen::Tensor data(dimdim.dimsCur[0], dimdim.dimsCur[1], - dimdim.dimsCur[2]); - interpolatedArrayCoord.readWithEigenTensor(data); - result.payloadArray.resize(dimdim.dimsCur[1], dimdim.dimsCur[2]); - - for (int i = 0; i < data.dimension(0); i++) { - for (int k = 0; k < data.dimension(2); k++) { - result.payloadArray(i, k) = data(i, i, k); + // Load a stack of full matrices + boost::multi_array fullMatrices( + boost::extents[dimdim.dimsCur[0]][dimdim.dimsCur[1]][dimdim.dimsCur[2]]); + interpolatedArrayCoord.read(gsl::span(fullMatrices.data(), + fullMatrices.num_elements())); + + // Extract their diagonals into the payload array + result.payloadArray.resize(boost::extents[dimdim.dimsCur[1]][dimdim.dimsCur[2]]); + for (int i = 0; i < dimdim.dimsCur[1]; i++) { + for (int k = 0; k < dimdim.dimsCur[2]; k++) { + result.payloadArray[i][k] = fullMatrices[i][i][k]; } } } else if (dimdim.dimensionality == 2) { - Eigen::ArrayXXf data; - interpolatedArrayCoord.readWithEigenRegular(data); - result.payloadArray.resize(dimdim.dimsCur[1], 1); - - for (int i = 0; i < data.rows(); i++) { - result.payloadArray(i, 0) = data(i, i); + // Load a full matrix + boost::multi_array fullMatrix( + boost::extents[dimdim.dimsCur[0]][dimdim.dimsCur[1]]); + interpolatedArrayCoord.read(gsl::span(fullMatrix.data(), + fullMatrix.num_elements())); + + // Extract its diagonal into the payload array + result.payloadArray.resize(boost::extents[dimdim.dimsCur[1]][1]); + for (int i = 0; i < dimdim.dimsCur[1]; i++) { + result.payloadArray[i][0] = fullMatrix[i][i]; } } else { throw eckit::Exception("Expecting 3D or 2D array for error covariance.", Here()); @@ -276,10 +283,16 @@ DataExtractorInput DataExtractorNetCDFBackend::loadData( dim2CoordMapping[dim].emplace_back(coord.first); // update our reverse lookup } } - } else if ((dimdim.dimensionality == 2) || (dimdim.dimensionality == 1)) { + } else if (dimdim.dimensionality == 2) { // This is just the diagonals - read directly into our container - // -------------------------------- - interpolatedArrayCoord.readWithEigenRegular(result.payloadArray); + result.payloadArray.resize(boost::extents[dimdim.dimsCur[0]][dimdim.dimsCur[1]]); + interpolatedArrayCoord.read(gsl::span(result.payloadArray.data(), + result.payloadArray.num_elements())); + } else if (dimdim.dimensionality == 1) { + // This is just the diagonals - read directly into our container + result.payloadArray.resize(boost::extents[dimdim.dimsCur[0]][1]); + interpolatedArrayCoord.read(gsl::span(result.payloadArray.data(), + result.payloadArray.num_elements())); } else { throw eckit::Exception("The array to be interpolated has an unsupported number of dimensions.", Here()); @@ -313,4 +326,9 @@ DataExtractorInput DataExtractorNetCDFBackend::loadData( return result; } +// Explicit instantiations +template class DataExtractorNetCDFBackend; +template class DataExtractorNetCDFBackend; +template class DataExtractorNetCDFBackend; + } // namespace ufo diff --git a/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h b/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h index 9b7f1321e..5e1e220c2 100644 --- a/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h +++ b/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h @@ -22,7 +22,8 @@ namespace ufo /// - It should contain exactly one variable whose name ends with the `@` suffix, where /// is the value of the `payloadGroup` parameter passed to the loadData() method. /// This is the variable containing the values to be extracted (also known as the _payload_). -/// It should be of type `float`. +/// Its type needs to match the template parameter `ExtractedValue`, which must be +/// set to either `float`, `int` or `std::string`. /// - For most types of data, the above array should be 1D or 2D. As a special case, /// if this class is used to extract variances, the file may contain a full 2D covariance /// matrix or a stack of such matrices stored as a 3D array, with variances located on the @@ -110,14 +111,15 @@ namespace ufo /// air_temperature:coordinates = "/MetaData/air_pressure /MetaData/observation_type" ; /// } /// \endcode -class DataExtractorNetCDFBackend : public DataExtractorBackend { +template +class DataExtractorNetCDFBackend : public DataExtractorBackend { public: /// \brief Create a new instance. /// /// \param filepath Path to the NetCDF file that will be read by loadData(). explicit DataExtractorNetCDFBackend(const std::string &filepath); - DataExtractorInput loadData(const std::string &payloadGroup) const override; + DataExtractorInput loadData(const std::string &payloadGroup) const override; private: std::string filepath_; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9ec4edfcf..05ceebb7c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -276,6 +276,8 @@ list( APPEND ufo_test_input testinput/unit_tests/composite_GsiSfcModel_SfcPCorrected.yaml testinput/unit_tests/composite_vertInterp_SfcPCorrected.yaml testinput/unit_tests/function_clwret_ssmis.yaml + testinput/unit_tests/function_conditional.yaml + testinput/unit_tests/function_drawvaluefromfile.yaml testinput/unit_tests/function_tropo_estimate.yaml testinput/unit_tests/function_lam_domaincheck.yaml testinput/unit_tests/function_lam_domaincheck_circle.yaml @@ -2622,6 +2624,20 @@ ecbuild_add_test( TARGET test_ufo_function_clwret_obsval_atms DEPENDS test_ObsFunction.x TEST_DEPENDS ufo_get_ufo_test_data) +ecbuild_add_test( TARGET test_ufo_function_conditional + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x + ARGS "testinput/unit_tests/function_conditional.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFunction.x + TEST_DEPENDS ufo_get_ufo_test_data) + +ecbuild_add_test( TARGET test_ufo_function_drawvaluefromfile + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x + ARGS "testinput/unit_tests/function_drawvaluefromfile.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFunction.x + TEST_DEPENDS ufo_get_ufo_test_data) + ecbuild_add_test( TARGET test_ufo_function_errfgrosschk COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x ARGS "testinput/function_errfgrosschk.yaml" diff --git a/test/testinput/obserror_assign_unittests.yaml b/test/testinput/obserror_assign_unittests.yaml index 8dd0c7d37..10b8b1047 100644 --- a/test/testinput/obserror_assign_unittests.yaml +++ b/test/testinput/obserror_assign_unittests.yaml @@ -341,7 +341,7 @@ Assign observation error from NetCDF variance; attempt at using 'nearest' for st interpolation: - name: StringVar@MetaData method: nearest - expectExceptionWithMessage: Nearest match not compatible with string type + expectExceptionWithMessage: The 'nearest' method cannot be used for string variables Assign observation error from NetCDF variance; attempt at using 'linear' for strings: window begin: 2000-01-01T00:00:00Z window end: 2030-01-01T00:00:00Z @@ -369,7 +369,7 @@ Assign observation error from NetCDF variance; attempt at using 'linear' for str interpolation: - name: StringVar@MetaData method: linear - expectExceptionWithMessage: linear interpolation not compatible with string type + expectExceptionWithMessage: Linear interpolation cannot be performed along coordinate axes indexed by string variables such as StringVar@MetaData Assign observation error from NetCDF variance; 1 least upper bound; 1 exact: window begin: 2000-01-01T00:00:00Z window end: 2030-01-01T00:00:00Z @@ -576,6 +576,93 @@ Assign observation error from NetCDF variance; attempt to use the 'greatest lowe - name: StringVar@MetaData method: greatest lower bound expectExceptionWithMessage: The 'greatest lower bound' method cannot be used for string variables +Assign observation error from NetCDF file; no variable with the @ErrorVariance suffix: + window begin: 2000-01-01T00:00:00Z + window end: 2030-01-01T00:00:00Z + obs space: + name: Aircraft + simulated variables: [air_temperature] + generate: + list: + lats: [ -2, -1, 0, 1, 2, 3 ] + lons: [ 178, 179, 180, 181, 182, 183 ] + datetimes: [ '2010-01-01T00:00:00Z', '2010-01-01T01:00:00Z', '2010-01-01T02:00:00Z', + '2010-01-01T03:00:00Z', '2010-01-01T04:00:00Z', '2010-01-01T04:00:00Z' ] + obs errors: [1.0] + air_pressure: [ 599, 599, 100, 600, 600, 600] + station_id: [0, 1, 2, 0, 1, 2] + ObsError assign: + filter variables: + - name: air_temperature + action: + name: assign error + error function: + name: DrawObsErrorFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/obserror_misspelled_variance.nc + interpolation: + - name: air_pressure@MetaData + method: greatest lower bound + - name: station_id@MetaData + method: exact + expectExceptionWithMessage: "No payload variable found: no variable name begins with 'ErrorVariance/' or ends with '@ErrorVariance'" +Assign observation error from NetCDF file; multiple variables with the @ErrorVariance suffix: + window begin: 2000-01-01T00:00:00Z + window end: 2030-01-01T00:00:00Z + obs space: + name: Aircraft + simulated variables: [air_temperature] + generate: + list: + lats: [ -2, -1, 0, 1, 2, 3 ] + lons: [ 178, 179, 180, 181, 182, 183 ] + datetimes: [ '2010-01-01T00:00:00Z', '2010-01-01T01:00:00Z', '2010-01-01T02:00:00Z', + '2010-01-01T03:00:00Z', '2010-01-01T04:00:00Z', '2010-01-01T04:00:00Z' ] + obs errors: [1.0] + air_pressure: [ 599, 599, 100, 600, 600, 600] + station_id: [0, 1, 2, 0, 1, 2] + ObsError assign: + filter variables: + - name: air_temperature + action: + name: assign error + error function: + name: DrawObsErrorFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/obserror_multiple_variances.nc + interpolation: + - name: air_pressure@MetaData + method: greatest lower bound + - name: station_id@MetaData + method: exact + expectExceptionWithMessage: "Multiple payload candidates found: more than one variable name begins with 'ErrorVariance/' or ends with '@ErrorVariance'" +Assign observation error from NetCDF file; 2D array of full covariance matrices: + window begin: 2000-01-01T00:00:00Z + window end: 2030-01-01T00:00:00Z + obs space: + name: Aircraft + simulated variables: [air_temperature] + generate: + list: + lats: [ -2, -1, 0, 1, 2 ] + lons: [ 178, 179, 180, 181, 182 ] + datetimes: [ '2010-01-01T00:00:00Z', '2010-01-01T01:00:00Z', '2010-01-01T02:00:00Z', + '2010-01-01T03:00:00Z', '2010-01-01T04:00:00Z' ] + obs errors: [1.0] + channel_number: [ 2, 0, 4, 7, 5 ] + ObsError assign: + filter variables: + - name: air_temperature + action: + name: assign error + error function: + name: DrawObsErrorFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/obserror_4d.nc + interpolation: + - name: channel_number@MetaData + method: exact + expectExceptionWithMessage: Expecting 3D or 2D array for error covariance Assign observation error from NetCDF variance; multi-channel variable support: window begin: 2000-01-01T00:00:00Z window end: 2030-01-01T00:00:00Z @@ -979,6 +1066,33 @@ Assign observation error from CSV file; no data lines: - name: StringVar@MetaData method: exact expectExceptionWithMessage: No data could be loaded from the file +Assign observation error from CSV file; empty data line: + window begin: 2000-01-01T00:00:00Z + window end: 2030-01-01T00:00:00Z + obs space: + name: Aircraft + simulated variables: [air_temperature] + generate: + list: + lats: [ 1 ] + lons: [ 1 ] + datetimes: [ '2010-01-01T00:00:00Z' ] + obs errors: [1.0] + air_pressure: [1] + StringVar: [abc] + ObsError assign: + filter variables: + - name: air_temperature + action: + name: assign error + error function: + name: DrawObsErrorFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/obserror_empty_data_line.csv + interpolation: + - name: StringVar@MetaData + method: exact + expectExceptionWithMessage: No data could be loaded from the file Assign observation error from CSV file; incorrect number of data types: window begin: 2000-01-01T00:00:00Z window end: 2030-01-01T00:00:00Z @@ -1060,6 +1174,33 @@ Assign observation error from CSV file; unsupported data type: - name: StringVar@MetaData method: exact expectExceptionWithMessage: Unsupported data type 'color' +Assign observation error from CSV file; criteria insufficient to identify a single matching row: + window begin: 2000-01-01T00:00:00Z + window end: 2030-01-01T00:00:00Z + obs space: + name: Aircraft + simulated variables: [air_temperature] + generate: + list: + lats: [ 0 ] + lons: [ 0 ] + datetimes: [ '2010-01-01T00:00:00Z' ] + obs errors: [1.0] + air_pressure: [100000] + StringVar: [south] + ObsError assign: + filter variables: + - name: air_temperature + action: + name: assign error + error function: + name: DrawObsErrorFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/obserror_stringvar_pressure_longitude.csv + interpolation: + - name: StringVar@MetaData + method: exact + expectExceptionWithMessage: Previous calls to extract() have failed to identify a single value to return Assign observation error from CSV file; data types float, int and string; missing values: window begin: 2000-01-01T00:00:00Z window end: 2030-01-01T00:00:00Z diff --git a/test/testinput/qc_variableassignment.yaml b/test/testinput/qc_variableassignment.yaml index 4c4afb3dc..fe2dec5f8 100644 --- a/test/testinput/qc_variableassignment.yaml +++ b/test/testinput/qc_variableassignment.yaml @@ -115,39 +115,36 @@ observations: reference: name: assigned_string_variable_2@TestReference -# We can't test datetime variables yet because there's no way to store reference data in the -# ObsSpace: the loader recognizes only a single variable of type DateTime: datetime@MetaData. -# -#- obs space: -# <<: *ObsSpace -# name: Assign a fixed value to a pre-existing datetime variable -# obs filters: -# - filter: Variable Assignment -# where: *Where -# assignments: -# - name: datetime@MetaData -# value: 2000-01-02T03:04:05Z -# compareVariables: -# - test: -# name: datetime@MetaData -# reference: -# name: assigned_datetime_variable_1@TestReference -# -#- obs space: -# <<: *ObsSpace -# name: Assign a fixed value to a new datetime variable -# obs filters: -# - filter: Variable Assignment -# where: *Where -# assignments: -# - name: new_variable@MetaData -# value: 2000-01-02T03:04:05Z -# type: string -# compareVariables: -# - test: -# name: new_variable@MetaData -# reference: -# name: assigned_datetime_variable_2@TestReference +- obs space: + <<: *ObsSpace + name: Assign a fixed value to a pre-existing datetime variable + obs filters: + - filter: Variable Assignment + where: *Where + assignments: + - name: datetime@MetaData + value: 2000-01-02T03:04:05Z + compareVariables: + - test: + name: datetime@MetaData + reference: + name: assigned_datetime_variable_1@TestReference + +- obs space: + <<: *ObsSpace + name: Assign a fixed value to a new datetime variable + obs filters: + - filter: Variable Assignment + where: *Where + assignments: + - name: new_variable@MetaData + value: 2000-01-02T03:04:05Z + type: datetime + compareVariables: + - test: + name: new_variable@MetaData + reference: + name: assigned_datetime_variable_2@TestReference - obs space: <<: *ObsSpace @@ -625,7 +622,29 @@ observations: x1: [10] err0: [0.25] err1: [5] - expectExceptionWithMessage: ObsFunction values cannot be assigned to non-numeric variables + expectExceptionWithMessage: ObsErrorModelRamp@ObsFunction is not a function producing values of type std::string + +- obs space: &ObsSpace + name: Try to assign values produced by a string-valued ObsFunction to a floating-point variable + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs filters: + - filter: Variable Assignment + where: *Where + assignments: + - name: float_variable_1@MetaData + function: + name: Conditional@StringObsFunction + options: + cases: + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 2-4 + value: XYZ + expectExceptionWithMessage: Conditional@StringObsFunction is not a function producing numeric values - obs space: <<: *ObsSpace diff --git a/test/testinput/unit_tests/function_conditional.yaml b/test/testinput/unit_tests/function_conditional.yaml new file mode 100644 index 000000000..466a79272 --- /dev/null +++ b/test/testinput/unit_tests/function_conditional.yaml @@ -0,0 +1,229 @@ +window begin: 2000-01-01T00:00:00Z +window end: 2030-01-01T00:00:00Z + +observations: +# 1. Tests of Conditional@ObsFunction +- obs space: &ObsSpace + name: Floating-point result, with default value + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: Conditional@ObsFunction + options: + defaultvalue: 75.5 + cases: + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 1 + value: 3.0 + - where: + - variable: + name: int_variable_1@MetaData + is_in: 5 + value: 9.0 + # Variable from the TestReference group storing the reference result, + # [3.0, 75.5, 75.5, 75.5, 9.0] + variables: [assigned_float_variable_1] + tolerance: 1e-8 +- obs space: &ObsSpace + <<: *ObsSpace + name: Floating-point result, without default value + obs function: + name: Conditional@ObsFunction + options: + cases: + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 2-4 + value: 75.5 + # Variable from the TestReference group storing the reference result, + # [missing, 75.5, 75.5, 75.5, missing] + variables: [assigned_float_variable_2] + tolerance: 1e-8 +- obs space: + <<: *ObsSpace + name: Floating-point result, firstmatchingcase flag set to false + obs function: + name: Conditional@ObsFunction + options: + firstmatchingcase: false + cases: + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 2-4 + value: 12345.0 # will be overriden by the value set below + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 2-4 + value: 75.5 # will override the value set above + # Variable from the TestReference group storing the reference result, + # [missing, 75.5, 75.5, 75.5, missing] + variables: [assigned_float_variable_2] + tolerance: 1e-8 +- obs space: + <<: *ObsSpace + name: Floating-point result, firstmatchingcase flag set to true + obs function: + name: Conditional@ObsFunction + options: + firstmatchingcase: true + cases: + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 2-4 + value: 75.5 + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 2-4 + value: 12345.0 # won't be overriden by the value set below + # Variable from the TestReference group storing the reference result, + # [missing, 75.5, 75.5, 75.5, missing] + variables: [assigned_float_variable_2] + tolerance: 1e-8 + +# 2. Tests of Conditional@IntObsFunction +- obs space: &ObsSpace + name: Integer result, with default value + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: Conditional@IntObsFunction + options: + defaultvalue: 75 + cases: + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 1 + value: 1 + - where: + - variable: + name: int_variable_1@MetaData + is_in: 5 + value: 5 + # Variable from the TestReference group storing the reference result, + # [1, 75, 75, 75, 1] + variables: [assigned_int_variable_1] +- obs space: &ObsSpace + <<: *ObsSpace + name: Integer result, without default value + obs function: + name: Conditional@IntObsFunction + options: + cases: + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 2-4 + value: 75 + # Variable from the TestReference group storing the reference result, + # [missing, 75, 75, 75, missing] + variables: [assigned_int_variable_2] + +# 3. Tests of Conditional@StringObsFunction +- obs space: &ObsSpace + name: String result, with default value + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: Conditional@StringObsFunction + options: + defaultvalue: XYZ + cases: + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 1 + value: ABC + - where: + - variable: + name: int_variable_1@MetaData + is_in: 5 + value: MNO + # Variable from the TestReference group storing the reference result, + # [ABC, XYZ, XYZ, XYZ, MNO] + variables: [assigned_string_variable_1] +- obs space: &ObsSpace + <<: *ObsSpace + name: String result, without default value + obs function: + name: Conditional@StringObsFunction + options: + cases: + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 2-4 + value: XYZ + # Variable from the TestReference group storing the reference result, + # [missing, XYZ, XYZ, XYZ, missing] + variables: [assigned_string_variable_2] + +# 4. Tests of Conditional@DateTimeObsFunction +- obs space: &ObsSpace + name: Datetime result, with default value + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: Conditional@DateTimeObsFunction + options: + defaultvalue: 2000-01-02T03:04:05Z + cases: + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 1 + value: 2018-04-15T06:00:00Z + - where: + - variable: + name: int_variable_1@MetaData + is_in: 5 + value: 2018-04-19T06:00:00Z + # Variable from the TestReference group storing the reference result, + # [2018-04-15T06:00:00Z, + # 2000-01-02T03:04:05Z + # 2000-01-02T03:04:05Z + # 2000-01-02T03:04:05Z + # 2018-04-19T06:00:00Z] + variables: [assigned_datetime_variable_1] +- obs space: &ObsSpace + <<: *ObsSpace + name: Datetime result, without default value + obs function: + name: Conditional@DateTimeObsFunction + options: + cases: + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 2-4 + value: 2000-01-02T03:04:05Z + # Variable from the TestReference group storing the reference result, + # [missing, + # 2000-01-02T03:04:05Z, + # 2000-01-02T03:04:05Z, + # 2000-01-02T03:04:05Z, + # missing] + variables: [assigned_datetime_variable_2] diff --git a/test/testinput/unit_tests/function_drawvaluefromfile.yaml b/test/testinput/unit_tests/function_drawvaluefromfile.yaml new file mode 100644 index 000000000..7c051ebbe --- /dev/null +++ b/test/testinput/unit_tests/function_drawvaluefromfile.yaml @@ -0,0 +1,368 @@ +window begin: 2000-01-01T00:00:00Z +window end: 2030-01-01T00:00:00Z + +observations: + +# 1. Tests of DrawValueFromFile@IntObsFunction using a CSV input file + +- obs space: &ObsSpace + name: Map a string variable to an int result by exact matching (with a fallback value), CSV + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@IntObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.csv + group: IntResult + interpolation: + # ["ABC", "DEF", "GHI", "JKL", "MNO"] + - name: string_variable_1@MetaData + method: exact + # Variable from the TestReference group storing the reference result, + # [1, 75, 75, 75, 5] + variables: [assigned_int_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Map an int variable to an int result by nearest-neighbor matching, CSV + obs function: + name: DrawValueFromFile@IntObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.csv + group: IntResult + interpolation: + # [1, 2, 3, 4, 5] + - name: int_variable_1@MetaData + method: nearest + # Variable from the TestReference group storing the reference result, + # [1, 75, 75, 75, 5] + variables: [assigned_int_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Map a float variable to an int result by least-upper-bound matching, CSV + obs function: + name: DrawValueFromFile@IntObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.csv + group: IntResult + interpolation: + # [-2.5, -1.25, 0, 1.25, 2.5] + - name: latitude@MetaData + method: least upper bound + # Variable from the TestReference group storing the reference result, + # [1, 75, 75, 75, 5] + variables: [assigned_int_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Map a float variable to an int result by greatest-lower-bound matching, CSV + obs function: + name: DrawValueFromFile@IntObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.csv + group: IntResult + interpolation: + # [11.25, 12.25, 13.5, 14.75, 15.75] + - name: longitude@MetaData + method: least upper bound + # Variable from the TestReference group storing the reference result, + # [1, 75, 75, 75, 5] + variables: [assigned_int_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Attempt to use linear interpolation to extract an int variable + obs function: + name: DrawValueFromFile@IntObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.csv + group: IntResult + interpolation: + # [11.25, 12.25, 13.5, 14.75, 15.75] + - name: longitude@MetaData + method: linear + variables: [assigned_string_variable_1] + expect compute to throw exception with message: Linear interpolation can be used when extracting floating-point values, but not integers or strings + +- obs space: &ObsSpace + <<: *ObsSpace + name: Attempt to produce integers from a payload column of type string + obs function: + name: DrawValueFromFile@IntObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.csv + group: StringResult + interpolation: + # [11.25, 12.25, 13.5, 14.75, 15.75] + - name: longitude@MetaData + method: linear + variables: [assigned_string_variable_1] + expect compute to throw exception with message: The payload column must contain numeric data + + +# 2. Tests of DrawValueFromFile@StringObsFunction using a CSV input file + +- obs space: &ObsSpace + name: Map a string variable to a string result by exact matching (with a fallback value), CSV + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@StringObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.csv + group: StringResult + interpolation: + # ["ABC", "DEF", "GHI", "JKL", "MNO"] + - name: string_variable_1@MetaData + method: exact + # Variable from the TestReference group storing the reference result, + # ["ABC", "XYZ", "XYZ", "XYZ", "MNO"] + variables: [assigned_string_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Map an int variable to a string result by nearest-neighbor matching, CSV + obs function: + name: DrawValueFromFile@StringObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.csv + group: StringResult + interpolation: + # [1, 2, 3, 4, 5] + - name: int_variable_1@MetaData + method: nearest + # Variable from the TestReference group storing the reference result, + # ["ABC", "XYZ", "XYZ", "XYZ", "MNO"] + variables: [assigned_string_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Map a float variable to a string result by least-upper-bound matching, CSV + obs function: + name: DrawValueFromFile@StringObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.csv + group: StringResult + interpolation: + # [-2.5, -1.25, 0, 1.25, 2.5] + - name: latitude@MetaData + method: least upper bound + # Variable from the TestReference group storing the reference result, + # ["ABC", "XYZ", "XYZ", "XYZ", "MNO"] + variables: [assigned_string_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Map a float variable to a string result by greatest-lower-bound matching, CSV + obs function: + name: DrawValueFromFile@StringObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.csv + group: StringResult + interpolation: + # [11.25, 12.25, 13.5, 14.75, 15.75] + - name: longitude@MetaData + method: least upper bound + # Variable from the TestReference group storing the reference result, + # ["ABC", "XYZ", "XYZ", "XYZ", "MNO"] + variables: [assigned_string_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Attempt to use linear interpolation to extract a string variable + obs function: + name: DrawValueFromFile@StringObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.csv + group: StringResult + interpolation: + # [11.25, 12.25, 13.5, 14.75, 15.75] + - name: longitude@MetaData + method: linear + variables: [assigned_string_variable_1] + expect compute to throw exception with message: Linear interpolation can be used when extracting floating-point values, but not integers or strings + +- obs space: &ObsSpace + <<: *ObsSpace + name: Attempt to produce strings from a payload column of type int + obs function: + name: DrawValueFromFile@StringObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.csv + group: IntResult + interpolation: + # [11.25, 12.25, 13.5, 14.75, 15.75] + - name: longitude@MetaData + method: linear + variables: [assigned_string_variable_1] + expect compute to throw exception with message: The payload column must contain strings or datetimes + +# 3. Tests of DrawValueFromFile@IntObsFunction using a NetCDF input file + +- obs space: &ObsSpace + name: Map a string variable to an int result by exact matching (with a fallback value), NetCDF + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@IntObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.nc + group: IntResult + interpolation: + # ["ABC", "DEF", "GHI", "JKL", "MNO"] + - name: string_variable_1@MetaData + method: exact + # Variable from the TestReference group storing the reference result, + # [1, 75, 75, 75, 5] + variables: [assigned_int_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Map an int variable to an int result by nearest-neighbor matching, NetCDF + obs function: + name: DrawValueFromFile@IntObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.nc + group: IntResult + interpolation: + # [1, 2, 3, 4, 5] + - name: int_variable_1@MetaData + method: nearest + # Variable from the TestReference group storing the reference result, + # [1, 75, 75, 75, 5] + variables: [assigned_int_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Map a float variable to an int result by least-upper-bound matching, NetCDF + obs function: + name: DrawValueFromFile@IntObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.nc + group: IntResult + interpolation: + # [-2.5, -1.25, 0, 1.25, 2.5] + - name: latitude@MetaData + method: least upper bound + # Variable from the TestReference group storing the reference result, + # [1, 75, 75, 75, 5] + variables: [assigned_int_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Map a float variable to an int result by greatest-lower-bound matching, NetCDF + obs function: + name: DrawValueFromFile@IntObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.nc + group: IntResult + interpolation: + # [11.25, 12.25, 13.5, 14.75, 15.75] + - name: longitude@MetaData + method: least upper bound + # Variable from the TestReference group storing the reference result, + # [1, 75, 75, 75, 5] + variables: [assigned_int_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Attempt to use linear interpolation to extract an int variable, NetCDF + obs function: + name: DrawValueFromFile@IntObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.nc + group: IntResult + interpolation: + # [11.25, 12.25, 13.5, 14.75, 15.75] + - name: longitude@MetaData + method: linear + variables: [assigned_string_variable_1] + expect compute to throw exception with message: Linear interpolation can be used when extracting floating-point values, but not integers or strings + + +# 4. Tests of DrawValueFromFile@StringObsFunction using a NetCDF input file + +- obs space: &ObsSpace + name: Map a string variable to a string result by exact matching (with a fallback value), NetCDF + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@StringObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.nc + group: StringResult + interpolation: + # ["ABC", "DEF", "GHI", "JKL", "MNO"] + - name: string_variable_1@MetaData + method: exact + # Variable from the TestReference group storing the reference result, + # ["ABC", "XYZ", "XYZ", "XYZ", "MNO"] + variables: [assigned_string_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Map an int variable to a string result by nearest-neighbor matching, NetCDF + obs function: + name: DrawValueFromFile@StringObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.nc + group: StringResult + interpolation: + # [1, 2, 3, 4, 5] + - name: int_variable_1@MetaData + method: nearest + # Variable from the TestReference group storing the reference result, + # ["ABC", "XYZ", "XYZ", "XYZ", "MNO"] + variables: [assigned_string_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Map a float variable to a string result by least-upper-bound matching, NetCDF + obs function: + name: DrawValueFromFile@StringObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.nc + group: StringResult + interpolation: + # [-2.5, -1.25, 0, 1.25, 2.5] + - name: latitude@MetaData + method: least upper bound + # Variable from the TestReference group storing the reference result, + # ["ABC", "XYZ", "XYZ", "XYZ", "MNO"] + variables: [assigned_string_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Map a float variable to a string result by greatest-lower-bound matching, NetCDF + obs function: + name: DrawValueFromFile@StringObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.nc + group: StringResult + interpolation: + # [11.25, 12.25, 13.5, 14.75, 15.75] + - name: longitude@MetaData + method: least upper bound + # Variable from the TestReference group storing the reference result, + # ["ABC", "XYZ", "XYZ", "XYZ", "MNO"] + variables: [assigned_string_variable_1] + +- obs space: &ObsSpace + <<: *ObsSpace + name: Attempt to use linear interpolation to extract a string variable + obs function: + name: DrawValueFromFile@StringObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_simple.nc + group: StringResult + interpolation: + # [11.25, 12.25, 13.5, 14.75, 15.75] + - name: longitude@MetaData + method: linear + variables: [assigned_string_variable_1] + expect compute to throw exception with message: Linear interpolation can be used when extracting floating-point values, but not integers or strings diff --git a/test/testinput/variables.yaml b/test/testinput/variables.yaml index 290918428..dc398693c 100644 --- a/test/testinput/variables.yaml +++ b/test/testinput/variables.yaml @@ -4,16 +4,52 @@ test variables: channels: 1, 4-5 reference names: [brightness_temperature_1, brightness_temperature_4, brightness_temperature_5] reference group: ObsValue + reference full name: brightness_temperature@ObsValue - name: latitude@MetaData reference names: [latitude] reference group: MetaData + reference full name: latitude@MetaData - name: optical_thickness_of_atmosphere_layer@ObsDiag channels: 23-25 reference names: [optical_thickness_of_atmosphere_layer_23, optical_thickness_of_atmosphere_layer_24, optical_thickness_of_atmosphere_layer_25] reference group: ObsDiag + reference full name: optical_thickness_of_atmosphere_layer@ObsDiag oops variables: - variables: ['var1', 'var2', 'var3'] reference names: ['var1', 'var2', 'var3'] - variables: ['var'] channels: 3-5,8 reference names: ['var_3', 'var_4', 'var_5', 'var_8'] +# Options passed to Conditional@ObsFunction. The test uses them to verify that when a Variables +# object contains a variable from the ObsFunction group, the results produced by its has() and +# allFromGroup() functions include variables needed by the ObsFunction itself (e.g. +# eastward_wind@ObsValue in the example below). +float conditional: + cases: + - where: + - variable: + name: eastward_wind@ObsValue + minvalue: 10 + value: 1.5 +# Same for Conditional@IntObsFunction, and so on. +int conditional: + cases: + - where: + - variable: + name: eastward_wind@ObsValue + minvalue: 10 + value: 1 +string conditional: + cases: + - where: + - variable: + name: eastward_wind@ObsValue + minvalue: 10 + value: XYZ +datetime conditional: + cases: + - where: + - variable: + name: eastward_wind@ObsValue + minvalue: 10 + value: 2000-01-01T00:00:00Z diff --git a/test/ufo/ObsFunction.h b/test/ufo/ObsFunction.h index a14d86dd0..6273b9dc5 100644 --- a/test/ufo/ObsFunction.h +++ b/test/ufo/ObsFunction.h @@ -22,6 +22,7 @@ #include "ioda/ObsVector.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" +#include "oops/util/Expect.h" #include "oops/util/missingValues.h" #include "test/interface/ObsTestsFixture.h" #include "test/TestEnvironment.h" @@ -32,11 +33,24 @@ #include "ufo/ObsDiagnostics.h" #include "ufo/ObsTraits.h" +namespace eckit +{ + // Don't use the contracted output for these types: the current implementation works only + // with integer types. + template <> struct VectorPrintSelector { typedef VectorPrintSimple selector; }; +} // namespace eckit + +// ----------------------------------------------------------------------------- + namespace ufo { namespace test { // ----------------------------------------------------------------------------- +const char *expectComputeToThrow = "expect compute to throw exception with message"; + +// ----------------------------------------------------------------------------- + void dataVectorDiff(const ioda::ObsSpace & ospace, ioda::ObsDataVector & vals, const ioda::ObsDataVector & ref, std::vector & rms_out) { float missing = util::missingValue(missing); @@ -58,80 +72,147 @@ void dataVectorDiff(const ioda::ObsSpace & ospace, ioda::ObsDataVector & // ----------------------------------------------------------------------------- -void testFunction() { - typedef ::test::ObsTestsFixture Test_; +/// Checks that `vals` and `vals_ofd` are equal to `ref`. +/// +/// \param vals +/// ObsFunction values calculated by calling ObsFunction::compute() directly. +/// \param vals_ofd +/// ObsFunction values calculated by calling ObsFilterData::get(). +/// \param ref +/// Reference values. +template +void checkResults(const ioda::ObsSpace &/*ospace*/, + const eckit::Configuration &/*obsfuncconf*/, + const ioda::ObsDataVector & vals, + const ioda::ObsDataVector & vals_ofd, + const ioda::ObsDataVector & ref) { + const size_t nvars = ref.nvars(); + EXPECT_EQUAL(vals.nvars(), nvars); + + for (size_t i = 0; i < nvars; ++i) + EXPECT_EQUAL(vals[i], ref[i]); + + for (size_t i = 0; i < nvars; ++i) + EXPECT_EQUAL(vals_ofd[i], ref[i]); +} - std::vector typeconfs; - ::test::TestEnvironment::config().get("observations", typeconfs); +/// Specialization for floats: expects the RMSs of the differences between `vals` and `ref` and +/// between `vals_ofd` and `ref` to be within a specified tolerance. +/// +/// The contents of `vals` and `vals_ofd` are destroyed. +void checkResults(const ioda::ObsSpace &ospace, + const eckit::Configuration &obsfuncconf, + ioda::ObsDataVector & vals, + ioda::ObsDataVector & vals_ofd, + const ioda::ObsDataVector & ref) { + const double tol = obsfuncconf.getDouble("tolerance"); + const size_t nvars = ref.nvars(); + EXPECT_EQUAL(vals.nvars(), nvars); + + /// Calculate rms(f(x) - ref) and compare to tolerance + std::vector rms_out(nvars); + dataVectorDiff(ospace, vals, ref, rms_out); + + oops::Log::info() << "Vector difference between reference and computed: " << std::endl; + oops::Log::info() << vals << std::endl; + for (size_t ivar = 0; ivar < nvars; ivar++) { + // FIXME(someone): whatever this does, it certainly doesn't convert percentages to fractions + EXPECT(rms_out[ivar] < 100*tol); // change tol from percent to actual value. + } - for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - ioda::ObsSpace &ospace = Test_::obspace()[jj].obsspace(); - const eckit::Configuration &conf = typeconfs[jj]; + dataVectorDiff(ospace, vals_ofd, ref, rms_out); + oops::Log::info() << "Vector difference between reference and computed via ObsFilterData: " + << std::endl; + oops::Log::info() << vals_ofd << std::endl; + for (size_t ivar = 0; ivar < nvars; ivar++) { + // FIXME(someone): whatever this does, it certainly doesn't convert percentages to fractions + EXPECT(rms_out[ivar] < 100*tol); // change tol from percent to actual value. + } +} -/// Setup ObsFilterData - ObsFilterData inputs(ospace); +// ----------------------------------------------------------------------------- -/// Get function name and which group to use for H(x) - const eckit::LocalConfiguration obsfuncconf(conf, "obs function"); - Variable funcname(obsfuncconf); +template +void doTestFunction(ioda::ObsSpace &ospace, const eckit::Configuration &conf) { +/// Get function name + const eckit::LocalConfiguration obsfuncconf(conf, "obs function"); + Variable funcname(obsfuncconf); /// Setup function - ObsFunction obsfunc(funcname); - ufo::Variables allfuncvars = obsfunc.requiredVariables(); + ObsFunction obsfunc(funcname); + ufo::Variables allfuncvars = obsfunc.requiredVariables(); + +/// Setup ObsFilterData + ObsFilterData inputs(ospace); /// Setup GeoVaLs - const oops::Variables geovars = allfuncvars.allFromGroup("GeoVaLs").toOopsVariables(); - std::unique_ptr gval; - if (geovars.size() > 0) { - const eckit::LocalConfiguration gconf(conf, "geovals"); - gval.reset(new GeoVaLs(gconf, ospace, geovars)); - inputs.associate(*gval); - } + const oops::Variables geovars = allfuncvars.allFromGroup("GeoVaLs").toOopsVariables(); + std::unique_ptr gval; + if (geovars.size() > 0) { + const eckit::LocalConfiguration gconf(conf, "geovals"); + gval.reset(new GeoVaLs(gconf, ospace, geovars)); + inputs.associate(*gval); + } /// Setup ObsDiags - const oops::Variables diagvars = allfuncvars.allFromGroup("ObsDiag").toOopsVariables(); - std::unique_ptr diags; - if (diagvars.size() > 0) { - const eckit::LocalConfiguration diagconf(conf, "obs diagnostics"); - diags.reset(new ObsDiagnostics(diagconf, ospace, diagvars)); - inputs.associate(*diags); - } + const oops::Variables diagvars = allfuncvars.allFromGroup("ObsDiag").toOopsVariables(); + std::unique_ptr diags; + if (diagvars.size() > 0) { + const eckit::LocalConfiguration diagconf(conf, "obs diagnostics"); + diags.reset(new ObsDiagnostics(diagconf, ospace, diagvars)); + inputs.associate(*diags); + } /// Get output variable names - const oops::Variables outputvars(obsfuncconf, "variables"); + const oops::Variables outputvars(obsfuncconf, "variables"); /// Compute function result - ioda::ObsDataVector vals(ospace, outputvars, "ObsFunction", false); + ioda::ObsDataVector vals(ospace, outputvars, funcname.group(), false); + if (!conf.has(expectComputeToThrow)) { obsfunc.compute(inputs, vals); - vals.save("TestResult"); - int nvars = vals.nvars(); + } else { + // The call to compute() is expected to throw an exception containing the specified string. + const std::string expectedMessage = conf.getString(expectComputeToThrow); + EXPECT_THROWS_MSG(obsfunc.compute(inputs, vals), expectedMessage.c_str()); + return; + } + vals.save("TestResult"); /// Compute function result through ObsFilterData - ioda::ObsDataVector vals_ofd(ospace, outputvars, "ObsFunction", false); - inputs.get(funcname, vals_ofd); + ioda::ObsDataVector vals_ofd(ospace, outputvars, funcname.group(), false); + inputs.get(funcname, vals_ofd); /// Read reference values from ObsSpace - ioda::ObsDataVector ref(ospace, outputvars, "TestReference"); + ioda::ObsDataVector ref(ospace, outputvars, "TestReference"); + checkResults(ospace, obsfuncconf, vals, vals_ofd, ref); +} - const double tol = obsfuncconf.getDouble("tolerance"); +// ----------------------------------------------------------------------------- -/// Calculate rms(f(x) - ref) and compare to tolerance - std::vector rms_out(nvars); - dataVectorDiff(ospace, vals, ref, rms_out); +void testFunction() { + typedef ::test::ObsTestsFixture Test_; - oops::Log::info() << "Vector difference between reference and computed: " << std::endl; - oops::Log::info() << vals << std::endl; - for (size_t ivar = 0; ivar < nvars; ivar++) { - EXPECT(rms_out[ivar] < 100*tol); // change tol from percent to actual value. - } + std::vector typeconfs; + ::test::TestEnvironment::config().get("observations", typeconfs); - dataVectorDiff(ospace, vals_ofd, ref, rms_out); - oops::Log::info() << "Vector difference between reference and computed via ObsFilterData: " - << std::endl; - oops::Log::info() << vals_ofd << std::endl; - for (size_t ivar = 0; ivar < nvars; ivar++) { - EXPECT(rms_out[ivar] < 100*tol); // change tol from percent to actual value. - } + for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + ioda::ObsSpace &ospace = Test_::obspace()[jj].obsspace(); + const eckit::Configuration &conf = typeconfs[jj]; + +/// Use the function group to determine the type of values produced by the function and thus +/// select the right specialization of doTestFunction(). + Variable funcname(conf.getSubConfiguration("obs function")); + if (funcname.group() == ObsFunctionTraits::groupName) + doTestFunction(ospace, conf); + else if (funcname.group() == ObsFunctionTraits::groupName) + doTestFunction(ospace, conf); + else if (funcname.group() == ObsFunctionTraits::groupName) + doTestFunction(ospace, conf); + else if (funcname.group() == ObsFunctionTraits::groupName) + doTestFunction(ospace, conf); + else + throw eckit::testing::TestException("Variable " + funcname.fullName() + + " is not an ObsFunction", Here()); } } diff --git a/test/ufo/Variables.h b/test/ufo/Variables.h index bc81aeb6b..42207979e 100644 --- a/test/ufo/Variables.h +++ b/test/ufo/Variables.h @@ -43,6 +43,9 @@ void testVariable() { for (std::size_t jvar = 0; jvar < var.size(); ++jvar) { EXPECT(var.variable(jvar) == refvars[jvar]); } + // test the fullName() method + const std::string refFullName = conf[jj].getString("reference full name"); + EXPECT_EQUAL(var.fullName(), refFullName); } } @@ -78,26 +81,54 @@ bool hasVariable(const Variables & vars, const Variable & var) { } +// ----------------------------------------------------------------------------- +void testAllFromGroupFor(const std::string &funcName, + const eckit::LocalConfiguration &funcConf, + const ufo::Variables &initialVars) { + ufo::Variables vars = initialVars; + vars += Variable(funcName, funcConf); + + Variables res = vars.allFromGroup("ObsValue"); + oops::Log::info() << res << std::endl; + + EXPECT(res.size() == 2); + EXPECT(hasVariable(res, Variable("eastward_wind@ObsValue"))); + EXPECT(hasVariable(res, Variable("temperature@ObsValue"))); +} + // ----------------------------------------------------------------------------- // Test that ufo::Variables::allFromGroup() gets variables from the functions void testAllFromGroup() { Variables vars; vars += Variable("height@GeoVaLs"); - vars += Variable("Velocity@ObsFunction"); vars += Variable("latitude@MetaData"); vars += Variable("temperature@ObsValue"); vars += Variable("longitude@MetaData"); - Variables res = vars.allFromGroup("ObsValue"); - oops::Log::info() << res << std::endl; - - EXPECT(res.size() == 3); - EXPECT(hasVariable(res, Variable("eastward_wind@ObsValue"))); - EXPECT(hasVariable(res, Variable("northward_wind@ObsValue"))); - EXPECT(hasVariable(res, Variable("temperature@ObsValue"))); + const eckit::Configuration &conf = ::test::TestEnvironment::config(); + testAllFromGroupFor("Conditional@ObsFunction", + eckit::LocalConfiguration(conf, "float conditional"), + vars); + testAllFromGroupFor("Conditional@IntObsFunction", + eckit::LocalConfiguration(conf, "int conditional"), + vars); + testAllFromGroupFor("Conditional@StringObsFunction", + eckit::LocalConfiguration(conf, "string conditional"), + vars); + testAllFromGroupFor("Conditional@DateTimeObsFunction", + eckit::LocalConfiguration(conf, "datetime conditional"), + vars); } // ----------------------------------------------------------------------------- +void testHasGroupFor(const std::string &funcName, + const eckit::LocalConfiguration &funcConf, + const ufo::Variables &initialVars) { + ufo::Variables vars = initialVars; + vars += Variable(funcName, funcConf); + EXPECT(vars.hasGroup("ObsValue")); +} + // Test that ufo::Variables::hasGroup() works for functions void testHasGroup() { ufo::Variables vars; @@ -107,12 +138,21 @@ void testHasGroup() { EXPECT(vars.hasGroup("MetaData")); EXPECT(!vars.hasGroup("ObsValue")); - vars += Variable("Velocity@ObsFunction"); - - EXPECT(vars.hasGroup("ObsValue")); + const eckit::Configuration &conf = ::test::TestEnvironment::config(); + testHasGroupFor("Conditional@ObsFunction", + eckit::LocalConfiguration(conf, "float conditional"), + vars); + testHasGroupFor("Conditional@IntObsFunction", + eckit::LocalConfiguration(conf, "int conditional"), + vars); + testHasGroupFor("Conditional@StringObsFunction", + eckit::LocalConfiguration(conf, "string conditional"), + vars); + testHasGroupFor("Conditional@DateTimeObsFunction", + eckit::LocalConfiguration(conf, "datetime conditional"), + vars); } - // ----------------------------------------------------------------------------- class Variables : public oops::Test { From fbe44fa9380c943ef45f1a2abcab0f9a7678b5ca Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Thu, 8 Jul 2021 13:04:26 -0600 Subject: [PATCH 28/98] update readme (#1323) Co-authored-by: Ryan Honeyager --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b17e64377..fdbf6715a 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ -GNU:[![AWS-gnu](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiVjRjeGRXc2EyQit2Z3J4YXNWYjl3dGpWaFMyc0V4UkxGR25PS29VSXN1Z3ExbjJwNHBjRVZCUUptV243dlNWUFFtSmdxS1k5VHBwK25HeHdNbXJmTlFRPSIsIml2UGFyYW1ldGVyU3BlYyI6InM4dW1jYjE1enUxU216Y3UiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/projects/automated-testing-ufo-gnu/history?region=us-east-1) +### Continuous integration: +| Platform | JCSDA-internal | JCSDA | +| ------------- | ------------- |------------- | +| GNU | [![AWS-gnu](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiVngyM2gvQ3d4dzRMb1c0ZmJKa2xnWmtTTHFhVEFoWFZNTHFBNWZTQ3ZrUlhVeldqNHFRQlVxbHJrRUs5MDlPRkhvZkt6K1kyaUs1UkJzaEpSSkZaRllNPSIsIml2UGFyYW1ldGVyU3BlYyI6IjJtd3F5dlk0WDhuUGRrWGkiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/469205354006/projects/ufo-internal-gnu/history) | [![AWS-gnu](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiVjRjeGRXc2EyQit2Z3J4YXNWYjl3dGpWaFMyc0V4UkxGR25PS29VSXN1Z3ExbjJwNHBjRVZCUUptV243dlNWUFFtSmdxS1k5VHBwK25HeHdNbXJmTlFRPSIsIml2UGFyYW1ldGVyU3BlYyI6InM4dW1jYjE1enUxU216Y3UiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/469205354006/projects/automated-testing-ufo-gnu/history) +| Intel | [![AWS-intel](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiSnVxeG5qNndXd3JyZk1MaHJKODRCVlJXRTNnV2NnSGNzUGMxcFNaY3NnK3oyS0hHaklCdG8vK0VDeFZlSVFRLzhDZVBOMExPM29ncVQ2Z255KzVXWmg4PSIsIml2UGFyYW1ldGVyU3BlYyI6ImRtQ09kb0RjVG5ObWI2Vm8iLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/469205354006/projects/ufo-internal-intel/history) | [![AWS-intel](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiVjRjeGRXc2EyQit2Z3J4YXNWYjl3dGpWaFMyc0V4UkxGR25PS29VSXN1Z3ExbjJwNHBjRVZCUUptV243dlNWUFFtSmdxS1k5VHBwK25HeHdNbXJmTlFRPSIsIml2UGFyYW1ldGVyU3BlYyI6InM4dW1jYjE1enUxU216Y3UiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/469205354006/projects/automated-testing-ufo-gnu/history) +| CLANG | [![AWS-clang](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiWEI0Z2xCa0NpdlpNVTdLNEJGRWR5aEJ5NkpKdjNTT2ZQMURJWG5GNFZQUjV4Mkc4R1I2M3NGbEtUYzM2MCthdzZDeDVjS0NVSjB1R3h1TUFCYkdNb0RrPSIsIml2UGFyYW1ldGVyU3BlYyI6ImdrWGZnaXhzSG9pcDBMa0IiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/469205354006/projects/ufo-internal-clang/history) | [![AWS-clang](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiVU1vbVR2Y0twa3NpWHRoMUhraGlreU9Da2t5RVpZQk9kV3NrL2hTc0szdXRlaCszVzhHRndrd0VqcnpSb2lyL2VCRXNtK1Y4WG5LRHVxbldteFArU2IwPSIsIml2UGFyYW1ldGVyU3BlYyI6IkpQUTRlaThyRWVkWEJjNEwiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/469205354006/projects/automated-testing-ufo-clang/history) +| Code Coverage | [![codecov](https://codecov.io/gh/JCSDA/ufo/branch/develop/graph/badge.svg?token=nxhUKP82Pd)](https://codecov.io/gh/JCSDA-internal/ufo) | -INTEL:[![AWS-intel](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiUGxUMXUrdkNTVURCcjlwaTFlVlZ5eklHWGNodVdlWko2M0phM0lTRVIrWS9OcWQraWpwN3FvL240N2FDZWdEZGF3dU9vSHZHS1lFbEVJd2M5M1NWWHUwPSIsIml2UGFyYW1ldGVyU3BlYyI6Ii8xWTZDL0VxV1hNSE1LVHMiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/projects/automated-testing-ufo-intel/history?region=us-east-1) - -CLANG: [![AWS_clang](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiVU1vbVR2Y0twa3NpWHRoMUhraGlreU9Da2t5RVpZQk9kV3NrL2hTc0szdXRlaCszVzhHRndrd0VqcnpSb2lyL2VCRXNtK1Y4WG5LRHVxbldteFArU2IwPSIsIml2UGFyYW1ldGVyU3BlYyI6IkpQUTRlaThyRWVkWEJjNEwiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://us-east-1.console.aws.amazon.com/codesuite/codebuild/projects/automated-testing-ufo-clang/history?region=us-east-1) - -[![codecov](https://codecov.io/gh/JCSDA/ufo/branch/develop/graph/badge.svg?token=nxhUKP82Pd)](https://codecov.io/gh/JCSDA/ufo) Unified Forward Operators for Joint Effort for Data assimilation Integration (JEDI) project. From cad646b5ab5fa25fe8390d1b908fe4d93891259b Mon Sep 17 00:00:00 2001 From: adammartins Date: Thu, 8 Jul 2021 21:32:16 +0100 Subject: [PATCH 29/98] Feature/model best fit pressure (#1258) * initial filter * working app, values different to testref * correct pressure_levels * corrections. Test passing * add apply and break in metoffice flag setting loop * tidy and add UV switch * rename variables and addd docs * use is_close_absolute * change data path * remove output data * use new accumulator * review comment changes * temporary comment unused variables * ensure correct geovals order, use obspressure for variable name * use reverse and fix some descriptions * clearer discriptions * . Co-authored-by: Ryan Honeyager --- src/ufo/filters/CMakeLists.txt | 2 + src/ufo/filters/ModelBestFitPressure.cc | 301 ++++++++++++++++++++ src/ufo/filters/ModelBestFitPressure.h | 109 +++++++ src/ufo/instantiateObsFilterFactory.h | 3 + src/ufo/utils/metoffice/MetOfficeQCFlags.h | 4 +- test/CMakeLists.txt | 10 +- test/testinput/qc_modelbestfitpressure.yaml | 40 +++ 7 files changed, 466 insertions(+), 3 deletions(-) create mode 100644 src/ufo/filters/ModelBestFitPressure.cc create mode 100644 src/ufo/filters/ModelBestFitPressure.h create mode 100644 test/testinput/qc_modelbestfitpressure.yaml diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index 2aad855c7..0143c6949 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -43,6 +43,8 @@ set ( filters_files MetOfficeBuddyPair.h MetOfficeBuddyPairFinder.cc MetOfficeBuddyPairFinder.h + ModelBestFitPressure.cc + ModelBestFitPressure.h ModelObThreshold.cc ModelObThreshold.h MWCLWCheck.cc diff --git a/src/ufo/filters/ModelBestFitPressure.cc b/src/ufo/filters/ModelBestFitPressure.cc new file mode 100644 index 000000000..ce8660cc4 --- /dev/null +++ b/src/ufo/filters/ModelBestFitPressure.cc @@ -0,0 +1,301 @@ +/* + * (C) Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/filters/ModelBestFitPressure.h" + +#include +#include +#include +#include +#include + +#include "ioda/distribution/Accumulator.h" +#include "ioda/ObsDataVector.h" +#include "ioda/ObsSpace.h" + +#include "oops/util/FloatCompare.h" +#include "oops/util/Logger.h" + +#include "ufo/GeoVaLs.h" +#include "ufo/utils/metoffice/MetOfficeQCFlags.h" + +namespace ufo { + +// ----------------------------------------------------------------------------- + +ModelBestFitPressure::ModelBestFitPressure(ioda::ObsSpace & obsdb, const Parameters_ & parameters, + std::shared_ptr > flags, + std::shared_ptr > obserr) + : FilterBase(obsdb, parameters, flags, obserr), + parameters_(parameters) +{ + oops::Log::trace() << "ModelBestFitPressure contructor starting" << std::endl; + // Get parameters from options + allvars_ += parameters_.obs_pressure; + // Include list of required data from GeoVals + allvars_ += Variable("air_pressure_levels@GeoVaLs"); + allvars_ += Variable("eastward_wind@GeoVaLs"); + allvars_ += Variable("northward_wind@GeoVaLs"); +} + +// ----------------------------------------------------------------------------- + +ModelBestFitPressure::~ModelBestFitPressure() { + oops::Log::trace() << "ModelBestFitPressure destructed" << std::endl; +} + +// ----------------------------------------------------------------------------- +/*! \brief A filter that calculates the pressure at which the AMV wind vector (u,v) + * is a best match to the model wind profile. The best-fit pressure is also checked + * to see if it is well-constrained (True/False) + * + * Example: + * \code{.unparsed} + * obs filter: + * - filter: Model Best Fit Pressure + * observation pressure: + * name: air_pressure_levels@MetaData + * top pressure: 10000 + * pressure band half-width: 10000 + * upper vector diff: 4 + * lower vector diff: 2 + * tolerance vector diff: 1.0e-8 + * tolerance pressure: 0.01 + * calculate bestfit winds: true + * \endcode + * + * \author A.Martins (Met Office) + * + * \date 18/05/2021: Created + */ +void ModelBestFitPressure::applyFilter(const std::vector & apply, + const Variables & filtervars, + std::vector> & flagged) const { + oops::Log::trace() << "ModelBestFitPressure applyFilter" << std::endl; + + const float missing = util::missingValue(missing); + const size_t nlocs = obsdb_.nlocs(); + + // Get parameters from options. + const float top_pressure = parameters_.top_pressure.value(); + const float pressure_band_half_width = parameters_.pressure_band_half_width.value(); + const float upper_vector_diff = parameters_.upper_vector_diff.value(); + const float lower_vector_diff = parameters_.lower_vector_diff.value(); + const float tolerance_vector_diff = parameters_.tolerance_vector_diff.value(); + const float tolerance_pressure = parameters_.tolerance_pressure.value(); + const bool calculate_best_fit_winds = parameters_.calculate_best_fit_winds.value(); + // get names of GeoVal variables + const std::string model_pressure_name = parameters_.obs_pressure.value().variable(); + const std::string model_eastvec_name = "eastward_wind"; + const std::string model_northvec_name = "northward_wind"; + + // Get GeoVaLs + const ufo::GeoVaLs * gvals = data_.getGeoVaLs(); + // Get number of vertical levels in GeoVaLs + const size_t num_level = data_.nlevs(Variable(model_eastvec_name + "@GeoVaLs")); + + std::vector satwind_best_fit_press(nlocs, missing); + std::vector satwind_best_fit_eastward_wind; + std::vector satwind_best_fit_northward_wind; + if (calculate_best_fit_winds) { + satwind_best_fit_eastward_wind.resize(nlocs, missing); + satwind_best_fit_northward_wind.resize(nlocs, missing); + } + + // Get the observation pressure + std::vector obs_pressure(nlocs); + data_.get(parameters_.obs_pressure, obs_pressure); + + // wind vector obs + std::vector obs_eastward(nlocs); + obsdb_.get_db("ObsValue", model_eastvec_name, obs_eastward); + std::vector obs_northward(nlocs); + obsdb_.get_db("ObsValue", model_northvec_name, obs_northward); + + // Get flags + std::vector u_flags(obsdb_.nlocs()); + std::vector v_flags(obsdb_.nlocs()); + if (obsdb_.has("QCFlags", model_eastvec_name) && + obsdb_.has("QCFlags", model_northvec_name)) { + obsdb_.get_db("QCFlags", model_eastvec_name, u_flags); + obsdb_.get_db("QCFlags", model_northvec_name, v_flags); + } else { + throw eckit::Exception("eastward_wind@QCFlags or northward_wind@QCFlags not initialised"); + } + + // Vectors storing GeoVaL column for each location. + std::vector model_pressure_profile(gvals->nlevs(model_pressure_name), 0.0); + std::vector model_eastvec_profile(gvals->nlevs(model_eastvec_name), 0.0); + std::vector model_northvec_profile(gvals->nlevs(model_northvec_name), 0.0); + std::vector vec_diff(num_level); + // diagnostic variable to be summed over all processors at the end of the routine + std::unique_ptr> countAccumulator = + obsdb_.distribution()->createAccumulator(); + + for (size_t idata = 0; idata < nlocs; ++idata) { + if (apply[idata]) { + // Get GeoVaLs at the specified location. + gvals->getAtLocation(model_pressure_profile, model_pressure_name, idata); + gvals->getAtLocation(model_eastvec_profile, model_eastvec_name, idata); + gvals->getAtLocation(model_northvec_profile, model_northvec_name, idata); + + // check ascending or decending (this code assume pressures are in descending order) + // if the model pressures are ascending, reverse order + if (std::is_sorted(model_pressure_profile.begin(), + model_pressure_profile.end()) || + model_pressure_profile.front() < model_pressure_profile.back()) { + std::reverse(model_pressure_profile.begin(), model_pressure_profile.end()); + std::reverse(model_eastvec_profile.begin(), model_eastvec_profile.end()); + std::reverse(model_northvec_profile.begin(), model_northvec_profile.end()); + } + + // check gvals are descending + ASSERT(!std::is_sorted(model_pressure_profile.begin(), + model_pressure_profile.end())); + + float min_vector_diff = std::numeric_limits::max(); + const size_t UNINITIALIZED = std::numeric_limits::max(); + size_t imin = UNINITIALIZED; + + // 1) Calculate vector difference between observed and background at all levels. + // Calculate best-fit pressure using vector difference. + // Find model level best-fit pressure (minimum vector difference). + // Use parabolic fit to find best-fit pressure. + for (size_t ilev = 0; ilev < num_level - 1; ++ilev) { + vec_diff[ilev] = std::hypot(obs_eastward[idata] - model_eastvec_profile[ilev], + obs_northward[idata] - model_northvec_profile[ilev]); + if (model_pressure_profile[ilev] < top_pressure) continue; + if (vec_diff[ilev] < min_vector_diff) { + min_vector_diff = vec_diff[ilev]; + imin = ilev; + } + } + // check if imin set + if (imin == UNINITIALIZED) { + throw eckit::Exception("No model level pressure above top_pressure", + Here()); + } + + // use parabolic fit to find best-fit pressure + float pressure_1; + const float pressure_2 = model_pressure_profile[imin]; + float pressure_3; + float vec_diff_1; + const float vec_diff_2 = vec_diff[imin]; + float vec_diff_3; + + // if bottom model level + if (imin == 0) { + satwind_best_fit_press[idata] = pressure_2; + } else { + pressure_1 = model_pressure_profile[imin - 1]; + pressure_3 = model_pressure_profile[imin + 1]; + vec_diff_1 = vec_diff[imin - 1]; + vec_diff_3 = vec_diff[imin + 1]; + + // if top of allowed region + if (pressure_3 < top_pressure) { + satwind_best_fit_press[idata] = pressure_2; + // if vec_diff_2 /= vec_diff_3 + } else if (!(std::fabs(vec_diff_2 - vec_diff_3) <= tolerance_vector_diff)) { + const float top = (((pressure_2 - pressure_1) * + (pressure_2 - pressure_1) * + (vec_diff_2 - vec_diff_3)) - + ((pressure_2 - pressure_3) * + (pressure_2 - pressure_3) * + (vec_diff_2 - vec_diff_1))); + const float bottom = (((pressure_2 - pressure_1) * + (vec_diff_2 - vec_diff_3)) - + ((pressure_2 - pressure_3) * + (vec_diff_2 - vec_diff_1))); + satwind_best_fit_press[idata] = pressure_2 - + (0.5f * (top / bottom)); + // if vec_diff_2 = vec_diff_3 set best fit pressure = pressure_2 + } else { + satwind_best_fit_press[idata] = pressure_2; + } + } + + // 2) Find bestfit eastward and northwards winds by linear interpolation, + // if calculate_best_fit_winds set to true (default: false) + if (calculate_best_fit_winds) { + size_t lev_below; + size_t lev_above; + float prop; + if (std::fabs(pressure_2 - satwind_best_fit_press[idata]) <= + tolerance_pressure) { + satwind_best_fit_eastward_wind[idata] = model_eastvec_profile[imin]; + satwind_best_fit_northward_wind[idata] = model_northvec_profile[imin]; + } else { + if (pressure_2 < satwind_best_fit_press[idata]) { + lev_below = imin - 1; + lev_above = imin; + prop = (satwind_best_fit_press[idata] - pressure_1) / + (pressure_2 - pressure_1); + } else { + lev_below = imin; + lev_above = imin + 1; + prop = (satwind_best_fit_press[idata] - pressure_2) / + (pressure_3 - pressure_2); + } + ASSERT(prop >= 0 && prop <= 1); + satwind_best_fit_eastward_wind[idata] = model_eastvec_profile[lev_below] * + (1.0f - prop) + model_eastvec_profile[lev_above] * prop; + satwind_best_fit_northward_wind[idata] = model_northvec_profile[lev_below] * + (1.0f - prop) + model_northvec_profile[lev_above] * prop; + } + } + + // 3) Check if best-fit pressure is well constrained then set flag SatwindPoorConstraint. + if (min_vector_diff <= upper_vector_diff) { + for (size_t ilev = 0; ilev < num_level - 1; ++ilev) { + if (model_pressure_profile[ilev] < top_pressure) continue; + if ((model_pressure_profile[ilev] < + satwind_best_fit_press[idata] - pressure_band_half_width || + model_pressure_profile[ilev] > + satwind_best_fit_press[idata] + pressure_band_half_width) && + vec_diff[ilev] <= min_vector_diff + lower_vector_diff) { + countAccumulator->addTerm(idata, 1); + u_flags[idata] |= ufo::MetOfficeQCFlags::SatWind::SatwindPoorConstraint; + v_flags[idata] |= ufo::MetOfficeQCFlags::SatWind::SatwindPoorConstraint; + break; + } + } + } else { + countAccumulator->addTerm(idata, 1); + u_flags[idata] |= ufo::MetOfficeQCFlags::SatWind::SatwindPoorConstraint; + v_flags[idata] |= ufo::MetOfficeQCFlags::SatWind::SatwindPoorConstraint; + } + } // apply + } // location loop + + const std::size_t iconstraint = countAccumulator->computeResult(); + if (iconstraint > 0) { + oops::Log::info() << "Satwind Poor constraint: "<< iconstraint + << " observations with modified pressure" << std::endl; + } + // write back flags and best-fit pressure/ winds + obsdb_.put_db("QCFlags", model_eastvec_name, u_flags); + obsdb_.put_db("QCFlags", model_northvec_name, v_flags); + obsdb_.put_db("DerivedValue", "model_bestfit_pressure", satwind_best_fit_press); + if (calculate_best_fit_winds) { + obsdb_.put_db("DerivedValue", "model_bestfit_eastward_wind", + satwind_best_fit_eastward_wind); + obsdb_.put_db("DerivedValue", "model_bestfit_northward_wind", + satwind_best_fit_northward_wind); + } +} + +// ----------------------------------------------------------------------------- + +void ModelBestFitPressure::print(std::ostream & os) const { + os << "ModelBestFitPressure filter" << parameters_ << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/filters/ModelBestFitPressure.h b/src/ufo/filters/ModelBestFitPressure.h new file mode 100644 index 000000000..87076812a --- /dev/null +++ b/src/ufo/filters/ModelBestFitPressure.h @@ -0,0 +1,109 @@ +/* + * (C) Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_FILTERS_MODELBESTFITPRESSURE_H_ +#define UFO_FILTERS_MODELBESTFITPRESSURE_H_ + +#include +#include +#include +#include + +#include "oops/util/ObjectCounter.h" +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/RequiredParameter.h" +#include "ufo/filters/FilterBase.h" +#include "ufo/filters/QCflags.h" +#include "ufo/filters/Variable.h" +#include "ufo/utils/parameters/ParameterTraitsVariable.h" + +namespace eckit { +} + +namespace ioda { + template class ObsDataVector; + class ObsSpace; +} + +namespace ufo { + +/// \brief Parameters controlling the operation of the ModelBestFitPressure filter. +class ModelBestFitPressureParameters : public FilterParametersBase { + OOPS_CONCRETE_PARAMETERS(ModelBestFitPressureParameters, FilterParametersBase) + + public: + /// Name of the observation pressure variable to correct. + oops::RequiredParameter obs_pressure{"observation pressure", this}; + /// Minimum allowed pressure region. Model levels will not be considered if their + /// pressure is below this value. + oops::Parameter top_pressure{"top pressure", 10000, this}; + /// Winds within this pressure-range of the best-fit pressure will be checked for + /// consistency within this range of the minimum speed. + oops::Parameter pressure_band_half_width{"pressure band half-width", 10000, this}; + /// Maximum vector difference allowed, for calculating constraint. + oops::Parameter upper_vector_diff{"upper vector diff", 4, this}; + /// Minimum vector difference allowed, for calculating constraint. + oops::Parameter lower_vector_diff{"lower vector diff", 2, this}; + /// Tolerance for vec_diff comparison. + /// Used when calculating bestfit pressure using parabolic fit. + oops::Parameter tolerance_vector_diff{"tolerance vector diff", 1.0e-8, this}; + /// Tolerance for pressure comparison. Used for calculating bestfit winds when comparing + /// pressure with bestfit pressure. Only used if calculate_best_fit_winds is true. + oops::Parameter tolerance_pressure{"tolerance pressure", 0.01, this}; + /// To calculate bestfit eastward/northward winds by linear interpolation. + oops::Parameter calculate_best_fit_winds{"calculate bestfit winds", false, this}; +}; + +/// \brief A filter to calculate the best fit pressure and if the pressure is well constrained. +/// Also can calculate the best fit eastward/northward winds. +/// +/// \details The model best-fit pressure is defined as the model pressure (Pa) with the +/// smallest vector difference between the AMV and model background wind, but +/// additionally is not allowed to be above top pressure (top_pressure) (can reasonably +/// expect that AMVs should not be above this level). Vertical interpolation is performed +/// between model levels to find the minimum vector difference. +/// +/// Checking if the pressure is well-constrained: +/// 1. Remove any winds where the minimum vector difference between the AMV eastward and +/// northward winds and the background column u and v is greater than upper_vector_diff. +/// This check aims to remove cases where there is no good agreement between the AMV +/// and the winds at any level in the background wind column. +/// 2. Remove any winds where the vector difference is less than the minimum vector +/// difference + lower_vector_diff outside of a band +/- pressure_band_half_width from the +/// best-fit pressure level. This aims to catch cases where there are secondary minima +/// or very broad minima. In both cases the best-fit pressure is not well constrained. +/// The default parameter values were chosen by eye-balling vector difference profiles and together +/// remove just over half the winds. +/// +/// See ModelBestFitPressureParameters for the documentation of +/// the parameters controlling this filter. +class ModelBestFitPressure : public FilterBase, + private util::ObjectCounter { + public: + /// The type of parameters accepted by the constructor of this filter. + /// This typedef is used by the FilterFactory. + typedef ModelBestFitPressureParameters Parameters_; + + static const std::string classname() {return "ufo::ModelBestFitPressure";} + + ModelBestFitPressure(ioda::ObsSpace &, const Parameters_ &, + std::shared_ptr >, + std::shared_ptr >); + ~ModelBestFitPressure(); + + private: + void print(std::ostream &) const override; + void applyFilter(const std::vector &, const Variables &, + std::vector> &) const override; + int qcFlag() const override {return QCflags::pass; } + + Parameters_ parameters_; +}; + +} // namespace ufo + +#endif // UFO_FILTERS_MODELBESTFITPRESSURE_H_ diff --git a/src/ufo/instantiateObsFilterFactory.h b/src/ufo/instantiateObsFilterFactory.h index 574e4c554..51a43b4db 100644 --- a/src/ufo/instantiateObsFilterFactory.h +++ b/src/ufo/instantiateObsFilterFactory.h @@ -22,6 +22,7 @@ #include "ufo/filters/HistoryCheck.h" #include "ufo/filters/ImpactHeightCheck.h" #include "ufo/filters/MetOfficeBuddyCheck.h" +#include "ufo/filters/ModelBestFitPressure.h" #include "ufo/filters/ModelObThreshold.h" #include "ufo/filters/MWCLWCheck.h" #include "ufo/filters/ObsBoundsCheck.h" @@ -80,6 +81,8 @@ template void instantiateObsFilterFactory() { differenceCheckMaker("Difference Check"); static oops::FilterMaker > historyCheckMaker("History Check"); + static oops::FilterMaker > + ModelBestFitPressureMaker("Model Best Fit Pressure"); static oops::FilterMaker > ModelObThresholdMaker("ModelOb Threshold"); static oops::FilterMaker > diff --git a/src/ufo/utils/metoffice/MetOfficeQCFlags.h b/src/ufo/utils/metoffice/MetOfficeQCFlags.h index 986c1c490..6398f64c5 100644 --- a/src/ufo/utils/metoffice/MetOfficeQCFlags.h +++ b/src/ufo/utils/metoffice/MetOfficeQCFlags.h @@ -36,7 +36,6 @@ namespace MetOfficeQCFlags { // Satellite wind data MissingDataReport = 1 << 14, ///< Missing data SatwindAltReport = 1 << 15, ///< Satwind alternative p/uv - SatwindGoodConstraint = 1 << 16, ///< Best-fit pressure is not well constrained // Other miscellaneous flags Thin4DFlag = 1 << 17, ///< Duplicate found StationListThinFlag = 1 << 18 ///< Rejected based on station list @@ -100,7 +99,8 @@ namespace MetOfficeQCFlags { SatwindConfFlag = 1 << 12, ///< Satwind product confidence SatwindInversionFlag = 1 << 13, ///< Inversion height corrected SatwindDryLayerFlag = 1 << 14, ///< Model dry layer QC - SatwindWrongLayerFlag = 1 << 15 ///< Wrong moist layer QC + SatwindWrongLayerFlag = 1 << 15, ///< Wrong moist layer QC + SatwindPoorConstraint = 1 << 16, ///< Best-fit pressure is not well constrained }; // Flags for scatterometers diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 05ceebb7c..0334fb211 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -186,6 +186,7 @@ list( APPEND ufo_test_input testinput/qc_met_office_buddy_check.yaml testinput/qc_met_office_buddy_check_unittests.yaml testinput/qc_met_office_buddy_pair_finder.yaml + testinput/qc_modelbestfitpressure.yaml testinput/qc_modelobthreshold.yaml testinput/qc_oceancolor_preqc.yaml testinput/qc_performaction.yaml @@ -1798,12 +1799,19 @@ ecbuild_add_test( TARGET test_ufo_qc_preqc_halo DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_qc_modelbestfitpressure + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x + ARGS "testinput/qc_modelbestfitpressure.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFilters.x + TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) + ecbuild_add_test( TARGET test_ufo_qc_modelobthreshold COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x ARGS "testinput/qc_modelobthreshold.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x - TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) + TEST_DEPENDS ufo_get_ufo_test_data ) ecbuild_add_test( TARGET test_ufo_qc_stuckcheck MPI 4 diff --git a/test/testinput/qc_modelbestfitpressure.yaml b/test/testinput/qc_modelbestfitpressure.yaml new file mode 100644 index 000000000..47378dc36 --- /dev/null +++ b/test/testinput/qc_modelbestfitpressure.yaml @@ -0,0 +1,40 @@ +window begin: 2020-10-01T03:00:00Z +window end: 2020-10-01T09:00:00Z + +observations: +- obs space: + name: Satwind + obsdatain: + obsfile: Data/ufo/testinput_tier_1/satwind_obs_1d_2020100106_noinv.nc4 + simulated variables: [eastward_wind, northward_wind] + geovals: + filename: Data/ufo/testinput_tier_1/satwind_geoval_20201001T0600Z_noinv.nc4 + obs filters: + - filter: Variable Assignment + assignments: + - name: eastward_wind@QCFlags + type: int + value: 0 + - name: northward_wind@QCFlags + type: int + value: 0 + - name: matching_bestfit_pressure@DerivedValue + type: float + value: -99999.0 + - filter: Model Best Fit Pressure + observation pressure: + name: air_pressure_levels@MetaData + top pressure: 10000 + pressure band half-width: 10000 + upper vector diff: 4 + lower vector diff: 2 + tolerance vector diff: 1.0e-8 + tolerance pressure: 0.01 + calculate bestfit winds: true + compareVariables: + - reference: + name: matching_bestfit_pressure@TestReference + test: + name: model_bestfit_pressure@DerivedValue + absTol: 0.5 + passedBenchmark: 182102 From bcc13d983139da804d21022a731b3d470b1bf386 Mon Sep 17 00:00:00 2001 From: Carwyn Pelley Date: Thu, 8 Jul 2021 22:19:55 +0100 Subject: [PATCH 30/98] ENH: Multi-level buddy check (#1270) * ENH: Sorting and pairs rough attempt looks reasonable * ENH: Minor chng * ENH: Added checkScalarUpperAirData function * ENH: Added checkVectorUpperAirData function * ENH: Using eigen arrays with existing tests passing * ENH: getValidObservationIds updated to pull out 1st level * ENH: Working scalar multi-level buddy check * ENH: Fully working * MAINT: Using added ufo-data * MAINT: Coding norms * ENH: Single level buddy check * MAINT: Minor review changes * ENH: Using records * ENH: Suporting extended ObsSpace * MAINT: Changes based on review * MAINT: Minor review changes * ENH: Added multi-level buddy check non-consecutive records test Co-authored-by: Ryan Honeyager --- src/ufo/filters/MetOfficeBuddyCheck.cc | 572 ++++++++++++------ src/ufo/filters/MetOfficeBuddyCheck.h | 80 ++- .../filters/MetOfficeBuddyCheckParameters.h | 5 + test/testinput/qc_met_office_buddy_check.yaml | 217 ++++++- 4 files changed, 641 insertions(+), 233 deletions(-) diff --git a/src/ufo/filters/MetOfficeBuddyCheck.cc b/src/ufo/filters/MetOfficeBuddyCheck.cc index 86a7ed025..ad906a064 100644 --- a/src/ufo/filters/MetOfficeBuddyCheck.cc +++ b/src/ufo/filters/MetOfficeBuddyCheck.cc @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #include "ufo/filters/MetOfficeBuddyPairFinder.h" #include "ufo/utils/PiecewiseLinearInterpolation.h" + namespace ufo { namespace { @@ -38,6 +40,87 @@ namespace { const double expArgMax = 80.0; // maximum argument of the exponential function const float maxGrossErrorProbability = 1.0; // maximum allowed value for PGE +/// brief Extract 1st level data from the provided flattened vector. +template +std::vector extract1stLev(const std::vector &flattenedArray, + const boost::optional &profileIndex) { + if ( !profileIndex ) + return flattenedArray; + + std::vector res; + res.resize((*profileIndex).rows()); + for (size_t row=0; row < (*profileIndex).rows(); row++) { + res[row] = flattenedArray[(*profileIndex)(row, 0)]; + } + return res; +} + +/// Return a 2D eigen array from the provided flattened vector. +Eigen::ArrayXXf unravel(const std::vector &flattenedArray, + const boost::optional &profileIndex) { + if (profileIndex) { + Eigen::ArrayXXf res((*profileIndex).rows(), (*profileIndex).cols()); + for (size_t row=0; row < (*profileIndex).rows(); row++) + for (size_t col=0; col < (*profileIndex).cols(); col++) + res(row, col) = flattenedArray[(*profileIndex)(row, col)]; + return res; + } else { + Eigen::ArrayXXf res(flattenedArray.size(), 1); + for (size_t row=0; row < flattenedArray.size(); row++) + res(row, 0) = flattenedArray[row]; + return res; + } +} + +/// Update the provided flat array with values from the provided eigen array +void updateFlatData(std::vector & flatArray, const Eigen::ArrayXXf &array, + const boost::optional &profileIndex) { + if (profileIndex) { + for (size_t row=0; row < (*profileIndex).rows(); row++) + for (size_t col=0; col < (*profileIndex).cols(); col++) + flatArray[(*profileIndex)(row, col)] = array(row, col); + } else { + ASSERT(array.cols() == 1); + for (size_t row=0; row < array.rows(); row++) + flatArray[row] = array(row, 0); + } +} + +/// Derive a mapping between the ObsSpace flat data and an unraveled 2D eigen array representation. +Eigen::ArrayXXi deriveIndices(const ioda::ObsSpace & obsdb, + const int numLevels) { + // Assume ObsSpace contains only the averaged profiles if this variable isn't present. + boost::optional> extended_obs_space; + if (obsdb.has("MetaData", "extended_obs_space")) { + extended_obs_space = std::vector(obsdb.nlocs()); + obsdb.get_db("MetaData", "extended_obs_space", *extended_obs_space); + } + Eigen::ArrayXXi profileIndex {obsdb.nrecs(), numLevels}; + + int recnum = 0; + for (ioda::ObsSpace::RecIdxIter irec = obsdb.recidx_begin(); irec != obsdb.recidx_end(); ++irec) { + int levnum = 0; + const std::vector &rSort = obsdb.recidx_vector(irec); + for (size_t ilocs = 0; ilocs < rSort.size(); ++ilocs) { + if (extended_obs_space && ((*extended_obs_space)[rSort[ilocs]] != 1)) + continue; + oops::Log::debug() << "MetOfficeBuddyCheck: profile index (record_number, location_number)"; + oops::Log::debug() << " = original obs index; (" << recnum << "," << ilocs << ") = "; + oops::Log::debug() << rSort[ilocs] << std::endl; + profileIndex(recnum, levnum) = rSort[ilocs]; + levnum++; + } + if (levnum != numLevels) { + std::stringstream msg; + msg << "Record (profile): " << recnum << " length: " << levnum+1 << " does not match the " + << "number of levels expected: " << numLevels; + throw eckit::UserError(msg.str(), Here()); + } + recnum++; + } + return profileIndex; +} + std::string fullVariableName(const Variable &var) { if (var.group().empty()) @@ -74,16 +157,19 @@ std::vector mapDistinctValuesToDistinctInts(const std::vector &values) return ints; } -struct ScalarSingleLevelVariableData { - const std::vector *varFlags = nullptr; - const std::vector *obsValues = nullptr; - const std::vector *obsBiases = nullptr; - const std::vector *obsErrors = nullptr; - std::vector bgValues; - std::vector bgErrors; - std::vector grossErrorProbabilities; + +struct ScalarVariableData { + std::vector varFlags; + Eigen::ArrayXXf obsValues; + Eigen::ArrayXXf obsBiases; + Eigen::ArrayXXf obsErrors; + Eigen::ArrayXXf bgValues; + Eigen::ArrayXXf bgErrors; + std::vector flatGrossErrorProbabilities; + Eigen::ArrayXXf grossErrorProbabilities; }; + } // namespace /// \brief Metadata of all observations processed by the filter. @@ -92,6 +178,7 @@ struct MetOfficeBuddyCheck::MetaData { std::vector longitudes; std::vector datetimes; boost::optional> pressures; + boost::optional pressuresML; std::vector stationIds; }; @@ -110,19 +197,38 @@ MetOfficeBuddyCheck::MetOfficeBuddyCheck(ioda::ObsSpace& obsdb, const Parameters MetOfficeBuddyCheck::~MetOfficeBuddyCheck() {} + + void MetOfficeBuddyCheck::applyFilter(const std::vector & apply, const Variables & filtervars, std::vector> & flagged) const { - // Identify observations to process and collect their data and metadata + // Fetch metadata required for identifying buddy pairs. + + const boost::optional &numLevels = options_.numLevels.value(); + + boost::optional profileIndex; + if (numLevels) + profileIndex = deriveIndices(obsdb_, *numLevels); + + const std::vector validObsIds = getValidObservationIds(apply, profileIndex); + MetaData obsData = collectMetaData(profileIndex); - const std::vector validObsIds = getValidObservationIds(apply); - MetaData obsData = collectMetaData(); const std::vector bgErrorHorizCorrScales = calcBackgroundErrorHorizontalCorrelationScales( validObsIds, obsData.latitudes); const std::vector verbose = flagAndPrintVerboseObservations( validObsIds, obsData.latitudes, obsData.longitudes, obsData.datetimes, obsData.pressures.get_ptr(), obsData.stationIds, bgErrorHorizCorrScales); + // Identify buddy pairs + + const std::vector *pressures = + options_.sortByPressure ? obsData.pressures.get_ptr() : nullptr; + MetOfficeBuddyPairFinder buddyPairFinder(options_, obsData.latitudes, obsData.longitudes, + obsData.datetimes, pressures, + obsData.stationIds); + const std::vector buddyPairs = buddyPairFinder.findBuddyPairs(validObsIds); + + // Fetch data/metadata required for buddy-check calculation. const oops::Variables &observedVars = obsdb_.obsvariables(); ioda::ObsDataVector obsValues(obsdb_, filtervars.toOopsVariables(), "ObsValue"); @@ -130,30 +236,30 @@ void MetOfficeBuddyCheck::applyFilter(const std::vector & apply, return filtervars.variable(filterVarIndex).variable(); }; - auto getScalarSingleLevelVariableData = [&] (size_t filterVarIndex) { + auto getScalarVariableData = [&] (size_t filterVarIndex) { + // Fetch raw data corresponding this the specific variable of interest. const size_t observedVarIndex = observedVars.find(getFilterVariableName(filterVarIndex)); - ScalarSingleLevelVariableData data; - data.varFlags = &(*flags_)[observedVarIndex]; - data.obsValues = &obsValues[filterVarIndex]; - data.obsErrors = &(*obserr_)[observedVarIndex]; - data_.get(ufo::Variable(filtervars[filterVarIndex], "HofX"), data.bgValues); - data_.get(backgroundErrorVariable(filtervars[filterVarIndex]), data.bgErrors); - // TODO(wsmigaj): How is this variable going to be initialized? + std::vector bgValues; + data_.get(ufo::Variable(filtervars[filterVarIndex], "HofX"), bgValues); + std::vector bgErrors; + data_.get(backgroundErrorVariable(filtervars[filterVarIndex]), bgErrors); + + // Store data for usage. + ScalarVariableData data; data_.get(ufo::Variable(filtervars[filterVarIndex], "GrossErrorProbability"), - data.grossErrorProbabilities); + data.flatGrossErrorProbabilities); + + // Unravel these flattened vectors. + data.varFlags = extract1stLev((*flags_)[observedVarIndex], profileIndex); + data.obsValues = unravel(obsValues[filterVarIndex], profileIndex); + data.obsErrors = unravel((*obserr_)[observedVarIndex], profileIndex); + data.bgValues = unravel(bgValues, profileIndex); + data.bgErrors = unravel(bgErrors, profileIndex); + data.grossErrorProbabilities = unravel(data.flatGrossErrorProbabilities, profileIndex); return data; }; - // Identify buddy pairs - - const std::vector *pressures = - options_.sortByPressure ? obsData.pressures.get_ptr() : nullptr; - MetOfficeBuddyPairFinder buddyPairFinder(options_, obsData.latitudes, obsData.longitudes, - obsData.datetimes, pressures, - obsData.stationIds); - const std::vector buddyPairs = buddyPairFinder.findBuddyPairs(validObsIds); - // Buddy-check all filter variables // Gross error probabilities updated by buddy check, indexed by variable name. @@ -164,62 +270,66 @@ void MetOfficeBuddyCheck::applyFilter(const std::vector & apply, for (size_t filterVarIndex = 0; filterVarIndex < filtervars.size(); ++filterVarIndex) { if (previousVariableWasFirstComponentOfTwo) { // Vector (two-component) variable - ScalarSingleLevelVariableData firstComponentData = - getScalarSingleLevelVariableData(filterVarIndex - 1); - ScalarSingleLevelVariableData secondComponentData = - getScalarSingleLevelVariableData(filterVarIndex); - - for (size_t i = 0; i < firstComponentData.grossErrorProbabilities.size(); ++i) - firstComponentData.grossErrorProbabilities[i] = - std::max(firstComponentData.grossErrorProbabilities[i], - secondComponentData.grossErrorProbabilities[i]); - - checkVectorSurfaceData(buddyPairs, - *firstComponentData.varFlags, - verbose, bgErrorHorizCorrScales, - obsData.stationIds, obsData.datetimes, - *firstComponentData.obsValues, - *secondComponentData.obsValues, - *firstComponentData.obsErrors, - firstComponentData.bgValues, secondComponentData.bgValues, - firstComponentData.bgErrors, - firstComponentData.grossErrorProbabilities); + ScalarVariableData firstComponentData = + getScalarVariableData(filterVarIndex - 1); + ScalarVariableData secondComponentData = + getScalarVariableData(filterVarIndex); + + for (Eigen::Index i = 0; i < firstComponentData.grossErrorProbabilities.rows(); ++i) { + for (Eigen::Index j = 0; j < firstComponentData.grossErrorProbabilities.cols(); ++j) { + firstComponentData.grossErrorProbabilities(i, j) = + std::max(firstComponentData.grossErrorProbabilities(i, j), + secondComponentData.grossErrorProbabilities(i, j)); + } + } + + checkVectorData(buddyPairs, firstComponentData.varFlags, verbose, + bgErrorHorizCorrScales, obsData.stationIds, obsData.datetimes, + obsData.pressuresML.get_ptr(), firstComponentData.obsValues, + secondComponentData.obsValues, firstComponentData.obsErrors, + firstComponentData.bgValues, secondComponentData.bgValues, + firstComponentData.bgErrors, firstComponentData.grossErrorProbabilities); // OPS doesn't update the gross error probabilities of the second component variable, // but it seems more consistent to do so (and it facilitates the implementation of // flagRejectedObservations(). // The implementation of checkVectorSurfaceData() still assumes that the *input* gross error // probabilities, flags and background error estimates are the same for both components. + + // Update the flat GPE values. + updateFlatData(firstComponentData.flatGrossErrorProbabilities, + firstComponentData.grossErrorProbabilities, profileIndex); calculatedGrossErrProbsByVarName[getFilterVariableName(filterVarIndex - 1)] = - firstComponentData.grossErrorProbabilities; + firstComponentData.flatGrossErrorProbabilities; calculatedGrossErrProbsByVarName[getFilterVariableName(filterVarIndex)] = - std::move(firstComponentData.grossErrorProbabilities); - + std::move(firstComponentData.flatGrossErrorProbabilities); previousVariableWasFirstComponentOfTwo = false; } else { if (filtervars[filterVarIndex].options().getBool("first_component_of_two", false)) { previousVariableWasFirstComponentOfTwo = true; } else { // Scalar variable - ScalarSingleLevelVariableData data = - getScalarSingleLevelVariableData(filterVarIndex); - checkScalarSurfaceData(buddyPairs, - *data.varFlags, verbose, bgErrorHorizCorrScales, - obsData.stationIds, obsData.datetimes, - *data.obsValues, *data.obsErrors, - data.bgValues, data.bgErrors, - data.grossErrorProbabilities); + ScalarVariableData data = + getScalarVariableData(filterVarIndex); + checkScalarData(buddyPairs, data.varFlags, verbose, bgErrorHorizCorrScales, + obsData.stationIds, obsData.datetimes, obsData.pressuresML.get_ptr(), + data.obsValues, data.obsErrors, + data.bgValues, data.bgErrors, + data.grossErrorProbabilities); + + // Update the flat GPE values. + updateFlatData(data.flatGrossErrorProbabilities, data.grossErrorProbabilities, + profileIndex); calculatedGrossErrProbsByVarName[getFilterVariableName(filterVarIndex)] = - std::move(data.grossErrorProbabilities); + std::move(data.flatGrossErrorProbabilities); } } } // Update observations flags and gross error probabilities - for (const auto &varNameAndGrossErrProbs : calculatedGrossErrProbsByVarName) { + for (const auto &varNameAndGrossErrProbs : calculatedGrossErrProbsByVarName) obsdb_.put_db("GrossErrorProbability", varNameAndGrossErrProbs.first, varNameAndGrossErrProbs.second); - } flagRejectedObservations(filtervars, calculatedGrossErrProbsByVarName, flagged); @@ -232,7 +342,8 @@ Variable MetOfficeBuddyCheck::backgroundErrorVariable(const Variable &filterVari return Variable(filterVariable.variable() + "_background_error@ObsDiag"); } -MetOfficeBuddyCheck::MetaData MetOfficeBuddyCheck::collectMetaData() const { +MetOfficeBuddyCheck::MetaData MetOfficeBuddyCheck::collectMetaData( + const boost::optional & profileIndex) const { MetaData obsData; obsData.latitudes.resize(obsdb_.nlocs()); @@ -247,10 +358,19 @@ MetOfficeBuddyCheck::MetaData MetOfficeBuddyCheck::collectMetaData() const { if (obsdb_.has("MetaData", "air_pressure")) { obsData.pressures = std::vector(obsdb_.nlocs()); obsdb_.get_db("MetaData", "air_pressure", *obsData.pressures); + obsData.pressuresML = unravel(*obsData.pressures, profileIndex); } - obsData.stationIds = getStationIds(); + if (profileIndex) { + obsData.latitudes = extract1stLev(obsData.latitudes, profileIndex); + obsData.longitudes = extract1stLev(obsData.longitudes, profileIndex); + obsData.datetimes = extract1stLev(obsData.datetimes, profileIndex); + obsData.stationIds = extract1stLev(obsData.stationIds, profileIndex); + if (obsdb_.has("MetaData", "air_pressure")) { + obsData.pressures = extract1stLev(*obsData.pressures, profileIndex); + } + } return obsData; } @@ -346,199 +466,253 @@ std::vector MetOfficeBuddyCheck::flagAndPrintVerboseObservations( return verbose; } -void MetOfficeBuddyCheck::checkScalarSurfaceData(const std::vector &pairs, - const std::vector &flags, - const std::vector &verbose, - const std::vector &bgErrorHorizCorrScales, - const std::vector &stationIds, - const std::vector &datetimes, - const std::vector &obsValues, - const std::vector &obsErrors, - const std::vector &bgValues, - const std::vector &bgErrors, - std::vector &pges) const { +void MetOfficeBuddyCheck::checkScalarData(const std::vector &pairs, + const std::vector &flags, + const std::vector &verbose, + const std::vector &bgErrorHorizCorrScales, + const std::vector &stationIds, + const std::vector &datetimes, + const Eigen::ArrayXXf *pressures, + const Eigen::ArrayXXf &obsValues, + const Eigen::ArrayXXf &obsErrors, + const Eigen::ArrayXXf &bgValues, + const Eigen::ArrayXXf &bgErrors, + Eigen::ArrayXXf &pges) const { using util::sqr; + const boost::optional &nolevs = options_.numLevels.value(); + const int numLevels = nolevs ? *nolevs : 0; // number of actual levels + const Eigen::Index cols = obsValues.cols(); // number of data columns (levels) to loop over const bool isMaster = obsdb_.comm().rank() == 0; if (isMaster) { oops::Log::trace() << __func__ << " " << " dampingFactor1 = " << options_.dampingFactor1 << ", dampingFactor2 = " << options_.dampingFactor2 << '\n'; - oops::Log::trace() << "ObsA ObsB StatIdA StatIdB DiffA DiffB " + oops::Log::trace() << "ObsA ObsB StatIdA StatIdB lev DiffA DiffB " "Dist Corr Agree PgeA PgeB Mult\n"; } const double invTemporalCorrScale = 1.0 / options_.temporalCorrelationScale.value().toSeconds(); + const float missing = util::missingValue(1.0f); + // Loop over buddy pairs for (const MetOfficeBuddyPair &pair : pairs) { const size_t jA = pair.obsIdA; const size_t jB = pair.obsIdB; // Check that observations are valid and buddy check is required - if (!(flags[jA] == QCflags::pass && flags[jB] == QCflags::pass && - pges[jA] < maxGrossErrorProbability && pges[jB] < maxGrossErrorProbability)) - continue; + if (flags[jA] != QCflags::pass || flags[jB] != QCflags::pass) + continue; // skip to next pair // eqn 3.9 + // - hcScale: horizontal error scale for the pair of obs + // - scaledDist: scaled Distance between observations const double hcScale = 0.5 * (bgErrorHorizCorrScales[jA] + bgErrorHorizCorrScales[jB]); const double scaledDist = pair.distanceInKm / hcScale; - // Background error correlation between ob positions. - // Surface data; treat vertical correlation as 1.0 + // Background error correlation between observation positions. // eqns 3.10, 3.11 - const double corr = (1.0 + scaledDist) * - std::exp(-scaledDist - sqr((datetimes[jA] - datetimes[jB]).toSeconds() * - invTemporalCorrScale)); - - if (corr < 0.1) + double corr; + if (numLevels == 1) { + // Single level data. + corr = (1.0 + scaledDist) * + std::exp(-scaledDist - + options_.verticalCorrelationScale.value() * + sqr(std::log((*pressures)(jA, 0)/(*pressures)(jB, 0))) - + sqr((datetimes[jA] - datetimes[jB]).toSeconds() * invTemporalCorrScale)); + } else { + // Multi-level/surface data; treat vertical correlation as 1.0 + corr = (1.0 + scaledDist) * + std::exp(-scaledDist - + sqr((datetimes[jA] - datetimes[jB]).toSeconds() * invTemporalCorrScale)); + } + if (corr < 0.1) // Check against minimum background error correlation. continue; // skip to next pair - // Differences from background - double diffA = obsValues[jA] - bgValues[jA]; - double diffB = obsValues[jB] - bgValues[jB]; - // Estimated error variances (ob+bk) (eqn 2.5) - double errVarA = sqr(obsErrors[jA]) + sqr(bgErrors[jA]); - double errVarB = sqr(obsErrors[jB]) + sqr(bgErrors[jB]); - // Background error covariance between ob positions (eqn 3.13) - double covar = corr * bgErrors[jA] * bgErrors[jB]; - // (Total error correlation between ob positions)**2 (eqn 3.14) - double rho2 = sqr(covar) / (errVarA * errVarB); - // Argument for exponents - double expArg = -(0.5 * rho2 / (1.0 - rho2)) * - (sqr(diffA) / errVarA + sqr(diffB) / errVarB - 2.0 * diffA * diffB / covar); - expArg = options_.dampingFactor1 * (-0.5 * std::log(1.0 - rho2) + expArg); // exponent of - expArg = std::min(expArgMax, std::max(-expArgMax, expArg)); // eqn 3.18 - // Z = P(OA)*P(OB)/P(OA and OB) - double z = 1.0 / (1.0 - (1.0 - pges[jA]) * (1.0 - pges[jB]) * (1.0 - std::exp(expArg))); - if (z <= 0.0) - z = 1.0; // rounding error control - z = std::pow(z, options_.dampingFactor2); // eqn 3.16 - pges[jA] *= z; // eqn 3.17 - pges[jB] *= z; // eqn 3.17 - if (isMaster && (verbose[jA] || verbose[jB])) { - oops::Log::trace() << boost::format("%5d %5d %8d %8d " - "%5.1f %5.1f %6.1f " - "%5.3f %6.3f %6.3f %6.3f %6.3f\n") % - jA % jB % stationIds[jA] % stationIds[jB] % - diffA % diffB % pair.distanceInKm % - corr % std::exp(expArg) % pges[jA] % pges[jB] % z; + // Loop over each level + for (Eigen::Index jlev=0; jlev < cols; jlev++) { + if (pges(jA, jlev) >= maxGrossErrorProbability || + pges(jB, jlev) >= maxGrossErrorProbability || + pges(jA, jlev) == missing || pges(jB, jlev) == missing) + continue; // skip to next level + + // Differences from background + const double diffA = obsValues(jA, jlev) - bgValues(jA, jlev); + const double diffB = obsValues(jB, jlev) - bgValues(jB, jlev); + // Estimated error variances (ob+bk) (eqn 2.5) + const double errVarA = sqr(obsErrors(jA, jlev)) + sqr(bgErrors(jA, jlev)); + const double errVarB = sqr(obsErrors(jB, jlev)) + sqr(bgErrors(jB, jlev)); + // Background error covariance between ob positions (eqn 3.13) + const double covar = corr * bgErrors(jA, jlev) * bgErrors(jB, jlev); + // (Total error correlation between ob positions)**2 (eqn 3.14) + const double rho2 = sqr(covar) / (errVarA * errVarB); + // Argument for exponents + double expArg = -(0.5 * rho2 / (1.0 - rho2)) * + (sqr(diffA) / errVarA + sqr(diffB) / errVarB - 2.0 * diffA * diffB / covar); + expArg = options_.dampingFactor1 * (-0.5 * std::log(1.0 - rho2) + expArg); // exponent of + expArg = std::min(expArgMax, std::max(-expArgMax, expArg)); // eqn 3.18 + // Z = P(OA)*P(OB)/P(OA and OB) + double z = 1.0 / (1.0 - (1.0 - pges(jA, jlev)) * (1.0 - pges(jB, jlev)) * + (1.0 - std::exp(expArg))); + if (z <= 0.0) + z = 1.0; // rounding error control + z = std::pow(z, options_.dampingFactor2); // eqn 3.16 + pges(jA, jlev) *= z; // eqn 3.17 + pges(jB, jlev) *= z; // eqn 3.17 + if (isMaster && (verbose[jA] || verbose[jB])) { + oops::Log::trace() << boost::format("%5d %5d %8d %8d %5d " + "%5.1f %5.1f %6.1f " + "%5.3f %6.3f %6.3f %6.3f %6.3f\n") % + jA % jB % stationIds[jA] % stationIds[jB] % jlev % + diffA % diffB % pair.distanceInKm % + corr % std::exp(expArg) % pges(jA, jlev) % pges(jB, jlev) % z; + } } } } -void MetOfficeBuddyCheck::checkVectorSurfaceData(const std::vector &pairs, - const std::vector &flags, - const std::vector &verbose, - const std::vector &bgErrorHorizCorrScales, - const std::vector &stationIds, - const std::vector &datetimes, - const std::vector &uObsValues, - const std::vector &vObsValues, - const std::vector &obsErrors, - const std::vector &uBgValues, - const std::vector &vBgValues, - const std::vector &bgErrors, - std::vector &pges) const { + +void MetOfficeBuddyCheck::checkVectorData(const std::vector &pairs, + const std::vector &flags, + const std::vector &verbose, + const std::vector &bgErrorHorizCorrScales, + const std::vector &stationIds, + const std::vector &datetimes, + const Eigen::ArrayXXf *pressures, + const Eigen::ArrayXXf &uObsValues, + const Eigen::ArrayXXf &vObsValues, + const Eigen::ArrayXXf &obsErrors, + const Eigen::ArrayXXf &uBgValues, + const Eigen::ArrayXXf &vBgValues, + const Eigen::ArrayXXf &bgErrors, + Eigen::ArrayXXf &pges) const { using util::sqr; + const boost::optional &nolevs = options_.numLevels.value(); + const int numLevels = nolevs ? *nolevs : 0; // number of actual levels + const Eigen::Index cols = uObsValues.cols(); // number of data columns (levels) to loop over const bool isMaster = obsdb_.comm().rank() == 0; if (isMaster) { oops::Log::trace() << __func__ << " " << " dampingFactor1 = " << options_.dampingFactor1 << ", dampingFactor2 = " << options_.dampingFactor2 << '\n'; - oops::Log::trace() << "ObsA ObsB StatIdA StatIdB LDiffA LDiffB TDiffA TDiffB " + oops::Log::trace() << "ObsA ObsB StatIdA StatIdB lev LDiffA LDiffB TDiffA TDiffB " "Dist Corr Agree PgeA PgeB Mult\n"; } - const double invTemporalCorrScale = 1.0 / options_.temporalCorrelationScale.value().toSeconds(); + const float missing = util::missingValue(1.0f); + // Loop over buddy pairs for (const MetOfficeBuddyPair &pair : pairs) { const size_t jA = pair.obsIdA; const size_t jB = pair.obsIdB; // Check that observations are valid and buddy check is required - if (!(flags[jA] == QCflags::pass && flags[jB] == QCflags::pass && - pges[jA] < maxGrossErrorProbability && pges[jB] < maxGrossErrorProbability)) - continue; + if (flags[jA] != QCflags::pass || flags[jB] != QCflags::pass) + continue; // skip to next pair // eqn 3.9 - double horizCorrScale = 0.5 * (bgErrorHorizCorrScales[jA] + bgErrorHorizCorrScales[jB]); - double scaleDist = pair.distanceInKm / horizCorrScale; - // Background error correlation between ob positions. - // Surface data; treat vertical correlation as 1.0 - // eqns 3.10, 3.11 - const double lCorr = std::exp(-scaleDist - sqr((datetimes[jA] - datetimes[jB]).toSeconds() * - invTemporalCorrScale)); + // - hcScale: horizontal error scale for the pair of obs + // - scaledDist: scaled Distance between observations + const double hcScale = 0.5 * (bgErrorHorizCorrScales[jA] + bgErrorHorizCorrScales[jB]); + const double scaledDist = pair.distanceInKm / hcScale; - if ((1.0 + scaleDist) * lCorr < 0.1) + // Background error correlation between observation positions. + // eqns 3.10, 3.11 + double lCorr; + if (numLevels == 1) { + lCorr = std::exp(-scaledDist - + options_.verticalCorrelationScale.value() * + sqr(std::log((*pressures)(jA, 0)/(*pressures)(jB, 0))) - + sqr((datetimes[jA] - datetimes[jB]).toSeconds() * invTemporalCorrScale)); + } else { + // Multi-level/surface data; treat vertical correlation as 1.0 + lCorr = std::exp(-scaledDist - + sqr((datetimes[jA] - datetimes[jB]).toSeconds() * invTemporalCorrScale)); + } + if ((1.0 + scaledDist) * lCorr < 0.1) // Check against minimum background error correlation. continue; // skip to next pair // Calculate longitudinal and transverse wind components - double sinRot = std::sin(pair.rotationAInRad); - double cosRot = std::cos(pair.rotationAInRad); - // Difference from background - longitudinal wind - double lDiffA = cosRot * (uObsValues[jA] - uBgValues[jA]) - + sinRot * (vObsValues[jA] - vBgValues[jA]); // eqn 3.19 - // Difference from background - transverse wind - double tDiffA = - sinRot * (uObsValues[jA] - uBgValues[jA]) - + cosRot * (vObsValues[jA] - vBgValues[jA]); // eqn 3.20 - sinRot = std::sin(pair.rotationBInRad); - cosRot = std::cos(pair.rotationBInRad); - // Difference from background - longitudinal wind - double lDiffB = cosRot * (uObsValues[jB] - uBgValues[jB]) - + sinRot * (vObsValues[jB] - vBgValues[jB]); // eqn 3.19 - // Difference from background - transverse wind - double tDiffB = - sinRot * (uObsValues[jB] - uBgValues[jB]) - + cosRot * (vObsValues[jB] - vBgValues[jB]); // eqn 3.20 - - // Estimated error variances (ob + bk; component wind variance) - double errVarA = sqr(obsErrors[jA]) + sqr(bgErrors[jA]); // eqn 2.5 - double errVarB = sqr(obsErrors[jB]) + sqr(bgErrors[jB]); // eqn 2.5 - - // Calculate covariances and probabilities - double lCovar = lCorr * bgErrors[jA] * bgErrors[jB]; // eqn 3.13 - double tCovar = (1.0 - options_.nonDivergenceConstraint * scaleDist) * lCovar; // eqn 3.12, 13 - // rho2 = (total error correlation between ob positions)**2 - double lRho2 = sqr(lCovar) / (errVarA * errVarB); // eqn 3.14 - double tRho2 = sqr(tCovar) / (errVarA * errVarB); // eqn 3.14 - // Argument for exponents - double expArg; - if (std::abs (tRho2) <= 0.00001) - expArg = 0.0; // prevent division by tCovar=0.0 - else - expArg = -(0.5 * tRho2 / (1.0 - tRho2)) * - (sqr(tDiffA) / errVarA + sqr(tDiffB) / errVarB - 2.0 * tDiffA * tDiffB / tCovar); - expArg = expArg - (0.5 * lRho2 / (1.0 - lRho2)) * - (sqr(lDiffA) / errVarA + sqr(lDiffB) / errVarB - 2.0 * lDiffA * lDiffB / lCovar); - expArg = options_.dampingFactor1 * (-0.5 * std::log((1.0 - lRho2) * (1.0 - lRho2)) + expArg); - expArg = std::min(expArgMax, std::max(-expArgMax, expArg)); // eqn 3.22 - // Z = P(OA)*P(OB)/P(OA and OB) - double z = 1.0 / (1.0 - (1.0 - pges[jA]) * (1.0 - pges[jB]) * (1.0 - std::exp(expArg))); - if (z <= 0.0) - z = 1.0; // rounding error control - z = std::pow(z, options_.dampingFactor2); // eqn 3.16 - pges[jA] *= z; // eqn 3.17 - pges[jB] *= z; // eqn 3.17 - - if (isMaster && (verbose[jA] || verbose[jB])) { - oops::Log::trace() << boost::format("%5d %5d %8d %8d " - "%6.1f %6.1f %6.1f %6.1f %6.1f " - "%5.3f %6.3f %6.3f %6.3f %6.3f\n") % - jA % jB % stationIds[jA] % stationIds[jB] % - lDiffA % lDiffB % tDiffA % tDiffB % pair.distanceInKm % - lCorr % std::exp(expArg) % pges[jA] % pges[jB] % z; + const double sinRotA = std::sin(pair.rotationAInRad); + const double cosRotA = std::cos(pair.rotationAInRad); + const double sinRotB = std::sin(pair.rotationBInRad); + const double cosRotB = std::cos(pair.rotationBInRad); + + // Loop over each level + for (Eigen::Index jlev=0; jlev < cols; jlev++) { + if (pges(jA, jlev) >= maxGrossErrorProbability || + pges(jB, jlev) >= maxGrossErrorProbability || + pges(jA, jlev) == missing || pges(jB, jlev) == missing) + continue; // skip to next level + + // Difference from background - longitudinal wind (eqn 3.19) + const double lDiffA = cosRotA * (uObsValues(jA, jlev) - uBgValues(jA, jlev)) + + sinRotA * (vObsValues(jA, jlev) - vBgValues(jA, jlev)); + // Difference from background - transverse wind (eqn 3.20) + const double tDiffA = - sinRotA * (uObsValues(jA, jlev) - uBgValues(jA, jlev)) + + cosRotA * (vObsValues(jA, jlev) - vBgValues(jA, jlev)); + // Difference from background - longitudinal wind (eqn 3.19) + const double lDiffB = cosRotB * (uObsValues(jB, jlev) - uBgValues(jB, jlev)) + + sinRotB * (vObsValues(jB, jlev) - vBgValues(jB, jlev)); + // Difference from background - transverse wind (eqn 3.20) + const double tDiffB = - sinRotB * (uObsValues(jB, jlev) - uBgValues(jB, jlev)) + + cosRotB * (vObsValues(jB, jlev) - vBgValues(jB, jlev)); + + // Estimated error variances (ob + bk; component wind variance; eqn 2.5) + const double errVarA = sqr(obsErrors(jA, jlev)) + sqr(bgErrors(jA, jlev)); + const double errVarB = sqr(obsErrors(jB, jlev)) + sqr(bgErrors(jB, jlev)); + + // Calculate covariances and probabilities eqn 3.12, 3.13 + const double lCovar = lCorr * bgErrors(jA, jlev) * bgErrors(jB, jlev); + const double tCovar = (1.0 - options_.nonDivergenceConstraint * scaledDist) * lCovar; + // rho2 = (total error correlation between ob positions)**2 + const double lRho2 = sqr(lCovar) / (errVarA * errVarB); // eqn 3.14 + const double tRho2 = sqr(tCovar) / (errVarA * errVarB); // eqn 3.14 + // Argument for exponents + double expArg; + if (std::abs (tRho2) <= 0.00001) + expArg = 0.0; // prevent division by tCovar=0.0 + else + expArg = -(0.5 * tRho2 / (1.0 - tRho2)) * + (sqr(tDiffA) / errVarA + sqr(tDiffB) / errVarB - 2.0 * tDiffA * tDiffB / tCovar); + expArg = expArg - (0.5 * lRho2 / (1.0 - lRho2)) * + (sqr(lDiffA) / errVarA + sqr(lDiffB) / errVarB - 2.0 * lDiffA * lDiffB / lCovar); + expArg = options_.dampingFactor1 * (-0.5 * std::log((1.0 - lRho2) * (1.0 - lRho2)) + expArg); + expArg = std::min(expArgMax, std::max(-expArgMax, expArg)); // eqn 3.22 + // Z = P(OA)*P(OB)/P(OA and OB) + double z = 1.0 / (1.0 - (1.0 - pges(jA, jlev)) * (1.0 - pges(jB, jlev)) * + (1.0 - std::exp(expArg))); + if (z <= 0.0) + z = 1.0; // rounding error control + z = std::pow(z, options_.dampingFactor2); // eqn 3.16 + pges(jA, jlev) *= z; // eqn 3.17 + pges(jB, jlev) *= z; // eqn 3.17 + + if (isMaster && (verbose[jA] || verbose[jB])) { + oops::Log::trace() << boost::format("%5d %5d %8d %8d %5d " + "%6.1f %6.1f %6.1f %6.1f %6.1f " + "%5.3f %6.3f %6.3f %6.3f %6.3f\n") % + jA % jB % stationIds[jA] % stationIds[jB] % jlev % + lDiffA % lDiffB % tDiffA % tDiffB % pair.distanceInKm % + lCorr % std::exp(expArg) % pges(jA, jlev) % pges(jB, jlev) % z; + } } } } + std::vector MetOfficeBuddyCheck::getValidObservationIds( - const std::vector & apply) const { + const std::vector & apply, const boost::optional & profileIndex) const { std::vector validObsIds; - for (size_t obsId = 0; obsId < apply.size(); ++obsId) - // TODO(wsmigaj): The second condition below may need reviewing. + + const auto lev1flags = extract1stLev((*flags_)[0], profileIndex); + const std::vector lev1apply = extract1stLev(apply, profileIndex); + for (size_t obsId = 0; obsId < lev1flags.size(); ++obsId) + // TODO(wsmigaj): The condition below may need reviewing. // Perhaps we should process only observations marked as passed in all filter variables? // Or those marked as passed in at least one filter variable? - if (apply[obsId] && (*flags_)[0][obsId] == QCflags::pass) + if (lev1apply[obsId] && lev1flags[obsId] == QCflags::pass) validObsIds.push_back(obsId); return validObsIds; } diff --git a/src/ufo/filters/MetOfficeBuddyCheck.h b/src/ufo/filters/MetOfficeBuddyCheck.h index 2d079f405..9e24d56ed 100644 --- a/src/ufo/filters/MetOfficeBuddyCheck.h +++ b/src/ufo/filters/MetOfficeBuddyCheck.h @@ -46,8 +46,8 @@ class MetOfficeBuddyPair; /// updating their gross error probabilities (PGEs) and rejecting observations whose PGE exceeds /// a threshold specified in the filter parameters. /// -/// Variables to be checked should be specified using the "filter variables" YAML option. Currently -/// only surface (single-level) variables are supported. Variables can be either scalar or vector +/// Variables to be checked should be specified using the "filter variables" YAML option, supporting +/// surface (single-level) and multi-level variables. Variables can be either scalar or vector /// (with two Cartesian components, such as the eastward and northward wind components). In the /// latter case the two components need to specified one after the other in the "filter variables" /// list, with the first component having the \c first_component_of_two option set to true. @@ -104,10 +104,12 @@ class MetOfficeBuddyCheck : public FilterBase, Variable backgroundErrorVariable(const Variable &filterVariable) const; /// \brief Returns a vector of IDs of all observations that should be buddy-checked. - std::vector getValidObservationIds(const std::vector &apply) const; + std::vector getValidObservationIds( + const std::vector & apply, + const boost::optional & profileIndex) const; - /// \brief Collects and return smetadata of all observations. - MetaData collectMetaData() const; + /// \brief Collects and returns metadata of all observations. + MetaData collectMetaData(const boost::optional & profileIndex) const; /// \brief Returns a vector of integer-valued station IDs, obtained from the source indicated by /// the filter parameters. @@ -136,7 +138,7 @@ class MetOfficeBuddyCheck : public FilterBase, const std::vector &stationIds, const std::vector &bgErrorHorizCorrScales) const; - /// \brief Buddy check for scalar surface quantities. + /// \brief Buddy check for scalar quantities. /// /// Method: see the OPS Scientific Documentation Paper 2, sections 3.6 and 3.7 /// @@ -152,6 +154,12 @@ class MetOfficeBuddyCheck : public FilterBase, /// Station IDs ("call signs"). /// \param datetimes /// Observation times. + /// \param pressures + /// Model average pressures can be null, representing surface data, single-level (num_levels + /// parameter == 1) data or multi-level (num_levels > 1) data. In all cases except + /// the single-level case, a vertical correlation of 1 is assumed. For single-level data, + /// the estimate of the background error correlation depends upon the ratio of pressures + /// between each pair of observations. /// \param obsValues /// Observed values. /// \param obsErrors @@ -162,19 +170,20 @@ class MetOfficeBuddyCheck : public FilterBase, /// Estimated errors of background values. /// \param[inout] pges /// Gross error probabilities. These values are updated by the buddy check. - void checkScalarSurfaceData(const std::vector &pairs, - const std::vector &flags, - const std::vector &verbose, - const std::vector &bgErrorHorizCorrScales, - const std::vector &stationIds, - const std::vector &datetimes, - const std::vector &obsValues, - const std::vector &obsErrors, - const std::vector &bgValues, - const std::vector &bgErrors, - std::vector &pges) const; - - /// \brief Buddy check for vector (two-dimensional) surface quantities. + void checkScalarData(const std::vector &pairs, + const std::vector &flags, + const std::vector &verbose, + const std::vector &bgErrorHorizCorrScales, + const std::vector &stationIds, + const std::vector &datetimes, + const Eigen::ArrayXXf *pressures, + const Eigen::ArrayXXf &obsValues, + const Eigen::ArrayXXf &obsErrors, + const Eigen::ArrayXXf &bgValues, + const Eigen::ArrayXXf &bgErrors, + Eigen::ArrayXXf &pges) const; + + /// \brief Buddy check for vector (two-dimensional) quantities. /// /// Method: see the OPS Scientific Documentation Paper 2, sections 3.6 and 3.7 /// @@ -190,6 +199,12 @@ class MetOfficeBuddyCheck : public FilterBase, /// Station IDs ("call signs"). /// \param datetimes /// Observation times. + /// \param pressures + /// Model average pressures can be null, represent surface data, single-level (num_levels + /// parameter == 1) data or multi-level (num_levels > 1) data. In all cases except + /// the single-level case, a vertical correlation of 1 is assumed. For single-level data, + /// the estimate of the background error correlation depends upon the ratio of pressures + /// between each pair of observations. /// \param uObsValues /// Observed values of the first component, u. /// \param vObsValues @@ -204,19 +219,20 @@ class MetOfficeBuddyCheck : public FilterBase, /// Estimated errors of background values (u or v). /// \param[inout] pges /// Probabilities of gross error in u or v. These values are updated by the buddy check. - void checkVectorSurfaceData(const std::vector &pairs, - const std::vector &flags, - const std::vector &verbose, - const std::vector &bgErrorHorizCorrScales, - const std::vector &stationIds, - const std::vector &datetimes, - const std::vector &uObsValues, - const std::vector &vObsValues, - const std::vector &obsErrors, - const std::vector &uBgValues, - const std::vector &vBgValues, - const std::vector &bgErrors, - std::vector &pges) const; + void checkVectorData(const std::vector &pairs, + const std::vector &flags, + const std::vector &verbose, + const std::vector &bgErrorHorizCorrScales, + const std::vector &stationIds, + const std::vector &datetimes, + const Eigen::ArrayXXf *pressures, + const Eigen::ArrayXXf &uObsValues, + const Eigen::ArrayXXf &vObsValues, + const Eigen::ArrayXXf &obsErrors, + const Eigen::ArrayXXf &uBgValues, + const Eigen::ArrayXXf &vBgValues, + const Eigen::ArrayXXf &bgErrors, + Eigen::ArrayXXf &pges) const; /// Marks observations whose gross error probability is >= options_->rejectionThreshold /// as rejected by the buddy check. diff --git a/src/ufo/filters/MetOfficeBuddyCheckParameters.h b/src/ufo/filters/MetOfficeBuddyCheckParameters.h index e0dfcf0fb..487d6011d 100644 --- a/src/ufo/filters/MetOfficeBuddyCheckParameters.h +++ b/src/ufo/filters/MetOfficeBuddyCheckParameters.h @@ -46,6 +46,8 @@ class MetOfficeBuddyCheckParameters : public FilterParametersBase { /// \name Parameters controlling buddy pair identification /// @{ + oops::OptionalParameter numLevels{"num_levels", this}; + /// Maximum distance between two observations that may be classified as buddies, in km. oops::Parameter searchRadius{"search_radius", 100, this}; @@ -123,6 +125,9 @@ class MetOfficeBuddyCheckParameters : public FilterParametersBase { oops::Parameter temporalCorrelationScale{"temporal_correlation_scale", util::Duration("PT6H"), this}; + /// Vertical correlation scale (relates to the ratio of pressures). + oops::Parameter verticalCorrelationScale{"vertical_correlation_scale", 6, this}; + /// Parameter used to "damp" gross error probability updates using method 1 described in section /// 3.8 of the OPS Scientific Documentation Paper 2 to make the buddy check /// better-behaved in data-dense areas. See the reference above for the full description. diff --git a/test/testinput/qc_met_office_buddy_check.yaml b/test/testinput/qc_met_office_buddy_check.yaml index d62ed609a..180ea3452 100644 --- a/test/testinput/qc_met_office_buddy_check.yaml +++ b/test/testinput/qc_met_office_buddy_check.yaml @@ -2,7 +2,220 @@ window begin: 2000-01-01T00:00:00Z window end: 2030-12-31T23:59:59Z observations: -- obs space: # Basic comparison against results produced by the Met Office OPS system +- obs space: # Comparison against Met Office OPS system (multi-level) + name: Sonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_buddy_check_multi_level.nc + obsgrouping: + group variables: [ "station_id" ] + simulated variables: [air_temperature, eastward_wind, northward_wind] + HofX: HofX + obs diagnostics: + filename: Data/ufo/testinput_tier_1/met_office_buddy_check_multi_level_diag.nc + obs filters: + - filter: Met Office Buddy Check + filter variables: + - name: air_temperature + - name: eastward_wind + options: + first_component_of_two: true + - name: northward_wind + sort_by_pressure: false # true for sat wind and aircraft (group:4), not sonde (group:5) + # Maps latitudes to kms + horizontal_correlation_scale: {"90": 7200, "30": 7200, "20": 8400, + "-20": 8400, "-30": 9600, "-90": 9600} + temporal_correlation_scale: PT6H + num_zonal_bands: 36 + num_levels: 4 + search_radius: 3000 # km + max_total_num_buddies: 9 + max_num_buddies_from_single_band: 6 + max_num_buddies_with_same_station_id: 0 + damping_factor_1: 1.0 + damping_factor_2: 0.5 + non_divergence_constraint: 1.0 + use_legacy_buddy_collector: true + traced_boxes: + - min_latitude: -90 + max_latitude: 90 + min_longitude: -180 + max_longitude: 180 + # 10 missing (final pressure level) (40-10) x3 for air_temperature, eastward_wind and northward_wind + passedBenchmark: 90 + compareVariables: + - reference: + name: air_temperature@GrossErrorProbabilityAfterOpsBuddyCheck + test: + name: air_temperature@GrossErrorProbability + absTol: 5.0e-5 # The relative difference in Earth radius assumed by OPS and JEDI is ~4e-5 + - reference: + name: eastward_wind@GrossErrorProbabilityAfterOpsBuddyCheck + test: + name: eastward_wind@GrossErrorProbability + absTol: 5.0e-5 +- obs space: # Comparison against Met Office OPS system (multi-level extended obs space) + name: Sonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_buddy_check_multi_level_extended.nc + obsgrouping: + group variables: [ "station_id" ] + simulated variables: [air_temperature, eastward_wind, northward_wind] + HofX: HofX + obs diagnostics: + filename: Data/ufo/testinput_tier_1/met_office_buddy_check_multi_level_diag_extended.nc + obs filters: + - filter: Met Office Buddy Check + filter variables: + - name: air_temperature + - name: eastward_wind + options: + first_component_of_two: true + - name: northward_wind + sort_by_pressure: false # true for sat wind and aircraft (group:4), not sonde (group:5) + # Maps latitudes to kms + horizontal_correlation_scale: {"90": 7200, "30": 7200, "20": 8400, + "-20": 8400, "-30": 9600, "-90": 9600} + temporal_correlation_scale: PT6H + num_zonal_bands: 36 + num_levels: 4 + search_radius: 3000 # km + max_total_num_buddies: 9 + max_num_buddies_from_single_band: 6 + max_num_buddies_with_same_station_id: 0 + damping_factor_1: 1.0 + damping_factor_2: 0.5 + non_divergence_constraint: 1.0 + use_legacy_buddy_collector: true + traced_boxes: + - min_latitude: -90 + max_latitude: 90 + min_longitude: -180 + max_longitude: 180 + # 10 missing (final pressure level) (42-10) x3 for air_temperature, eastward_wind and northward_wind + passedBenchmark: 96 + compareVariables: + - reference: + name: air_temperature@GrossErrorProbabilityAfterOpsBuddyCheck + test: + name: air_temperature@GrossErrorProbability + absTol: 5.0e-5 # The relative difference in Earth radius assumed by OPS and JEDI is ~4e-5 + - reference: + name: eastward_wind@GrossErrorProbabilityAfterOpsBuddyCheck + test: + name: eastward_wind@GrossErrorProbability + absTol: 5.0e-5 +- obs space: # Comparison against Met Office OPS system (multi-level non-consecutive records) + name: Sonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_buddy_check_multi_level_nonconsec.nc + obsgrouping: + group variables: [ "station_id" ] + simulated variables: [air_temperature, eastward_wind, northward_wind] + HofX: HofX + obs diagnostics: + filename: Data/ufo/testinput_tier_1/met_office_buddy_check_multi_level_diag_nonconsec.nc + obs filters: + - filter: Met Office Buddy Check + filter variables: + - name: air_temperature + - name: eastward_wind + options: + first_component_of_two: true + - name: northward_wind + sort_by_pressure: false # true for sat wind and aircraft (group:4), not sonde (group:5) + # Maps latitudes to kms + horizontal_correlation_scale: {"90": 7200, "30": 7200, "20": 8400, + "-20": 8400, "-30": 9600, "-90": 9600} + temporal_correlation_scale: PT6H + num_zonal_bands: 36 + num_levels: 4 + search_radius: 3000 # km + max_total_num_buddies: 9 + max_num_buddies_from_single_band: 6 + max_num_buddies_with_same_station_id: 0 + damping_factor_1: 1.0 + damping_factor_2: 0.5 + non_divergence_constraint: 1.0 + use_legacy_buddy_collector: true + traced_boxes: + - min_latitude: -90 + max_latitude: 90 + min_longitude: -180 + max_longitude: 180 + # 10 missing (final pressure level) (40-10) x3 for air_temperature, eastward_wind and northward_wind + passedBenchmark: 90 + compareVariables: + - reference: + name: air_temperature@GrossErrorProbabilityAfterOpsBuddyCheck + test: + name: air_temperature@GrossErrorProbability + absTol: 5.0e-5 # The relative difference in Earth radius assumed by OPS and JEDI is ~4e-5 + - reference: + name: eastward_wind@GrossErrorProbabilityAfterOpsBuddyCheck + test: + name: eastward_wind@GrossErrorProbability + absTol: 5.0e-5 +- obs space: # Comparison against Met Office OPS system (single-level: num_levels == 1) + name: Sonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_buddy_check_single_level.nc + obsgrouping: + group variables: [ "station_id" ] + simulated variables: [air_temperature, eastward_wind, northward_wind] + HofX: HofX + obs diagnostics: + filename: Data/ufo/testinput_tier_1/met_office_buddy_check_single_level_diag.nc + obs operator: + name: Composite + components: + # operator used to evaluate H(x) + - name: Identity + # operator used to evaluate background errors + - name: BackgroundErrorIdentity + obs filters: + - filter: Met Office Buddy Check + filter variables: + - name: air_temperature + - name: eastward_wind + options: + first_component_of_two: true + - name: northward_wind + sort_by_pressure: false # true for sat wind and aircraft (group:4), not sonde (group:5) + # Maps latitudes to kms + horizontal_correlation_scale: {"90": 7200, "30": 7200, "20": 8400, + "-20": 8400, "-30": 9600, "-90": 9600} + temporal_correlation_scale: PT6H + vertical_correlation_scale: 6 + num_zonal_bands: 36 + num_levels: 1 + search_radius: 3000 # km + max_total_num_buddies: 9 + max_num_buddies_from_single_band: 6 + max_num_buddies_with_same_station_id: 0 + damping_factor_1: 1.0 + damping_factor_2: 0.5 + non_divergence_constraint: 1.0 + use_legacy_buddy_collector: true + traced_boxes: + - min_latitude: -90 + max_latitude: 90 + min_longitude: -180 + max_longitude: 180 + geovals: + filename: Data/ufo/testinput_tier_1/met_office_buddy_check_single_level_geovals.nc + passedBenchmark: 30 # (3x10) + compareVariables: + - reference: + name: air_temperature@GrossErrorProbabilityAfterOpsBuddyCheck + test: + name: air_temperature@GrossErrorProbability + absTol: 5.0e-5 # The relative difference in Earth radius assumed by OPS and JEDI is ~4e-5 + - reference: + name: eastward_wind@GrossErrorProbabilityAfterOpsBuddyCheck + test: + name: eastward_wind@GrossErrorProbability + absTol: 5.0e-5 +- obs space: # Comparison against Met Office OPS system (surface data) name: Aircraft obsdatain: obsfile: Data/ufo/testinput_tier_1/met_office_buddy_check.nc4 @@ -78,8 +291,8 @@ observations: options: first_component_of_two: true - name: northward_wind - # Maps latitudes to kms sort_by_pressure: true + # Maps latitudes to kms horizontal_correlation_scale: {"90": 7200, "30": 7200, "20": 8400, "-20": 8400, "-30": 9600, "-90": 9600} temporal_correlation_scale: PT6H From 26ee1da660dd0d15133eb228d8fde538b23b750b Mon Sep 17 00:00:00 2001 From: Michael Cooke <48374999+mikecooke77@users.noreply.github.com> Date: Fri, 9 Jul 2021 14:32:06 +0100 Subject: [PATCH 31/98] Add a Satellite Selector predictor (#1318) * First attempt * First attempt at wrapper predictor * Update tests to point to new files with two different satids in. * Tidy up and make name of metadata item flexible * Update copyright to correct statement. * Add order to name to be consistent with scan angle and lapse rate predictors * Coding norms * First commit following review of @wsmigaj * Move the PredictorParametersWrapper back to ObsBiasParameters Co-authored-by: Ryan Honeyager --- src/ufo/predictors/CMakeLists.txt | 2 + src/ufo/predictors/Legendre.cc | 2 +- src/ufo/predictors/OrbitalAngle.cc | 2 +- src/ufo/predictors/SatelliteSelector.cc | 64 +++++++++++++++ src/ufo/predictors/SatelliteSelector.h | 82 +++++++++++++++++++ test/CMakeLists.txt | 8 ++ test/testinput/constant_predictor.yaml | 4 +- test/testinput/legendre_predictor.yaml | 2 - test/testinput/orbital_angle_predictor.yaml | 6 +- .../satellite_selector_predictor.yaml | 42 ++++++++++ test/testinput/thickness_predictor.yaml | 16 ++-- 11 files changed, 210 insertions(+), 20 deletions(-) create mode 100644 src/ufo/predictors/SatelliteSelector.cc create mode 100644 src/ufo/predictors/SatelliteSelector.h create mode 100644 test/testinput/satellite_selector_predictor.yaml diff --git a/src/ufo/predictors/CMakeLists.txt b/src/ufo/predictors/CMakeLists.txt index 275c5eff5..d9ee0c8a7 100644 --- a/src/ufo/predictors/CMakeLists.txt +++ b/src/ufo/predictors/CMakeLists.txt @@ -22,6 +22,8 @@ set ( predictor_files Legendre.cc OrbitalAngle.h OrbitalAngle.cc + SatelliteSelector.h + SatelliteSelector.cc ScanAngle.h ScanAngle.cc SineOfLatitude.h diff --git a/src/ufo/predictors/Legendre.cc b/src/ufo/predictors/Legendre.cc index 184a8a705..8fabb3b31 100644 --- a/src/ufo/predictors/Legendre.cc +++ b/src/ufo/predictors/Legendre.cc @@ -23,7 +23,7 @@ Legendre::Legendre(const Parameters_ & parameters, const oops::Variables & vars) nscan_(parameters.numScanPositions) { if (parameters.order.value() != boost::none) { // override the predictor name to distinguish between Legendre predictors of different orders - name() = name() + "_" + std::to_string(order_); + name() = name() + "_order_" + std::to_string(order_); } } diff --git a/src/ufo/predictors/OrbitalAngle.cc b/src/ufo/predictors/OrbitalAngle.cc index be8e3e468..f1045f7e4 100644 --- a/src/ufo/predictors/OrbitalAngle.cc +++ b/src/ufo/predictors/OrbitalAngle.cc @@ -28,7 +28,7 @@ OrbitalAngle::OrbitalAngle(const Parameters_ & parameters, const oops::Variables component_(parameters.component) { // override the predictor name to distinguish between Orbital angle predictors of // different orders as well as the two components, sine and cosine. - name() = name() + "_" + std::to_string(order_)+ "_" + + name() = name() + "_order_" + std::to_string(order_)+ "_" + (component_ == FourierTermType::SIN ? "sin" : "cos"); } diff --git a/src/ufo/predictors/SatelliteSelector.cc b/src/ufo/predictors/SatelliteSelector.cc new file mode 100644 index 000000000..f3e3d65f7 --- /dev/null +++ b/src/ufo/predictors/SatelliteSelector.cc @@ -0,0 +1,64 @@ +/* + * (C) Crown copyright 2021, MetOffice + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include +#include +#include + +#include "ufo/predictors/SatelliteSelector.h" +#include "ufo/utils/Constants.h" + +#include "ioda/ObsSpace.h" + +namespace ufo { + +static PredictorMaker makerFuncSatelliteSelector_("satellite_selector"); + +// ----------------------------------------------------------------------------- + +SatelliteSelector::SatelliteSelector(const Parameters_ & parameters, const oops::Variables & vars) + : PredictorBase(parameters, vars), predictor_(), satid_(parameters.satelliteId), + metadata_name_(parameters.metadataName) { + // Setup the predictor that will be run + std::string predname = parameters.predictor.value().getString("name"); + std::unique_ptr params(PredictorFactory::createParameters(predname)); + params->validateAndDeserialize(parameters.predictor.value()); + std::unique_ptr pred(PredictorFactory::create(*params, vars)); + predictor_ = std::move(pred); + + // Setup the name and variables for external access + name() = predictor_->name() + "_satid_" + std::to_string(satid_); + geovars_ = predictor_->requiredGeovars(); + hdiags_ = predictor_->requiredHdiagnostics(); +} + +// ----------------------------------------------------------------------------- + +void SatelliteSelector::compute(const ioda::ObsSpace & odb, + const GeoVaLs & gv, + const ObsDiagnostics & obsdiags, + ioda::ObsVector & out) const { + const size_t nlocs = out.nlocs(); + const size_t nvars = out.nvars(); + + predictor_->compute(odb, gv, obsdiags, out); + + std::vector satid(nlocs); + odb.get_db("MetaData", metadata_name_, satid); + + for (std::size_t jloc = 0; jloc < nlocs; ++jloc) { + if (satid[jloc] != satid_) { + for (std::size_t jvar = 0; jvar < nvars; ++jvar) { + out[jloc*nvars+jvar] = Constants::zero; + } // jvar + } + } // jloc +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/predictors/SatelliteSelector.h b/src/ufo/predictors/SatelliteSelector.h new file mode 100644 index 000000000..660a5c66b --- /dev/null +++ b/src/ufo/predictors/SatelliteSelector.h @@ -0,0 +1,82 @@ +/* + * (C) Crown copyright 2021, MetOffice + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_PREDICTORS_SATELLITESELECTOR_H_ +#define UFO_PREDICTORS_SATELLITESELECTOR_H_ + +#include +#include +#include + +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/RequiredParameter.h" + +#include "ufo/predictors/PredictorBase.h" + +namespace eckit { + class LocalConfiguration; +} + +namespace oops { + class Variables; +} + +namespace ioda { + class ObsSpace; +} + +namespace ufo { + +/// Configuration parameters of the SatelliteSelector wrapper for a predictor. +class SatelliteSelectorParameters : public PredictorParametersBase { + OOPS_CONCRETE_PARAMETERS(SatelliteSelectorParameters, PredictorParametersBase); + + public: + /// A wrapper to allow for satellite selection when applying any of the + /// predictors. Where the satid is not in the list the predictor is set + /// to zero. + /// + /// The satellite id of the satellite which the predictor should apply to. + oops::RequiredParameter satelliteId{"satellite id", this}; + + /// The configuration for a specific predictor + oops::RequiredParameter predictor{"owned predictor", this}; + + /// Name for the metadata item which will be compared to allow for: + /// satellite_id@MetaData - the default + /// satellite_identifier@MetaData + /// This will hopefully be removed in the future. + oops::Parameter metadataName{"metadata name", "satellite_id", this}; +}; + +// ----------------------------------------------------------------------------- + +class SatelliteSelector : public PredictorBase { + public: + /// The type of parameters accepted by the constructor of this predictor. + /// This typedef is used by the PredictorFactory. + typedef SatelliteSelectorParameters Parameters_; + + SatelliteSelector(const Parameters_ &, const oops::Variables &); + + void compute(const ioda::ObsSpace &, + const GeoVaLs &, + const ObsDiagnostics &, + ioda::ObsVector &) const override; + + private: + /// The local predictor specified from yaml + std::unique_ptr predictor_; + const int satid_; + const std::string metadata_name_; +}; + +// ----------------------------------------------------------------------------- + +} // namespace ufo + +#endif // UFO_PREDICTORS_SATELLITESELECTOR_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0334fb211..a647978d7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -212,6 +212,7 @@ list( APPEND ufo_test_input testinput/radiosonde.yaml testinput/radialvelocity.yaml testinput/reflectivity.yaml + testinput/satellite_selector_predictor.yaml testinput/satname.yaml testinput/sattcwv.yaml testinput/satwind.yaml @@ -3260,6 +3261,13 @@ ecbuild_add_test( TARGET test_interpolate_data_from_file_predictor DEPENDS test_Predictor.x TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_satellite_selector_predictor + COMMAND ${CMAKE_BINARY_DIR}/bin/test_Predictor.x + ARGS "testinput/satellite_selector_predictor.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_Predictor.x + TEST_DEPENDS ufo_get_ufo_test_data ) + ##################################################################### # Files for CRTM tests ##################################################################### diff --git a/test/testinput/constant_predictor.yaml b/test/testinput/constant_predictor.yaml index dbef717cc..ccf68bd14 100644 --- a/test/testinput/constant_predictor.yaml +++ b/test/testinput/constant_predictor.yaml @@ -5,9 +5,7 @@ observations: - obs space: name: atms_n20 obsdatain: - obsfile: Data/ufo/testinput_tier_1/atms_n20_obs_20191230T0000_rttov_predictors.nc4 - obsdataout: - obsfile: Data/atms_npp_obs_2019123000_m_rttov_out_bias.nc4 + obsfile: Data/ufo/testinput_tier_1/atms_obs_20191230T0000_rttov_predictors.nc4 simulated variables: [brightness_temperature] channels: &channels 1-22 geovals: diff --git a/test/testinput/legendre_predictor.yaml b/test/testinput/legendre_predictor.yaml index 0eacd8ad4..4380e7e54 100644 --- a/test/testinput/legendre_predictor.yaml +++ b/test/testinput/legendre_predictor.yaml @@ -6,8 +6,6 @@ observations: name: atms_n20 obsdatain: obsfile: Data/ufo/testinput_tier_1/atms_n20_obs_20191230T0000_rttov_biaspred_redo_scan.nc4 - obsdataout: - obsfile: Data/atms_npp_obs_2019123000_m_rttov_out_bias.nc4 simulated variables: [brightness_temperature] channels: &channels 1-22 geovals: diff --git a/test/testinput/orbital_angle_predictor.yaml b/test/testinput/orbital_angle_predictor.yaml index 715e6a51e..146e66660 100644 --- a/test/testinput/orbital_angle_predictor.yaml +++ b/test/testinput/orbital_angle_predictor.yaml @@ -17,7 +17,7 @@ observations: - name: orbital_angle order: 1 component: "cos" - - name: orbital_angle + - name: orbital_angle order: 1 component: "sin" - name: orbital_angle @@ -29,7 +29,7 @@ observations: - name: orbital_angle order: 3 component: "cos" - - name: orbital_angle + - name: orbital_angle order: 3 component: "sin" - name: orbital_angle @@ -41,7 +41,7 @@ observations: - name: orbital_angle order: 5 component: "cos" - - name: orbital_angle + - name: orbital_angle order: 5 component: "sin" - name: orbital_angle diff --git a/test/testinput/satellite_selector_predictor.yaml b/test/testinput/satellite_selector_predictor.yaml new file mode 100644 index 000000000..295c945cf --- /dev/null +++ b/test/testinput/satellite_selector_predictor.yaml @@ -0,0 +1,42 @@ +window begin: 2019-12-29T21:00:00Z +window end: 2019-12-30T03:00:00Z + +observations: +- obs space: + name: atms_n20 + obsdatain: + obsfile: Data/ufo/testinput_tier_1/atms_obs_20191230T0000_rttov_predictors.nc4 + simulated variables: [brightness_temperature] + channels: &channels 1-22 + geovals: + filename: Data/ufo/testinput_tier_1/geovals_atms_20191230T0000Z_predictors.nc4 + obs bias: + variational bc: + predictors: +# Predictors for 224 S-NPP + - name: satellite_selector + satellite id: 224 + owned predictor: + name: constant + - name: satellite_selector + satellite id: 224 + owned predictor: + name: thickness + layer top: 30000 + layer base: 85000 + mean: 7.6 + standard deviation: 0.4 +# Predictors for 225 NOAA-20 + - name: satellite_selector + satellite id: 225 + owned predictor: + name: constant + - name: satellite_selector + satellite id: 225 + owned predictor: + name: thickness + layer top: 30000 + layer base: 85000 + mean: 7.6 + standard deviation: 0.4 + tolerance: 5.0e-5 diff --git a/test/testinput/thickness_predictor.yaml b/test/testinput/thickness_predictor.yaml index b4d65e8b7..7c233c2ed 100644 --- a/test/testinput/thickness_predictor.yaml +++ b/test/testinput/thickness_predictor.yaml @@ -2,14 +2,14 @@ window begin: 2019-12-29T21:00:00Z window end: 2019-12-30T03:00:00Z observations: -- obs space: +- obs space: &ObsSpace name: atms_n20 obsdatain: - obsfile: Data/ufo/testinput_tier_1/atms_n20_obs_20191230T0000_rttov_predictors.nc4 + obsfile: Data/ufo/testinput_tier_1/atms_obs_20191230T0000_rttov_predictors.nc4 simulated variables: [brightness_temperature] - channels: &channels 1-22 + channels: 1-22 geovals: - filename: Data/ufo/testinput_tier_1/geovals_atms_20191230T0000Z_predictors.nc4 + filename: &geovalsin Data/ufo/testinput_tier_1/geovals_atms_20191230T0000Z_predictors.nc4 obs bias: variational bc: predictors: @@ -21,13 +21,9 @@ observations: expectExceptionWithMessage: layer top is greater than largest model pressure level - obs space: - name: atms_n20 - obsdatain: - obsfile: Data/ufo/testinput_tier_1/atms_n20_obs_20191230T0000_rttov_predictors.nc4 - simulated variables: [brightness_temperature] - channels: &channels 1-22 + <<: *ObsSpace geovals: - filename: Data/ufo/testinput_tier_1/geovals_atms_20191230T0000Z_predictors.nc4 + filename: *geovalsin obs bias: variational bc: predictors: From 6ddb57e219f8946eece0452dc4636edda69cd4d8 Mon Sep 17 00:00:00 2001 From: david-rundle <37152257+david-rundle@users.noreply.github.com> Date: Fri, 9 Jul 2021 15:34:50 +0100 Subject: [PATCH 32/98] Feature/rttov interface jun21 (#1288) * change expected u and v wind geovals to uwind_at_10m output emissivity from RTTOV change ColdSurfaceCheck default from true to false * update to change coldsurfacecheck update to use var_sfc_u10/v10 rather than var_u/v update to keep on rolling if rttov returns an error during calculation * more don't abort when there's a problem with rttov_direct and rttov_k * finished twiddling with the warnings for rttov run failures potential fix to hofxdiags * use some more rttov constants * missed line to correctly allocate obsdiags for predictors * change print level to info from debug * remove mandatory var_prsi * fix for obsdiags and chunking * fix profile mis-indexing there was a missing call to a routine to zero the jacobians which is now fixed * found another instance of var_[uv] to update upon pulling in develop Co-authored-by: Ryan Honeyager --- ...ttovonedvarcheck_minimize_jacobian_mod.f90 | 12 +- ...o_rttovonedvarcheck_minimize_utils_mod.f90 | 22 +-- src/ufo/rttov/ufo_radiancerttov_mod.F90 | 53 +++--- src/ufo/rttov/ufo_radiancerttov_tlad_mod.F90 | 53 +++--- src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 | 160 +++++++++++------- .../ufo_metoffice_bmatrixstatic_mod.f90 | 2 +- .../amsr2_rttov_ops_qc_rttovonedvarcheck.yaml | 6 +- 7 files changed, 183 insertions(+), 125 deletions(-) diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_jacobian_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_jacobian_mod.f90 index 8ee6466c8..e79a01217 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_jacobian_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_jacobian_mod.f90 @@ -287,22 +287,22 @@ subroutine ufo_rttovonedvarcheck_GetHmatrixRTTOVsimobs(geovals, ob, obsdb, & end do end if -! 2.4) Windspeed - var_u = "eastward_wind" -! - var_v = "northward_wind" +! 2.4) Windspeed - var_sfc_u10 = "uwind_at_10m" +! - var_sfc_v10 = "vwind_at_10m" ! - windsp = sqrt (u*u + v*v) if (profindex % windspeed > 0) then - call ufo_geovals_get_var(geovals, trim(var_u), geoval) + call ufo_geovals_get_var(geovals, trim(var_sfc_u10), geoval) u = geoval % vals(1, 1) - call ufo_geovals_get_var(geovals, trim(var_v), geoval) + call ufo_geovals_get_var(geovals, trim(var_sfc_v10), geoval) v = geoval % vals(1, 1) windsp = sqrt (u ** 2 + v ** 2) do i = 1, nchans - write(varname,"(3a,i0)") "brightness_temperature_jacobian_",trim(var_u),"_",channels(i) + write(varname,"(3a,i0)") "brightness_temperature_jacobian_",trim(var_sfc_u10),"_",channels(i) call ufo_geovals_get_var(hofxdiags, varname, geoval) dBT_du = geoval % vals(1,1) - write(varname,"(3a,i0)") "brightness_temperature_jacobian_",trim(var_v),"_",channels(i) + write(varname,"(3a,i0)") "brightness_temperature_jacobian_",trim(var_sfc_v10),"_",channels(i) call ufo_geovals_get_var(hofxdiags, varname, geoval) dBT_dv = geoval % vals(1,1) if (windsp > zero) then diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_utils_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_utils_mod.f90 index faf9b286b..172ca5320 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_utils_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_minimize_utils_mod.f90 @@ -167,13 +167,13 @@ subroutine ufo_rttovonedvarcheck_GeoVaLs2ProfVec( geovals, & ! in ! prof_x(profindex % cloudfrac) = ob % cloudfrac !end if -! Windspeed - var_u = "eastward_wind" -! - var_v = "northward_wind" +! Windspeed - var_sfc_u10 = "uwind_at_10m" +! - var_sfc_v10 = "vwind_at_10m" ! - windsp = sqrt (u*u + v*v) if (profindex % windspeed > 0) then - call ufo_geovals_get_var(geovals, trim(var_u), geoval) + call ufo_geovals_get_var(geovals, trim(var_sfc_u10), geoval) u = geoval % vals(1, 1) - call ufo_geovals_get_var(geovals, trim(var_v), geoval) + call ufo_geovals_get_var(geovals, trim(var_sfc_v10), geoval) v = geoval % vals(1, 1) prof_x(profindex % windspeed) = sqrt(u ** 2 + v ** 2) end if @@ -392,9 +392,9 @@ subroutine ufo_rttovonedvarcheck_ProfVec2GeoVaLs(geovals, & ! inout ! windspeed if (profindex % windspeed > 0) then - call ufo_geovals_get_var(geovals, trim(var_u), geoval) + call ufo_geovals_get_var(geovals, trim(var_sfc_u10), geoval) u = geoval % vals(1, 1) - call ufo_geovals_get_var(geovals, trim(var_v), geoval) + call ufo_geovals_get_var(geovals, trim(var_sfc_v10), geoval) v = geoval % vals(1, 1) windsp = sqrt (u ** 2 + v ** 2) @@ -411,14 +411,14 @@ subroutine ufo_rttovonedvarcheck_ProfVec2GeoVaLs(geovals, & ! inout ! Write back updated u component gv_index = 0 do i=1,geovals%nvar - if (trim(var_u) == trim(geovals%variables(i))) gv_index = i + if (trim(var_sfc_u10) == trim(geovals%variables(i))) gv_index = i end do geovals%geovals(gv_index)%vals(1,1) = u ! Write back updated v component gv_index = 0 do i=1,geovals%nvar - if (trim(var_v) == trim(geovals%variables(i))) gv_index = i + if (trim(var_sfc_v10) == trim(geovals%variables(i))) gv_index = i end do geovals%geovals(gv_index)%vals(1,1) = v end if @@ -1265,11 +1265,11 @@ subroutine ufo_rttovonedvarcheck_hofxdiags_levels(retrieval_vars, nlevels, ret_n trim(varname) == trim(var_sfc_q2m) .or. & trim(varname) == trim(var_sfc_p2m) .or. & trim(varname) == trim(var_sfc_tskin) .or. & - trim(varname) == trim(var_u) .or. & - trim(varname) == trim(var_v)) then + trim(varname) == trim(var_sfc_u10) .or. & + trim(varname) == trim(var_sfc_v10)) then ret_nlevs(i) = 1 else - write(message, *) trim(varlist(i)), "not setup for retrieval yet: aborting" + write(message, *) trim(varlist(i)), " not setup for retrieval yet: aborting" call abor1_ftn(message) end if end do diff --git a/src/ufo/rttov/ufo_radiancerttov_mod.F90 b/src/ufo/rttov/ufo_radiancerttov_mod.F90 index a2d9eee96..39f37439e 100644 --- a/src/ufo/rttov/ufo_radiancerttov_mod.F90 +++ b/src/ufo/rttov/ufo_radiancerttov_mod.F90 @@ -224,22 +224,28 @@ subroutine ufo_radiancerttov_simobs(self, geovals, obss, nvars, nlocs, & ! Build the list of profile/channel indices in chanprof do iprof_rttov = 1, nprof_sim + errorstatus = errorstatus_success ! iprof is the index for the full set of RTTOV profiles iprof = prof_start + iprof_rttov - 1 - do ichan = 1, nchan_inst - ichan_sim = ichan_sim + 1_jpim - chanprof(ichan_sim) % prof = iprof_rttov ! this refers to the slice of the RTprofile array passed to RTTOV - chanprof(ichan_sim) % chan = self % channels(ichan) - self % RTprof % chanprof(nchan_total + ichan_sim) % prof = iprof ! this refers to the index of the profile from the geoval - self % RTprof % chanprof(nchan_total + ichan_sim) % chan = self % channels(ichan) - end do - nchan_sim = ichan_sim - - if(self % conf % RTTOV_profile_checkinput) call self % RTprof % check(self % conf, iprof, i_inst) + ! print profile information if requested if(any(self % conf % inspect == iprof)) call self % RTprof % print(self % conf, iprof, i_inst) - + + ! check RTTOV profile and flag it if it fails the check + if(self % conf % RTTOV_profile_checkinput) call self % RTprof % check(self % conf, iprof, i_inst, errorstatus) + + if (errorstatus == errorstatus_success) then + do ichan = 1, nchan_inst + ichan_sim = ichan_sim + 1_jpim + chanprof(ichan_sim) % prof = iprof_rttov ! this refers to the slice of the RTprofile array passed to RTTOV + chanprof(ichan_sim) % chan = self % channels(ichan) + self % RTprof % chanprof(nchan_total + ichan_sim) % prof = iprof ! this refers to the index of the profile from the geoval + self % RTprof % chanprof(nchan_total + ichan_sim) % chan = self % channels(ichan) + end do + nchan_sim = ichan_sim + endif + end do ! Set surface emissivity @@ -267,8 +273,10 @@ subroutine ufo_radiancerttov_simobs(self, geovals, obss, nvars, nlocs, & emissivity_k = self % RTProf % emissivity_k(1:nchan_sim))!, &! inout input/output emissivities per channel if ( errorstatus /= errorstatus_success ) then - write(message,'(A, 2I6)') 'after rttov_k: error ', errorstatus, i_inst - call abor1_ftn(message) + write(message,'(A, A, 2I6)') trim(routine_name), 'after rttov_k: error ', errorstatus, i_inst, & + ' skipping profiles ', prof_start, ' -- ', prof_start + nprof_sim - 1 + call fckit_log%info(message) + cycle end if else @@ -284,19 +292,23 @@ subroutine ufo_radiancerttov_simobs(self, geovals, obss, nvars, nlocs, & emissivity = self % RTProf % emissivity(1:nchan_sim))!, &! inout input/output emissivities per channel if ( errorstatus /= errorstatus_success ) then - write(message,'(A, 2I6)') 'after rttov_direct: error ', errorstatus, i_inst - call abor1_ftn(message) + write(message,'(A, A, 2I6)') trim(routine_name), 'after rttov_direct: error ', errorstatus, i_inst, & + ' skipping profiles ', prof_start, ' -- ', prof_start + nprof_sim - 1 + call fckit_log%info(message) + cycle end if endif ! jacobian_needed ! Put simulated brightness temperature into hofx - do ichan = 1, nchan_sim, size(self%channels) - iprof = self % RTProf % chanprof(nchan_total + ichan)%prof - hofx(1:size(self%channels),iprof) = self % RTprof % radiance % bt(ichan:ichan+size(self%channels)-1) - enddo + if ( errorstatus == errorstatus_success ) then + do ichan = 1, nchan_sim, size(self%channels) + iprof = self % RTProf % chanprof(nchan_total + ichan)%prof + hofx(1:size(self%channels),iprof) = self % RTprof % radiance % bt(ichan:ichan+size(self%channels)-1) + enddo ! Put simulated diagnostics into hofxdiags - if(hofxdiags%nvar > 0) call populate_hofxdiags(self % RTProf, chanprof, self % conf, hofxdiags) + if(hofxdiags%nvar > 0) call populate_hofxdiags(self % RTProf, chanprof, self % conf, prof_start, hofxdiags) + endif ! increment profile and channel counters nchan_total = nchan_total + nchan_sim @@ -304,6 +316,7 @@ subroutine ufo_radiancerttov_simobs(self, geovals, obss, nvars, nlocs, & ! deallocate local chanprof so it can be re-allocated with a different number of channels if reqd. deallocate(chanprof) + if (jacobian_needed) call self % RTprof % zero_K() end do RTTOV_loop diff --git a/src/ufo/rttov/ufo_radiancerttov_tlad_mod.F90 b/src/ufo/rttov/ufo_radiancerttov_tlad_mod.F90 index 856e9f37a..621d17fba 100644 --- a/src/ufo/rttov/ufo_radiancerttov_tlad_mod.F90 +++ b/src/ufo/rttov/ufo_radiancerttov_tlad_mod.F90 @@ -102,9 +102,9 @@ subroutine ufo_radiancerttov_tlad_setup(self, f_confOper, channels) ind = ind + 1 self%varin(ind) = var_sfc_tskin ind = ind + 1 - self%varin(ind) = var_u + self%varin(ind) = var_sfc_u10 ind = ind + 1 - self%varin(ind) = var_v + self%varin(ind) = var_sfc_v10 ! save channels allocate(self%channels(size(channels))) @@ -242,22 +242,28 @@ subroutine ufo_radiancerttov_tlad_settraj(self, geovals, obss, hofxdiags) ! Build the list of profile/channel indices in chanprof do iprof_rttov = 1, nprof_sim + errorstatus = errorstatus_success ! iprof is the index for the full set of RTTOV profiles iprof = prof_start + iprof_rttov - 1 - do ichan = 1, nchan_inst - ichan_sim = ichan_sim + 1_jpim - chanprof(ichan_sim) % prof = iprof_rttov ! this refers to the slice of the RTprofile array passed to RTTOV - chanprof(ichan_sim) % chan = self % channels(ichan) - self % RTprof_K % chanprof(nchan_total + ichan_sim) % prof = iprof - self % RTprof_K % chanprof(nchan_total + ichan_sim) % chan = self % channels(ichan) - end do - nchan_sim = ichan_sim - - if(self % conf % RTTOV_profile_checkinput) call self % RTprof_K % check(self % conf, iprof, i_inst) + ! print profile information if requested if(any(self % conf % inspect == iprof)) call self % RTprof_K % print(self % conf, iprof, i_inst) - + + ! check RTTOV profile and flag it if it fails the check + if(self % conf % RTTOV_profile_checkinput) call self % RTprof_K % check(self % conf, iprof, i_inst, errorstatus) + + if (errorstatus == errorstatus_success) then + do ichan = 1, nchan_inst + ichan_sim = ichan_sim + 1_jpim + chanprof(ichan_sim) % prof = iprof_rttov ! this refers to the slice of the RTprofile array passed to RTTOV + chanprof(ichan_sim) % chan = self % channels(ichan) + self % RTprof_K % chanprof(nchan_total + ichan_sim) % prof = iprof ! this refers to the index of the profile from the geoval + self % RTprof_K % chanprof(nchan_total + ichan_sim) % chan = self % channels(ichan) + end do + nchan_sim = ichan_sim + endif + end do ! Set surface emissivity @@ -283,14 +289,15 @@ subroutine ufo_radiancerttov_tlad_settraj(self, geovals, obss, hofxdiags) emissivity_k = self % RTprof_K % emissivity_k(1:nchan_sim))!, &! inout input/output emissivities per channel if ( errorstatus /= errorstatus_success ) then - write(message,'(A, A, 2I6)') trim(routine_name), 'after rttov_k: error ', errorstatus, i_inst - call abor1_ftn(message) + write(message,'(A, A, 2I6)') trim(routine_name), 'after rttov_k: error ', errorstatus, i_inst, & + ' skipping profiles ', prof_start, ' -- ', prof_start + nprof_sim - 1 + call fckit_log%info(message) + else + ! Put simulated diagnostics into hofxdiags + ! ---------------------------------------------- + if(hofxdiags%nvar > 0) call populate_hofxdiags(self % RTprof_K, chanprof, self % conf, prof_start, hofxdiags) end if - ! Put simulated diagnostics into hofxdiags - ! ---------------------------------------------- - if(hofxdiags%nvar > 0) call populate_hofxdiags(self % RTprof_K, chanprof, self % conf, hofxdiags) - ! increment profile and channel counters nchan_total = nchan_total + nchan_sim prof_start = prof_start + nprof_sim @@ -435,8 +442,8 @@ subroutine ufo_radiancerttov_simobs_tl(self, geovals, obss, nvars, nlocs, hofx) end do !windspeed - call ufo_geovals_get_var(geovals, var_u, geoval_d) - call ufo_geovals_get_var(geovals, var_v, geoval_d2) + call ufo_geovals_get_var(geovals, var_sfc_u10, geoval_d) + call ufo_geovals_get_var(geovals, var_sfc_v10, geoval_d2) do ichan = 1, self % nchan_total, size(self%channels) prof = self % RTprof_K % chanprof(ichan) % prof @@ -579,8 +586,8 @@ subroutine ufo_radiancerttov_simobs_ad(self, geovals, obss, nvars, nlocs, hofx) end do !windspeed - call ufo_geovals_get_var(geovals, var_u, geoval_d) - call ufo_geovals_get_var(geovals, var_v, geoval_d2) + call ufo_geovals_get_var(geovals, var_sfc_u10, geoval_d) + call ufo_geovals_get_var(geovals, var_sfc_v10, geoval_d2) do ichan = 1, self % nchan_total, size(self%channels) prof = self % RTprof_K % chanprof(ichan) % prof diff --git a/src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 b/src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 index 29684f0a5..ce3c4e306 100644 --- a/src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 +++ b/src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 @@ -70,7 +70,7 @@ module ufo_radiancerttov_utils_mod character(len=maxvarlen), dimension(9), target :: varin_default_satrad = & (/var_prs, var_ts, var_q, var_sfc_t2m, & - var_u, var_v, var_sfc_p2m, var_sfc_q2m, & + var_sfc_u10, var_sfc_v10, var_sfc_p2m, var_sfc_q2m, & var_sfc_tskin /) character(len=maxvarlen), pointer, public :: varin_default(:) @@ -142,6 +142,7 @@ module ufo_radiancerttov_utils_mod procedure :: alloc_k => ufo_rttov_alloc_k procedure :: alloc_profs => ufo_rttov_alloc_profiles procedure :: alloc_profs_K => ufo_rttov_alloc_profiles_K + procedure :: zero_K => ufo_rttov_zero_K procedure :: init_emissivity => ufo_rttov_init_emissivity procedure :: setup => ufo_rttov_setup_rtprof procedure :: check => ufo_rttov_check_rtprof @@ -169,7 +170,7 @@ module ufo_radiancerttov_utils_mod logical :: SatRad_compatibility = .true. logical :: UseRHwaterForQC = .true. ! only used with SatRad compatibility - logical :: UseColdSurfaceCheck = .true. ! only used with SatRad compatibility + logical :: UseColdSurfaceCheck = .false. ! to replicate pre-PS45 results logical :: SplitQtotal = .false. ! true for SatRad compatibility with MW logical :: UseQtsplitRain = .false. logical :: RTTOV_profile_checkinput = .false. @@ -770,7 +771,7 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) !DAR: This will be extended for RTTOV_SCATT !profiles_scatt = > self % profiles_scatt - if (present(ob_info)) then + if(present(ob_info)) then nlocs_total = 1 else nlocs_total = obsspace_get_nlocs(obss) @@ -935,14 +936,14 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) enddo endif - varname = var_u ! Eastward-wind in m/s + varname = var_sfc_u10 ! Eastward-wind in m/s if (ufo_vars_getindex(geovals%variables, varname) > 0) then call ufo_geovals_get_var(geovals, varname, geoval) profiles(1:nprofiles)%s2m%u = geoval%vals(1,1:nprofiles) !assume if eastward then northward too - varname = var_v ! Northward-wind in m/s + varname = var_sfc_v10 ! Northward-wind in m/s call ufo_geovals_get_var(geovals, varname, geoval) profiles(1:nprofiles)%s2m%v = geoval%vals(1,1:nprofiles) @@ -964,7 +965,7 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) if (.not. allocated(tmpvar)) allocate(TmpVar(nlocs_total)) !Hard code watertype to ocean. Only used to determine BRDF in visible calculations - profiles(1:nprofiles) % skin % watertype = 1 ! always assume ocean + profiles(1:nprofiles) % skin % watertype = watertype_ocean_water ! always assume ocean !Get Skin (surface) temperature (K) varname = var_sfc_tskin @@ -1168,6 +1169,8 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) call fckit_log%info(message) end if +!Set RTTOV viewing geometry + ! sensor zenith - RTTOV convention 0-max (coef dependent). Nadir = 0 deg ! mandatory so assume it's present call obsspace_get_db(obss, "MetaData", "sensor_zenith_angle", TmpVar) @@ -1226,26 +1229,27 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) end subroutine ufo_rttov_setup_rtprof - subroutine ufo_rttov_check_rtprof(self, conf, iprof, i_inst) + subroutine ufo_rttov_check_rtprof(self, conf, iprof, i_inst, errorstatus) implicit none class(ufo_rttov_io), target, intent(inout) :: self type(rttov_conf), intent(in) :: conf integer, intent(in) :: iprof integer, intent(in) :: i_inst + integer, intent(out) :: errorstatus character(10) :: prof_str include 'rttov_print_profile.interface' include 'rttov_user_profile_checkinput.interface' - call rttov_user_profile_checkinput(rttov_errorstatus, & + call rttov_user_profile_checkinput(errorstatus, & conf % rttov_opts, & conf % rttov_coef_array(i_inst), & self % profiles(iprof)) ! print erroneous profile to stderr - if(rttov_errorstatus /= errorstatus_success) then + if(errorstatus /= errorstatus_success .and. debug) then write(prof_str,'(i0)') iprof self % profiles(iprof) % id = prof_str call rttov_print_profile(self % profiles(iprof), lu = stderr) @@ -1264,11 +1268,10 @@ subroutine ufo_rttov_print_rtprof(self, conf, iprof, i_inst) character(10) :: prof_str include 'rttov_print_profile.interface' - if (any(conf % inspect == iprof)) then - write(prof_str,'(i0)') iprof - self % profiles(iprof) % id = prof_str - call rttov_print_profile(self % profiles(iprof), lu = stdout) - endif + write(*,*) 'profile ', iprof + write(prof_str,'(i0)') iprof + self % profiles(iprof) % id = prof_str + call rttov_print_profile(self % profiles(iprof), lu = stdout) end subroutine ufo_rttov_print_rtprof @@ -1484,6 +1487,22 @@ subroutine ufo_rttov_alloc_profiles_k(self, errorstatus, conf, nprofiles, nlevel end subroutine ufo_rttov_alloc_profiles_k + subroutine ufo_rttov_zero_k(self) + implicit none + + class(ufo_rttov_io), target, intent(inout) :: self + + include 'rttov_init_prof.interface' + + call rttov_init_prof(self % profiles_k) + self % emissivity_k(:) % emis_in = 0.0 + self % emissivity_k(:) % emis_out = 0.0 + self % emissivity(:) % emis_out = 0.0 + self % radiance_k % bt(:) = 1.0 + self % radiance_k % total(:) = 1.0 + + end subroutine ufo_rttov_zero_k + subroutine ufo_rttov_init_emissivity(self, conf, prof_start) class(ufo_rttov_io), intent(inout) :: self type(rttov_conf), intent(in) :: conf @@ -1620,7 +1639,7 @@ subroutine set_defaults_rttov(self, default_opts_set) self % rttov_opts % rt_mw % fastem_version = 6 !< FASTEM version (0-6); 0 => TESSEM2 self % rttov_opts % rt_mw % supply_foam_fraction = .false. !< Supply a foam fraction to FASTEM self % rttov_opts % rt_mw % clw_data = .false. !< Switch to enable input of cloud liquid water profile - self % rttov_opts % rt_mw % clw_scheme = 1 !< MW CLW scheme: 1 => Liebe, 2 => Rosenkranz, 3 => TKC + self % rttov_opts % rt_mw % clw_scheme = mw_clw_scheme_liebe !< MW CLW scheme: 1 => Liebe, 2 => Rosenkranz, 3 => TKC self % rttov_opts % rt_mw % clw_calc_on_coef_lev = .true. !< Apply MW CLW calculations on coef/user levels (true/false resp.) self % rttov_opts % rt_mw % clw_cloud_top = 322 !< Lower pressure limit for MW CLW calculations (hPa) self % rttov_opts % rt_mw % apply_band_correction = .true. !< Apply band-correction for Planck radiance and BT calculations @@ -1654,7 +1673,7 @@ subroutine set_defaults_rttov(self, default_opts_set) self % rttov_opts % rt_mw % clw_data = .true. ! Set to true for allocation purposes self % rttov_opts % interpolation % addinterp = .true. ! Allow interpolation of input profile - self % rttov_opts % interpolation % interp_mode = 4 ! Set interpolation method (4 for all insts at PS44) + self % rttov_opts % interpolation % interp_mode = interp_rochon_wfn ! Set interpolation method (4 for all insts at PS44) endif !RTTOV options that are different from RTTOV defaults at and before PS44 @@ -1681,7 +1700,7 @@ subroutine set_defaults_rttov(self, default_opts_set) if (PS_Number == 45) then self % rttov_opts % rt_all % addrefrac = .true. ! This is an RTTOV 13 default - self % rttov_opts % rt_mw % clw_scheme = 2 ! This is an RTTOV 13 default + self % rttov_opts % rt_mw % clw_scheme = mw_clw_scheme_rosenkranz ! This is an RTTOV 13 default self % rttov_opts % interpolation % reg_limit_extrap = .true. ! This is an RTTOV 13 default endif @@ -1689,17 +1708,19 @@ subroutine set_defaults_rttov(self, default_opts_set) end subroutine set_defaults_rttov - subroutine populate_hofxdiags(RTProf, chanprof, conf, hofxdiags) + subroutine populate_hofxdiags(RTProf, chanprof, conf, prof_start, hofxdiags) use ufo_constants_mod, only : g_to_kg type(ufo_rttov_io), intent(in) :: RTProf type(rttov_chanprof), intent(in) :: chanprof(:) type(rttov_conf), intent(in) :: conf + integer, intent(in) :: prof_start type(ufo_geovals), intent(inout) :: hofxdiags !non-h(x) diagnostics - integer :: jvar, chan, prof, ichan - integer :: nlayers, nchanprof, nlevels, nprofiles - real(kind_real), allocatable :: od_level(:), wfunc(:) + integer :: jvar, chan, prof, ichan, rttov_prof + integer :: nchanprof, nlevels, nprofiles + real(kind_real), allocatable :: od_level(:), wfunc(:) + logical, save :: firsttime = .true. include 'rttov_calc_weighting_fn.interface' @@ -1710,7 +1731,7 @@ subroutine populate_hofxdiags(RTProf, chanprof, conf, hofxdiags) nchanprof = size(chanprof) nlevels = size(RTProf % profiles(1) % p) - nprofiles = maxval(chanprof(:)%prof) !SIZE(RTProf % profiles) + nprofiles = nlocs_total do jvar = 1, hofxdiags%nvar if (len(trim(hofxdiags%variables(jvar))) < 1) cycle @@ -1728,16 +1749,18 @@ subroutine populate_hofxdiags(RTProf, chanprof, conf, hofxdiags) ! variable: weightingfunction_of_atmosphere_layer_CH case (var_opt_depth, var_lvl_transmit,var_lvl_weightfunc) - nlayers = nlevels - 1 hofxdiags%geovals(jvar)%nval = nlevels - if(.not. allocated(hofxdiags%geovals(jvar)%vals)) & - allocate(hofxdiags%geovals(jvar)%vals(hofxdiags%geovals(jvar)%nval,nprofiles)) - hofxdiags%geovals(jvar)%vals = missing + if(.not. allocated(hofxdiags%geovals(jvar)%vals)) then + allocate(hofxdiags%geovals(jvar)%vals(hofxdiags%geovals(jvar)%nval,nprofiles)) + hofxdiags%geovals(jvar)%vals = missing + endif + ! get channel/profile do ichan = 1, nchanprof chan = chanprof(ichan)%chan - prof = chanprof(ichan)%prof - + rttov_prof = chanprof(ichan)%prof + prof = prof_start + chanprof(ichan)%prof - 1 + if(chan == ch_diags(jvar)) then ! if profile not skipped if(cmp_strings(ystr_diags(jvar), var_opt_depth)) then @@ -1748,11 +1771,10 @@ subroutine populate_hofxdiags(RTProf, chanprof, conf, hofxdiags) RTProf % transmission%tau_levels(2:,chan) else if (cmp_strings(ystr_diags(jvar), var_lvl_weightfunc)) then od_level(:) = log(RTProf % transmission%tau_levels(:,chan)) !level->TOA transmittances -> od - call rttov_calc_weighting_fn(rttov_errorstatus, RTProf % profiles(prof)%p, od_level(:), & + call rttov_calc_weighting_fn(rttov_errorstatus, RTProf % profiles(rttov_prof)%p, od_level(:), & hofxdiags%geovals(jvar)%vals(:,prof)) endif - !endif endif enddo @@ -1760,16 +1782,19 @@ subroutine populate_hofxdiags(RTProf, chanprof, conf, hofxdiags) ! variable: brightness_temperature_assuming_clear_sky_CH ! variable: pressure_level_at_peak_of_weightingfunction_CH ! variable: toa_total_transmittance_CH - case (var_radiance, var_tb_clr, var_tb, var_pmaxlev_weightfunc, var_total_transmit) + ! variable: surface_emissivity_CH + case (var_radiance, var_tb_clr, var_tb, var_sfc_emiss, var_pmaxlev_weightfunc, var_total_transmit) ! always returned hofxdiags%geovals(jvar)%nval = 1 - if(.not. allocated(hofxdiags%geovals(jvar)%vals)) & - allocate(hofxdiags%geovals(jvar)%vals(hofxdiags%geovals(jvar)%nval,nprofiles)) - hofxdiags%geovals(jvar)%vals = missing + if(.not. allocated(hofxdiags%geovals(jvar)%vals)) then + allocate(hofxdiags%geovals(jvar)%vals(hofxdiags%geovals(jvar)%nval,nprofiles)) + hofxdiags%geovals(jvar)%vals = missing + endif do ichan = 1, nchanprof chan = chanprof(ichan)%chan - prof = chanprof(ichan)%prof + rttov_prof = chanprof(ichan)%prof + prof = prof_start + chanprof(ichan)%prof - 1 if(chan == ch_diags(jvar)) then if(cmp_strings(ystr_diags(jvar), var_radiance)) then @@ -1779,11 +1804,13 @@ subroutine populate_hofxdiags(RTProf, chanprof, conf, hofxdiags) else if(cmp_strings(ystr_diags(jvar), var_tb)) then hofxdiags%geovals(jvar)%vals(1,prof) = RTProf % radiance % bt(ichan) else if(cmp_strings(ystr_diags(jvar), var_pmaxlev_weightfunc)) then - call rttov_calc_weighting_fn(rttov_errorstatus, RTProf % profiles(prof)%p, od_level(:), & + call rttov_calc_weighting_fn(rttov_errorstatus, RTProf % profiles(rttov_prof)%p, od_level(:), & Wfunc(:)) hofxdiags%geovals(jvar)%vals(1,prof) = maxloc(Wfunc(:), DIM=1) ! scalar not array(1) else if(cmp_strings(ystr_diags(jvar), var_total_transmit)) then hofxdiags%geovals(jvar)%vals(1,prof) = RTProf % transmission % tau_total(ichan) + else if(cmp_strings(ystr_diags(jvar), var_sfc_emiss)) then + hofxdiags%geovals(jvar)%vals(1,prof) = RTProf % emissivity(ichan) % emis_out end if endif end do @@ -1791,13 +1818,17 @@ subroutine populate_hofxdiags(RTProf, chanprof, conf, hofxdiags) case default ! not a supported obsdiag but we allocate and initialise here anyway for use later on hofxdiags%geovals(jvar)%nval = 1 - allocate(hofxdiags%geovals(jvar)%vals(hofxdiags%geovals(jvar)%nval,nprofiles)) - hofxdiags%geovals(jvar)%vals = missing + if(.not. allocated(hofxdiags%geovals(jvar)%vals)) then + allocate(hofxdiags%geovals(jvar)%vals(hofxdiags%geovals(jvar)%nval,nprofiles)) + hofxdiags%geovals(jvar)%vals = missing + endif - write(message,*) 'ufo_radiancerttov_simobs: //& - & ObsDiagnostic is unsupported but allocating anyway, ', & - & hofxdiags%variables(jvar), shape(hofxdiags%geovals(jvar)%vals) - call fckit_log%info(message) + if(firsttime) then + write(message,*) 'ufo_radiancerttov_simobs: //& + & ObsDiagnostic is unsupported but allocating anyway, ', & + & hofxdiags%variables(jvar), shape(hofxdiags%geovals(jvar)%vals) + call fckit_log%info(message) + endif end select @@ -1807,15 +1838,16 @@ subroutine populate_hofxdiags(RTProf, chanprof, conf, hofxdiags) case (var_ts,var_mixr,var_q,var_clw,var_cli) - nlayers = nlevels - 1 hofxdiags%geovals(jvar)%nval = nlevels - if(.not. allocated(hofxdiags%geovals(jvar)%vals)) & + if(.not. allocated(hofxdiags%geovals(jvar)%vals)) then allocate(hofxdiags%geovals(jvar)%vals(hofxdiags%geovals(jvar)%nval,nprofiles)) - hofxdiags%geovals(jvar)%vals = missing + hofxdiags%geovals(jvar)%vals = missing + endif do ichan = 1, nchanprof chan = chanprof(ichan)%chan - prof = chanprof(ichan)%prof + rttov_prof = chanprof(ichan)%prof + prof = prof_start + chanprof(ichan)%prof - 1 if(chan == ch_diags(jvar)) then if(xstr_diags(jvar) == var_ts) then @@ -1836,15 +1868,17 @@ subroutine populate_hofxdiags(RTProf, chanprof, conf, hofxdiags) endif enddo - case (var_sfc_t2m, var_sfc_tskin, var_sfc_emiss, var_sfc_q2m, var_sfc_p2m, var_u, var_v) + case (var_sfc_t2m, var_sfc_tskin, var_sfc_emiss, var_sfc_q2m, var_sfc_p2m, var_sfc_u10, var_sfc_v10) hofxdiags%geovals(jvar)%nval = 1 - if(.not. allocated(hofxdiags%geovals(jvar)%vals)) & + if(.not. allocated(hofxdiags%geovals(jvar)%vals)) then allocate(hofxdiags%geovals(jvar)%vals(hofxdiags%geovals(jvar)%nval,nprofiles)) - hofxdiags%geovals(jvar)%vals = missing + hofxdiags%geovals(jvar)%vals = missing + endif do ichan = 1, nchanprof chan = chanprof(ichan)%chan - prof = chanprof(ichan)%prof + rttov_prof = chanprof(ichan)%prof + prof = prof_start + chanprof(ichan)%prof - 1 if(chan == ch_diags(jvar)) then if(xstr_diags(jvar) == var_sfc_tskin) then @@ -1859,10 +1893,10 @@ subroutine populate_hofxdiags(RTProf, chanprof, conf, hofxdiags) else if (xstr_diags(jvar) == var_sfc_q2m) then hofxdiags%geovals(jvar)%vals(1,prof) = & RTProf % profiles_k(ichan) % s2m % q * conf%scale_fac(gas_id_watervapour) - else if (xstr_diags(jvar) == var_u) then + else if (xstr_diags(jvar) == var_sfc_u10) then hofxdiags%geovals(jvar)%vals(1,prof) = & RTProf % profiles_k(ichan) % s2m % u - else if (xstr_diags(jvar) == var_v) then + else if (xstr_diags(jvar) == var_sfc_v10) then hofxdiags%geovals(jvar)%vals(1,prof) = & RTProf % profiles_k(ichan) % s2m % v else if (xstr_diags(jvar) == var_sfc_emiss) then @@ -1873,22 +1907,26 @@ subroutine populate_hofxdiags(RTProf, chanprof, conf, hofxdiags) end do case default + if (firsttime) then + write(message,*) 'ufo_radiancerttov_simobs: //& + & Jacobian ObsDiagnostic is unsupported, ', & + & hofxdiags%variables(jvar) + call fckit_log%info(message) + endif + end select + else + if (firsttime) then write(message,*) 'ufo_radiancerttov_simobs: //& - & Jacobian ObsDiagnostic is unsupported, ', & + & ObsDiagnostic is not recognised, ', & & hofxdiags%variables(jvar) call fckit_log%info(message) - end select - else - write(message,*) 'ufo_radiancerttov_simobs: //& - & ObsDiagnostic is not recognised, ', & - & hofxdiags%variables(jvar) - call fckit_log%info(message) + endif end if enddo deallocate(od_level,wfunc) - + firsttime = .false. end subroutine populate_hofxdiags subroutine parse_hofxdiags(hofxdiags, jacobian_needed) @@ -1899,7 +1937,7 @@ subroutine parse_hofxdiags(hofxdiags, jacobian_needed) character(10), parameter :: jacobianstr = "_jacobian_" integer :: str_pos(4) - character(len=maxvarlen) :: varstr + character(len=maxvarlen) :: varstr integer :: jvar character(len=max_string) :: err_msg diff --git a/src/ufo/utils/metoffice/ufo_metoffice_bmatrixstatic_mod.f90 b/src/ufo/utils/metoffice/ufo_metoffice_bmatrixstatic_mod.f90 index 1a9c88fed..1e8ffa9d4 100644 --- a/src/ufo/utils/metoffice/ufo_metoffice_bmatrixstatic_mod.f90 +++ b/src/ufo/utils/metoffice/ufo_metoffice_bmatrixstatic_mod.f90 @@ -704,7 +704,7 @@ subroutine rttovonedvarcheck_create_fields_in(fields_in, variables, qtotal_flag) call abor1_ftn("rttovonedvarcheck not setup for independent clw yet") end if - case (var_u, var_v) + case (var_sfc_u10, var_sfc_v10) fields_in(counter) = 11 ! surface wind speed ! 12 - o3profile is not implmented yet diff --git a/test/testinput/amsr2_rttov_ops_qc_rttovonedvarcheck.yaml b/test/testinput/amsr2_rttov_ops_qc_rttovonedvarcheck.yaml index 4f46d743d..23e844e40 100644 --- a/test/testinput/amsr2_rttov_ops_qc_rttovonedvarcheck.yaml +++ b/test/testinput/amsr2_rttov_ops_qc_rttovonedvarcheck.yaml @@ -14,7 +14,7 @@ observations: SatRad_compatibility: true RTTOV_GasUnitConv: true UseRHwaterForQC: &UseRHwaterForQC true # default - UseColdSurfaceCheck: &UseColdSurfaceCheck true # default + UseColdSurfaceCheck: &UseColdSurfaceCheck true # not default Sensor_ID: &sensor_id gcom-w_1_amsr2 CoefficientPath: Data/ obs space: @@ -52,8 +52,8 @@ observations: - specific_humidity_at_two_meters_above_surface # 4 - skin_temperature # 5 - air_pressure_at_two_meters_above_surface # 6 - - eastward_wind # 11 - required for windspeed retrieval - - northward_wind # 11 - required for windspeed retrieval + - uwind_at_10m # 11 - required for windspeed retrieval + - vwind_at_10m # 11 - required for windspeed retrieval nlevels: 70 qtotal: true UseQtSplitRain: true From c6882a7e822412e089b68fb877ce1f2dc981c683 Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Fri, 9 Jul 2021 19:03:25 +0100 Subject: [PATCH 33/98] Observation operator for average profile u and v (#1301) * modifications to algorithm * Modifications to algorithm; fill H(x) * refactor to use data handling class * improve documentation * coding norms * more unit tests * more error checking; improved error messages * coding norms * coding norms * add extra tests to trigger exceptions * Trigger CI * clear stringstreams correctly * improve comments; change function name * no longer clear stringstream Co-authored-by: Ryan Honeyager --- src/ufo/profile/CMakeLists.txt | 2 + src/ufo/profile/ObsProfileAverage.cc | 160 +++------------ src/ufo/profile/ObsProfileAverage.h | 23 +-- src/ufo/profile/ObsProfileAverageData.cc | 182 ++++++++++++++++++ src/ufo/profile/ObsProfileAverageData.h | 75 ++++++++ .../profileconsistencychecks_opr_average.yaml | 149 ++++++++++++-- 6 files changed, 426 insertions(+), 165 deletions(-) create mode 100644 src/ufo/profile/ObsProfileAverageData.cc create mode 100644 src/ufo/profile/ObsProfileAverageData.h diff --git a/src/ufo/profile/CMakeLists.txt b/src/ufo/profile/CMakeLists.txt index cc3470978..7ea79bfa8 100644 --- a/src/ufo/profile/CMakeLists.txt +++ b/src/ufo/profile/CMakeLists.txt @@ -14,6 +14,8 @@ set ( profile_files ObsProfileAverage.cc ObsProfileAverage.h ObsProfileAverageParameters.h + ObsProfileAverageData.cc + ObsProfileAverageData.h ProfileAveragePressure.cc ProfileAveragePressure.h ProfileAverageRelativeHumidity.cc diff --git a/src/ufo/profile/ObsProfileAverage.cc b/src/ufo/profile/ObsProfileAverage.cc index ce2e5edfe..b8baf7764 100644 --- a/src/ufo/profile/ObsProfileAverage.cc +++ b/src/ufo/profile/ObsProfileAverage.cc @@ -15,9 +15,7 @@ #include "ioda/ObsVector.h" #include "oops/base/Variables.h" -#include "oops/util/FloatCompare.h" #include "oops/util/Logger.h" -#include "oops/util/missingValues.h" #include "ufo/GeoVaLs.h" #include "ufo/Locations.h" @@ -31,36 +29,9 @@ static ObsOperatorMaker obsProfileAverageMaker_("ProfileAvera ObsProfileAverage::ObsProfileAverage(const ioda::ObsSpace & odb, const eckit::Configuration & config) - : ObsOperatorBase(odb, config), odb_(odb) + : ObsOperatorBase(odb, config), odb_(odb), data_(odb, config) { - oops::Log::trace() << "ObsProfileAverage constructor starting" << std::endl; - - // Ensure observations have been grouped into profiles. - if (odb_.obs_group_vars().empty()) - throw eckit::UserError("Group variables configuration is empty", Here()); - - // Check the ObsSpace has been extended. If this is not the case - // then it will not be possible to access profiles in the original and - // extended sections of the ObsSpace. - if (!odb_.has("MetaData", "extended_obs_space")) - throw eckit::UserError("The extended obs space has not been produced", Here()); - - // Check the observed pressure is present. - if (!odb_.has("MetaData", "air_pressure")) - throw eckit::UserError("air_pressure@MetaData not present", Here()); - - // Set up configuration options. - options_.validateAndDeserialize(config); - - // Add air_pressure_levels to the list of variables used in this operator. - // todo(ctgh): obtain the required simulated variables in a future PR. - requiredVars_ += oops::Variables({"air_pressure_levels"}); - - // If required, set up vectors for OPS comparison. - if (options_.compareWithOPS.value()) - setUpAuxiliaryReferenceVariables(); - - oops::Log::trace() << "ObsProfileAverage constructor finished" << std::endl; + oops::Log::trace() << "ObsProfileAverage constructed" << std::endl; } // ----------------------------------------------------------------------------- @@ -73,20 +44,11 @@ ObsProfileAverage::~ObsProfileAverage() { void ObsProfileAverage::simulateObs(const GeoVaLs & gv, ioda::ObsVector & ovec, ObsDiagnostics & ydiags) const { - oops::Log::trace() << "ObsProfileAverage: simulateObs entered" << std::endl; - - const float missing = util::missingValue(missing); - - // Get variable indicating which section of the ObsSpace a profile is in. - std::vector extended_obs_space(odb_.nlocs()); - odb_.get_db("MetaData", "extended_obs_space", extended_obs_space); - - // Get observed pressure. - std::vector pressure_obs(odb_.nlocs()); - odb_.get_db("MetaData", "air_pressure", pressure_obs); + oops::Log::trace() << "ObsProfileAverage: simulateObs started" << std::endl; // Get correspondence between record numbers and indices in the total sample. const std::vector &recnums = odb_.recidx_all_recnums(); + // Number of profiles in the original ObsSpace. const std::size_t nprofs = recnums.size() / 2; @@ -97,110 +59,42 @@ void ObsProfileAverage::simulateObs(const GeoVaLs & gv, ioda::ObsVector & ovec, // Get locations of profile in the original ObsSpace and // the corresponding profile in the extended ObsSpace. // Assuming the extended ObsSpace has been configured correctly, which is - // checked above, the profile in the extended ObsSpace is always located + // checked in the constructor of the data_ member variable, + // the profile in the extended ObsSpace is always located // nprofs positions further on than the profile in the original ObsSpace. const std::vector &locsOriginal = odb_.recidx_vector(recnums[jprof]); const std::vector &locsExtended = odb_.recidx_vector(recnums[jprof + nprofs]); - // Observed pressures for this profile. - std::vector pObs; - for (const std::size_t loc : locsOriginal) - pObs.push_back(pressure_obs[loc]); - - // Set up GeoVaLs and H(x) vectors. - // Number of levels for air_pressure_levels. - const std::size_t nlevs = gv.nlevs("air_pressure_levels"); - // Vector storing location for each level along the slant path. - // Initially the first location in the profile is used everywhere. - std::vector slant_path_location(nlevs, 0); - // Vector used to store different pressure GeoVaLs. - std::vector pressure_gv(nlevs); - // Loop over model levels and find intersection of profile with model layer boundary. - for (std::size_t mlev = 0; mlev < nlevs; ++mlev) { - for (int iter = 0; iter < options_.numIntersectionIterations.value(); ++iter) { - for (std::size_t jloc = slant_path_location[mlev]; jloc < pObs.size(); ++jloc) { - gv.getAtLocation(pressure_gv, "air_pressure_levels", jloc); - // If pressure has not been recorded, move to the next level. - if (pObs[jloc] == missing) continue; - // Break from the loop if the observed pressure is lower than - // the pressure of this model level. - if (pObs[jloc] <= pressure_gv[mlev]) break; - // Record the value of this location at this level and all above. - // This ensures that missing values are dealt with correctly. - for (std::size_t mlevcolumn = mlev; mlevcolumn < nlevs; ++mlevcolumn) - slant_path_location[mlevcolumn] = jloc; - } + // Retrieve slant path locations. + const std::vector& slant_path_location = + data_.getSlantPathLocations(locsOriginal, locsExtended, gv); + + // Fill H(x) vector for each variable. + for (size_t jvar = 0; jvar < ovec.nvars(); ++jvar) { + const auto& variable = ovec.varnames().variables()[jvar]; + // Number of levels for this variable. + const std::size_t nlevs_var = gv.nlevs(variable); + // GeoVaL vector for this variable. + std::vector var_gv(nlevs_var); + // For each level: + // - get the relevant slant path location, + // - retrieve the GeoVaL at that location, + // - fill H(x) with the relevant level in the GeoVaL. + for (std::size_t mlev = 0; mlev < nlevs_var; ++mlev) { + const std::size_t jloc = slant_path_location[mlev]; + gv.getAtLocation(var_gv, variable, jloc); + ovec[locsExtended[mlev] * ovec.nvars() + jvar] = var_gv[mlev]; } } - - // Fill slanted pressure vector. - // todo(ctgh): fill other simulated variables in a future PR. - std::vector slant_pressure; - for (std::size_t mlev = 0; mlev < nlevs; ++mlev) { - const std::size_t jloc = slant_path_location[mlev]; - gv.getAtLocation(pressure_gv, "air_pressure_levels", jloc); - slant_pressure.push_back(pressure_gv[mlev]); - } - - // Fill H(x) in the extended ObsSpace. - // todo(ctgh): do this in a future PR. - // for (std::size_t jloc = 0; jloc < locsExtended.size(); ++jloc) { - // for (std::size_t jvar = 0; jvar < ovec.nvars(); ++jvar) { - // ovec[(jloc + locsExtended.front()) * ovec.nvars() + jvar] = slant_variable[jloc]; - // } - // } - - // If required, compare slant path locations and slant pressure with OPS. - if (options_.compareWithOPS.value()) - compareAuxiliaryVariables(locsExtended, - slant_path_location, - slant_pressure); } - oops::Log::trace() << "ObsProfileAverage: simulateObs exit " << std::endl; -} -// ----------------------------------------------------------------------------- - -void ObsProfileAverage::setUpAuxiliaryReferenceVariables() { - if (!(odb_.has("MetOfficeHofX", "slant_path_location") && - odb_.has("MetOfficeHofX", "slant_pressure"))) - throw eckit::UserError("At least one reference variable is not present", Here()); - // Get reference values of the slant path locations and pressures. - slant_path_location_ref_.resize(odb_.nlocs()); - slant_pressure_ref_.resize(odb_.nlocs()); - odb_.get_db("MetOfficeHofX", "slant_path_location", slant_path_location_ref_); - odb_.get_db("MetOfficeHofX", "slant_pressure", slant_pressure_ref_); -} - -// ----------------------------------------------------------------------------- - -void ObsProfileAverage::compareAuxiliaryVariables -(const std::vector &locsExtended, - const std::vector &slant_path_location, - const std::vector &slant_pressure) const { - std::vector slant_path_location_ref_profile; - std::vector slant_pressure_ref_profile; - for (const std::size_t loc : locsExtended) { - slant_path_location_ref_profile.push_back(slant_path_location_ref_[loc]); - slant_pressure_ref_profile.push_back(slant_pressure_ref_[loc]); - } - for (std::size_t jloccomp = 0; jloccomp < locsExtended.size(); ++jloccomp) { - if (slant_path_location[jloccomp] != slant_path_location_ref_profile[jloccomp]) - throw eckit::BadValue("Mismatch for slant_path_location, jloccomp = " + - jloccomp, Here()); - if (!oops::is_close_relative(slant_pressure[jloccomp], - slant_pressure_ref_profile[jloccomp], - 1e-9f)) - throw eckit::BadValue("Mismatch for slant_pressure, jloccomp = " + - jloccomp, Here()); - } + oops::Log::trace() << "ObsProfileAverage: simulateObs finished" << std::endl; } // ----------------------------------------------------------------------------- void ObsProfileAverage::print(std::ostream & os) const { - os << "ObsProfileAverage operator" << std::endl; - os << "config = " << options_ << std::endl; + data_.print(os); } // ----------------------------------------------------------------------------- diff --git a/src/ufo/profile/ObsProfileAverage.h b/src/ufo/profile/ObsProfileAverage.h index 092578787..8e515e226 100644 --- a/src/ufo/profile/ObsProfileAverage.h +++ b/src/ufo/profile/ObsProfileAverage.h @@ -18,6 +18,7 @@ #include "ufo/ObsOperatorBase.h" +#include "ufo/profile/ObsProfileAverageData.h" #include "ufo/profile/ObsProfileAverageParameters.h" /// Forward declarations @@ -85,31 +86,17 @@ class ObsProfileAverage : public ObsOperatorBase, void simulateObs(const GeoVaLs &, ioda::ObsVector &, ObsDiagnostics &) const override; - const oops::Variables & requiredVars() const override { return requiredVars_; } + const oops::Variables & requiredVars() const override { return data_.requiredVars(); } private: - /// Set up auxiliary reference variables that are used for comparison with OPS. - void setUpAuxiliaryReferenceVariables(); - - /// Compare auxiliary reference variables with those obtained in OPS. - void compareAuxiliaryVariables(const std::vector &locsExtended, - const std::vector &slant_path_location, - const std::vector &slant_pressure) const; - void print(std::ostream &) const override; private: + /// ObsSpace. const ioda::ObsSpace& odb_; - oops::Variables requiredVars_; - - /// Options for this observation operator. - ObsProfileAverageParameters options_; - - /// Reference values of slant path locations. - std::vector slant_path_location_ref_; - /// Reference values of slant path pressures. - std::vector slant_pressure_ref_; + /// Data handler for the ProfileAverage operator and TL/AD code. + ObsProfileAverageData data_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/profile/ObsProfileAverageData.cc b/src/ufo/profile/ObsProfileAverageData.cc new file mode 100644 index 000000000..a3727e593 --- /dev/null +++ b/src/ufo/profile/ObsProfileAverageData.cc @@ -0,0 +1,182 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/util/FloatCompare.h" +#include "oops/util/missingValues.h" + +#include "ufo/profile/ObsProfileAverageData.h" + +namespace ufo { + + ObsProfileAverageData::ObsProfileAverageData(const ioda::ObsSpace & odb, + const eckit::Configuration & config) + : odb_(odb) + { + // Ensure observations have been grouped into profiles. + if (odb_.obs_group_vars().empty()) + throw eckit::UserError("Group variables configuration is empty", Here()); + + // Ensure observations have been sorted by air pressure in descending order. + if (odb_.obs_sort_var() != "air_pressure") + throw eckit::UserError("Sort variable must be air_pressure", Here()); + if (odb_.obs_sort_order() != "descending") + throw eckit::UserError("Profiles must be sorted in descending order", Here()); + + // Check the ObsSpace has been extended. If this is not the case + // then it will not be possible to access profiles in the original and + // extended sections of the ObsSpace. + if (!odb_.has("MetaData", "extended_obs_space")) + throw eckit::UserError("The extended obs space has not been produced", Here()); + + // Check the observed pressure is present. This is necessary in order to + // determine the slant path locations. + if (!odb_.has("MetaData", "air_pressure")) + throw eckit::UserError("air_pressure@MetaData not present", Here()); + + // Add air_pressure_levels to the list of variables used in this operator. + // This GeoVaL is used to determine the slant path locations. + requiredVars_ += oops::Variables({"air_pressure_levels"}); + + // Add any simulated variables to the list of variables used in this operator. + requiredVars_ += odb_.obsvariables(); + + // Set up configuration options. + options_.validateAndDeserialize(config); + + // If required, set up vectors for OPS comparison. + if (options_.compareWithOPS.value()) + this->setUpAuxiliaryReferenceVariables(); + } + + const oops::Variables & ObsProfileAverageData::requiredVars() const { + return requiredVars_; + } + + std::vector ObsProfileAverageData::getSlantPathLocations + (const std::vector & locsOriginal, + const std::vector & locsExtended, + const GeoVaLs & gv) const + { + const float missing = util::missingValue(missing); + + // Get observed pressure. + std::vector pressure_obs(odb_.nlocs()); + odb_.get_db("MetaData", "air_pressure", pressure_obs); + + // Set up GeoVaLs and H(x) vectors. + // Number of levels for air_pressure_levels. + const std::size_t nlevs_p = gv.nlevs("air_pressure_levels"); + // Vector storing location for each level along the slant path. + // Initially the first location in the profile is used everywhere. + std::vector slant_path_location(nlevs_p, locsOriginal.front()); + // Vector of slanted pressures, used for comparisons with OPS. + std::vector slant_pressure; + // Vector used to store different pressure GeoVaLs. + std::vector pressure_gv(nlevs_p); + + // Loop over model levels and find intersection of profile with model layer boundary. + // This can be performed multiple times in order to account for slanted model levels. + std::size_t idxstart = 0; // Starting index for loop over levels. + // This counter records locations in the slanted profile. + // It is initialised at the first location in the original profile. + std::size_t jlocslant = locsOriginal.front(); + // Maximum iteration value. + const int itermax = options_.numIntersectionIterations.value() - 1; + // Loop over each model level in turn. + for (std::size_t mlev = 0; mlev < nlevs_p; ++mlev) { + for (int iter = 0; iter <= itermax; ++iter) { + // Get the GeoVaL that corresponds to the current slanted profile location. + gv.getAtLocation(pressure_gv, "air_pressure_levels", jlocslant); + // Define an iteration-specific location that is initialised to the + // current slanted profile location. + std::size_t jlociter = jlocslant; + // Determine the intersection of the observed profile with the current model level. + // The intersection is taken to be the location with the largest observed pressure + // that is less than or equal to the model pressure at this level. + for (std::size_t idx = idxstart; idx < locsOriginal.size(); ++idx) { + // Intersection location. + const std::size_t jlocintersect = locsOriginal[idx]; + // If pressure has not been recorded, move to the next level. + if (pressure_obs[jlocintersect] == missing) continue; + // Break from the loop if the observed pressure is lower than + // the pressure of this model level. + if (pressure_obs[jlocintersect] <= pressure_gv[mlev]) break; + // Update the iteration-specific location with the new intersection location. + jlociter = jlocintersect; + // Update the loop starting index when the last iteration is reached. + if (iter == itermax) idxstart = idx; + } + // Modify the slanted location in the original profile. + jlocslant = jlociter; + if (iter == itermax) { + // Record the value of the slant path location at this model level and all above. + // This ensures that missing values are dealt with correctly. + for (std::size_t mlevcolumn = mlev; mlevcolumn < nlevs_p; ++mlevcolumn) + slant_path_location[mlevcolumn] = jlocslant; + // Fill the slant pressure if the comparison with OPS will be performed. + if (options_.compareWithOPS.value()) + slant_pressure.push_back(pressure_gv[mlev]); + } + } + } + + // If required, compare slant path locations and slant pressure with OPS output. + if (options_.compareWithOPS.value()) + this->compareAuxiliaryReferenceVariables(locsExtended, + slant_path_location, + slant_pressure); + + return slant_path_location; + } + + void ObsProfileAverageData::setUpAuxiliaryReferenceVariables() { + if (!(odb_.has("MetOfficeHofX", "slant_path_location") && + odb_.has("MetOfficeHofX", "slant_pressure"))) + throw eckit::UserError("At least one reference variable is not present", Here()); + // Get reference values of the slant path locations and pressures. + slant_path_location_ref_.resize(odb_.nlocs()); + slant_pressure_ref_.resize(odb_.nlocs()); + odb_.get_db("MetOfficeHofX", "slant_path_location", slant_path_location_ref_); + odb_.get_db("MetOfficeHofX", "slant_pressure", slant_pressure_ref_); + } + + void ObsProfileAverageData::compareAuxiliaryReferenceVariables + (const std::vector & locsExtended, + const std::vector & slant_path_location, + const std::vector & slant_pressure) const { + std::vector slant_path_location_ref_profile; + std::vector slant_pressure_ref_profile; + for (const std::size_t loc : locsExtended) { + slant_path_location_ref_profile.push_back(slant_path_location_ref_[loc]); + slant_pressure_ref_profile.push_back(slant_pressure_ref_[loc]); + } + std::stringstream errmsg; + for (std::size_t jloccomp = 0; jloccomp < locsExtended.size(); ++jloccomp) { + if (slant_path_location[jloccomp] != slant_path_location_ref_profile[jloccomp]) { + errmsg << "Mismatch for slant_path_location, location = " << jloccomp + << " (this code, OPS): " + << slant_path_location[jloccomp] << ", " + << slant_path_location_ref_profile[jloccomp]; + throw eckit::BadValue(errmsg.str(), Here()); + } + if (!oops::is_close_relative(slant_pressure[jloccomp], + slant_pressure_ref_profile[jloccomp], + 1e-5f)) { + errmsg << "Mismatch for slant_pressure, location = " << jloccomp + << " (this code, OPS): " + << slant_pressure[jloccomp] << ", " + << slant_pressure_ref_profile[jloccomp]; + throw eckit::BadValue(errmsg.str(), Here()); + } + } + } + + void ObsProfileAverageData::print(std::ostream & os) const { + os << "ObsProfileAverage operator" << std::endl; + os << "config = " << options_ << std::endl; + } +} // namespace ufo diff --git a/src/ufo/profile/ObsProfileAverageData.h b/src/ufo/profile/ObsProfileAverageData.h new file mode 100644 index 000000000..e552584e8 --- /dev/null +++ b/src/ufo/profile/ObsProfileAverageData.h @@ -0,0 +1,75 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_PROFILE_OBSPROFILEAVERAGEDATA_H_ +#define UFO_PROFILE_OBSPROFILEAVERAGEDATA_H_ + +#include +#include + +#include "ioda/ObsSpace.h" + +#include "oops/base/Variables.h" + +#include "ufo/GeoVaLs.h" +#include "ufo/profile/ObsProfileAverageParameters.h" + +namespace eckit { + class Configuration; +} + +namespace ufo { + + /// \brief Data handling class for the ProfileAverage observation operator and TL/AD code. + class ObsProfileAverageData { + public: + ObsProfileAverageData(const ioda::ObsSpace & odb, + const eckit::Configuration & config); + + /// Return required variables for the operator. + const oops::Variables & requiredVars() const; + + /// Get slant path locations. This determines, for each model level, the location that + /// corresponds to the intersection of the observed profile with that level. + std::vector getSlantPathLocations(const std::vector & locsOriginal, + const std::vector & locsExtended, + const GeoVaLs & gv) const; + + /// Print operator configuration options. + void print(std::ostream & os) const; + + private: + /// Set up auxiliary reference variables that are used for comparison with OPS. + /// These reference variables are called MetOfficeHofX/slant_path_location and + /// MetOfficeHofX/slant_pressure. If a comparison with OPS is to be performed + /// then these variables must be present in the input data set. + void setUpAuxiliaryReferenceVariables(); + + /// Compare auxiliary reference variables with those obtained in OPS. + void compareAuxiliaryReferenceVariables(const std::vector & locsExtended, + const std::vector & slant_path_location, + const std::vector & slant_pressure) const; + + private: + /// ObsSpace. + const ioda::ObsSpace & odb_; + + /// Options for this observation operator. + ObsProfileAverageParameters options_; + + /// Required variables. + oops::Variables requiredVars_; + + /// Reference values of slant path locations. + std::vector slant_path_location_ref_; + + /// Reference values of slant path pressures. + std::vector slant_pressure_ref_; + }; +} // namespace ufo + +#endif // UFO_PROFILE_OBSPROFILEAVERAGEDATA_H_ diff --git a/test/testinput/profileconsistencychecks_opr_average.yaml b/test/testinput/profileconsistencychecks_opr_average.yaml index a605c3354..c2a305015 100644 --- a/test/testinput/profileconsistencychecks_opr_average.yaml +++ b/test/testinput/profileconsistencychecks_opr_average.yaml @@ -1,12 +1,8 @@ window begin: 2019-06-14T20:30:00Z window end: 2019-06-15T03:30:00Z -# rms ref is zero because only the auxiliary levels are tested -# todo(ctgh): in a future PR, the simulated variables will be treated correctly. -# When this occurs, use 'vector ref' with the correct reference values. - observations: -# Standard case. +# Standard case: eastward_wind. - obs space: name: Radiosonde obsdatain: @@ -15,7 +11,7 @@ observations: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" - simulated variables: [air_temperature] + simulated variables: [eastward_wind] extension: average profiles onto model levels: 71 obs operator: @@ -28,29 +24,102 @@ observations: coef TL: 0.1 tolerance TL: 1.0e-11 tolerance AD: 1.0e-13 - rms ref: 0 + vector ref: MetOfficeHofX tolerance: 1.0e-06 -# Group variables configuration is empty, throwing an exception. + +# Standard case: eastward_wind and northward_wind. - obs space: name: Radiosonde obsdatain: obsfile: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_obs.nc4 - simulated variables: [air_temperature] + obsgrouping: + group variables: [ "station_id" ] + sort variable: "air_pressure" + sort order: "descending" + simulated variables: [eastward_wind, northward_wind] extension: average profiles onto model levels: 71 - expect constructor to throw exception with message: Group variables configuration is empty obs operator: name: ProfileAverage compareWithOPS: true + numIntersectionIterations: 3 + geovals: + filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 + linear obs operator test: + coef TL: 0.1 + tolerance TL: 1.0e-11 + tolerance AD: 1.0e-13 + vector ref: MetOfficeHofX + tolerance: 1.0e-06 + +# The group variable is not used, throwing an exception. +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_obs.nc4 + simulated variables: [eastward_wind] + extension: + average profiles onto model levels: 71 + expect constructor to throw exception with message: Group variables configuration is empty + obs operator: + name: ProfileAverage + geovals: + filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 + linear obs operator test: + coef TL: 0.1 + tolerance TL: 1.0e-11 + tolerance AD: 1.0e-13 + vector ref: MetOfficeHofX + tolerance: 1.0e-06 + +# The sort variable is not air_pressure, throwing an exception +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_obs.nc4 + obsgrouping: + group variables: [ "station_id" ] + sort order: "descending" + simulated variables: [eastward_wind] + extension: + average profiles onto model levels: 71 + expect constructor to throw exception with message: Sort variable must be air_pressure + obs operator: + name: ProfileAverage + geovals: + filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 + linear obs operator test: + coef TL: 0.1 + tolerance TL: 1.0e-11 + tolerance AD: 1.0e-13 + vector ref: MetOfficeHofX + tolerance: 1.0e-06 + +# The sort order is not descending, throwing an exception +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_obs.nc4 + obsgrouping: + group variables: [ "station_id" ] + sort variable: "air_pressure" + sort order: "ascending" + simulated variables: [eastward_wind] + extension: + average profiles onto model levels: 71 + expect constructor to throw exception with message: Profiles must be sorted in descending order + obs operator: + name: ProfileAverage geovals: filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 linear obs operator test: coef TL: 0.1 tolerance TL: 1.0e-11 tolerance AD: 1.0e-13 - rms ref: 0 + vector ref: MetOfficeHofX tolerance: 1.0e-06 -# Extended ObsSpace is not used, throwing an exception. + +# The extended ObsSpace is not used, throwing an exception. - obs space: name: Radiosonde obsdatain: @@ -59,7 +128,7 @@ observations: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" - simulated variables: [air_temperature] + simulated variables: [eastward_wind] obs operator: name: ProfileAverage compareWithOPS: true @@ -70,5 +139,57 @@ observations: coef TL: 0.1 tolerance TL: 1.0e-11 tolerance AD: 1.0e-13 - rms ref: 0 + vector ref: MetOfficeHofX + tolerance: 1.0e-06 + +# Use deliberately wrong reference values of slant_path_location, throwing an exception. +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_wrong_slant_locs_obs.nc4 + obsgrouping: + group variables: [ "station_id" ] + sort variable: "air_pressure" + sort order: "descending" + simulated variables: [eastward_wind] + extension: + average profiles onto model levels: 71 + obs operator: + name: ProfileAverage + compareWithOPS: true + numIntersectionIterations: 3 + expect simulateObs to throw exception with message: "Mismatch for slant_path_location, location = 0 (this code, OPS): 2659, 0" + geovals: + filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 + linear obs operator test: + coef TL: 0.1 + tolerance TL: 1.0e-11 + tolerance AD: 1.0e-13 + vector ref: MetOfficeHofX + tolerance: 1.0e-06 + +# Use deliberately wrong reference values of slant_pressure, throwing an exception. +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_wrong_slant_pressure_obs.nc4 + obsgrouping: + group variables: [ "station_id" ] + sort variable: "air_pressure" + sort order: "descending" + simulated variables: [eastward_wind] + extension: + average profiles onto model levels: 71 + obs operator: + name: ProfileAverage + compareWithOPS: true + numIntersectionIterations: 3 + expect simulateObs to throw exception with message: "Mismatch for slant_pressure, location = 0 (this code, OPS): 100958, 0" + geovals: + filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 + linear obs operator test: + coef TL: 0.1 + tolerance TL: 1.0e-11 + tolerance AD: 1.0e-13 + vector ref: MetOfficeHofX tolerance: 1.0e-06 From d955132a82e00e2346ecfdc9e8fbc1624aa59d8e Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Fri, 9 Jul 2021 13:06:59 -0600 Subject: [PATCH 34/98] check that obserrors are not missing (instead of assert) --- src/ufo/filters/BackgroundCheck.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ufo/filters/BackgroundCheck.cc b/src/ufo/filters/BackgroundCheck.cc index a927748c1..0002014c9 100644 --- a/src/ufo/filters/BackgroundCheck.cc +++ b/src/ufo/filters/BackgroundCheck.cc @@ -92,8 +92,8 @@ void BackgroundCheck::applyFilter(const std::vector & apply, std::vector hofx; data_.get(varhofx.variable(jv), hofx); for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass) { - ASSERT((*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])); + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass && + (*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])) { ASSERT(obs[jv][jobs] != util::missingValue(obs[jv][jobs])); ASSERT(hofx[jobs] != util::missingValue(hofx[jobs])); @@ -129,8 +129,8 @@ void BackgroundCheck::applyFilter(const std::vector & apply, bc_factor = getScalarOrFilterData(*parameters_.BiasCorrectionFactor.value(), data_); for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass) { - ASSERT((*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])); + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass && + (*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])) { ASSERT(obs[jv][jobs] != util::missingValue(obs[jv][jobs])); if (parameters_.BiasCorrectionFactor.value()) { ASSERT(obsbias[jv][jobs] != util::missingValue(obsbias[jv][jobs])); From ea5c4e72673889c2bec45b55385ff63580979f07 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Fri, 9 Jul 2021 13:08:56 -0600 Subject: [PATCH 35/98] Revert "check that obserrors are not missing (instead of assert)" This reverts commit d955132a82e00e2346ecfdc9e8fbc1624aa59d8e. --- src/ufo/filters/BackgroundCheck.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ufo/filters/BackgroundCheck.cc b/src/ufo/filters/BackgroundCheck.cc index 0002014c9..a927748c1 100644 --- a/src/ufo/filters/BackgroundCheck.cc +++ b/src/ufo/filters/BackgroundCheck.cc @@ -92,8 +92,8 @@ void BackgroundCheck::applyFilter(const std::vector & apply, std::vector hofx; data_.get(varhofx.variable(jv), hofx); for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass && - (*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])) { + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass) { + ASSERT((*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])); ASSERT(obs[jv][jobs] != util::missingValue(obs[jv][jobs])); ASSERT(hofx[jobs] != util::missingValue(hofx[jobs])); @@ -129,8 +129,8 @@ void BackgroundCheck::applyFilter(const std::vector & apply, bc_factor = getScalarOrFilterData(*parameters_.BiasCorrectionFactor.value(), data_); for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass && - (*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])) { + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass) { + ASSERT((*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])); ASSERT(obs[jv][jobs] != util::missingValue(obs[jv][jobs])); if (parameters_.BiasCorrectionFactor.value()) { ASSERT(obsbias[jv][jobs] != util::missingValue(obsbias[jv][jobs])); From 1787974f888253d23d9234cb35554f5547e39deb Mon Sep 17 00:00:00 2001 From: Michael Cooke <48374999+mikecooke77@users.noreply.github.com> Date: Fri, 9 Jul 2021 20:13:26 +0100 Subject: [PATCH 36/98] Add channel support for linear combination obs function (#1303) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial attemp * Its working but needs tidying * Tidy up and add multichannel test * Changes for coding norms * Improve the testing * remove logging * Add comments and initialize to zero * Remove index calculation as its not needed * Change the way the variables are read in * Changes needed * Test case computing a linear combination of obs functions (#1306) * Added a test case verifying it is possible to calculate a linear combination of ObsFunctions. * Added a comment. * Unified yaml file * Trigger build * Updated test data Co-authored-by: Wojciech Śmigaj Co-authored-by: Ryan Honeyager --- .../ObsFunctionLinearCombination.cc | 49 +++++------ .../ObsFunctionLinearCombination.h | 26 ++++-- test/CMakeLists.txt | 8 ++ test/testinput/atms_rttov_qc.yaml | 17 +++- ...ction_linear_combination_multichannel.yaml | 81 +++++++++++++++++++ 5 files changed, 151 insertions(+), 30 deletions(-) create mode 100644 test/testinput/unit_tests/function_linear_combination_multichannel.yaml diff --git a/src/ufo/filters/obsfunctions/ObsFunctionLinearCombination.cc b/src/ufo/filters/obsfunctions/ObsFunctionLinearCombination.cc index 189ce9b12..08995aeb5 100644 --- a/src/ufo/filters/obsfunctions/ObsFunctionLinearCombination.cc +++ b/src/ufo/filters/obsfunctions/ObsFunctionLinearCombination.cc @@ -7,9 +7,12 @@ #include "ufo/filters/obsfunctions/ObsFunctionLinearCombination.h" +#include +#include #include #include "ioda/ObsDataVector.h" +#include "oops/util/IntSetParser.h" #include "oops/util/missingValues.h" namespace ufo { @@ -22,15 +25,11 @@ static ObsFunctionMaker LinearCombination::LinearCombination(const eckit::LocalConfiguration & conf) : invars_() { // Check options - options_.deserialize(conf); + options_.validateAndDeserialize(conf); - // number of input variables - int nv = options_.variables.value().size(); - - // get variable informations - std::vector variables = options_.variables.value(); - for (size_t ii = 0; ii < nv; ++ii) { - invars_ += variables[ii]; + // Create variable and add to invars_ + for (const Variable & var : options_.variables.value()) { + invars_ += var; } } @@ -46,27 +45,31 @@ void LinearCombination::compute(const ObsFilterData & in, const size_t nlocs = in.nlocs(); // number of input variables - int nv = invars_.size(); + const size_t nv = invars_.size(); // get coefs for linear combination const std::vector coefs = options_.coefs.value(); - // sanity check - ASSERT(coefs.size() == invars_.size() ); + // sanity checks / initialize + ASSERT(coefs.size() == nv); + out.zero(); - // compute linear conbination of input variables + // compute linear combination of input variables const float missing = util::missingValue(missing); - std::vector varin(nlocs); - for (size_t ii = 0; ii < nv; ++ii) { - in.get(invars_[ii], varin); - for (size_t jj = 0; jj < nlocs; ++jj) { - if ( varin[jj] == missing || out[0][jj] == missing ) { - out[0][jj] = missing; - } else { - out[0][jj] += coefs[ii]*varin[jj]; - } - } - } + for (size_t ivar = 0; ivar < nv; ++ivar) { + ioda::ObsDataVector varin(in.obsspace(), invars_[ivar].toOopsVariables()); + in.get(invars_[ivar], varin); + ASSERT(varin.nvars() == out.nvars()); + for (size_t iloc = 0; iloc < nlocs; ++iloc) { + for (size_t ichan = 0; ichan < out.nvars(); ++ichan) { + if ( varin[ichan][iloc] == missing || out[ichan][iloc] == missing ) { + out[ichan][iloc] = missing; + } else { + out[ichan][iloc] += coefs[ivar] * varin[ichan][iloc]; + } + } // ichan + } // nlocs + } // nvars } // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/obsfunctions/ObsFunctionLinearCombination.h b/src/ufo/filters/obsfunctions/ObsFunctionLinearCombination.h index b9d279b92..1c1e14bd6 100644 --- a/src/ufo/filters/obsfunctions/ObsFunctionLinearCombination.h +++ b/src/ufo/filters/obsfunctions/ObsFunctionLinearCombination.h @@ -11,8 +11,6 @@ #include #include -#include "oops/util/parameters/OptionalParameter.h" -#include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/Parameters.h" #include "oops/util/parameters/RequiredParameter.h" @@ -30,7 +28,7 @@ class LinearCombinationParameters : public oops::Parameters { public: /// Input variables of the linear combination - oops::RequiredParameter> variables{"variables", this}; + oops::RequiredParameter> variables{"variables", this}; /// coefficient associated with the above variables oops::RequiredParameter> coefs{"coefs", this}; }; @@ -39,7 +37,7 @@ class LinearCombinationParameters : public oops::Parameters { /// \brief Outputs a linear combination of variables /// -/// For example, the following +/// Example 1 /// /// obs function: /// name: LinearCombination@ObsFunction @@ -49,10 +47,26 @@ class LinearCombinationParameters : public oops::Parameters { /// coefs: [0.1, /// 1.0] /// -/// will return 0.1 *representation_error@GeoVaLs + +/// will return 0.1 * representation_error@GeoVaLs + /// 1.0 * sea_water_temperature@ObsError /// - +/// Example 2 - multi-channel +/// +/// obs function: +/// name: LinearCombination@ObsFunction +/// channels: &select_chans 6-15, 18-22 # this line may be needed depending on the filter used +/// options: +/// variables: +/// - name: brightness_temperature@ObsValue +/// channels: *select_chans +/// - name: brightness_temperature@ObsError +/// channels: *select_chans +/// coefs: [1.0, +/// 0.5] +/// +/// will return 1.0 * brightness_temperature_@ObsValue + +/// 0.5 * brightness_temperature_@ObsError +/// class LinearCombination : public ObsFunctionBase { public: diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a647978d7..9400bf0ab 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -283,6 +283,7 @@ list( APPEND ufo_test_input testinput/unit_tests/function_tropo_estimate.yaml testinput/unit_tests/function_lam_domaincheck.yaml testinput/unit_tests/function_lam_domaincheck_circle.yaml + testinput/unit_tests/function_linear_combination_multichannel.yaml testinput/unit_tests/function_ModelSurfaceHeightAdjust.yaml testinput/unit_tests/function_obserrorfactorquotient.yaml testinput/unit_tests/function_satwindsLNVDcheck.yaml @@ -2809,6 +2810,13 @@ ecbuild_add_test( TARGET test_ufo_function_linear_combination TEST_DEPENDS test_ufo_function_representation_err_synthetic_data) +ecbuild_add_test( TARGET test_ufo_function_linear_combination_multichannel + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x + ARGS "testinput/unit_tests/function_linear_combination_multichannel.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFunction.x + TEST_DEPENDS ufo_get_ufo_test_data) + ecbuild_add_test( TARGET test_ufo_function_setsurfacetype COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x ARGS "testinput/function_setsurfacetype.yaml" diff --git a/test/testinput/atms_rttov_qc.yaml b/test/testinput/atms_rttov_qc.yaml index 1b4a4870e..f48eb7f82 100644 --- a/test/testinput/atms_rttov_qc.yaml +++ b/test/testinput/atms_rttov_qc.yaml @@ -20,10 +20,25 @@ observations: obsdatain: obsfile: Data/ufo/testinput_tier_1/atms_n20_obs_20191230T0000_rttov.nc4 simulated variables: [brightness_temperature] - channels: 1-22 + channels: &all_channels 1-22 geovals: filename: Data/ufo/testinput_tier_1/geovals_atms_20191230T0000Z_benchmark.nc4 obs filters: +### Assign BiasCorrObsValue ### + - filter: Variable Assignment + assignments: + - name: brightness_temperature@BiasCorrObsValue + type: float + channels: *all_channels + function: + name: LinearCombination@ObsFunction + options: + variables: + - name: brightness_temperature@ObsValue + channels: *all_channels + - name: brightness_temperature@ObsBias + channels: *all_channels + coefs: [1.0,-1.0] ### Bennartz scattering test [over land] ### - filter: Bounds Check filter variables: diff --git a/test/testinput/unit_tests/function_linear_combination_multichannel.yaml b/test/testinput/unit_tests/function_linear_combination_multichannel.yaml new file mode 100644 index 000000000..2207d003f --- /dev/null +++ b/test/testinput/unit_tests/function_linear_combination_multichannel.yaml @@ -0,0 +1,81 @@ +window begin: 2017-12-31T21:00:00Z +window end: 2018-01-01T03:00:00Z + +observations: +- obs space: &ObsSpace + name: Calculate a bias corrected obs value for all channels + obsdatain: + obsfile: Data/ufo/testinput_tier_1/linear_combination_multichannel_unittest.nc4 + simulated variables: [brightness_temperature] + channels: &all_channels 1-3 + obs function: &ObsFunction + name: LinearCombination@ObsFunction + options: + variables: + - name: brightness_temperature@ObsValue + channels: *all_channels + - name: brightness_temperature@ObsBias + channels: *all_channels + coefs: [ 1.0, -1.0] + variables: [bias_corr_obs_value] + channels: *all_channels + tolerance: 1.0e-8 + +- obs space: + <<: *ObsSpace + name: Calculate a bias corrected obs value for a selection of channels + obs function: + name: LinearCombination@ObsFunction + options: + variables: + - name: brightness_temperature@ObsValue + channels: &select_chans 1,3 + - name: brightness_temperature@ObsBias + channels: *select_chans + coefs: [ 1.0, -1.0] + variables: [bias_corr_obs_value] + channels: *select_chans + tolerance: 1.0e-8 + +- obs space: + <<: *ObsSpace + name: Calculate a bias corrected obs value for a single channel + obs function: + name: LinearCombination@ObsFunction + options: + variables: + - name: brightness_temperature@ObsValue + channels: &single_chan 2 + - name: brightness_temperature@ObsBias + channels: *single_chan + coefs: [ 1.0, -1.0] + variables: [bias_corr_obs_value] + channels: *single_chan + tolerance: 1.0e-8 + +- obs space: + <<: *ObsSpace + name: LinearCombination of terms, some of which are also ObsFunctions + # Computes 0.5 * (2.0 * brightness_temperature@ObsValue) - 0.25 * (4.0 * brightness_temperature@ObsBias) + obs function: &ObsFunction + name: LinearCombination@ObsFunction + options: + variables: + - name: LinearCombination@ObsFunction + channels: *all_channels + options: + variables: + - name: brightness_temperature@ObsValue + channels: *all_channels + coefs: [2.0] + - name: LinearCombination@ObsFunction + channels: *all_channels + options: + variables: + - name: brightness_temperature@ObsBias + channels: *all_channels + coefs: [4.0] + coefs: [0.5, -0.25] + variables: [bias_corr_obs_value] + channels: *all_channels + tolerance: 1.0e-8 From 3e980ce11ed1c2cf354283e52547170663e22443 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Sat, 10 Jul 2021 06:55:53 -0600 Subject: [PATCH 37/98] check that obserrors are not missing (instead of assert) (#1333) --- src/ufo/filters/BackgroundCheck.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ufo/filters/BackgroundCheck.cc b/src/ufo/filters/BackgroundCheck.cc index a927748c1..0002014c9 100644 --- a/src/ufo/filters/BackgroundCheck.cc +++ b/src/ufo/filters/BackgroundCheck.cc @@ -92,8 +92,8 @@ void BackgroundCheck::applyFilter(const std::vector & apply, std::vector hofx; data_.get(varhofx.variable(jv), hofx); for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass) { - ASSERT((*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])); + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass && + (*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])) { ASSERT(obs[jv][jobs] != util::missingValue(obs[jv][jobs])); ASSERT(hofx[jobs] != util::missingValue(hofx[jobs])); @@ -129,8 +129,8 @@ void BackgroundCheck::applyFilter(const std::vector & apply, bc_factor = getScalarOrFilterData(*parameters_.BiasCorrectionFactor.value(), data_); for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass) { - ASSERT((*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])); + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass && + (*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])) { ASSERT(obs[jv][jobs] != util::missingValue(obs[jv][jobs])); if (parameters_.BiasCorrectionFactor.value()) { ASSERT(obsbias[jv][jobs] != util::missingValue(obsbias[jv][jobs])); From f73c74401d7b58c4c5149893f8aa5ee1c035461b Mon Sep 17 00:00:00 2001 From: JoanneAWaller <52206323+JoanneAWaller@users.noreply.github.com> Date: Mon, 12 Jul 2021 15:42:04 +0100 Subject: [PATCH 38/98] Add obs function to calculate adjusted surface wind vectors (#1298) * Begin adding obs function for surface wind adjustment. * Added obs function to adjust surfaced wind to model surface height. * Change name of temperatue adjust yaml. * Altered ObsFunction to be compatible with Variable Assignment filter. * Adhered to coding norms. * Corrected typo! * Select only required wind in constructor. * Reponded to PR comments. * Updated to be compatable with templated obs functions. Co-authored-by: Ryan Honeyager --- src/ufo/filters/CMakeLists.txt | 2 + .../ModelHeightAdjustedWindVectorComponent.cc | 96 +++++++++++++++++++ .../ModelHeightAdjustedWindVectorComponent.h | 59 ++++++++++++ test/CMakeLists.txt | 13 ++- ..._ModelSurfaceHeightAdjustTemperature.yaml} | 0 ...on_ModelSurfaceHeightAdjustWindVector.yaml | 36 +++++++ 6 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 src/ufo/filters/obsfunctions/ModelHeightAdjustedWindVectorComponent.cc create mode 100644 src/ufo/filters/obsfunctions/ModelHeightAdjustedWindVectorComponent.h rename test/testinput/unit_tests/{function_ModelSurfaceHeightAdjust.yaml => function_ModelSurfaceHeightAdjustTemperature.yaml} (100%) create mode 100755 test/testinput/unit_tests/function_ModelSurfaceHeightAdjustWindVector.yaml diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index 0143c6949..c5dc1479e 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -185,6 +185,8 @@ set ( filters_files obsfunctions/CloudDetectMinResidualAVHRR.h obsfunctions/CloudDetectMinResidualIR.cc obsfunctions/CloudDetectMinResidualIR.h + obsfunctions/ModelHeightAdjustedWindVectorComponent.cc + obsfunctions/ModelHeightAdjustedWindVectorComponent.h obsfunctions/ModelHeightAdjustedAirTemperature.cc obsfunctions/ModelHeightAdjustedAirTemperature.h obsfunctions/ObsErrorBoundIR.cc diff --git a/src/ufo/filters/obsfunctions/ModelHeightAdjustedWindVectorComponent.cc b/src/ufo/filters/obsfunctions/ModelHeightAdjustedWindVectorComponent.cc new file mode 100644 index 000000000..7b4cca8b0 --- /dev/null +++ b/src/ufo/filters/obsfunctions/ModelHeightAdjustedWindVectorComponent.cc @@ -0,0 +1,96 @@ +/* ----------------------------------------------------------------------------- + * (C) British Crown Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * ----------------------------------------------------------------------------- + */ + +#include +#include + +#include "ioda/ObsDataVector.h" +#include "oops/util/missingValues.h" +#include "ufo/filters/obsfunctions/ModelHeightAdjustedWindVectorComponent.h" +#include "ufo/filters/Variable.h" +#include "ufo/utils/Constants.h" + +namespace ufo { + +// instantiation of the template with northwardWind = false +static ObsFunctionMaker> + eastwardMaker_("ModelHeightAdjustedEastwardWind"); +// instantiation of the template with northwardWind = true +static ObsFunctionMaker> + northwardMaker_("ModelHeightAdjustedNorthwardWind"); + +// ----------------------------------------------------------------------------- +template +ModelHeightAdjustedWindVectorComponent::ModelHeightAdjustedWindVectorComponent( + const eckit::LocalConfiguration & conf): invars_() { + // Required observation data + if (northwardWind) { + invars_ += Variable("northward_wind@ObsValue"); + } else { + invars_ += Variable("eastward_wind@ObsValue"); + } + + // Required model surface altitude + invars_ += Variable("surface_altitude@GeoVaLs"); + + // Required observation station height + parameters_.validateAndDeserialize(conf); + const Variable elevation = parameters_.elevation.value(); + invars_ += elevation; +} + +// ----------------------------------------------------------------------------- + +template +void ModelHeightAdjustedWindVectorComponent::compute(const ObsFilterData & in, + ioda::ObsDataVector & out) const { + const size_t nlocs = in.nlocs(); + std::vector WindComponent(nlocs); + std::vector ModelHeight(nlocs); + std::vector StationHeight(nlocs); + + if (northwardWind) { + in.get(Variable("northward_wind@ObsValue"), WindComponent); + } else { + in.get(Variable("eastward_wind@ObsValue"), WindComponent); + } + in.get(Variable("surface_altitude@GeoVaLs"), ModelHeight); + in.get(parameters_.elevation.value(), StationHeight); + + const float missing = util::missingValue(missing); + + // compute wind correction and adjusted winds. + for (size_t jj = 0; jj < nlocs; ++jj) { + if (StationHeight[jj] == missing || ModelHeight[jj] == missing + || WindComponent[jj] == missing) { + out[0][jj] = missing; + } else { + float HeightDiff; + HeightDiff = StationHeight[jj] - ModelHeight[jj]; + if (HeightDiff > 100.0) { + float ScaleFactor; + ScaleFactor = 1.0/(1.0 + std::min(2.0, (HeightDiff - 100.0) * 0.002)); + out[0][jj] = WindComponent[jj]*ScaleFactor; + } else { + out[0][jj] = WindComponent[jj]; + } + } + } +} + +// ----------------------------------------------------------------------------- + +template +const ufo::Variables & + ModelHeightAdjustedWindVectorComponent::requiredVariables() const { + return invars_; +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/filters/obsfunctions/ModelHeightAdjustedWindVectorComponent.h b/src/ufo/filters/obsfunctions/ModelHeightAdjustedWindVectorComponent.h new file mode 100644 index 000000000..1f2a59f02 --- /dev/null +++ b/src/ufo/filters/obsfunctions/ModelHeightAdjustedWindVectorComponent.h @@ -0,0 +1,59 @@ +/* ----------------------------------------------------------------------------- + * (C) British Crown Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * ----------------------------------------------------------------------------- + */ + + +#ifndef UFO_FILTERS_OBSFUNCTIONS_MODELHEIGHTADJUSTEDWINDVECTORCOMPONENT_H_ +#define UFO_FILTERS_OBSFUNCTIONS_MODELHEIGHTADJUSTEDWINDVECTORCOMPONENT_H_ + +#include + +#include "ufo/filters/ObsFilterData.h" +#include "ufo/filters/obsfunctions/ObsFunctionBase.h" +#include "ufo/filters/Variables.h" + +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" +#include "ufo/utils/parameters/ParameterTraitsVariable.h" + +namespace ufo { + +/// \brief Options controlling ModelHeightAdjustedWindVector ObsFunction +class ModelHeightAdjustedWindVectorParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ModelHeightAdjustedWindVectorParameters, Parameters) + + public: + /// Input observation station height to be used + oops::RequiredParameter elevation{"elevation", this}; +}; + +/// \brief Function to calculate surface wind observation values adjusted from +/// station height to model surface height. Outputs a derived eastward and northward wind adjusted +/// to the model surface height. Observed winds from stations above the model +/// surface are divided by a scaling factor (S) based only on height difference (dh). +/// Scaling factors are: +/// - S = 1.0 for dh < 100m +/// - S = 1 + 0.002*(dh - 100) for 100 < dh < 1100 +/// - S = 3 for dh > 1100 + +template // true for the northward wind, false for the eastward wind +class ModelHeightAdjustedWindVectorComponent : public ObsFunctionBase { + public: + explicit ModelHeightAdjustedWindVectorComponent(const eckit::LocalConfiguration & + = eckit::LocalConfiguration()); + + void compute(const ObsFilterData &, + ioda::ObsDataVector &) const; + const ufo::Variables & requiredVariables() const; + + private: + ModelHeightAdjustedWindVectorParameters parameters_; + ufo::Variables invars_; +}; +} // namespace ufo + +#endif // UFO_FILTERS_OBSFUNCTIONS_MODELHEIGHTADJUSTEDWINDVECTORCOMPONENT_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9400bf0ab..3fcccf366 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -283,8 +283,9 @@ list( APPEND ufo_test_input testinput/unit_tests/function_tropo_estimate.yaml testinput/unit_tests/function_lam_domaincheck.yaml testinput/unit_tests/function_lam_domaincheck_circle.yaml + testinput/unit_tests/function_ModelSurfaceHeightAdjustWindVector.yaml + testinput/unit_tests/function_ModelSurfaceHeightAdjustTemperature.yaml testinput/unit_tests/function_linear_combination_multichannel.yaml - testinput/unit_tests/function_ModelSurfaceHeightAdjust.yaml testinput/unit_tests/function_obserrorfactorquotient.yaml testinput/unit_tests/function_satwindsLNVDcheck.yaml testinput/unit_tests/function_satwindsSPDBcheck.yaml @@ -2824,10 +2825,16 @@ ecbuild_add_test( TARGET test_ufo_function_setsurfacetype DEPENDS test_ObsFunction.x TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_function_modelsurfaceheightadjust_windvector + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x + ARGS "testinput/unit_tests/function_ModelSurfaceHeightAdjustWindVector.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFunction.x + TEST_DEPENDS ufo_get_ufo_test_data) -ecbuild_add_test( TARGET test_ufo_function_modelsurfaceheightadjust +ecbuild_add_test( TARGET test_ufo_function_modelsurfaceheightadjust_temperature COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x - ARGS "testinput/unit_tests/function_ModelSurfaceHeightAdjust.yaml" + ARGS "testinput/unit_tests/function_ModelSurfaceHeightAdjustTemperature.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFunction.x TEST_DEPENDS ufo_get_ufo_test_data) diff --git a/test/testinput/unit_tests/function_ModelSurfaceHeightAdjust.yaml b/test/testinput/unit_tests/function_ModelSurfaceHeightAdjustTemperature.yaml similarity index 100% rename from test/testinput/unit_tests/function_ModelSurfaceHeightAdjust.yaml rename to test/testinput/unit_tests/function_ModelSurfaceHeightAdjustTemperature.yaml diff --git a/test/testinput/unit_tests/function_ModelSurfaceHeightAdjustWindVector.yaml b/test/testinput/unit_tests/function_ModelSurfaceHeightAdjustWindVector.yaml new file mode 100755 index 000000000..fcd180154 --- /dev/null +++ b/test/testinput/unit_tests/function_ModelSurfaceHeightAdjustWindVector.yaml @@ -0,0 +1,36 @@ +window begin: 2021-05-21T00:00:00Z +window end: 2021-05-21T23:00:00Z + +observations: +- obs space: + name: Surface + obsdatain: + obsfile: Data/ufo/testinput_tier_1/surface_obs_20210521T1200Z.nc4 + simulated variables: [eastward_wind] + geovals: + filename: Data/ufo/testinput_tier_1/surface_geovals_20210521T1200Z.nc4 + + obs function: + name: ModelHeightAdjustedEastwardWind@ObsFunction + options: + elevation: + name: station_elevation@MetaData + variables: [eastward_wind] + tolerance: 1.0e-6 + +- obs space: + name: Surface + obsdatain: + obsfile: Data/ufo/testinput_tier_1/surface_obs_20210521T1200Z.nc4 + simulated variables: [northward_wind] + geovals: + filename: Data/ufo/testinput_tier_1/surface_geovals_20210521T1200Z.nc4 + + obs function: + name: ModelHeightAdjustedNorthwardWind@ObsFunction + options: + elevation: + name: station_elevation@MetaData + variables: [northward_wind] + tolerance: 1.0e-6 + From 9cf89dfb2360323f56d0b3321356cb57c17c5170 Mon Sep 17 00:00:00 2001 From: JoanneAWaller <52206323+JoanneAWaller@users.noreply.github.com> Date: Mon, 12 Jul 2021 21:15:15 +0100 Subject: [PATCH 39/98] Add obs function to calculate adjusted surface relative humidity (#1313) * Begin adding surface adjust relative humidity filter. * Conform to coding norms. * Tidied comments. * Responded to comments. * Followed coding norms. * Make compatable with develop and updated ObsFunctionBase. * Resynced test data Co-authored-by: Ryan Honeyager --- src/ufo/filters/CMakeLists.txt | 2 + .../ModelHeightAdjustedAirTemperature.cc | 4 - .../ModelHeightAdjustedAirTemperature.h | 2 - .../ModelHeightAdjustedRelativeHumidity.cc | 88 +++++++++++++++++++ .../ModelHeightAdjustedRelativeHumidity.h | 57 ++++++++++++ test/CMakeLists.txt | 8 ++ ...elSurfaceHeightAdjustRelativeHumidity.yaml | 22 +++++ 7 files changed, 177 insertions(+), 6 deletions(-) create mode 100644 src/ufo/filters/obsfunctions/ModelHeightAdjustedRelativeHumidity.cc create mode 100644 src/ufo/filters/obsfunctions/ModelHeightAdjustedRelativeHumidity.h create mode 100755 test/testinput/unit_tests/function_ModelSurfaceHeightAdjustRelativeHumidity.yaml diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index c5dc1479e..fc043567e 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -189,6 +189,8 @@ set ( filters_files obsfunctions/ModelHeightAdjustedWindVectorComponent.h obsfunctions/ModelHeightAdjustedAirTemperature.cc obsfunctions/ModelHeightAdjustedAirTemperature.h + obsfunctions/ModelHeightAdjustedRelativeHumidity.cc + obsfunctions/ModelHeightAdjustedRelativeHumidity.h obsfunctions/ObsErrorBoundIR.cc obsfunctions/ObsErrorBoundIR.h obsfunctions/ObsErrorBoundMW.cc diff --git a/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.cc b/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.cc index e93eafe17..dffb1a905 100644 --- a/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.cc +++ b/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.cc @@ -36,10 +36,6 @@ ModelHeightAdjustedAirTemperature::ModelHeightAdjustedAirTemperature( // ----------------------------------------------------------------------------- -ModelHeightAdjustedAirTemperature::~ModelHeightAdjustedAirTemperature() {} - -// ----------------------------------------------------------------------------- - void ModelHeightAdjustedAirTemperature::compute(const ObsFilterData & in, ioda::ObsDataVector & out) const { const size_t nlocs = in.nlocs(); diff --git a/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.h b/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.h index 5df56edf7..669b9d05e 100644 --- a/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.h +++ b/src/ufo/filters/obsfunctions/ModelHeightAdjustedAirTemperature.h @@ -40,8 +40,6 @@ class ModelHeightAdjustedAirTemperature : public ObsFunctionBase { public: explicit ModelHeightAdjustedAirTemperature(const eckit::LocalConfiguration & = eckit::LocalConfiguration()); - ~ModelHeightAdjustedAirTemperature(); - void compute(const ObsFilterData &, ioda::ObsDataVector &) const; const ufo::Variables & requiredVariables() const; diff --git a/src/ufo/filters/obsfunctions/ModelHeightAdjustedRelativeHumidity.cc b/src/ufo/filters/obsfunctions/ModelHeightAdjustedRelativeHumidity.cc new file mode 100644 index 000000000..684b2565a --- /dev/null +++ b/src/ufo/filters/obsfunctions/ModelHeightAdjustedRelativeHumidity.cc @@ -0,0 +1,88 @@ +/* ----------------------------------------------------------------------------- + * (C) British Crown Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * ----------------------------------------------------------------------------- + */ + +#include +#include + +#include "ioda/ObsDataVector.h" +#include "oops/util/missingValues.h" +#include "ufo/filters/obsfunctions/ModelHeightAdjustedRelativeHumidity.h" +#include "ufo/filters/Variable.h" +#include "ufo/utils/Constants.h" + +namespace ufo { + +static ObsFunctionMaker + makerModelHeightAdjustedRelativeHumidity_("ModelHeightAdjustedRelativeHumidity"); + +// ----------------------------------------------------------------------------- + +ModelHeightAdjustedRelativeHumidity::ModelHeightAdjustedRelativeHumidity( + const eckit::LocalConfiguration & conf): invars_() { + // Retrieve observation data // Required observation data + invars_ += Variable("relative_humidity_at_2m@ObsValue"); + // Required model surface altitude + invars_ += Variable("surface_altitude@GeoVaLs"); + + // Required observation station height and temperature + parameters_.validateAndDeserialize(conf); + const Variable elevation = parameters_.elevation.value(); + const Variable temperature = parameters_.temperature.value(); + invars_ += elevation; + invars_ += temperature; +} + +// ----------------------------------------------------------------------------- + +void ModelHeightAdjustedRelativeHumidity::compute(const ObsFilterData & in, + ioda::ObsDataVector & out) const { + const size_t nlocs = in.nlocs(); + std::vector rh(nlocs); + std::vector T(nlocs); + std::vector ModelHeight(nlocs); + std::vector StationHeight(nlocs); + + in.get(Variable("relative_humidity_at_2m@ObsValue"), rh); + in.get(parameters_.temperature.value(), T); + in.get(Variable("surface_altitude@GeoVaLs"), ModelHeight); + in.get(parameters_.elevation.value(), StationHeight); + + const float missing = util::missingValue(missing); + + // Maximum values of RH_ice for temperatures 0 to -40 deg C + const std::vector rhmax{100.00, 100.98, 101.97, 102.96, 103.97, 104.99, + 106.01, 107.05, 108.10, 109.16, 110.23, 111.31, + 112.40, 113.51, 114.62, 115.75, 116.88, 118.03, + 119.19, 120.36, 121.54, 122.74, 123.94, 125.15, + 126.38, 127.62, 128.87, 130.12, 131.39, 132.67, + 133.96, 135.26, 136.58, 137.90, 139.23, 140.57, + 141.92, 143.27, 144.64, 146.02, 147.40}; + + // compute relative humidity correction and adjusted relative humidity. + for (size_t jj = 0; jj < nlocs; ++jj) { + if (StationHeight[jj] == missing || ModelHeight[jj] == missing || rh[jj] == missing) { + out[0][jj] = missing; + } else { + int Tbin = std::ceil(Constants::t0c - T[jj]); + Tbin = std::max(0, std::min(40, Tbin)); + float CorrectedRH = std::max(rh[jj] - 0.01*(StationHeight[jj] - ModelHeight[jj]), 0.0); + CorrectedRH = std::min(CorrectedRH, rhmax[Tbin]); + out[0][jj] = CorrectedRH; + } + } +} + +// ----------------------------------------------------------------------------- + +const ufo::Variables & ModelHeightAdjustedRelativeHumidity::requiredVariables() const { + return invars_; +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/filters/obsfunctions/ModelHeightAdjustedRelativeHumidity.h b/src/ufo/filters/obsfunctions/ModelHeightAdjustedRelativeHumidity.h new file mode 100644 index 000000000..05339f53d --- /dev/null +++ b/src/ufo/filters/obsfunctions/ModelHeightAdjustedRelativeHumidity.h @@ -0,0 +1,57 @@ +/* ----------------------------------------------------------------------------- + * (C) British Crown Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * ----------------------------------------------------------------------------- + */ + +#ifndef UFO_FILTERS_OBSFUNCTIONS_MODELHEIGHTADJUSTEDRELATIVEHUMIDITY_H_ +#define UFO_FILTERS_OBSFUNCTIONS_MODELHEIGHTADJUSTEDRELATIVEHUMIDITY_H_ + +#include + +#include "ufo/filters/ObsFilterData.h" +#include "ufo/filters/obsfunctions/ObsFunctionBase.h" +#include "ufo/filters/Variables.h" + +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" +#include "ufo/utils/parameters/ParameterTraitsVariable.h" + +namespace ufo { + +/// \brief Options controlling ModelHeightAdjustedRelativeHumidity ObsFunction +class ModelHeightAdjustedRelativeHumidityParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ModelHeightAdjustedRelativeHumidityParameters, Parameters) + + public: + /// Input observation station height to be used + oops::RequiredParameter elevation{"elevation", this}; + /// Temperature to be used + oops::RequiredParameter temperature{"temperature", this}; +}; + +/// \brief Function to calculate surface relative humidity observation value adjusted from +/// station height to model surface height. Outputs a derived 2m relative humidity adjusted +/// to the model surface height. Observed relative humidity is adjusted from station level +/// to model surface using an empirical vertical gradient LRH = -0.01 %/m. The adjusted +/// humidity value is then constrained to lie between zero and supersaturation with respect +/// to liquid water. + +class ModelHeightAdjustedRelativeHumidity : public ObsFunctionBase { + public: + explicit ModelHeightAdjustedRelativeHumidity(const eckit::LocalConfiguration & + = eckit::LocalConfiguration()); + + void compute(const ObsFilterData &, + ioda::ObsDataVector &) const; + const ufo::Variables & requiredVariables() const; + + private: + ModelHeightAdjustedRelativeHumidityParameters parameters_; + ufo::Variables invars_; +}; +} // namespace ufo + +#endif // UFO_FILTERS_OBSFUNCTIONS_MODELHEIGHTADJUSTEDRELATIVEHUMIDITY_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3fcccf366..9adac1c7f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -283,6 +283,7 @@ list( APPEND ufo_test_input testinput/unit_tests/function_tropo_estimate.yaml testinput/unit_tests/function_lam_domaincheck.yaml testinput/unit_tests/function_lam_domaincheck_circle.yaml + testinput/unit_tests/function_ModelSurfaceHeightAdjustRelativeHumidity.yaml testinput/unit_tests/function_ModelSurfaceHeightAdjustWindVector.yaml testinput/unit_tests/function_ModelSurfaceHeightAdjustTemperature.yaml testinput/unit_tests/function_linear_combination_multichannel.yaml @@ -2839,6 +2840,13 @@ ecbuild_add_test( TARGET test_ufo_function_modelsurfaceheightadjust_temperature DEPENDS test_ObsFunction.x TEST_DEPENDS ufo_get_ufo_test_data) +ecbuild_add_test( TARGET test_ufo_function_modelsurfaceheightadjust_relativehumidity + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x + ARGS "testinput/unit_tests/function_ModelSurfaceHeightAdjustRelativeHumidity.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFunction.x + TEST_DEPENDS ufo_get_ufo_test_data) + # Test Diagnostics ecbuild_add_test( TARGET test_ufo_obsdiag_background_error_vert_interp_air_pressure diff --git a/test/testinput/unit_tests/function_ModelSurfaceHeightAdjustRelativeHumidity.yaml b/test/testinput/unit_tests/function_ModelSurfaceHeightAdjustRelativeHumidity.yaml new file mode 100755 index 000000000..59329ebe9 --- /dev/null +++ b/test/testinput/unit_tests/function_ModelSurfaceHeightAdjustRelativeHumidity.yaml @@ -0,0 +1,22 @@ +window begin: 2021-05-21T00:00:00Z +window end: 2021-05-21T23:00:00Z + +observations: +- obs space: + name: Surface + obsdatain: + obsfile: Data/ufo/testinput_tier_1/surface_obs_20210521T1200Z.nc4 + simulated variables: [relative_humidity_at_2m] + geovals: + filename: Data/ufo/testinput_tier_1/surface_geovals_20210521T1200Z.nc4 + + obs function: + name: ModelHeightAdjustedRelativeHumidity@ObsFunction + options: + elevation: + name: station_elevation@MetaData + temperature: + name: air_temperature_at_2m@TestReference + variables: [relative_humidity_at_2m] + tolerance: 1.0e-6 + From db4c1cd56c65b40445f8079a72d90474a5d4bc0e Mon Sep 17 00:00:00 2001 From: Greg Thompson <63315545+gthompsnJCSDA@users.noreply.github.com> Date: Wed, 14 Jul 2021 07:25:31 -0600 Subject: [PATCH 40/98] do not allow negative ust (ustar) variable (#1340) Co-authored-by: gthompsnJCSDA --- src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 b/src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 index afd83e458..5018148f3 100644 --- a/src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 +++ b/src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 @@ -222,7 +222,7 @@ subroutine atmsfcinterp_simobs_(self, geovals_in, obss, nvars, nlocs, hofx) hofx(ivar,iobs) = profile%vals(1,iobs) * redfac end if case("specific_humidity") - ust = von_karman * sqrt(V2) / (gzsoz0 - psim) + ust = max(0.01_kind_real, von_karman * sqrt(V2) / (gzsoz0 - psim)) psiq = log(von_karman*ust*zbot/ka + zbot/zq0) - psih psiqz = log(von_karman*ust*agl/ka + agl/zq0) - psihz hofx(ivar,iobs) = qg + (q%vals(1,iobs) - qg)*psiqz/psiq From 330c02897142d90c06ff2c4428b0d9c79ad12b3c Mon Sep 17 00:00:00 2001 From: Fabio L R Diniz <45880035+fabiolrdiniz@users.noreply.github.com> Date: Wed, 14 Jul 2021 11:36:34 -0300 Subject: [PATCH 41/98] Adding the surface_pressure variable to the radiosonde ewok templated yaml files for the JEDI-GDAS application (#1334) * syncing sondes YAML between instrumentTests and ewok folders * fix for typo Co-authored-by: Ryan Honeyager --- ewok/jedi-gdas/sondes.yaml | 119 ++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 53 deletions(-) diff --git a/ewok/jedi-gdas/sondes.yaml b/ewok/jedi-gdas/sondes.yaml index b79b7ad48..175f4a02a 100644 --- a/ewok/jedi-gdas/sondes.yaml +++ b/ewok/jedi-gdas/sondes.yaml @@ -8,9 +8,22 @@ obs space: sort order: "descending" obsdataout: obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).sondes.{{window_begin}}.nc4 - simulated variables: [air_temperature, specific_humidity, eastward_wind, northward_wind] # surface_pressure + simulated variables: [air_temperature, specific_humidity, eastward_wind, northward_wind, surface_pressure] obs operator: - name: VertInterp + name: Composite + components: + - name: VertInterp + variables: + - name: air_temperature + - name: specific_humidity + - name: eastward_wind + - name: northward_wind + - name: SfcPCorrected + variables: + - name: surface_pressure + da_psfc_scheme: UKMO +# geovar_geomz: geopotential_height +# geovar_sfc_geomz: surface_geopotential_height obs filters: # Reject all obs with PreQC mark already set above 3 - filter: PreQC @@ -27,13 +40,13 @@ obs filters: action: name: reject # -#- filter: Bounds Check -# filter variables: -# - name: surface_pressure -# minvalue: 37499 -# maxvalue: 106999 -# action: -# name: reject +- filter: Bounds Check + filter variables: + - name: surface_pressure + minvalue: 37499 + maxvalue: 106999 + action: + name: reject # - filter: Bounds Check filter variables: @@ -73,7 +86,7 @@ obs filters: name: reject # # Assign the initial observation error, based on height/pressure -- filter: BlackList +- filter: Perform Action filter variables: - name: air_temperature action: @@ -86,32 +99,32 @@ obs filters: xvals: [100000, 95000, 90000, 85000, 35000, 30000, 25000, 20000, 15000, 10000, 7500, 5000, 4000, 3000, 2000, 1000] errors: [1.2, 1.1, 0.9, 0.8, 0.8, 0.9, 1.2, 1.2, 1.0, 0.8, 0.8, 0.9, 0.95, 1.0, 1.25, 1.5] # -#- filter: BlackList -# filter variables: -# - name: surface_pressure -# action: -# name: assign error -# error function: -# name: ObsErrorModelStepwiseLinear@ObsFunction -# options: -# xvar: -# name: surface_pressure@ObsValue -# xvals: [80000, 75000] -# errors: [110, 120] # 1.1 mb below 800 mb and 1.2 mb agove 750 mb +- filter: Perform Action + filter variables: + - name: surface_pressure + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: surface_pressure@ObsValue + xvals: [80000, 75000] + errors: [110, 120] # 1.1 mb below 800 mb and 1.2 mb agove 750 mb # -#- filter: BlackList -# filter variables: -# - name: surface_pressure -# action: -# name: inflate error -# inflation variable: -# name: ObsErrorFactorSfcPressure@ObsFunction -# options: -# error_min: 100 # 1 mb -# error_max: 300 # 3 mb -# geovar_sfc_geomz: surface_geopotential_height +- filter: Perform Action + filter variables: + - name: surface_pressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSfcPressure@ObsFunction + options: + error_min: 100 # 1 mb + error_max: 300 # 3 mb + geovar_sfc_geomz: surface_geopotential_height # -- filter: BlackList +- filter: Perform Action filter variables: - name: specific_humidity action: @@ -125,7 +138,7 @@ obs filters: errors: [0.2, 0.4, 0.8] # 20% RH up to 250 mb, then increased rapidly above scale_factor_var: specific_humidity@ObsValue # -- filter: BlackList +- filter: Perform Action filter variables: - name: eastward_wind - name: northward_wind @@ -140,7 +153,7 @@ obs filters: errors: [1.4, 1.5, 1.6, 1.8, 1.9, 2.0, 2.1, 2.3, 2.6, 2.8, 3.0, 3.2, 2.7, 2.4, 2.1] # # Inflate obserror when multiple obs exist inside vertical model layers. -- filter: BlackList +- filter: Perform Action filter variables: - name: specific_humidity action: @@ -151,7 +164,7 @@ obs filters: test QCflag: PreQC inflate variables: [specific_humidity] defer to post: true -- filter: BlackList +- filter: Perform Action filter variables: - name: air_temperature action: @@ -162,7 +175,7 @@ obs filters: test QCflag: PreQC inflate variables: [air_temperature] defer to post: true -- filter: BlackList +- filter: Perform Action filter variables: - name: eastward_wind action: @@ -174,7 +187,7 @@ obs filters: inflate variables: [eastward_wind] defer to post: true # -- filter: BlackList +- filter: Perform Action filter variables: - name: northward_wind action: @@ -266,17 +279,17 @@ obs filters: name: northward_wind@ObsError defer to post: true # -#- filter: Bounds Check -# filter variables: -# - name: surface_pressure -# action: -# name: reject -# maxvalue: 4.0 -# test variables: -# - name: ObsErrorFactorQuotient@ObsFunction -# options: -# numerator: -# name: surface_pressure@ObsErrorData # After inflation step -# denominator: -# name: surface_pressure@ObsError -# defer to post: true +- filter: Bounds Check + filter variables: + - name: surface_pressure + action: + name: reject + maxvalue: 4.0 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: surface_pressure@ObsErrorData # After inflation step + denominator: + name: surface_pressure@ObsError + defer to post: true From c7361e443e29b0849ff48643e70cf1b95e6c563e Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 14 Jul 2021 11:12:08 -0600 Subject: [PATCH 42/98] Add R with cross-variable correlations (#1327) * Add R with cross-variable correlations * throw exception instead of assert + update comments * more comment changes * use Cholesky decomposition for inverse of R * preallocate vectors and matrices before the loop on nlocs * solveInPlace + simpler * Restart CI Co-authored-by: Ryan Honeyager --- src/ufo/errors/CMakeLists.txt | 2 + src/ufo/errors/ObsErrorCrossVarCov.cc | 193 +++++++++++++++++++++++ src/ufo/errors/ObsErrorCrossVarCov.h | 97 ++++++++++++ src/ufo/instantiateObsErrorFactory.h | 3 + test/CMakeLists.txt | 11 ++ test/testinput/obserrorcrossvarcorr.yaml | 28 ++++ 6 files changed, 334 insertions(+) create mode 100644 src/ufo/errors/ObsErrorCrossVarCov.cc create mode 100644 src/ufo/errors/ObsErrorCrossVarCov.h create mode 100644 test/testinput/obserrorcrossvarcorr.yaml diff --git a/src/ufo/errors/CMakeLists.txt b/src/ufo/errors/CMakeLists.txt index bc13c7168..9175e0d03 100644 --- a/src/ufo/errors/CMakeLists.txt +++ b/src/ufo/errors/CMakeLists.txt @@ -4,6 +4,8 @@ # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. set ( errors_files + ObsErrorCrossVarCov.cc + ObsErrorCrossVarCov.h ObsErrorDiagonal.cc ObsErrorDiagonal.h ) diff --git a/src/ufo/errors/ObsErrorCrossVarCov.cc b/src/ufo/errors/ObsErrorCrossVarCov.cc new file mode 100644 index 000000000..014ebc6f8 --- /dev/null +++ b/src/ufo/errors/ObsErrorCrossVarCov.cc @@ -0,0 +1,193 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/errors/ObsErrorCrossVarCov.h" + +#include + +#include "ioda/Engines/Factory.h" +#include "ioda/Engines/HH.h" +#include "ioda/Layout.h" +#include "ioda/ObsGroup.h" + +namespace ufo { + +// ----------------------------------------------------------------------------- + +ObsErrorCrossVarCov::ObsErrorCrossVarCov(const eckit::Configuration & conf, + ioda::ObsSpace & obspace) + : stddev_(obspace, "ObsError"), + varcorrelations_(Eigen::MatrixXd::Identity(stddev_.nvars(), stddev_.nvars())) +{ + ObsErrorCrossVarCovParameters options; + options.deserialize(conf); + + // Open and read error correlations from the hdf5 file + ioda::Engines::BackendNames backendName = ioda::Engines::BackendNames::Hdf5File; + ioda::Engines::BackendCreationParameters backendParams; + backendParams.fileName = options.inputFile; + backendParams.action = ioda::Engines::BackendFileActions::Open; + backendParams.openMode = ioda::Engines::BackendOpenModes::Read_Only; + + ioda::Group backend = constructBackend(backendName, backendParams); + ioda::ObsGroup obsgroup = ioda::ObsGroup(backend, + ioda::detail::DataLayoutPolicy::generate( + ioda::detail::DataLayoutPolicy::Policies::None)); + + ioda::Variable corrvar = obsgroup.vars["obserror_correlations"]; + corrvar.readWithEigenRegular(varcorrelations_); + // Check that the sizes are correct + const size_t nvars = stddev_.nvars(); + if ((varcorrelations_.rows() != nvars) || (varcorrelations_.cols() != nvars)) { + std::string errormsg = std::string("Correlation matrix for R, specified in ") + + options.inputFile.value() + std::string(" should be size ") + + std::to_string(nvars) + std::string(" by ") + std::to_string(nvars); + throw eckit::UserError(errormsg, Here()); + } +} + +// ----------------------------------------------------------------------------- + +void ObsErrorCrossVarCov::update(const ioda::ObsVector & obserr) { + stddev_ = obserr; +} + +// ----------------------------------------------------------------------------- + +void ObsErrorCrossVarCov::multiply(ioda::ObsVector & dy) const { + // R * dy = D^{1/2} * C * D^{1/2} * dy + // where D^{1/2} - diagonal matrix with stddev_ on the diagonal + // C - correlations + + // D^{1/2] * dy + dy *= stddev_; + + // C * D^{1/2} * dy + const size_t nlocs = dy.nlocs(); + const size_t nvars = dy.nvars(); + const double missing = util::missingValue(double()); + // preallocate data + std::vector usedobs_indices(nvars); + Eigen::VectorXd dy_at_loc(nvars); + Eigen::MatrixXd corr = Eigen::MatrixXd::Identity(nvars, nvars); + + // loop over all observations locations + for (size_t jloc = 0; jloc < nlocs; ++jloc) { + // find values to be used (the ones that passed QC), and create subset of used values + // at this location (dy_at_loc) and submatrix of correlations for the used values (corr) + size_t nused = 0; + for (size_t jvar = 0; jvar < nvars; ++jvar) { + if (dy[jloc*nvars + jvar] != missing) usedobs_indices[nused++] = jvar; + } + for (size_t jvar = 0; jvar < nused; ++jvar) { + int ind = usedobs_indices[jvar]; + dy_at_loc(jvar) = dy[jloc*nvars + ind]; + for (size_t jvar2 = jvar+1; jvar2 < nused; ++jvar2) { + int ind2 = usedobs_indices[jvar2]; + corr(jvar, jvar2) = varcorrelations_(ind, ind2); + corr(jvar2, jvar) = varcorrelations_(ind2, ind); + } + } + // multiply by C + dy_at_loc.head(nused) = corr.block(0, 0, nused, nused) * dy_at_loc.head(nused); + // save results in dy + for (size_t jvar = 0; jvar < nused; ++jvar) { + dy[jloc*nvars + usedobs_indices[jvar]] = dy_at_loc[jvar]; + } + } + + // D^{1/2} * C * D^{1/2} * dy + dy *= stddev_; +} + +// ----------------------------------------------------------------------------- + +void ObsErrorCrossVarCov::inverseMultiply(ioda::ObsVector & dy) const { + // R^{-1} * dy = D^{-1/2} * C^{-1} * D^{-1/2} * dy + // where D^{1/2} - diagonal matrix with stddev_ on the diagonal + // C - correlations + + // D^{-1/2] * dy + dy /= stddev_; + + // C^{-1} * D^{-1/2} * dy + const size_t nlocs = dy.nlocs(); + const size_t nvars = dy.nvars(); + const double missing = util::missingValue(double()); + // preallocate data + std::vector usedobs_indices(nvars); + Eigen::VectorXd dy_at_loc(nvars); + Eigen::MatrixXd corr = Eigen::MatrixXd::Identity(nvars, nvars); + // loop over all observations locations + for (size_t jloc = 0; jloc < nlocs; ++jloc) { + // find values to be used (the ones that passed QC), and create subset of used values + // at this location (dy_at_loc) and submatrix of correlations for the used values (corr) + size_t nused = 0; + for (size_t jvar = 0; jvar < nvars; ++jvar) { + if (dy[jloc*nvars + jvar] != missing) usedobs_indices[nused++] = jvar; + } + for (size_t jvar = 0; jvar < nused; ++jvar) { + int ind = usedobs_indices[jvar]; + dy_at_loc(jvar) = dy[jloc*nvars + ind]; + for (size_t jvar2 = jvar+1; jvar2 < nused; ++jvar2) { + int ind2 = usedobs_indices[jvar2]; + // only need the lower triangle for llt() below; not filling upper triangle + corr(jvar2, jvar) = varcorrelations_(ind2, ind); + } + } + // Multiply by inverse of C, using standard Cholesky decomposition from Eigen library + // https://eigen.tuxfamily.org/dox/classEigen_1_1LLT.html + corr.topLeftCorner(nused, nused).llt().solveInPlace(dy_at_loc.head(nused)); + // save results in dy + for (size_t jvar = 0; jvar < nused; ++jvar) { + dy[jloc*nvars + usedobs_indices[jvar]] = dy_at_loc[jvar]; + } + } + + // D^{-1/2} * C^{-1} * D^{-1/2} * dy + dy /= stddev_; +} + +// ----------------------------------------------------------------------------- + +void ObsErrorCrossVarCov::randomize(ioda::ObsVector & dy) const { + dy.random(); + multiply(dy); +} + +// ----------------------------------------------------------------------------- + +void ObsErrorCrossVarCov::save(const std::string & name) const { + stddev_.save(name); +} + +// ----------------------------------------------------------------------------- + +ioda::ObsVector ObsErrorCrossVarCov::obserrors() const { + return stddev_; +} + +// ----------------------------------------------------------------------------- + +ioda::ObsVector ObsErrorCrossVarCov::inverseVariance() const { + ioda::ObsVector inverseVariance = stddev_; + inverseVariance *= stddev_; + inverseVariance.invert(); + return inverseVariance; +} + +// ----------------------------------------------------------------------------- +void ObsErrorCrossVarCov::print(std::ostream & os) const { + os << "Observation error covariance with cross-variable correlations." << std::endl; + os << " Obs error stddev: " << stddev_ << std::endl; + os << " Cross-variable correlations: " << std::endl << varcorrelations_; +} + +// ----------------------------------------------------------------------------- + + +} // namespace ufo diff --git a/src/ufo/errors/ObsErrorCrossVarCov.h b/src/ufo/errors/ObsErrorCrossVarCov.h new file mode 100644 index 000000000..51cf278bc --- /dev/null +++ b/src/ufo/errors/ObsErrorCrossVarCov.h @@ -0,0 +1,97 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_ERRORS_OBSERRORCROSSVARCOV_H_ +#define UFO_ERRORS_OBSERRORCROSSVARCOV_H_ + +#include + +#include +#include + +#include "ioda/ObsVector.h" + +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" +#include "oops/util/Printable.h" + +namespace eckit { + class Configuration; +} + +namespace ioda { + class ObsSpace; +} + +namespace ufo { + +/// \brief Parameters for obs errors with cross-variable correlations +class ObsErrorCrossVarCovParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsErrorCrossVarCovParameters, Parameters) + public: + /// Input file containing correlations + oops::RequiredParameter inputFile{"input file", this}; +}; + +// ----------------------------------------------------------------------------- +/// \brief Observation error covariance matrix with cross-variable +/// correlations. +/// \details Correlations are the same at all locations and are read +/// from the file specified in the configuration. Obs error standard +/// deviations are read from ObsSpace as ObsError group. +/// Full observation error covariance matrix is R = D^{1/2} * C * D^{1/2} +/// where D^{1/2} is a diagonal matrix with stddev_ (ObsError group) +/// on the diagonal, and C is the correlation matrix. +class ObsErrorCrossVarCov : public util::Printable { + public: + /// Initialize observation errors + ObsErrorCrossVarCov(const eckit::Configuration &, ioda::ObsSpace &); + + /// Update obs error standard deviations to be equal to \p stddev + void update(const ioda::ObsVector & stddev); + + /// Multiply \p y by this observation error covariance + /// Computed as R * dy = D^{1/2} * C * D^{1/2} * dy + /// where D^{1/2} - diagonal matrix with stddev_ on the diagonal + /// C - correlations + void multiply(ioda::ObsVector & y) const; + + /// Multiply \p y by inverse of this observation error covariance + /// Computed as R^{-1} * dy = D^{-1/2} * C^{-1] * D^{-1/2} * dy + /// where D^{1/2} - diagonal matrix with stddev_ on the diagonal + /// C - correlations + void inverseMultiply(ioda::ObsVector & y) const; + + /// Generate \p y as a random perturbation + void randomize(ioda::ObsVector & y) const; + + /// Save obs error standard deviations under \p name group name + void save(const std::string & name) const; + + /// Return RMS of obs error standard deviations + double getRMSE() const {return stddev_.rms();} + + /// Return obs errors std deviation + ioda::ObsVector obserrors() const; + + /// Return inverse of obs error variance + ioda::ObsVector inverseVariance() const; + + private: + /// Print covariance details (for logging) + void print(std::ostream &) const override; + /// Observation error standard deviations + ioda::ObsVector stddev_; + /// Correlations between variables + Eigen::MatrixXd varcorrelations_; +}; + +// ----------------------------------------------------------------------------- + +} // namespace ufo + +#endif // UFO_ERRORS_OBSERRORCROSSVARCOV_H_ diff --git a/src/ufo/instantiateObsErrorFactory.h b/src/ufo/instantiateObsErrorFactory.h index c9dac7cce..32928f430 100644 --- a/src/ufo/instantiateObsErrorFactory.h +++ b/src/ufo/instantiateObsErrorFactory.h @@ -11,6 +11,7 @@ #include "oops/generic/instantiateObsErrorFactory.h" #include "oops/interface/ObsErrorCovariance.h" +#include "ufo/errors/ObsErrorCrossVarCov.h" #include "ufo/errors/ObsErrorDiagonal.h" namespace ufo { @@ -18,6 +19,8 @@ template void instantiateObsErrorFactory() { oops::instantiateObsErrorFactory(); static oops::ObsErrorMaker > makerDiagUFO("diagonal ufo"); + static oops::ObsErrorMaker > + makerCrossVarCov("cross variable covariances"); } } // namespace ufo diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9adac1c7f..97573b212 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -151,6 +151,7 @@ list( APPEND ufo_test_input testinput/obsdiag_crtm_iasi_optics.yaml testinput/obserror_assign_unittests.yaml testinput/obserrordiagonal.yaml + testinput/obserrorcrossvarcorr.yaml testinput/obsfilterdata.yaml testinput/omi_aura.yaml testinput/omi_aura_flipz.yaml @@ -534,6 +535,8 @@ ecbuild_add_test( TARGET test_ufo_locations LIBS ufo TEST_DEPENDS ufo_get_ufo_test_data ) +# Test ObsError implementations + ecbuild_add_test( TARGET test_ufo_obserrordiagonal SOURCES mains/TestObsErrorCovariance.cc ARGS "testinput/obserrordiagonal.yaml" @@ -542,6 +545,14 @@ ecbuild_add_test( TARGET test_ufo_obserrordiagonal LIBS ufo TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_obserrorcrossvarcorr + SOURCES mains/TestObsErrorCovariance.cc + ARGS "testinput/obserrorcrossvarcorr.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + MPI 4 + LIBS ufo + TEST_DEPENDS ufo_get_ufo_test_data ) + # Test UFO-specific classes: ecbuild_add_test( TARGET test_ufo_obsfilterdata diff --git a/test/testinput/obserrorcrossvarcorr.yaml b/test/testinput/obserrorcrossvarcorr.yaml new file mode 100644 index 000000000..65c15db49 --- /dev/null +++ b/test/testinput/obserrorcrossvarcorr.yaml @@ -0,0 +1,28 @@ +window begin: 2018-04-14T21:00:00Z +window end: 2018-04-15T03:00:00Z + +observations: +- obs space: + name: Sondes (cross-var corr, one var) + obsdatain: + obsfile: Data/ufo/testinput_tier_1/sondes_obs_2018041500_s.nc4 + simulated variables: [air_temperature] + obs error: + covariance model: cross variable covariances + input file: Data/ufo/testinput_tier_1/obserror_corr_1var.nc4 +- obs space: + name: Sondes (cross-var corr, two vars) + obsdatain: + obsfile: Data/ufo/testinput_tier_1/sondes_obs_2018041500_s.nc4 + simulated variables: [eastward_wind, northward_wind] + obs error: + covariance model: cross variable covariances + input file: Data/ufo/testinput_tier_1/obserror_corr_2var.nc4 +- obs space: + name: Sondes (cross-var corr, three vars) + obsdatain: + obsfile: Data/ufo/testinput_tier_1/sondes_obs_2018041500_s.nc4 + simulated variables: [air_temperature, eastward_wind, northward_wind] + obs error: + covariance model: cross variable covariances + input file: Data/ufo/testinput_tier_1/obserror_corr_3var.nc4 From b128f806050d91cf867e802864e14bb5cde67b9e Mon Sep 17 00:00:00 2001 From: Claude Gibert Date: Thu, 15 Jul 2021 17:33:37 +0100 Subject: [PATCH 43/98] first commit (#1350) --- ewok/aircraft.yaml | 13 ------------- ewok/r2d2.yaml | 7 +++++++ ewok/radiosonde.yaml | 12 ------------ ewok/ship.yaml | 11 ----------- 4 files changed, 7 insertions(+), 36 deletions(-) delete mode 100644 ewok/aircraft.yaml create mode 100644 ewok/r2d2.yaml delete mode 100644 ewok/radiosonde.yaml delete mode 100644 ewok/ship.yaml diff --git a/ewok/aircraft.yaml b/ewok/aircraft.yaml deleted file mode 100644 index 9582995dc..000000000 --- a/ewok/aircraft.yaml +++ /dev/null @@ -1,13 +0,0 @@ -obs space: - name: Aircraft - obsdatain: - obsfile: $(experiment_dir)/{{current_cycle}}/aircraft.{{window_begin}}.nc4 - obsdataout: - obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).aircraft.{{window_begin}}.nc4 - simulated variables: - - eastward_wind - - northward_wind - - air_temperature - - specific_humidity -obs operator: - name: VertInterp diff --git a/ewok/r2d2.yaml b/ewok/r2d2.yaml new file mode 100644 index 000000000..7ae817731 --- /dev/null +++ b/ewok/r2d2.yaml @@ -0,0 +1,7 @@ +fetch: + bc: + - file_type: satbias + target_file: $(current_dir)/$(obs_type).$(date).$(file_type).nc4 + - file_type: tlapse + target_file: $(current_dir)/$(obs_type).$(date).$(file_type).txt + diff --git a/ewok/radiosonde.yaml b/ewok/radiosonde.yaml deleted file mode 100644 index 98c937f80..000000000 --- a/ewok/radiosonde.yaml +++ /dev/null @@ -1,12 +0,0 @@ -obs space: - name: Radiosonde - obsdatain: - obsfile: $(experiment_dir)/{{current_cycle}}/radiosonde.{{window_begin}}.nc4 - obsdataout: - obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).radiosonde.{{window_begin}}.nc4 - simulated variables: - - eastward_wind - - northward_wind - - air_temperature -obs operator: - name: VertInterp diff --git a/ewok/ship.yaml b/ewok/ship.yaml deleted file mode 100644 index 501eb32bf..000000000 --- a/ewok/ship.yaml +++ /dev/null @@ -1,11 +0,0 @@ -obs space: - name: Ship - obsdatain: - obsfile: $(experiment_dir)/{{current_cycle}}/ship.{{window_begin}}.nc4 - obsdataout: - obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).ship.{{window_begin}}.nc4 - simulated variables: - - surface_pressure -obs operator: - name: SfcPCorrected - da_psfc_scheme: UKMO From 332137be9a350631115228654aa848d0fa1045b8 Mon Sep 17 00:00:00 2001 From: Neill Bowler <45592137+neill-b@users.noreply.github.com> Date: Thu, 15 Jul 2021 19:32:46 +0100 Subject: [PATCH 44/98] Introduce code and tests for Met Office TL/AD of GNSS-RO refractivity (#1271) * First version of new refractivity calculation. * First version of unified refractivity code. * Integration of forward operator with new code First version of changes to allow forward operator to work with the unified refractivity code. * First working version Runs through 1DVar, and gives same results as current routines. * Adjusting to allow for a unfified refractivity. * First version which has added new directory * Changes to make use of the unified refractivity code * Adjusting the alignments. * More aligning lines. * Move NBAM refractivity operator to a differently-named directory. First working version of forward model for Met Office refractivity. * Add optional arguments needed for ground GNSS interaction, and remove duplicate QC test for bending angle. * Minor bug fixes * Mostly white-space changes - this version compares with the updated version on develop. * Remove TL and AD code, since this has not yet been implemented. Change name of header guards to fit in with naming convention. * Moved obs file for gnssro refractivity from ioda to ufo. * Remove redundant code. * Remove Met Office refractivity operator, since this is separate to the original set of changes. * Bug fixes from removing refractivity operator. * Bug fix - make sure that nstate is initialised * Various minor changes in response to Chris's review. * Revert "Merge branch 'feature/unify_refractivity' into feature/gnssro_refractivity_operator" This reverts commit 794d6dc1a01b03f4c1c47beeac5f915d908fdc10, reversing changes made to 0f818775e18773eb3cee1ac0d88f7d8d8df8f4b0. * Bug fix and review comments from unify_refractivity branch. * Minor changes from previous PR review. * Various changes following review. * Fix typo. * Remove unnecessary declarations in header files. * First attempt at a TL/AD. * Bug fixes in my routines and zorder. * Put the code to calculate partial derivatives in a separate routine. * Extra comments in the code. * Remove print which is called in normal running, and add second test. * Tests now encompass pseudo- and non-pseudo-levels, so remove redundant suffix. * Slightly reduce the perturbation size for one linear test. * Remove obs bias from set trajectory, as required by #1219 * Various changes following Chris's review, and change from using configuration to parameters. * Pass coding norms. * Bug fixes and improvements following review. * Rename obs options parameters and give default value. Remove allocation of geovals which is no longer needed. * Bug fix on obs options Co-authored-by: orlewis Co-authored-by: Stephen Herbener Co-authored-by: Ryan Honeyager --- src/ufo/gnssro/RefMetOffice/CMakeLists.txt | 8 +- .../RefMetOffice/ObsGnssroRefMetOffice.cc | 14 +- .../RefMetOffice/ObsGnssroRefMetOffice.h | 6 +- .../ObsGnssroRefMetOffice.interface.F90 | 17 +- .../ObsGnssroRefMetOffice.interface.h | 4 +- .../ObsGnssroRefMetOfficeParameters.h | 49 ++ .../RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc | 87 +++ .../RefMetOffice/ObsGnssroRefMetOfficeTLAD.h | 67 ++ .../ObsGnssroRefMetOfficeTLAD.interface.F90 | 123 +++ .../ObsGnssroRefMetOfficeTLAD.interface.h | 39 + .../ufo_gnssro_refmetoffice_mod.F90 | 66 +- .../ufo_gnssro_refmetoffice_tlad_mod.F90 | 725 ++++++++++++++++++ src/ufo/ufo_geovals_mod.F90 | 4 +- src/ufo/utils/RefractivityCalculator.F90 | 292 ++++--- test/CMakeLists.txt | 9 +- test/testinput/gnssrorefmetoffice.yaml | 2 +- 16 files changed, 1367 insertions(+), 145 deletions(-) create mode 100644 src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeParameters.h create mode 100644 src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc create mode 100644 src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.h create mode 100644 src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.interface.F90 create mode 100644 src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.interface.h create mode 100644 src/ufo/gnssro/RefMetOffice/ufo_gnssro_refmetoffice_tlad_mod.F90 diff --git a/src/ufo/gnssro/RefMetOffice/CMakeLists.txt b/src/ufo/gnssro/RefMetOffice/CMakeLists.txt index 29852e138..5c605bf63 100644 --- a/src/ufo/gnssro/RefMetOffice/CMakeLists.txt +++ b/src/ufo/gnssro/RefMetOffice/CMakeLists.txt @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2020 Met Office +# (C) British Crown Copyright 2021 Met Office # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -8,6 +8,12 @@ set ( refmetoffice_src_files ObsGnssroRefMetOffice.cc ObsGnssroRefMetOffice.interface.h ObsGnssroRefMetOffice.interface.F90 + ObsGnssroRefMetOfficeTLAD.h + ObsGnssroRefMetOfficeTLAD.cc + ObsGnssroRefMetOfficeTLAD.interface.h + ObsGnssroRefMetOfficeTLAD.interface.F90 ufo_gnssro_refmetoffice_mod.F90 + ufo_gnssro_refmetoffice_tlad_mod.F90 + ObsGnssroRefMetOfficeParameters.h PARENT_SCOPE ) diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc index c77c08445..08f209254 100644 --- a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc @@ -1,5 +1,5 @@ /* - * (C) British Crown Copyright 2020 Met Office + * (C) British Crown Copyright 2021 Met Office * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -29,13 +29,17 @@ ObsGnssroRefMetOffice::ObsGnssroRefMetOffice(const ioda::ObsSpace & odb, const eckit::Configuration & config) : ObsOperatorBase(odb, config), keyOperGnssroRefMetOffice_(0), odb_(odb), varin_() { + parameters_.validateAndDeserialize(config); + ObsGnssroRefMetOfficeOptions obsOptions = parameters_.obsOptions.value(); + const std::vector vv{"air_pressure_levels", "specific_humidity", "geopotential_height", "geopotential_height_levels"}; varin_.reset(new oops::Variables(vv)); - const eckit::LocalConfiguration obsOptions(config, "obs options"); - const eckit::Configuration *configc = &obsOptions; - ufo_gnssro_refmetoffice_setup_f90(keyOperGnssroRefMetOffice_, &configc); + ufo_gnssro_refmetoffice_setup_f90(keyOperGnssroRefMetOffice_, + obsOptions.vertInterpOPS, + obsOptions.pseudoLevels, + obsOptions.minTempGrad); oops::Log::trace() << "ObsGnssroRefMetOffice created." << std::endl; } @@ -58,7 +62,7 @@ void ObsGnssroRefMetOffice::simulateObs(const GeoVaLs & gom, ioda::ObsVector & o // ----------------------------------------------------------------------------- void ObsGnssroRefMetOffice::print(std::ostream & os) const { - os << "ObsGnssroRefMetOffice::print not implemented"; + os << "ObsGnssroRefMetOffice: config = " << parameters_ << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h index 869958aa4..023adfe91 100644 --- a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h @@ -1,5 +1,5 @@ /* - * (C) British Crown Copyright 2020 Met Office + * (C) British Crown Copyright 2021 Met Office * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -16,6 +16,7 @@ #include "oops/util/ObjectCounter.h" #include "ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.h" #include "ufo/ObsOperatorBase.h" +#include "ObsGnssroRefMetOfficeParameters.h" namespace eckit { class Configuration; @@ -31,8 +32,8 @@ namespace ufo { class ObsDiagnostics; // ----------------------------------------------------------------------------- - /// GnssroRefMetOffice observation operator +// ----------------------------------------------------------------------------- class ObsGnssroRefMetOffice : public ObsOperatorBase, private util::ObjectCounter { public: @@ -55,6 +56,7 @@ class ObsGnssroRefMetOffice : public ObsOperatorBase, F90hop keyOperGnssroRefMetOffice_; const ioda::ObsSpace& odb_; std::unique_ptr varin_; + ObsGnssroRefMetOfficeParameters parameters_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.F90 b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.F90 index f4fdbd411..1ba17852c 100644 --- a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.F90 +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.F90 @@ -1,4 +1,4 @@ -! (C) British Crown Copyright 2020 Met Office +! (C) British Crown Copyright 2021 Met Office ! ! This software is licensed under the terms of the Apache Licence Version 2.0 ! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -33,18 +33,21 @@ module ufo_gnssro_refmetoffice_mod_c ! ------------------------------------------------------------------------------ -subroutine ufo_gnssro_refmetoffice_setup_c(c_key_self, c_conf) bind(c,name='ufo_gnssro_refmetoffice_setup_f90') +subroutine ufo_gnssro_refmetoffice_setup_c(c_key_self, & + vert_interp_ops, & + pseudo_ops, & + min_temp_grad) bind(c,name='ufo_gnssro_refmetoffice_setup_f90') implicit none integer(c_int), intent(inout) :: c_key_self -type(c_ptr), intent(in) :: c_conf - +logical(c_bool), intent(in) :: vert_interp_ops +logical(c_bool), intent(in) :: pseudo_ops +real(c_float), intent(in) :: min_temp_grad + type(ufo_gnssro_RefMetOffice), pointer :: self -type(fckit_configuration) :: f_conf call ufo_gnssro_refmetoffice_registry%setup(c_key_self, self) -f_conf = fckit_configuration(c_conf) -call self%setup(f_conf) +call self%setup(vert_interp_ops, pseudo_ops, min_temp_grad) end subroutine ufo_gnssro_refmetoffice_setup_c diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.h b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.h index 99b1a7647..73261ece1 100644 --- a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.h +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.interface.h @@ -1,5 +1,5 @@ /* - * (C) British Crown Copyright 2020 Met Office + * (C) British Crown Copyright 2021 Met Office * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -23,7 +23,7 @@ extern "C" { // ----------------------------------------------------------------------------- // Gnssro refractivity observation operators - (Met Office 1D) // ----------------------------------------------------------------------------- - void ufo_gnssro_refmetoffice_setup_f90(F90hop &, const eckit::Configuration * const *); + void ufo_gnssro_refmetoffice_setup_f90(F90hop &, const bool &, const bool &, const float &); void ufo_gnssro_refmetoffice_delete_f90(F90hop &); void ufo_gnssro_refmetoffice_simobs_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, const int &, double &, const F90goms &); diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeParameters.h b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeParameters.h new file mode 100644 index 000000000..b1927db37 --- /dev/null +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeParameters.h @@ -0,0 +1,49 @@ +/* + * (C) Copyright 2021 UK Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICEPARAMETERS_H_ +#define UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICEPARAMETERS_H_ + +#include +#include + +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" + +namespace ufo { + +/// Configuration options recognized by the refractivity operator. +class ObsGnssroRefMetOfficeOptions : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsGnssroRefMetOfficeOptions, Parameters) + + public: + /// If true assume that pressure varies exponentially with height when + /// interpolating. Otherwise assume that exner varies linearly with height, + /// and derive pressure from this. + oops::Parameter vertInterpOPS{"vert_interp_ops", true, this}; + /// Whether to use pseudo-levels in the calculation. + oops::Parameter pseudoLevels{"pseudo_ops", true, this}; + /// The minimum temperature gradient permitted before a profile is considered + /// isothermal. Only used if pseudo-levels are also used. + oops::Parameter minTempGrad{"min_temp_grad", 1.0e-6, this}; +}; + +/// Configuration options recognized by the refractivity operator. +class ObsGnssroRefMetOfficeParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsGnssroRefMetOfficeParameters, Parameters) + + public: + /// Operator name. In future will be moved to a base class for parameters of all ObsOperators. + oops::OptionalParameter name{"name", this}; + /// Obs Options - settings for the observation operator + oops::Parameter obsOptions{"obs options", + ObsGnssroRefMetOfficeOptions(), this}; +}; + +} // namespace ufo +#endif // UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICEPARAMETERS_H_ diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc new file mode 100644 index 000000000..18efa6752 --- /dev/null +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc @@ -0,0 +1,87 @@ +/* + * (C) British Crown Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.h" + +#include +#include +#include + + +#include "ioda/ObsSpace.h" +#include "ioda/ObsVector.h" + +#include "oops/base/Variables.h" +#include "oops/util/Logger.h" + +#include "ufo/GeoVaLs.h" + +namespace ufo { + +// ----------------------------------------------------------------------------- +static LinearObsOperatorMaker + makerGnssroRefMetOfficeTL_("GnssroRefMetOffice"); +// ----------------------------------------------------------------------------- + +ObsGnssroRefMetOfficeTLAD::ObsGnssroRefMetOfficeTLAD(const ioda::ObsSpace & odb, + const eckit::Configuration & config) + : LinearObsOperatorBase(odb), keyOperGnssroRefMetOffice_(0), varin_() +{ + parameters_.validateAndDeserialize(config); + ObsGnssroRefMetOfficeOptions obsOptions = parameters_.obsOptions.value(); + + ufo_gnssro_refmetoffice_tlad_setup_f90(keyOperGnssroRefMetOffice_, + obsOptions.vertInterpOPS, + obsOptions.pseudoLevels, + obsOptions.minTempGrad); + const std::vector vv{"air_pressure_levels", "specific_humidity", + "geopotential_height", "geopotential_height_levels"}; + + varin_.reset(new oops::Variables(vv)); + oops::Log::info() << "ObsGnssroRefMetOfficeTLAD vars: " << *varin_ << std::endl; + oops::Log::trace() << "ObsGnssroRefMetOfficeTLAD created" << std::endl; +} + +// ----------------------------------------------------------------------------- + +ObsGnssroRefMetOfficeTLAD::~ObsGnssroRefMetOfficeTLAD() { + ufo_gnssro_refmetoffice_tlad_delete_f90(keyOperGnssroRefMetOffice_); + oops::Log::trace() << "ObsGnssroRefMetOfficeTLAD destructed" << std::endl; +} + +// ----------------------------------------------------------------------------- + +void ObsGnssroRefMetOfficeTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { + ufo_gnssro_refmetoffice_tlad_settraj_f90(keyOperGnssroRefMetOffice_, geovals.toFortran(), + obsspace()); +} + +// ----------------------------------------------------------------------------- + +void ObsGnssroRefMetOfficeTLAD::simulateObsTL( + const GeoVaLs & geovals, ioda::ObsVector & ovec) const { + ufo_gnssro_refmetoffice_simobs_tl_f90(keyOperGnssroRefMetOffice_, geovals.toFortran(), + obsspace(), ovec.size(), ovec.toFortran()); +} + +// ----------------------------------------------------------------------------- + +void ObsGnssroRefMetOfficeTLAD::simulateObsAD( + GeoVaLs & geovals, const ioda::ObsVector & ovec) const { + ufo_gnssro_refmetoffice_simobs_ad_f90(keyOperGnssroRefMetOffice_, geovals.toFortran(), + obsspace(), ovec.size(), ovec.toFortran()); +} + +// ----------------------------------------------------------------------------- + +void ObsGnssroRefMetOfficeTLAD::print(std::ostream & os) const { + os << "ObsGnssroRefMetOfficeTLAD: config = " << parameters_ << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.h b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.h new file mode 100644 index 000000000..b482b6053 --- /dev/null +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.h @@ -0,0 +1,67 @@ +/* + * (C) British Crown Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICETLAD_H_ +#define UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICETLAD_H_ + +#include +#include +#include + +#include "oops/base/Variables.h" +#include "oops/util/ObjectCounter.h" +#include "ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.interface.h" +#include "ufo/LinearObsOperatorBase.h" +#include "ObsGnssroRefMetOfficeParameters.h" + +// Forward declarations +namespace eckit { + class Configuration; +} + +namespace ioda { + class ObsSpace; + class ObsVector; +} + +namespace ufo { + class GeoVaLs; + class ObsDiagnostics; + +// ----------------------------------------------------------------------------- +/// GnssroRefMetOffice observation operator - TL/AD +// ----------------------------------------------------------------------------- +class ObsGnssroRefMetOfficeTLAD : public LinearObsOperatorBase, + private util::ObjectCounter { + public: + static const std::string classname() {return "ufo::ObsGnssroRefMetOfficeTLAD";} + + ObsGnssroRefMetOfficeTLAD(const ioda::ObsSpace &, const eckit::Configuration &); + virtual ~ObsGnssroRefMetOfficeTLAD(); + + // Obs Operators + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; + void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; + void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; + + // Other + const oops::Variables & requiredVars() const override {return *varin_;} + + int & toFortran() {return keyOperGnssroRefMetOffice_;} + const int & toFortran() const {return keyOperGnssroRefMetOffice_;} + + private: + void print(std::ostream &) const override; + F90hop keyOperGnssroRefMetOffice_; + std::unique_ptr varin_; + ObsGnssroRefMetOfficeParameters parameters_; +}; + +// ----------------------------------------------------------------------------- + +} // namespace ufo +#endif // UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICETLAD_H_ diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.interface.F90 b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.interface.F90 new file mode 100644 index 000000000..4ad25880b --- /dev/null +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.interface.F90 @@ -0,0 +1,123 @@ +! (C) British Crown Copyright 2021 Met Office +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +!> Fortran module to handle GNSS-RO refractivity observations + +module ufo_gnssro_refmetoffice_tlad_mod_c + + use fckit_configuration_module, only: fckit_configuration + use ufo_gnssro_refmetoffice_tlad_mod + implicit none + private + +#define LISTED_TYPE ufo_gnssro_refmetoffice_tlad + + !> Linked list interface - defines registry_t type +#include "oops/util/linkedList_i.f" + + !> Global registry + type(registry_t) :: ufo_gnssro_refmetoffice_tlad_registry + + ! ------------------------------------------------------------------------------ +contains + ! ------------------------------------------------------------------------------ + !> Linked list implementation +#include "oops/util/linkedList_c.f" + +! ------------------------------------------------------------------------------ + +subroutine ufo_gnssro_refmetoffice_tlad_setup_c(c_key_self, & + vert_interp_ops, & + pseudo_ops, & + min_temp_grad) bind(c,name='ufo_gnssro_refmetoffice_tlad_setup_f90') +implicit none +integer(c_int), intent(inout) :: c_key_self +logical(c_bool), intent(in) :: vert_interp_ops +logical(c_bool), intent(in) :: pseudo_ops +real(c_float), intent(in) :: min_temp_grad + +type(ufo_gnssro_refmetoffice_tlad), pointer :: self + +call ufo_gnssro_refmetoffice_tlad_registry%setup(c_key_self, self) + +call self%setup(vert_interp_ops, pseudo_ops, min_temp_grad) + +end subroutine ufo_gnssro_refmetoffice_tlad_setup_c + +! ------------------------------------------------------------------------------ + +subroutine ufo_gnssro_refmetoffice_tlad_delete_c(c_key_self) bind(c,name='ufo_gnssro_refmetoffice_tlad_delete_f90') +implicit none +integer(c_int), intent(inout) :: c_key_self + +type(ufo_gnssro_refmetoffice_tlad), pointer :: self + +call ufo_gnssro_refmetoffice_tlad_registry%get(c_key_self, self) +call self%opr_delete() +call ufo_gnssro_refmetoffice_tlad_registry%remove(c_key_self) + +end subroutine ufo_gnssro_refmetoffice_tlad_delete_c + +! ------------------------------------------------------------------------------ + +subroutine ufo_gnssro_refmetoffice_tlad_settraj_c(c_key_self, c_key_geovals, c_obsspace) bind(c,name='ufo_gnssro_refmetoffice_tlad_settraj_f90') + +implicit none +integer(c_int), intent(in) :: c_key_self +integer(c_int), intent(in) :: c_key_geovals +type(c_ptr), value, intent(in) :: c_obsspace + +type(ufo_gnssro_refmetoffice_tlad), pointer :: self + +character(len=*), parameter :: myname_="ufo_gnssro_refmetoffice_tlad_settraj_c" + +call ufo_gnssro_refmetoffice_tlad_registry%get(c_key_self, self) +call self%opr_settraj(c_key_geovals, c_obsspace) + +end subroutine ufo_gnssro_refmetoffice_tlad_settraj_c + +! ------------------------------------------------------------------------------ + +subroutine ufo_gnssro_refmetoffice_simobs_tl_c(c_key_self, c_key_geovals, c_obsspace, c_nobs, c_hofx) bind(c,name='ufo_gnssro_refmetoffice_simobs_tl_f90') + +implicit none +integer(c_int), intent(in) :: c_key_self +integer(c_int), intent(in) :: c_key_geovals +type(c_ptr), value, intent(in) :: c_obsspace +integer(c_int), intent(in) :: c_nobs +real(c_double), intent(inout) :: c_hofx(c_nobs) + +type(ufo_gnssro_refmetoffice_tlad), pointer :: self + +character(len=*), parameter :: myname_="ufo_gnssro_refmetoffice_simobs_tl_c" + +call ufo_gnssro_refmetoffice_tlad_registry%get(c_key_self, self) +call self%opr_simobs_tl(c_key_geovals, c_obsspace, c_hofx) + +end subroutine ufo_gnssro_refmetoffice_simobs_tl_c + +! ------------------------------------------------------------------------------ + +subroutine ufo_gnssro_refmetoffice_simobs_ad_c(c_key_self, c_key_geovals, c_obsspace, c_nobs, c_hofx) bind(c,name='ufo_gnssro_refmetoffice_simobs_ad_f90') + +implicit none +integer(c_int), intent(in) :: c_key_self +integer(c_int), intent(in) :: c_key_geovals +type(c_ptr), value, intent(in) :: c_obsspace +integer(c_int), intent(in) :: c_nobs +real(c_double), intent(in) :: c_hofx(c_nobs) + +type(ufo_gnssro_refmetoffice_tlad), pointer :: self + +character(len=*), parameter :: myname_="ufo_gnssro_refmetoffice_simobs_ad_c" + +call ufo_gnssro_refmetoffice_tlad_registry%get(c_key_self, self) +call self%opr_simobs_ad(c_key_geovals, c_obsspace, c_hofx) + +end subroutine ufo_gnssro_refmetoffice_simobs_ad_c + +! ------------------------------------------------------------------------------ + +end module ufo_gnssro_refmetoffice_tlad_mod_c diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.interface.h b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.interface.h new file mode 100644 index 000000000..4476c8927 --- /dev/null +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.interface.h @@ -0,0 +1,39 @@ +/* + * (C) British Crown Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICETLAD_INTERFACE_H_ +#define UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICETLAD_INTERFACE_H_ + +#include "ioda/ObsSpace.h" +#include "ufo/Fortran.h" + +namespace ufo { + +/// Interface to Fortran UFO routines +/*! + * The core of the UFO is coded in Fortran. + * Here we define the interfaces to the Fortran code. + */ + +extern "C" { +// ----------------------------------------------------------------------------- +// Met Office GNSS-RO refractivity TL/AD observation operators +// ----------------------------------------------------------------------------- + void ufo_gnssro_refmetoffice_tlad_setup_f90(F90hop &, const bool &, const bool &, const float &); + void ufo_gnssro_refmetoffice_tlad_delete_f90(F90hop &); + void ufo_gnssro_refmetoffice_tlad_settraj_f90(const F90hop &, const F90goms &, + const ioda::ObsSpace &); + void ufo_gnssro_refmetoffice_simobs_tl_f90( + const F90hop &, const F90goms &, const ioda::ObsSpace &, const int &, double &); + void ufo_gnssro_refmetoffice_simobs_ad_f90( + const F90hop &, const F90goms &, const ioda::ObsSpace &, const int &, const double &); +// ----------------------------------------------------------------------------- + +} // extern C + +} // namespace ufo +#endif // UFO_GNSSRO_REFMETOFFICE_OBSGNSSROREFMETOFFICETLAD_INTERFACE_H_ diff --git a/src/ufo/gnssro/RefMetOffice/ufo_gnssro_refmetoffice_mod.F90 b/src/ufo/gnssro/RefMetOffice/ufo_gnssro_refmetoffice_mod.F90 index 20ceecada..758fa1493 100644 --- a/src/ufo/gnssro/RefMetOffice/ufo_gnssro_refmetoffice_mod.F90 +++ b/src/ufo/gnssro/RefMetOffice/ufo_gnssro_refmetoffice_mod.F90 @@ -1,5 +1,5 @@ !------------------------------------------------------------------------------- -! (C) British Crown Copyright 2020 Met Office +! (C) British Crown Copyright 2021 Met Office ! ! This software is licensed under the terms of the Apache Licence Version 2.0 ! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -45,26 +45,44 @@ module ufo_gnssro_refmetoffice_mod contains -! ------------------------------------------------------------------------------ -! Get the optional settings for the forward model, and save them in the object -! so that they can be used in the code. -! ------------------------------------------------------------------------------ -subroutine ufo_gnssro_refmetoffice_setup(self, f_conf) +!------------------------------------------------------------------------------- +!> \brief Set up the Met Office GNSS-RO refractivity operator +!! +!! \details **ufo_gnssro_refmetoffice_setup** +!! * Get the optional settings for the forward model, and save them in the +!! object so that they can be used in the code. +!! +!! \author Neill Bowler (Met Office) +!! +!! \date 20 March 2021 +!! +!------------------------------------------------------------------------------- +subroutine ufo_gnssro_refmetoffice_setup(self, vert_interp_ops, pseudo_ops, min_temp_grad) -use fckit_configuration_module, only: fckit_configuration implicit none -class(ufo_gnssro_RefMetOffice), intent(inout) :: self -type(fckit_configuration), intent(in) :: f_conf -call f_conf%get_or_die("vert_interp_ops", self % vert_interp_ops) -call f_conf%get_or_die("pseudo_ops", self % pseudo_ops) -call f_conf%get_or_die("min_temp_grad", self % min_temp_grad) +class(ufo_gnssro_refmetoffice), intent(inout) :: self +logical(c_bool), intent(in) :: vert_interp_ops +logical(c_bool), intent(in) :: pseudo_ops +real(c_float), intent(in) :: min_temp_grad + +self % vert_interp_ops = vert_interp_ops +self % pseudo_ops = pseudo_ops +self % min_temp_grad = min_temp_grad end subroutine ufo_gnssro_refmetoffice_setup -! ------------------------------------------------------------------------------ -! 1-dimensional GNSS-RO forward operator for the Met Office system -! ------------------------------------------------------------------------------ +!------------------------------------------------------------------------------- +!> \brief Calculate the model forecast of the observations +!! +!! \details **ufo_gnssro_refmetoffice_simobs** +!! * 1-dimensional GNSS-RO forward operator for the Met Office system +!! +!! \author Neill Bowler (Met Office) +!! +!! \date 20 March 2021 +!! +!------------------------------------------------------------------------------- subroutine ufo_gnssro_refmetoffice_simobs(self, geovals_in, obss, hofx, obs_diags) implicit none @@ -124,10 +142,10 @@ subroutine ufo_gnssro_refmetoffice_simobs(self, geovals_in, obss, hofx, obs_diag call ufo_geovals_copy(geovals_in, geovals) ! dont want to change geovals_in call ufo_geovals_reorderzdir(geovals, var_prsi, "bottom2top") - write(message, *) myname_, ' Running Met Office GNSS-RO forward operator with' + write(message, *) myname_, ' Running Met Office GNSS-RO forward operator with:' call fckit_log%info(message) write(message, *) 'vert_interp_ops =', self % vert_interp_ops, & - 'pseudo_ops =', self % pseudo_ops + 'pseudo_ops =', self % pseudo_ops, 'min_temp_grad =', self % min_temp_grad call fckit_log%info(message) ! get variables from geovals @@ -216,9 +234,19 @@ subroutine ufo_gnssro_refmetoffice_simobs(self, geovals_in, obss, hofx, obs_diag call fckit_log%info(err_msg) end subroutine ufo_gnssro_refmetoffice_simobs -! ------------------------------------------------------------------------------ - +!------------------------------------------------------------------------------- +!> \brief Interface routine for the GNSS-RO refractivity forward operator +!! +!! \details **RefMetOffice_ForwardModel** +!! * Calculate the refractivity on model or pseudo levels +!! * Vertically interpolate the model refractivity to the observation locations +!! +!! \author Neill Bowler (Met Office) +!! +!! \date 20 March 2021 +!! +!------------------------------------------------------------------------------- SUBROUTINE RefMetOffice_ForwardModel(nlevp, & nlevq, & za, & diff --git a/src/ufo/gnssro/RefMetOffice/ufo_gnssro_refmetoffice_tlad_mod.F90 b/src/ufo/gnssro/RefMetOffice/ufo_gnssro_refmetoffice_tlad_mod.F90 new file mode 100644 index 000000000..2d9581d0f --- /dev/null +++ b/src/ufo/gnssro/RefMetOffice/ufo_gnssro_refmetoffice_tlad_mod.F90 @@ -0,0 +1,725 @@ +!------------------------------------------------------------------------------- +! (C) British Crown Copyright 2021 Met Office +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +!------------------------------------------------------------------------------- +!> Fortran module for gnssro refractivity Met Office's tangent linear and adjoint + +module ufo_gnssro_refmetoffice_tlad_mod + +use iso_c_binding +use kinds +use ufo_vars_mod +use ufo_geovals_mod +use ufo_geovals_mod_c, only: ufo_geovals_registry +use vert_interp_mod +use ufo_basis_tlad_mod, only: ufo_basis_tlad +use obsspace_mod +use gnssro_mod_conf +use missing_values_mod +use fckit_log_module, only : fckit_log +use ufo_utils_refractivity_calculator, only: & + ufo_calculate_refractivity, ufo_refractivity_partial_derivatives +use ufo_constants_mod, only: & + rd, & ! Gas constant for dry air + cp, & ! Heat capacity at constant pressure for air + rd_over_cp, & ! Ratio of gas constant to heat capacity + grav, & ! Gravitational field strength + pref, & ! Reference pressure for calculating exner + mw_ratio, & ! Ratio of molecular weights of water and dry air + c_virtual, & ! Related to mw_ratio + n_alpha, & ! Refractivity constant a + n_beta ! Refractivity constant b + + +integer, parameter :: max_string=800 + +!> Fortran derived type for gnssro trajectory +type, extends(ufo_basis_tlad) :: ufo_gnssro_refmetoffice_tlad + private + logical :: vert_interp_ops !< Do vertical interpolation using ln(p) or exner? + logical :: pseudo_ops !< Use pseudo-levels in the refractivity calculation + logical :: flip_it !< Do the outputs need to be flipped in order + real(kind_real) :: min_temp_grad !< The minimum temperature gradient before a profile is considered isothermal + ! Used in pseudo-levels calculation + integer :: nlevp !< The number of pressure levels + integer :: nlevq !< The number of specific humidity (or temperature) levels + integer :: nlocs !< The number of observations + real(kind_real), allocatable :: K(:,:) !< The K-matrix (Jacobian) + contains + procedure :: setup => ufo_gnssro_refmetoffice_tlad_setup + procedure :: delete => ufo_gnssro_refmetoffice_tlad_delete + procedure :: settraj => ufo_gnssro_refmetoffice_tlad_settraj + procedure :: simobs_tl => ufo_gnssro_refmetoffice_simobs_tl + procedure :: simobs_ad => ufo_gnssro_refmetoffice_simobs_ad +end type ufo_gnssro_refmetoffice_tlad + +contains + +!------------------------------------------------------------------------------- +!> \brief Set up the Met Office GNSS-RO refractivity TL/AD +!! +!! \details **ufo_gnssro_refmetoffice_tlad_setup** +!! * Get the optional settings for the forward model and its linear, and save +!! them in the object so that they can be used in the code. +!! +!! \author Neill Bowler (Met Office) +!! +!! \date 26 May 2021 +!! +!------------------------------------------------------------------------------- +subroutine ufo_gnssro_refmetoffice_tlad_setup(self, vert_interp_ops, pseudo_ops, min_temp_grad) + +implicit none + +class(ufo_gnssro_refmetoffice_tlad), intent(inout) :: self +logical(c_bool), intent(in) :: vert_interp_ops +logical(c_bool), intent(in) :: pseudo_ops +real(c_float), intent(in) :: min_temp_grad + +self % vert_interp_ops = vert_interp_ops +self % pseudo_ops = pseudo_ops +self % min_temp_grad = min_temp_grad + +end subroutine ufo_gnssro_refmetoffice_tlad_setup + +!------------------------------------------------------------------------------- +!> \brief Set up the K-matrix for (Jacobian) for the Met Office's GNSS-RO +!! refractivity operator +!! +!! \details **ufo_gnssro_refmetoffice_tlad_settraj** +!! * It is necessary to run this routine before calling the TL or AD routines. +!! * Get the geovals specifying the state around which to linearise, flipping +!! the vertical order if required. +!! * Call the helper function to calculate the K-matrix for each observation. +!! +!! \author Neill Bowler (Met Office) +!! +!! \date 26 May 2021 +!! +!------------------------------------------------------------------------------- +subroutine ufo_gnssro_refmetoffice_tlad_settraj(self, geovals, obss) + + implicit none +! Subroutine arguments + class(ufo_gnssro_refmetoffice_tlad), intent(inout) :: self !< The object that we use to save data in + type(ufo_geovals), intent(in) :: geovals !< The input geovals + type(c_ptr), value, intent(in) :: obss !< The input observations + +! Local parameters + character(len=*), parameter :: myname_="ufo_gnssro_refmetoffice_tlad_settraj" + +! Local variables + character(max_string) :: err_msg ! Messages to be output to the user + type(ufo_geoval), pointer :: q ! The model geovals - specific humidity + type(ufo_geoval), pointer :: prs ! The model geovals - atmospheric pressure + type(ufo_geoval), pointer :: rho_heights ! The model geovals - heights of the pressure-levels + type(ufo_geoval), pointer :: theta_heights ! The model geovals - heights of the theta-levels (stores q) + type(ufo_geoval), pointer :: p_temp ! The model geovals - atmospheric pressure before flipping + integer :: iobs ! Loop variable, observation number + + real(kind_real), allocatable :: obs_height(:) ! Geopotential height of the observation + type(ufo_geovals) :: geovals_local ! The model values, interpolated to the observation locations + ! and flipped in the vertical (if required) + + write(err_msg,*) "TRACE: ufo_gnssro_refmetoffice_tlad_settraj: begin" + call fckit_log%info(err_msg) + +! Make sure that any previous values of geovals don't get carried over + call self%delete() + +! make sure that the geovals are in the correct vertical order (surface first) + call ufo_geovals_copy(geovals, geovals_local) ! dont want to change geovals input + call ufo_geovals_get_var(geovals_local, var_prsi, p_temp) + if( p_temp%vals(1,1) < p_temp%vals(p_temp%nval,1) ) then + self%flip_it = .true. + else + self%flip_it = .false. + endif + call ufo_geovals_reorderzdir(geovals_local, var_prsi, "bottom2top") + +! get model state variables from geovals + call ufo_geovals_get_var(geovals_local, var_q, q) ! specific humidity + call ufo_geovals_get_var(geovals_local, var_prsi, prs) ! pressure + call ufo_geovals_get_var(geovals_local, var_z, theta_heights) ! Geopotential height of the normal model levels + call ufo_geovals_get_var(geovals_local, var_zi, rho_heights) ! Geopotential height of the pressure levels + +! Keep copy of dimensions + self % nlevp = prs % nval + self % nlevq = q % nval + self % nlocs = obsspace_get_nlocs(obss) + +! Get the meta-data from the observations + allocate(obs_height(self%nlocs)) + call obsspace_get_db(obss, "MetaData", "obs_height", obs_height) + allocate(self % K(1:self%nlocs, 1:prs%nval + q%nval)) + +! For each observation, calculate the K-matrix + obs_loop: do iobs = 1, self % nlocs + CALL jacobian_interface(prs % nval, & ! Number of pressure levels + q % nval, & ! Number of specific humidity levels + rho_heights % vals(:,iobs), & ! Heights of the pressure levels + theta_heights % vals(:,iobs), & ! Heights of the specific humidity levels + prs % vals(:,iobs), & ! Values of the pressure + q % vals(:,iobs), & ! Values of the specific humidity + self % pseudo_ops, & ! Whether to use pseudo-levels in the calculation + self % vert_interp_ops, & ! Whether to interpolate using log(pressure) + self % min_temp_grad, & ! Minimum allowed vertical temperature gradient + 1, & ! Number of observations in the profile + obs_height(iobs:iobs), & ! Impact parameter for this observation + self % K(iobs:iobs,1:prs%nval+q%nval)) ! K-matrix (Jacobian of the observation with respect to the inputs) + end do obs_loop + +! Note that this routine has been run. + self%ltraj = .true. + + deallocate(obs_height) + +end subroutine ufo_gnssro_refmetoffice_tlad_settraj + +!------------------------------------------------------------------------------- +!> \brief Given an increment to the model state, calculate an increment to the +!! observation +!! +!! \details **ufo_gnssro_refmetoffice_simobs_tl** +!! * Check that set trajectory has been previously called. +!! * Get the geovals for the increment. +!! * For each observation apply the K-matrix to calculate the increment to the +!! observation. +!! +!! \author Neill Bowler (Met Office) +!! +!! \date 26 May 2021 +!! +!------------------------------------------------------------------------------- + +subroutine ufo_gnssro_refmetoffice_simobs_tl(self, geovals, hofx, obss) + + implicit none + +! Subroutine arguments + class(ufo_gnssro_refmetoffice_tlad), intent(in) :: self !< Object which is being used to transfer information + type(ufo_geovals), intent(in) :: geovals !< Model perturbations + real(kind_real), intent(inout) :: hofx(:) !< Increment to the observations + type(c_ptr), value, intent(in) :: obss !< Input - the observations + +! Local parameters + character(len=*), parameter :: myname_="ufo_gnssro_refmetoffice_simobs_tl" + +! Local variables + integer :: iobs ! Loop variable, observation number + integer :: nlocs ! Number of observations + character(max_string) :: err_msg ! Message to be output + type(ufo_geoval), pointer :: q_d ! Increment to the specific humidity + type(ufo_geoval), pointer :: prs_d ! Increment to the air pressure + real(kind_real), allocatable :: x_d(:) ! Increment to the complete state + + write(err_msg,*) "TRACE: ufo_gnssro_refmetoffice_simobs_tl: begin" + call fckit_log%info(err_msg) + +! Check if trajectory was set + if (.not. self%ltraj) then + write(err_msg,*) myname_, ' trajectory wasnt set!' + call abor1_ftn(err_msg) + endif + +! Check if nlocs is consistent in geovals & hofx + if (geovals%nlocs /= size(hofx)) then + write(err_msg,*) myname_, ' error: nlocs inconsistent!' + call abor1_ftn(err_msg) + endif + +! Get variables from geovals + call ufo_geovals_get_var(geovals, var_q, q_d) ! specific humidity + call ufo_geovals_get_var(geovals, var_prsi, prs_d) ! pressure on rho levels + + nlocs = self % nlocs ! number of observations + + allocate(x_d(1:prs_d%nval+q_d%nval)) +! Loop through the obs, calculating the increment to the observation + obs_loop: do iobs = 1, nlocs ! order of loop doesn't matter + + x_d(1:prs_d%nval) = prs_d % vals(:,iobs) + x_d(prs_d%nval+1:prs_d%nval+q_d%nval) = q_d % vals(:,iobs) + hofx(iobs) = SUM(self % K(iobs,:) * x_d) + + end do obs_loop + + deallocate(x_d) + + write(err_msg,*) "TRACE: ufo_gnssro_refmetoffice_simobs_tl: complete" + call fckit_log%info(err_msg) + + return + +end subroutine ufo_gnssro_refmetoffice_simobs_tl + +!------------------------------------------------------------------------------- +!> \brief Given an increment to the observation, find the equivalent increment +!! to the model state +!! +!! \details **ufo_gnssro_refmetoffice_simobs_ad** +!! * Check that set trajectory has previously been called. +!! * Get the geovals for the model increment, and allocate these if they have +!! not already been allocated. +!! * For each observation calculate the observation increment using the K-matrix +!! +!! \author Neill Bowler (Met Office) +!! +!! \date 26 May 2021 +!! +!------------------------------------------------------------------------------- + +subroutine ufo_gnssro_refmetoffice_simobs_ad(self, geovals, hofx, obss) + + use typesizes, only: wp => EightByteReal + + implicit none + +! Subroutine arguments + class(ufo_gnssro_refmetoffice_tlad), intent(in) :: self !< Object which is being used to transfer information + type(ufo_geovals), intent(inout) :: geovals !< Calculated perturbations to model state + real(kind_real), intent(in) :: hofx(:) !< Increment to the observations + type(c_ptr), value, intent(in) :: obss !< Input - the observations + +! Local parameters + character(len=*), parameter :: myname_="ufo_gnssro_refmetoffice_simobs_ad" + +! Local variables + real(c_double) :: missing ! Missing data values + type(ufo_geoval), pointer :: q_d ! Pointer to the specific humidity perturbations + type(ufo_geoval), pointer :: prs_d ! Pointer to the pressure perturbations + integer :: iobs ! Loop variable, observation number + real(kind_real), allocatable :: x_d(:) ! Perturbation to the full model state + character(max_string) :: err_msg ! Message to be output + + write(err_msg,*) "TRACE: ufo_gnssro_refmetoffice_simobs_ad: begin" + call fckit_log%info(err_msg) + +! Check if trajectory was set + if (.not. self%ltraj) then + write(err_msg,*) myname_, ' trajectory wasnt set!' + call abor1_ftn(err_msg) + endif + +! Check if nlocs is consistent in geovals & hofx + if (geovals%nlocs /= size(hofx)) then + write(err_msg,*) myname_, ' error: nlocs inconsistent!' + call abor1_ftn(err_msg) + endif + +! Get variables from geovals + call ufo_geovals_get_var(geovals, var_q, q_d) ! specific humidity + call ufo_geovals_get_var(geovals, var_prsi, prs_d) ! pressure + + missing = missing_value(missing) + allocate(x_d(1:prs_d%nval + q_d%nval)) + +! Loop through the obs, calculating the increment to the model state + obs_loop: do iobs = 1, self % nlocs + + if (hofx(iobs) /= missing) then + x_d = self % K(iobs,:) * hofx(iobs) + if (self%flip_it) then + prs_d % vals(:,iobs) = prs_d % vals(:,iobs) + x_d(prs_d%nval:1:-1) + q_d % vals(:,iobs) = q_d % vals(:,iobs) + x_d(prs_d%nval+q_d%nval:prs_d%nval+1:-1) + else + prs_d % vals(:,iobs) = prs_d % vals(:,iobs) + x_d(1:prs_d%nval) + q_d % vals(:,iobs) = q_d % vals(:,iobs) + x_d(prs_d%nval+1:prs_d%nval+q_d%nval) + end if + end if + + end do obs_loop + + deallocate(x_d) + + write(err_msg,*) "TRACE: ufo_gnssro_refmetoffice_simobs_ad: complete" + call fckit_log%info(err_msg) + + return + +end subroutine ufo_gnssro_refmetoffice_simobs_ad + +!------------------------------------------------------------------------------- +!> \brief Tidy up the variables that are used for passing information +!! +!! \details **ufo_gnssro_refmetoffice_tlad_delete** +!! * Set lengths to zero, and deallocate K-matrix +!! +!! \author Neill Bowler (Met Office) +!! +!! \date 26 May 2021 +!! +!------------------------------------------------------------------------------- + +subroutine ufo_gnssro_refmetoffice_tlad_delete(self) + + implicit none + class(ufo_gnssro_refmetoffice_tlad), intent(inout) :: self + character(len=*), parameter :: myname_="ufo_gnssro_refmetoffice_tlad_delete" + + self%nlocs = 0 + self%nlevp = 0 + self%nlevq = 0 + if (allocated(self%K)) deallocate(self%K) + self%ltraj = .false. + +end subroutine ufo_gnssro_refmetoffice_tlad_delete + +!------------------------------------------------------------------------------- +!> \brief Calculate the K-matrix used in the TL/AD +!! +!! \details **jacobian_interface** +!! * Allocate temporary arrays +!! * Call partial-derivatives code which calculates various quantities on model +!! levels +!! * Loop over each observation, doing the following +!! * If using pseudo-levels, calculate specific humidity, pressure and +!! temperature on intermediate levels, and interpolate these appropriately. +!! Then calculate the refractivity gradients for each observation, +!! interpolated from the pseudo-levels. +!! * If not using pseudo-levels, assume that refractivity varies exponentially +!! with height and calculate gradients. +!! * Calculate K-matrix from the component gradients. +!! +!! \author Neill Bowler (Met Office) +!! +!! \date 26 May 2021 +!! +!------------------------------------------------------------------------------- + +SUBROUTINE jacobian_interface(nlevp, & + nlevq, & + za, & + zb, & + p, & + q, & + pseudo_ops, & + vert_interp_ops, & + min_temp_grad, & + nobs, & + zobs, & + Kmat) + +IMPLICIT NONE + +! Subroutine arguments: +integer, intent(in) :: nlevp !< Number of pressure levels +integer, intent(in) :: nlevq !< Number of specific humidity levels +real(kind_real), intent(in) :: za(:) !< Height of the pressure levels +real(kind_real), intent(in) :: zb(:) !< Height of the specific humidity levels +real(kind_real), intent(in) :: p(:) !< Input pressure +real(kind_real), intent(in) :: q(:) !< Input specific humidity +logical, intent(in) :: vert_interp_ops !< Use log(p) for vertical interpolation? +logical, intent(in) :: pseudo_ops !< Use pseudo-levels to reduce errors? +real(kind_real), intent(in) :: min_temp_grad !< Minimum value for the vertical temperature gradient +integer, intent(in) :: nobs !< Number of observations +real(kind_real), intent(in) :: zobs(:) !< Height of the observations +real(kind_real), intent(out) :: Kmat(:,:) !< K-matrix (Jacobian) + +! Local declarations: +integer :: n ! Loop variable, observation number +integer :: i ! Loop variable +real(kind_real) :: Exner(nlevp) ! Exner pressure, calculated on model pressure levels +real(kind_real) :: Pb(nlevq) ! Pressure on model theta levels +real(kind_real) :: Tv(nlevq) ! Virtual temperature on model theta levels +real(kind_real) :: T(nlevq) ! Temperature on model theta levels +real(kind_real) :: dExtheta_dPb(nlevq,nlevq) ! Partial derivative of theta wrt pressure (on model theta levels) +real(kind_real), ALLOCATABLE :: drefob_dPob(:,:) ! Partial derivative of refractivity wrt pressure (at ob location) +real(kind_real), ALLOCATABLE :: drefob_dTob(:,:) ! Partial derivative of refractivity wrt temperature (at ob location) +real(kind_real), ALLOCATABLE :: drefob_dqob(:,:) ! Partial derivative of refractivity wrt specific humidity (at ob location) +real(kind_real), ALLOCATABLE :: dqob_dqb(:,:) ! Partial derivative of specific humidity at ob location wrt specific humidity on model theta levels +real(kind_real), ALLOCATABLE :: dTob_dTb(:,:) ! Partial derivative of temperature at ob location wrt temperature on model theta levels +real(kind_real), ALLOCATABLE :: dPob_dT(:,:) ! Partial derivative of pressure at ob location wrt temperature on model theta levels +real(kind_real), ALLOCATABLE :: dPob_dPb(:,:) ! Partial derivative of pressure at ob location wrt pressure on model theta levels +real(kind_real), ALLOCATABLE :: dPob_dP(:,:) ! Partial derivative of pressure at ob location wrt pressure on model pressure levels +real(kind_real), ALLOCATABLE :: dTob_dP(:,:) ! Partial derivative of temperature at ob location wrt pressure on model pressure levels +real(kind_real), ALLOCATABLE :: dTb_dp(:,:) ! Partial derivative of temperature on model theta levels wrt pressure on model pressure levels +real(kind_real) :: c ! Continuity constant for hydrostatic pressure +real(kind_real) :: P_ob ! Model pressure at the ob location +real(kind_real) :: q_ob ! Model specific humidity at the ob location +real(kind_real) :: T_ob ! Model temperature at the ob location +real(kind_real) :: gamma ! Vertical gradient of log(q) +real(kind_real) :: beta ! Vertical temperature gradient +real(kind_real) :: Ndry ! Component of refractivity due to dry terms +real(kind_real) :: Nwet ! Component of refractivity due to wet terms +real(kind_real) :: refrac(nlevq) ! Refractivity on model theta levels +real(kind_real) :: dEx_dP(nlevp,nlevp) ! Partial derivative of exner wrt pressure +real(kind_real) :: dPb_dp(nlevq,nlevp) ! Partial derivative of pressure on theta levels wrt pressure on pressure levels +real(kind_real) :: dTv_dExtheta(nlevq,nlevq) ! Virtual temperature divided by exner on theta levels +real(kind_real) :: dTv_dEx(nlevq,nlevp) ! Partial derivative of virtual temperature wrt exner +real(kind_real) :: dT_dTv(nlevq,nlevq) ! Partial derivative of temperature wrt virtual temperature +real(kind_real) :: dT_dq(nlevq,nlevq) ! Partial derivative of temperature wrt specific humidity +real(kind_real) :: dref_dPb(nlevq,nlevq) ! Partial derivative of refractivity wrt pressure on theta levels +real(kind_real) :: dref_dT(nlevq,nlevq) ! Partial derivative of refractivity wrt temperature +real(kind_real) :: dref_dq(nlevq,nlevq) ! Partial derivative of refractivity wrt specific humidity +real(kind_real) :: dNref_dref(nobs,nlevq) ! Partial derivative of refractivity at the ob loation wrt model refractivity +real(kind_real) :: kmatP(nobs,nlevp) ! K-matrix contribution for pressure terms +real(kind_real) :: kmatq(nobs,nlevq) ! K-matrix contribution for specific humidity terms +real(kind_real) :: Nref(nobs) ! Model refractivity at observation locations +real(kind_real) :: m1(nobs,nlevq) ! Temporary matrix product used in calculation +real(kind_real) :: m2(nobs,nlevp) ! Temporary matrix product used in calculation +real(kind_real) :: m3(nobs,nlevq) ! Temporary matrix product used in calculation +real(kind_real) :: m4(nobs,nlevq) ! Temporary matrix product used in calculation +real(kind_real) :: g_RB ! Frequently used term +real(kind_real) :: c_ZZ ! Frequently used term +real(kind_real) :: dPo_dT1 ! dP_ob / dT_below +real(kind_real) :: dPo_dTo ! dP_ob / dT_ob +real(kind_real) :: dTo_dT1 ! dT_ob / dT_below +real(kind_real) :: dPo_dbeta ! dP_ob / dbeta +real(kind_real) :: dbeta_dT1 ! dbeta / dT_below +real(kind_real) :: dTo_dbeta ! dT_ob / dbeta +real(kind_real) :: dbeta_dT2 ! dbeta / dT_above +real(kind_real) :: dPo_dc ! dP_ob / dc +real(kind_real) :: dc_dT1 ! dc / dT_below +real(kind_real) :: dc_dbeta ! dc / dbeta +real(kind_real) :: dc_dt2 ! dc / dT_above +real(kind_real) :: dPo_dP1 ! dP_ob / dP_below +real(kind_real) :: dc_dP1 ! dc / dP_below +real(kind_real) :: dc_dP2 ! dc / dP_above +real(kind_real) :: model_height_diff ! Difference in height between two model levels +real(kind_real) :: obs_height_diff ! Difference in height between the observation and the model level below it +real(kind_real) :: height_diff_ratio ! obs_height_diff / model_height_diff + +!----------------------- +! 1. Allocate/initialise matrices +!----------------------- + +IF (pseudo_ops) THEN + ALLOCATE (drefob_dPob(nobs,nobs)) + ALLOCATE (drefob_dTob(nobs,nobs)) + ALLOCATE (drefob_dqob(nobs,nobs)) + ALLOCATE (dqob_dqb(nobs,nlevq)) + ALLOCATE (dTob_dTb(nobs,nlevq)) + ALLOCATE (dPob_dT(nobs,nlevq)) + ALLOCATE (dPob_dPb(nobs,nlevq)) + ALLOCATE (dPob_dP(nobs,nlevp)) + ALLOCATE (dTob_dP(nobs,nlevp)) + ALLOCATE (dTb_dP(nlevq,nlevp)) + + drefob_dPob(:,:) = 0.0 + drefob_dTob(:,:) = 0.0 + drefob_dqob(:,:) = 0.0 + dqob_dqb(:,:) = 0.0 + dTob_dTb(:,:) = 0.0 + dPob_dT(:,:) = 0.0 + dPob_dPb(:,:) = 0.0 + dPob_dP(:,:) = 0.0 + dTob_dP(:,:) = 0.0 + dTb_dP(:,:) = 0.0 +END IF + +call ufo_refractivity_partial_derivatives(nlevP, & + nlevq, & + za, & + zb, & + P, & + q, & + vert_interp_ops, & + dT_dTv, & + dT_dq, & + dref_dPb, & + dref_dT, & + dref_dq, & + refrac, & + T, & + Pb, & + dEx_dP, & + dExtheta_dPb, & + dTv_dExtheta, & + dPb_dP, & + dTv_dEx) + + +dNref_dref(:,:) = 0.0 +kmatP(:,:) = 0.0 +kmatq(:,:) = 0.0 +kmat(:,:) = 0.0 + +!------------------------------------------------- +! 2. Calculate the partial derivatives at the observation locations +!------------------------------------------------- + +DO n = 1, nobs + i = 1 + + if (zobs(n) >= zb(nlevq) .or. zobs(n) == missing_value(zobs(n))) cycle + DO + ! Find model layer containing ob + IF (zobs(n) < zb(i + 1)) EXIT + + i = i + 1 + + END DO + + IF (pseudo_ops) THEN + ! Calculate some quantities that will be re-used + model_height_diff = zb(i + 1) - zb(i) + obs_height_diff = zobs(n) - zb(i) + height_diff_ratio = obs_height_diff / model_height_diff + + ! Interpolate P,T,q separately + IF (MIN (q(i), q(i + 1)) > 0.0) THEN + ! q varies exponentially with height + gamma = LOG (q(i) / q(i + 1)) / model_height_diff + q_ob = q(i) * EXP (-gamma * obs_height_diff) + + dqob_dqb(n,i) = (q_ob / q(i)) * (1.0 - height_diff_ratio) + dqob_dqb(n,i + 1) = (q_ob / q(i + 1)) * height_diff_ratio + + ! Assume linear variation if humidities are -ve to avoid singularity + ELSE + q_ob = q(i) + (q(i + 1) - q(i)) * height_diff_ratio + + dqob_dqb(n,i) = (zb(i + 1) - zobs(n)) / model_height_diff + dqob_dqb(n,i + 1) = height_diff_ratio + END IF + + ! T varies linearly with height + beta = (T(i + 1) - T(i)) / model_height_diff + T_ob = T(i) + beta * obs_height_diff + + dTob_dTb(n,i) = (zb(i + 1) - zobs(n)) / model_height_diff + dTob_dTb(n,i + 1) = height_diff_ratio + + ! P varies to maintain hydrostatic balance + IF (ABS (T(i + 1) - T(i)) > min_temp_grad) THEN + c = ((Pb(i + 1) / Pb(i)) * (T(i + 1)/T(i)) ** (grav / (rd * beta)) - 1.0) / model_height_diff + P_ob = (Pb(i) * (T_ob / T(i)) ** & + (-grav / (rd * beta))) * (1.0 + c * obs_height_diff) + ELSE + ! If layer is isothermal, assume exponential variation to + ! avoid singularity + P_ob = Pb(i) * EXP(LOG(Pb(i + 1) / pb(i)) * height_diff_ratio) + END IF + + ! Temporary variables to keep equations neater + g_RB = grav / (rd * beta) + c_ZZ = c + 1.0 / model_height_diff + + dPo_dT1 = (g_RB / t(i)) * P_ob + dPo_dTo = -(g_RB / t_ob) * p_ob + dTo_dT1 = dTob_dTb(n,i) + dPo_dbeta = (g_RB / beta) * LOG (T_ob / T(i)) * P_ob + dbeta_dT1 = -1.0 / model_height_diff + dTo_dbeta = obs_height_diff + dbeta_dT2 = 1.0 / model_height_diff + + ! Pressure Jacobians for hydrostatic/exponential cases + IF (ABS (T(i + 1) - T(i)) > min_temp_grad) THEN + dPo_dc = obs_height_diff * Pb(i) * (T_ob / T(i)) ** (-g_RB) + dc_dT1 = -(g_RB / (T(i))) * (c_ZZ) + dc_dbeta = -(g_RB / beta) * LOG (T(i + 1) / T(i)) * c_ZZ + dc_dT2 = (g_RB / T(i + 1)) * c_ZZ + dPo_dP1 = P_ob / Pb(i) + dPob_dT(n,i) = dPo_dT1 + dPo_dTo * dTo_dT1 + dPo_dbeta * dbeta_dT1 + dPo_dc * & + (dc_dT1 + dc_dbeta * dbeta_dT1) + dPob_dT(n,i+1) = (dPo_dbeta + dPo_dTo * dTo_dbeta + dPo_dc * dc_dbeta) * & + dbeta_dT2 + dPo_dc * dc_dT2 + + dc_dP1 = -(1.0 / Pb(i)) * c_ZZ + dc_dP2 = (1.0 / Pb(i + 1)) * c_ZZ + + dPob_dPb(n,i) = dPo_dP1 + dPo_dc * dc_dP1 + dPob_dPb(n,i + 1) = dPo_dc * dc_dP2 + ELSE + dPob_dPb(n,i) = EXP (LOG (Pb(i + 1) / Pb(i)) * height_diff_ratio) * & + (1.0 - height_diff_ratio) + dPob_dPb(n,i + 1) = (Pb(i) / Pb(i + 1)) * height_diff_ratio * & + EXP(LOG(Pb(i + 1) / Pb(i)) * height_diff_ratio) + END IF + + ! Calculate refractivity + Ndry = n_alpha * P_ob / T_ob + Nwet = n_beta * P_ob * q_ob / (T_ob ** 2 * (mw_ratio + (1.0 - mw_ratio) * q_ob)) + Nref(n) = Ndry + Nwet + + drefob_dPob(n,n) = Nref(n) / P_ob + drefob_dTob(n,n) = -(Ndry + 2.0 * Nwet) / T_ob + drefob_dqob(n,n) = n_beta * p_ob * mw_ratio / (T_ob * (mw_ratio + (1.0 - mw_ratio) * q_ob)) ** 2 + + ELSE + + ! Use simple assumption of exponentially varying refractivity + + gamma = (zb(i + 1) - zobs(n)) / (zb(i + 1) - zb(i)) + + beta = 1.0 - gamma + + Nref(n) = EXP (gamma * LOG (refrac(i)) + beta * LOG (refrac(i + 1))) + + dNref_dref(n,i) = Nref(n) * gamma / refrac(i) + + dNref_dref(n,i + 1) = Nref(n) * beta / refrac(i + 1) + END IF + +END DO + +!------------------------------------------------- +! 3. Evaluate the Kmatrix by matrix multiplication +!------------------------------------------------- + +IF (pseudo_ops) THEN + + ! Derivatives: + ! dPob/dP = (dPob/dPb * dPb/dP) + (dPob/dT * dT/dTv * dTv/dEx * dEx/dP) + dPob_dP = MATMUL (dPob_dPb, dPb_dP) + MATMUL (dPob_dT, MATMUL (dT_dTv, MATMUL (dTv_dEx, dEx_dP))) + ! dTob/dP = (dTob/dT * dT/dTv * dTv/dEx * dEx/dP) + dTob_dP = MATMUL (dTob_dTb, MATMUL (dT_dTv, MATMUL (dTv_dEx, dEx_dP))) + + ! calc K matrix for p on rho levels + ! dNob/dP = (dNob/dPob * dPob/dP) + (dNob/dTob *dTob/dP) + KmatP(:,:) = MATMUL (drefob_dPob, dPob_dP) + MATMUL (drefob_dTob, dTob_dP) + + ! calc Kmatrix for q on theta levels + ! dNob/dq = (dNob/dqob * dqob/dq) + (dNob/dTob * dTob/dT * dT/dq) + (dNob/dPob * dPob/dT * dT/dq) + Kmatq(:,:) = MATMUL (drefob_dqob, dqob_dqb) + MATMUL (MATMUL (drefob_dTob, dTob_dTb), dT_dq) + & + MATMUL (MATMUL (drefob_dPob, dPob_dT), dT_dq) + +ELSE + + ! calc K matrix for p on rho levels + ! dNob/dP = (dNob/dN * dN/dPb * dPb/dP) + .... + KmatP(:,:) = MATMUL (dNref_dref, MATMUL (dref_dPb, dPb_dP)) + + ! .... (dNob/dN * dN/dT * dT/dTv * dTv/dEx * dEx/dP) + ..... + m1(:,:) = MATMUL (dNref_dref, MATMUL (dref_dT, dT_dTv)) + m2(:,:) = MATMUL (m1, dTv_dEx) + KmatP(:,:) = KmatP(:,:) + MATMUL (m2, dEx_dP) + + ! .... (dNob/dN * dN/dT * dT/dTv * dTv/dExtheta * dExtheta/dPb * dPb/dP) + m3(:,:) = MATMUL (m1, dTv_dExtheta) + m4(:,:) = MATMUL (m3, dExtheta_dPb) + KmatP(:,:) = KmatP(:,:) + MATMUL (m4,dPb_dP) + + ! calc Kmatrix for q on theta levels + ! dNob/dq = (dNob/dN * dN/dq) + (dNob/dN * dN/dT * dT/dq) + Kmatq(:,:) = Kmatq(:,:) + MATMUL (dNref_dref, MATMUL (dref_dT, dT_dq)) + +END IF + +! Calculate the full kmatrix in correct units + +Kmat(1:nobs, 1:nlevp) = 1.0E2 * Kmatp(1:nobs, 1:nlevp) ! h/Pa + +Kmat(1:nobs, nlevp+1:nlevp+nlevq) = 1.0E-3 * Kmatq(1:nobs, 1:nlevq) ! g/kg + +!------------------------------------------------------------------------------- +! 4. Deallocate the temporary arrays +!------------------------------------------------------------------------------- + +IF (ALLOCATED (drefob_dPob)) DEALLOCATE (drefob_dPob) +IF (ALLOCATED (drefob_dTob)) DEALLOCATE (drefob_dTob) +IF (ALLOCATED (drefob_dqob)) DEALLOCATE (drefob_dqob) +IF (ALLOCATED (dqob_dqb)) DEALLOCATE (dqob_dqb) +IF (ALLOCATED (dTob_dTb)) DEALLOCATE (dTob_dTb) +IF (ALLOCATED (dPob_dT)) DEALLOCATE (dPob_dT) +IF (ALLOCATED (dPob_dPb)) DEALLOCATE (dPob_dPb) +IF (ALLOCATED (dPob_dP)) DEALLOCATE (dPob_dP) +IF (ALLOCATED (dTob_dP)) DEALLOCATE (dTob_dP) +IF (ALLOCATED (dTb_dp)) DEALLOCATE (dTb_dp) + +END SUBROUTINE jacobian_interface + +end module ufo_gnssro_refmetoffice_tlad_mod diff --git a/src/ufo/ufo_geovals_mod.F90 b/src/ufo/ufo_geovals_mod.F90 index e5fb35ab9..ef0c89bf1 100644 --- a/src/ufo/ufo_geovals_mod.F90 +++ b/src/ufo/ufo_geovals_mod.F90 @@ -403,7 +403,9 @@ subroutine ufo_geovals_reorderzdir(self, varname, zdir) type(ufo_geoval), pointer :: geoval character(max_string) :: err_msg integer:: iobs, ivar, ival, kval -logical :: do_flip = .false. !< .true. if all the ufo_geoval arrays inside geovals +logical :: do_flip !< .true. if all the ufo_geoval arrays inside geovals + +do_flip = .false. if (.not. self%linit) then call abor1_ftn("ufo_geovals_reorderzdir: geovals not allocated") diff --git a/src/ufo/utils/RefractivityCalculator.F90 b/src/ufo/utils/RefractivityCalculator.F90 index f413b1782..4a9f08ca8 100644 --- a/src/ufo/utils/RefractivityCalculator.F90 +++ b/src/ufo/utils/RefractivityCalculator.F90 @@ -143,14 +143,14 @@ SUBROUTINE ufo_calculate_refractivity (nlevP, & DO i = 1, nlevq IF (P(i) == missing_value(P(i))) THEN ! pressure missing refracerr = .TRUE. - WRITE(message, *) RoutineName, "Input pressure missing", i + WRITE(message, *) RoutineName, " Input pressure missing", i CALL fckit_log % warning(message) EXIT END IF IF (P(i) - P(i + 1) < 0.0) THEN ! or non-monotonic pressure refracerr = .TRUE. - WRITE(message, *) RoutineName, "Input pressure non-monotonic", i + WRITE(message, *) RoutineName, " Input pressure non-monotonic", i, P(i), P(i+1) CALL fckit_log % warning(message) EXIT END IF @@ -158,7 +158,7 @@ SUBROUTINE ufo_calculate_refractivity (nlevP, & IF (ANY (P(:) <= 0.0)) THEN ! pressure zero or negative refracerr = .TRUE. - WRITE(message, *) RoutineName, "Input pressure not physical" + WRITE(message, *) RoutineName, " Input pressure not physical" CALL fckit_log % warning(message) END IF @@ -331,13 +331,12 @@ SUBROUTINE ufo_refractivity_kmat(nlevP, & REAL(kind_real), OPTIONAL, INTENT(OUT), ALLOCATABLE :: refractivity(:) !< Calculated refractivity ! Local declarations: -CHARACTER(len=*), PARAMETER :: RoutineName = "ufo_refractivity_kmat" INTEGER :: i INTEGER :: counter -REAL(kind_real) :: Exner(nlevP) REAL(kind_real) :: Pb(nlevq) REAL(kind_real) :: T_virtual(nlevq) REAL(kind_real) :: T(nlevq) +REAL(kind_real) :: refrac(nlevq) ! Refractivity on model levels REAL(kind_real) :: Extheta REAL(kind_real) :: pwt1 REAL(kind_real) :: pwt2 @@ -352,6 +351,7 @@ SUBROUTINE ufo_refractivity_kmat(nlevP, & REAL(kind_real) :: dT_dq(nlevq,nlevq) REAL(kind_real) :: dref_dPb(nlevq,nlevq) REAL(kind_real) :: dref_dT(nlevq,nlevq) +REAL(kind_real) :: dref_dq_model(nlevq,nlevq) ! Gradient of refractivity wrt specific humidity on model levels REAL(kind_real) :: m1(nRefLevels,nlevq) REAL(kind_real) :: m2(nRefLevels,nlevP) REAL(kind_real) :: m3(nRefLevels,nlevq) @@ -427,107 +427,28 @@ SUBROUTINE ufo_refractivity_kmat(nlevP, & ! 1. Initialise matrices !----------------------- -dPb_dP_local(:,:) = 0.0 -dExtheta_dPb(:,:) = 0.0 -dEx_dP(:,:) = 0.0 -dTv_dExtheta(:,:) = 0.0 -dTv_dEx(:,:) = 0.0 -dT_dTv(:,:) = 0.0 -dT_dq(:,:) = 0.0 -dref_dpb(:,:) = 0.0 -dref_dT(:,:) = 0.0 -dref_dq(:,:) = 0.0 dref_dp(:,:) = 0.0 -! Calculate exner on rho levels. - -Exner(:) = (P(:) / Pref) ** rd_over_cp - -DO i = 1,nlevp - - dEx_dP(i,i) = rd_over_cp / Pref * (P(i) / Pref) ** (rd_over_cp - 1.0) - -END DO - -!---------------------------------------------- -! 2. Calculate the refractivity on the temperature/theta levels -!---------------------------------------------- - -DO i = 1, nlevq - - ! Calc. pressure on b levels - - pwt1 = (za(i + 1) - zb(i)) / (za(i + 1) - za(i)) - - pwt2 = 1.0 - pwt1 - - ! calculate the pressure on the theta level. - IF (vert_interp_ops) THEN - Pb(i) = EXP (pwt1 * LOG (P(i)) + pwt2 * LOG (P(i + 1))) - - dPb_dP_local(i,i) = Pb(i) * pwt1 / P(i) - dPb_dP_local(i,i + 1) = Pb(i) * pwt2 / P(i + 1) - ELSE - ! Assume Exner varies linearly with height - Pb(i) = Pref * (pwt1 * (P(i) / Pref) ** rd_over_cp + pwt2 * (P(i + 1) / Pref) ** rd_over_cp) ** (1.0 / rd_over_cp) - - dPb_dP_local(i,i) = pwt1 * (pwt1 * (P(i) / Pref) ** rd_over_cp + pwt2 * & - (P(i + 1) / Pref) ** rd_over_cp) ** (1.0 / rd_over_cp - 1.0) * (P(i) / Pref) ** (rd_over_cp - 1.0) - dPb_dP_local(i,i + 1) = pwt2 * (pwt1 * (P(i) / Pref) ** rd_over_cp + pwt2 * & - (P(i + 1) / Pref) ** rd_over_cp) ** (1.0 / rd_over_cp - 1.0) * (P(i + 1) / Pref) ** (rd_over_cp-1.0) - END IF - - ! calculate Exner on the theta level. - - Extheta = (Pb(i) / Pref) ** rd_over_cp - - dExtheta_dPb(i,i) = rd_over_cp * (Pb(i) ** (rd_over_cp - 1.0)) / (Pref ** rd_over_cp) - - ! Calculate mean layer T_virtual on staggered vertical levels - - T_virtual(i) = grav * (za(i + 1) - za(i)) * Extheta / (Cp * (Exner(i) - Exner(i + 1))) - - dTv_dExtheta(i,i) = T_virtual(i) / Extheta - - dTv_dEx(i,i) = -T_virtual(i) / (Exner(i) - Exner(i + 1)) - - dTv_dEx(i,i + 1) = T_virtual(i) / (Exner(i) - Exner(i + 1)) - - IF (i > nlevq) THEN - - T(i) = T_virtual(i) - - dT_dTv(i,i) = 1.0 - - ! no wet component - - Nwet = 0.0 - - ELSE - - T(i) = T_virtual(i) / (1.0 + C_virtual * q(i)) - - dT_dTv(i,i) = 1.0 / (1.0 + C_virtual * q(i)) - - dT_dq(i,i) = -C_virtual * T(i) / (1.0 + C_virtual * q(i)) - - ! wet component - - Nwet = n_beta * Pb(i) * q(i) / (T(i) ** 2 * (mw_ratio + (1.0 - mw_ratio) * q(i))) - - dref_dq(i,i) = n_beta * Pb(i) * mw_ratio / (T(i) * (mw_ratio + (1.0 - mw_ratio) * q(i))) ** 2 - - END IF - - Ndry = n_alpha * Pb(i) / T(i) - - if (PRESENT(refractivity) .AND. .NOT. pseudo_ops) refractivity(i) = Ndry + Nwet - - dref_dPb(i,i) = (Ndry + Nwet) / Pb(i) - - dref_dT(i,i) = -(Ndry + 2.0 * Nwet) / T(i) - -END DO +call ufo_refractivity_partial_derivatives(nlevP, & + nlevq, & + za, & + zb, & + P, & + q, & + vert_interp_ops, & + dT_dTv, & + dT_dq, & + dref_dPb, & + dref_dT, & + dref_dq_model, & + refrac, & + T, & + Pb, & + dEx_dP, & + dExtheta_dPb, & + dTv_dExtheta, & + dPb_dP_local, & + dTv_dEx) IF (pseudo_ops) THEN !----------------------------------! @@ -548,7 +469,7 @@ SUBROUTINE ufo_refractivity_kmat(nlevP, & dref_dPpseudo(i,i) = dref_dPb(counter,counter) dref_dTpseudo(i,i) = dref_dT(counter,counter) - dref_dqpseudo(i,i) = dref_dq(counter,counter) + dref_dqpseudo(i,i) = dref_dq_model(counter,counter) dPpseudo_dPb(i,counter) = 1.0 dTpseudo_dTb(i,counter) = 1.0 @@ -667,6 +588,8 @@ SUBROUTINE ufo_refractivity_kmat(nlevP, & ! Normal model levels ELSE + if (PRESENT(refractivity)) refractivity = refrac + !------------------------------------------------- ! 3. Evaluate the Kmatrix by matrix multiplication !------------------------------------------------- @@ -687,7 +610,7 @@ SUBROUTINE ufo_refractivity_kmat(nlevP, & ! calc Kmatrix for q on theta levels ! dNmod/dq = (dNmod/dq) + (dNmod/dT*dT/dq) - dref_dq(:,:) = dref_dq(:,:) + MATMUL (dref_dT, dT_dq) + dref_dq(:,:) = dref_dq_model(:,:) + MATMUL (dref_dT, dT_dq) END IF @@ -710,4 +633,161 @@ SUBROUTINE ufo_refractivity_kmat(nlevP, & END SUBROUTINE ufo_refractivity_kmat +!------------------------------------------------------------------------------- +!> \brief Calculate some partial derivatives of refractivity on model levels +!! +!! \details **ufo_refractivity_partial_derivatives** +!! * Calculate the pressure on model theta levels +!! * Calculate exner on model theta level +!! * Calculate mean layer T_virtual, and then various partial derivatives +!! +!! \author Neill Bowler (Met Office) +!! +!! \date 26 May 2021 +!! +!------------------------------------------------------------------------------- + +subroutine ufo_refractivity_partial_derivatives(nlevP, & + nlevq, & + za, & + zb, & + P, & + q, & + vert_interp_ops, & + dT_dTv, & + dT_dq, & + dref_dPb, & + dref_dT, & + dref_dq, & + refractivity, & + T, & + Pb, & + dEx_dP, & + dExtheta_dPb, & + dTv_dExtheta, & + dPb_dP_local, & + dTv_dEx) + +IMPLICIT NONE + +! Subroutine arguments: +INTEGER, INTENT(IN) :: nlevP !< no. of pressure levels +INTEGER, INTENT(IN) :: nlevq !< no. of specific humidity levels +REAL(kind_real), INTENT(IN) :: za(nlevp) !< Height of the pressure levels +REAL(kind_real), INTENT(IN) :: zb(nlevq) !< Height of the specific humidity levels +REAL(kind_real), INTENT(IN) :: P(nlevp) !< Pressure +REAL(kind_real), INTENT(IN) :: q(nlevq) !< Specific humidity +LOGICAL, INTENT(IN) :: vert_interp_ops !< Whether to interpolate vertically using exner or ln(p) +REAL(kind_real), INTENT(OUT) :: dT_dTv(nlevq,nlevq) !< Partial derivative of temperature wrt virtual temperature +REAL(kind_real), INTENT(OUT) :: dT_dq(nlevq,nlevq) !< Partial derivative of temperature wrt specific humidity +REAL(kind_real), INTENT(OUT) :: dref_dPb(nlevq,nlevq) !< Partial derivative of refractivity wrt pressure on model theta levels +REAL(kind_real), INTENT(OUT) :: dref_dT(nlevq,nlevq) !< Partial derivative of refractivity wrt temperature +REAL(kind_real), INTENT(OUT) :: dref_dq(nlevq,nlevq) !< Partial derivative of refractivity wrt specific humidity +REAL(kind_real), INTENT(OUT) :: refractivity(nlevq) !< Calculated refractivity +REAL(kind_real), INTENT(OUT) :: T(nlevq) !< Calculated temperature +REAL(kind_real), INTENT(OUT) :: Pb(nlevq) !< Pressure on model theta levels +REAL(kind_real), INTENT(OUT) :: dEx_dP(nlevP,nlevP) !< Partial derivative of exner wrt pressure +REAL(kind_real), INTENT(OUT) :: dExtheta_dPb(nlevq,nlevq) !< Partial derivative of refractivity wrt pressure (at ob location) +REAL(kind_real), INTENT(OUT) :: dTv_dExtheta(nlevq,nlevq) !< Virtual temperature divided by exner on theta levels +REAL(kind_real), INTENT(OUT) :: dPb_dP_local(nlevq,nlevP) !< Partial derivative of pressure on theta levels wrt pressure on pressure levels +REAL(kind_real), INTENT(OUT) :: dTv_dEx(nlevq,nlevP) !< Partial derivative of virtual temperature wrt exner + +! Local declarations: +INTEGER :: i ! Loop variable +REAL(kind_real) :: Exner(nlevP) ! Exner on model pressure levels +REAL(kind_real) :: T_virtual(nlevq) ! Virtual temperature +REAL(kind_real) :: Extheta ! Exner on model theta levels +REAL(kind_real) :: pwt1 ! Weight given to the lower model level in interpolation +REAL(kind_real) :: pwt2 ! Weight given to the upper model level in interpolation +REAL(kind_real) :: Ndry ! Contribution to refractivity from dry terms +REAL(kind_real) :: Nwet ! Contribution to refractivity from wet terms + +!----------------------- +! 1. Initialise matrices +!----------------------- + +dPb_dP_local(:,:) = 0.0 +dExtheta_dPb(:,:) = 0.0 +dTv_dExtheta(:,:) = 0.0 +dTv_dEx(:,:) = 0.0 +dT_dTv(:,:) = 0.0 +dT_dq(:,:) = 0.0 +dref_dpb(:,:) = 0.0 +dref_dT(:,:) = 0.0 +dref_dq(:,:) = 0.0 +dEx_dP(:,:) = 0.0 + +! Calculate exner on rho levels. +Exner(:) = (P(:) / Pref) ** rd_over_cp +DO i = 1,nlevp + dEx_dP(i,i) = rd_over_cp / Pref * (P(i) / Pref) ** (rd_over_cp - 1.0) +END DO + +!---------------------------------------------- +! 2. Calculate the refractivity on the temperature/theta levels +!---------------------------------------------- + +DO i = 1, nlevq + + pwt1 = (za(i + 1) - zb(i)) / (za(i + 1) - za(i)) + + pwt2 = 1.0 - pwt1 + + ! calculate the pressure on the theta level. + IF (vert_interp_ops) THEN + Pb(i) = EXP (pwt1 * LOG (P(i)) + pwt2 * LOG (P(i + 1))) + + dPb_dP_local(i,i) = Pb(i) * pwt1 / P(i) + dPb_dP_local(i,i + 1) = Pb(i) * pwt2 / P(i + 1) + ELSE + ! Assume Exner varies linearly with height + Pb(i) = Pref * (pwt1 * (P(i) / Pref) ** rd_over_cp + pwt2 * (P(i + 1) / Pref) ** rd_over_cp) ** (1.0 / rd_over_cp) + + dPb_dP_local(i,i) = pwt1 * (pwt1 * (P(i) / Pref) ** rd_over_cp + pwt2 * & + (P(i + 1) / Pref) ** rd_over_cp) ** (1.0 / rd_over_cp - 1.0) * (P(i) / Pref) ** (rd_over_cp - 1.0) + dPb_dP_local(i,i + 1) = pwt2 * (pwt1 * (P(i) / Pref) ** rd_over_cp + pwt2 * & + (P(i + 1) / Pref) ** rd_over_cp) ** (1.0 / rd_over_cp - 1.0) * (P(i + 1) / Pref) ** (rd_over_cp-1.0) + END IF + + ! calculate Exner on the theta level. + + Extheta = (Pb(i) / Pref) ** rd_over_cp + + dExtheta_dPb(i,i) = rd_over_cp * (Pb(i) ** (rd_over_cp - 1.0)) / (Pref ** rd_over_cp) + + ! Calculate mean layer T_virtual on staggered vertical levels + + T_virtual(i) = grav * (za(i + 1) - za(i)) * Extheta / (Cp * (Exner(i) - Exner(i + 1))) + + dTv_dExtheta(i,i) = T_virtual(i) / Extheta + + dTv_dEx(i,i) = -T_virtual(i) / (Exner(i) - Exner(i + 1)) + + dTv_dEx(i,i + 1) = T_virtual(i) / (Exner(i) - Exner(i + 1)) + + T(i) = T_virtual(i) / (1.0 + C_virtual * q(i)) + + dT_dTv(i,i) = 1.0 / (1.0 + C_virtual * q(i)) + + dT_dq(i,i) = -C_virtual * T(i) / (1.0 + C_virtual * q(i)) + + ! wet component + + Nwet = n_beta * Pb(i) * q(i) / (T(i) ** 2 * (mw_ratio + (1.0 - mw_ratio) * q(i))) + + dref_dq(i,i) = n_beta * Pb(i) * mw_ratio / (T(i) * (mw_ratio + (1.0 - mw_ratio) * q(i))) ** 2 + + Ndry = n_alpha * Pb(i) / T(i) + + refractivity(i) = Ndry + Nwet + + dref_dPb(i,i) = (Ndry + Nwet) / Pb(i) + + dref_dT(i,i) = -(Ndry + 2.0 * Nwet) / T(i) + +END DO + +end subroutine ufo_refractivity_partial_derivatives + + end module ufo_utils_refractivity_calculator diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 97573b212..3fa23358c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1347,13 +1347,20 @@ ecbuild_add_test( TARGET test_ufo_linopr_gnssroBendMetOffice_nopseudo DEPENDS test_ObsOperatorTLAD.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_opr_gnssroRefMetOffice_pseudo +ecbuild_add_test( TARGET test_ufo_opr_gnssroRefMetOffice COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x ARGS "testinput/gnssrorefmetoffice.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsOperator.x TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_linopr_gnssroRefMetOffice + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperatorTLAD.x + ARGS "testinput/gnssrorefmetoffice.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsOperatorTLAD.x + TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) + if( ${ropp-ufo_FOUND} ) ecbuild_add_test( TARGET test_ufo_opr_gnssroBndROPP1D COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x diff --git a/test/testinput/gnssrorefmetoffice.yaml b/test/testinput/gnssrorefmetoffice.yaml index 416f7ab47..403a6e117 100644 --- a/test/testinput/gnssrorefmetoffice.yaml +++ b/test/testinput/gnssrorefmetoffice.yaml @@ -23,7 +23,7 @@ observations: norm ref: MetOfficeHofX tolerance: 1.0e-5 linear obs operator test: - coef TL: 1.0e-3 + coef TL: 1.0e-4 iterations TL: 10 tolerance TL: 1.0e-14 tolerance AD: 1.0e-14 From 57a44d04feccbe4225f1ba62dbf7831c5342d5fd Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Thu, 15 Jul 2021 20:22:30 +0100 Subject: [PATCH 45/98] Observation operator for average profile T and RH (#1338) * modifications to algorithm * Modifications to algorithm; fill H(x) * refactor to use data handling class * improve documentation * coding norms * more unit tests * more error checking; improved error messages * coding norms * coding norms * add extra tests to trigger exceptions * Trigger CI * clear stringstreams correctly * improve comments; change function name * no longer clear stringstream * allow for different vertical coordinate; test for T * add test with RH * add variables option * coding norms * Trigger CI * correct include path * update doxygen comments * change 'location' to 'level' in a couple of places * correct treatment of operator variable indices * Trigger CI Co-authored-by: Ryan Honeyager --- src/ufo/profile/ObsProfileAverage.cc | 2 +- src/ufo/profile/ObsProfileAverage.h | 22 ++-- src/ufo/profile/ObsProfileAverageData.cc | 49 +++++--- src/ufo/profile/ObsProfileAverageData.h | 16 +++ src/ufo/profile/ObsProfileAverageParameters.h | 13 +- .../profileconsistencychecks_opr_average.yaml | 118 ++++++++++++++++-- 6 files changed, 179 insertions(+), 41 deletions(-) diff --git a/src/ufo/profile/ObsProfileAverage.cc b/src/ufo/profile/ObsProfileAverage.cc index b8baf7764..b7aa6eff7 100644 --- a/src/ufo/profile/ObsProfileAverage.cc +++ b/src/ufo/profile/ObsProfileAverage.cc @@ -70,7 +70,7 @@ void ObsProfileAverage::simulateObs(const GeoVaLs & gv, ioda::ObsVector & ovec, data_.getSlantPathLocations(locsOriginal, locsExtended, gv); // Fill H(x) vector for each variable. - for (size_t jvar = 0; jvar < ovec.nvars(); ++jvar) { + for (int jvar : data_.operatorVarIndices()) { const auto& variable = ovec.varnames().variables()[jvar]; // Number of levels for this variable. const std::size_t nlevs_var = gv.nlevs(variable); diff --git a/src/ufo/profile/ObsProfileAverage.h b/src/ufo/profile/ObsProfileAverage.h index 8e515e226..62c560d2b 100644 --- a/src/ufo/profile/ObsProfileAverage.h +++ b/src/ufo/profile/ObsProfileAverage.h @@ -40,11 +40,11 @@ namespace ufo { /// /// This observation operator produces H(x) vectors which correspond to vertically-averaged /// profiles. The algorithm determines the locations at which reported-level profiles -/// intersect each model pressure level (based on the air_pressure_levels GeoVaL). -/// This is done by stepping through the observation locations from the lowest-altitude value -/// upwards. For each model level, the location of the observation whose pressure is larger than, -/// and closest to, the model pressure is recorded. -/// +/// intersect each model pressure level. The intersections are found by stepping through the +/// observation locations from the lowest-altitude value upwards. For each model level, +/// the location of the observation whose pressure is larger than, and closest to, the model +/// pressure is recorded. The \p parameter controls the model pressure +/// GeoVaLs that are used in this procedure. /// If there are no observations in a model level, which can occur for (e.g.) sondes reporting in /// low-frequency TAC format, the location corresponding to the last filled level is used. /// (If there are some model levels closer to the surface than the lowest-altitude observation, @@ -52,7 +52,7 @@ namespace ufo { /// /// This procedure is iterated multiple times in order to account for the fact that model pressures /// can be slanted close to the Earth's surface. -/// The number of iterations is configured with the \p numIntersectionIterations parameter. +/// The number of iterations is configured with the \p parameter. /// /// Having obtained the profile boundaries, values of model pressure and any simulated variables /// are obtained as in the locations that were determined in the procedure above. @@ -73,9 +73,15 @@ namespace ufo { /// values across each layer is not performed; a single model value is used in each case. /// This follows what is used in OPS. Alternatives could be considered in the future. /// -/// A comparison with OPS is be performed if the option \p compareWithOPS is set to true. +/// A comparison with OPS is be performed if the option \p is set to true. /// This checks values of the locations and pressure values associated with the slant path. /// All other comparisons are performed with the standard 'vector ref' option in the yaml file. +/// +/// This operator also accepts an optional `variables` parameter, which controls which ObsSpace +/// variables will be simulated. This option should only be set if this operator is used as a +/// component of the Composite operator. If `variables` is not set, the operator will simulate +/// all ObsSpace variables. Please see the documentation of the Composite operator for further +/// details. class ObsProfileAverage : public ObsOperatorBase, private util::ObjectCounter { public: @@ -88,6 +94,8 @@ class ObsProfileAverage : public ObsOperatorBase, const oops::Variables & requiredVars() const override { return data_.requiredVars(); } + oops::Variables simulatedVars() const override { return data_.simulatedVars(); } + private: void print(std::ostream &) const override; diff --git a/src/ufo/profile/ObsProfileAverageData.cc b/src/ufo/profile/ObsProfileAverageData.cc index a3727e593..784334c12 100644 --- a/src/ufo/profile/ObsProfileAverageData.cc +++ b/src/ufo/profile/ObsProfileAverageData.cc @@ -9,6 +9,7 @@ #include "oops/util/missingValues.h" #include "ufo/profile/ObsProfileAverageData.h" +#include "ufo/utils/OperatorUtils.h" // for getOperatorVariables namespace ufo { @@ -37,15 +38,17 @@ namespace ufo { if (!odb_.has("MetaData", "air_pressure")) throw eckit::UserError("air_pressure@MetaData not present", Here()); - // Add air_pressure_levels to the list of variables used in this operator. + // Set up configuration options. + options_.validateAndDeserialize(config); + + // Add model air pressure to the list of variables used in this operator. // This GeoVaL is used to determine the slant path locations. - requiredVars_ += oops::Variables({"air_pressure_levels"}); + modelVerticalCoord_ = options_.modelVerticalCoordinate; + requiredVars_ += oops::Variables({modelVerticalCoord_}); // Add any simulated variables to the list of variables used in this operator. - requiredVars_ += odb_.obsvariables(); - - // Set up configuration options. - options_.validateAndDeserialize(config); + getOperatorVariables(config, odb_.obsvariables(), operatorVars_, operatorVarIndices_); + requiredVars_ += operatorVars_; // If required, set up vectors for OPS comparison. if (options_.compareWithOPS.value()) @@ -56,6 +59,14 @@ namespace ufo { return requiredVars_; } + const oops::Variables & ObsProfileAverageData::simulatedVars() const { + return operatorVars_; + } + + const std::vector & ObsProfileAverageData::operatorVarIndices() const { + return operatorVarIndices_; + } + std::vector ObsProfileAverageData::getSlantPathLocations (const std::vector & locsOriginal, const std::vector & locsExtended, @@ -68,8 +79,8 @@ namespace ufo { odb_.get_db("MetaData", "air_pressure", pressure_obs); // Set up GeoVaLs and H(x) vectors. - // Number of levels for air_pressure_levels. - const std::size_t nlevs_p = gv.nlevs("air_pressure_levels"); + // Number of levels for model pressure. + const std::size_t nlevs_p = gv.nlevs(modelVerticalCoord_); // Vector storing location for each level along the slant path. // Initially the first location in the profile is used everywhere. std::vector slant_path_location(nlevs_p, locsOriginal.front()); @@ -90,7 +101,7 @@ namespace ufo { for (std::size_t mlev = 0; mlev < nlevs_p; ++mlev) { for (int iter = 0; iter <= itermax; ++iter) { // Get the GeoVaL that corresponds to the current slanted profile location. - gv.getAtLocation(pressure_gv, "air_pressure_levels", jlocslant); + gv.getAtLocation(pressure_gv, modelVerticalCoord_, jlocslant); // Define an iteration-specific location that is initialised to the // current slanted profile location. std::size_t jlociter = jlocslant; @@ -155,21 +166,21 @@ namespace ufo { slant_pressure_ref_profile.push_back(slant_pressure_ref_[loc]); } std::stringstream errmsg; - for (std::size_t jloccomp = 0; jloccomp < locsExtended.size(); ++jloccomp) { - if (slant_path_location[jloccomp] != slant_path_location_ref_profile[jloccomp]) { - errmsg << "Mismatch for slant_path_location, location = " << jloccomp + for (std::size_t mlev = 0; mlev < locsExtended.size(); ++mlev) { + if (slant_path_location[mlev] != slant_path_location_ref_profile[mlev]) { + errmsg << "Mismatch for slant_path_location, level = " << mlev << " (this code, OPS): " - << slant_path_location[jloccomp] << ", " - << slant_path_location_ref_profile[jloccomp]; + << slant_path_location[mlev] << ", " + << slant_path_location_ref_profile[mlev]; throw eckit::BadValue(errmsg.str(), Here()); } - if (!oops::is_close_relative(slant_pressure[jloccomp], - slant_pressure_ref_profile[jloccomp], + if (!oops::is_close_relative(slant_pressure[mlev], + slant_pressure_ref_profile[mlev], 1e-5f)) { - errmsg << "Mismatch for slant_pressure, location = " << jloccomp + errmsg << "Mismatch for slant_pressure, level = " << mlev << " (this code, OPS): " - << slant_pressure[jloccomp] << ", " - << slant_pressure_ref_profile[jloccomp]; + << slant_pressure[mlev] << ", " + << slant_pressure_ref_profile[mlev]; throw eckit::BadValue(errmsg.str(), Here()); } } diff --git a/src/ufo/profile/ObsProfileAverageData.h b/src/ufo/profile/ObsProfileAverageData.h index e552584e8..b38820b8b 100644 --- a/src/ufo/profile/ObsProfileAverageData.h +++ b/src/ufo/profile/ObsProfileAverageData.h @@ -9,6 +9,7 @@ #define UFO_PROFILE_OBSPROFILEAVERAGEDATA_H_ #include +#include #include #include "ioda/ObsSpace.h" @@ -33,6 +34,12 @@ namespace ufo { /// Return required variables for the operator. const oops::Variables & requiredVars() const; + /// Return simulated variables for the operator. + const oops::Variables & simulatedVars() const; + + /// Return operator variable indices for the operator. + const std::vector & operatorVarIndices() const; + /// Get slant path locations. This determines, for each model level, the location that /// corresponds to the intersection of the observed profile with that level. std::vector getSlantPathLocations(const std::vector & locsOriginal, @@ -61,9 +68,18 @@ namespace ufo { /// Options for this observation operator. ObsProfileAverageParameters options_; + /// Name of model vertical coordinate. + std::string modelVerticalCoord_; + /// Required variables. oops::Variables requiredVars_; + /// Operator variables. + oops::Variables operatorVars_; + + /// Indices of operator variables. + std::vector operatorVarIndices_; + /// Reference values of slant path locations. std::vector slant_path_location_ref_; diff --git a/src/ufo/profile/ObsProfileAverageParameters.h b/src/ufo/profile/ObsProfileAverageParameters.h index ef70599c8..54a777bc6 100644 --- a/src/ufo/profile/ObsProfileAverageParameters.h +++ b/src/ufo/profile/ObsProfileAverageParameters.h @@ -16,7 +16,8 @@ #include "oops/util/parameters/Parameters.h" #include "oops/util/parameters/RequiredParameter.h" -#include "ufo/profile/DataHandlerParameters.h" +#include "ufo/filters/Variable.h" +#include "ufo/utils/parameters/ParameterTraitsVariable.h" namespace ufo { @@ -28,13 +29,19 @@ class ObsProfileAverageParameters : public oops::Parameters { /// Operator name. In future will be moved to a base class for parameters of all ObsOperators. oops::OptionalParameter name{"name", this}; + /// List of variables to be used by this operator. + oops::Parameter> variables{"variables", {}, this}; + + /// Name of model vertical coordinate. + oops::RequiredParameter modelVerticalCoordinate{"vertical coordinate", this}; + /// Number of iterations that are used to find the intersection between /// the observed profile and each model level. oops::Parameter numIntersectionIterations - {"numIntersectionIterations", 3, this, {oops::minConstraint(1)}}; + {"number of intersection iterations", 3, this, {oops::minConstraint(1)}}; /// Perform comparisons of auxiliary variables with OPS? - oops::Parameter compareWithOPS{"compareWithOPS", false, this}; + oops::Parameter compareWithOPS{"compare with OPS", false, this}; }; } // namespace ufo diff --git a/test/testinput/profileconsistencychecks_opr_average.yaml b/test/testinput/profileconsistencychecks_opr_average.yaml index c2a305015..936b2886c 100644 --- a/test/testinput/profileconsistencychecks_opr_average.yaml +++ b/test/testinput/profileconsistencychecks_opr_average.yaml @@ -16,8 +16,9 @@ observations: average profiles onto model levels: 71 obs operator: name: ProfileAverage - compareWithOPS: true - numIntersectionIterations: 3 + vertical coordinate: "air_pressure_levels" + number of intersection iterations: 3 + compare with OPS: true geovals: filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 linear obs operator test: @@ -41,8 +42,9 @@ observations: average profiles onto model levels: 71 obs operator: name: ProfileAverage - compareWithOPS: true - numIntersectionIterations: 3 + vertical coordinate: "air_pressure_levels" + number of intersection iterations: 3 + compare with OPS: true geovals: filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 linear obs operator test: @@ -52,6 +54,94 @@ observations: vector ref: MetOfficeHofX tolerance: 1.0e-06 +# Standard case: air_temperature. +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_obs.nc4 + obsgrouping: + group variables: [ "station_id" ] + sort variable: "air_pressure" + sort order: "descending" + simulated variables: [air_temperature] + extension: + average profiles onto model levels: 71 + obs operator: + name: ProfileAverage + vertical coordinate: "air_pressure" + number of intersection iterations: 3 + compare with OPS: false + geovals: + filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 + linear obs operator test: + coef TL: 0.1 + tolerance TL: 1.0e-11 + tolerance AD: 1.0e-13 + vector ref: MetOfficeHofX + tolerance: 1.0e-05 + +# Standard case: air_temperature and relative_humidity. +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_obs.nc4 + obsgrouping: + group variables: [ "station_id" ] + sort variable: "air_pressure" + sort order: "descending" + simulated variables: [air_temperature, relative_humidity] + extension: + average profiles onto model levels: 71 + obs operator: + name: ProfileAverage + vertical coordinate: "air_pressure" + number of intersection iterations: 3 + compare with OPS: false + geovals: + filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 + linear obs operator test: + coef TL: 0.1 + tolerance TL: 1.0e-11 + tolerance AD: 1.0e-13 + vector ref: MetOfficeHofX + tolerance: 1.0e-05 + +# Standard case: eastward_wind, northward_wind, air_temperature, and relative_humidity. +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_obs.nc4 + obsgrouping: + group variables: [ "station_id" ] + sort variable: "air_pressure" + sort order: "descending" + simulated variables: [eastward_wind, northward_wind, air_temperature, relative_humidity] + extension: + average profiles onto model levels: 71 + obs operator: + name: Composite + components: + - name: ProfileAverage + variables: + - name: eastward_wind + - name: northward_wind + vertical coordinate: "air_pressure_levels" + number of intersection iterations: 3 + - name: ProfileAverage + variables: + - name: air_temperature + - name: relative_humidity + vertical coordinate: "air_pressure" + number of intersection iterations: 3 + geovals: + filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 + linear obs operator test: + coef TL: 0.1 + tolerance TL: 1.0e-11 + tolerance AD: 1.0e-13 + vector ref: MetOfficeHofX + tolerance: 1.0e-04 + # The group variable is not used, throwing an exception. - obs space: name: Radiosonde @@ -63,6 +153,7 @@ observations: expect constructor to throw exception with message: Group variables configuration is empty obs operator: name: ProfileAverage + vertical coordinate: "air_pressure_levels" geovals: filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 linear obs operator test: @@ -86,6 +177,7 @@ observations: expect constructor to throw exception with message: Sort variable must be air_pressure obs operator: name: ProfileAverage + vertical coordinate: "air_pressure_levels" geovals: filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 linear obs operator test: @@ -110,6 +202,7 @@ observations: expect constructor to throw exception with message: Profiles must be sorted in descending order obs operator: name: ProfileAverage + vertical coordinate: "air_pressure_levels" geovals: filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 linear obs operator test: @@ -131,7 +224,8 @@ observations: simulated variables: [eastward_wind] obs operator: name: ProfileAverage - compareWithOPS: true + vertical coordinate: "air_pressure_levels" + compare with OPS: true expect constructor to throw exception with message: The extended obs space has not been produced geovals: filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 @@ -156,9 +250,10 @@ observations: average profiles onto model levels: 71 obs operator: name: ProfileAverage - compareWithOPS: true - numIntersectionIterations: 3 - expect simulateObs to throw exception with message: "Mismatch for slant_path_location, location = 0 (this code, OPS): 2659, 0" + vertical coordinate: "air_pressure_levels" + number of intersection iterations: 3 + compare with OPS: true + expect simulateObs to throw exception with message: "Mismatch for slant_path_location, level = 0 (this code, OPS): 2659, 0" geovals: filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 linear obs operator test: @@ -182,9 +277,10 @@ observations: average profiles onto model levels: 71 obs operator: name: ProfileAverage - compareWithOPS: true - numIntersectionIterations: 3 - expect simulateObs to throw exception with message: "Mismatch for slant_pressure, location = 0 (this code, OPS): 100958, 0" + vertical coordinate: "air_pressure_levels" + number of intersection iterations: 3 + compare with OPS: true + expect simulateObs to throw exception with message: "Mismatch for slant_pressure, level = 0 (this code, OPS): 100958, 0" geovals: filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 linear obs operator test: From 2f3e97b045a5f6370a2b3d680588a15a112c7306 Mon Sep 17 00:00:00 2001 From: Greg Thompson <63315545+gthompsnJCSDA@users.noreply.github.com> Date: Thu, 15 Jul 2021 14:14:57 -0600 Subject: [PATCH 46/98] Bugfix: rapidly exit the simulate-obs subroutine if a CPU has no observations to simulate (#1351) * need to return quickly if no obs * ensure wind factor is bounded zero to one Co-authored-by: gthompsnJCSDA Co-authored-by: Ryan Honeyager --- src/ufo/atmsfcinterp/ufo_atmsfc_mod.F90 | 2 +- src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ufo/atmsfcinterp/ufo_atmsfc_mod.F90 b/src/ufo/atmsfcinterp/ufo_atmsfc_mod.F90 index cd77beaea..2fe999f83 100644 --- a/src/ufo/atmsfcinterp/ufo_atmsfc_mod.F90 +++ b/src/ufo/atmsfcinterp/ufo_atmsfc_mod.F90 @@ -148,7 +148,7 @@ subroutine sfc_wind_fact_gsi(u, v, tsen, q, psfc, prsi1, prsi2,& fm = fm - pm fh = fh - ph fm10 = fm10 - pm10 - f10m = fm10 / fm + f10m = max(zero, min(fm10/fm, one)) return diff --git a/src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 b/src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 index 5018148f3..d0b1d7a7f 100644 --- a/src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 +++ b/src/ufo/atmsfcinterp/ufo_atmsfcinterp_mod.F90 @@ -111,6 +111,10 @@ subroutine atmsfcinterp_simobs_(self, geovals_in, obss, nvars, nlocs, hofx) real(kind_real), parameter :: ka = 2.4e-5_kind_real character(1024) :: debug_msg + ! Quickly exit if nlocs is less than one, which happens on many CPUs and + ! no observations sent to one of them. + if (nlocs .lt. 1) return + ! Notes: ! (1) Set desired vertical coordinate direction (top2bottom or bottom2top) based ! on vertical coodinate variable and reload geovals according to the set From d26d20c85d3a323ea014a718dc3cd4b9f08284ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Fri, 16 Jul 2021 15:00:31 +0100 Subject: [PATCH 47/98] Check it is possible to load an ObsSpace with a simulated variable not accompanied by a variable in the ObsError group (#1347) * Check it is possible to load an ObsSpace with a simulated variable not accompanied by a variable in the ObsError group. * Empty commit to run the test suite again Co-authored-by: Stephen Herbener <32968781+srherbener@users.noreply.github.com> Co-authored-by: Ryan Honeyager --- test/testinput/qc_finalcheck.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/testinput/qc_finalcheck.yaml b/test/testinput/qc_finalcheck.yaml index 1e8b0ea55..9d770081b 100644 --- a/test/testinput/qc_finalcheck.yaml +++ b/test/testinput/qc_finalcheck.yaml @@ -117,3 +117,33 @@ observations: # Add a dummy filter to force the QCmanager and Final Check to be run. - filter: Domain Check failedObservationsBenchmark: [0, 1, 3, 4] + +# The following tests checks if a non-derived simulated variable not accompanied by an ObsError +# array in the input file is handled appropriately + +- obs space: + name: Simulated variable without initial ObsErrors; errors not assigned by any filter + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + # The input file contains no relative_humidity@ObsError variable + simulated variables: [relative_humidity, air_temperature] + obs filters: + # Add a dummy filter to force the QCmanager and Final Check to be run. + - filter: Domain Check + passedBenchmark: 5 # air_temperature + failedBenchmark: 5 # relative_humidity -- its error estimates are missing + +- obs space: + name: Simulated variable without initial ObsErrors; errors assigned by a filter + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + # The input file contains no relative_humidity@ObsError variable + simulated variables: [relative_humidity, air_temperature] + obs filters: + - filter: Perform Action + filter variables: [relative_humidity] + action: + name: assign error + error parameter: 0.1 + passedBenchmark: 10 + failedBenchmark: 0 From 4a887cf731652f9f53728da692edb587cdd354a6 Mon Sep 17 00:00:00 2001 From: david-rundle <37152257+david-rundle@users.noreply.github.com> Date: Fri, 16 Jul 2021 21:08:39 +0100 Subject: [PATCH 48/98] fix a couple of memory leaks (#1345) * fix a couple of memory leaks - rttov coef wasn't being deallocated - arrays inside profiles/profiles_k wasn't being deallocated either because -1 was being passed as nprofiles and so the deallocation loop wasn't running! * definitely allocate and definitely deallocate tmpvar --- src/ufo/rttov/ufo_radiancerttov_mod.F90 | 13 ++-- src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 | 62 ++++++++++--------- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/src/ufo/rttov/ufo_radiancerttov_mod.F90 b/src/ufo/rttov/ufo_radiancerttov_mod.F90 index 39f37439e..9a35eab5f 100644 --- a/src/ufo/rttov/ufo_radiancerttov_mod.F90 +++ b/src/ufo/rttov/ufo_radiancerttov_mod.F90 @@ -198,7 +198,7 @@ subroutine ufo_radiancerttov_simobs(self, geovals, obss, nvars, nlocs, & trim(routine_name), ': Allocating resources for RTTOV K code: ', nprof_sim, ' and ', nchan_sim, ' channels' call fckit_log%debug(message) - call self % RTprof % alloc_profs_K(errorstatus, self % conf, nchan_sim, nlevels, init=.true., asw=1) + call self % RTprof % alloc_profs_k(errorstatus, self % conf, nchan_sim, nlevels, init=.true., asw=1) call self % RTprof % alloc_k(errorstatus, self % conf, nprof_sim, nchan_sim, nlevels, init=.true., asw=1) endif @@ -276,7 +276,6 @@ subroutine ufo_radiancerttov_simobs(self, geovals, obss, nvars, nlocs, & write(message,'(A, A, 2I6)') trim(routine_name), 'after rttov_k: error ', errorstatus, i_inst, & ' skipping profiles ', prof_start, ' -- ', prof_start + nprof_sim - 1 call fckit_log%info(message) - cycle end if else @@ -295,7 +294,6 @@ subroutine ufo_radiancerttov_simobs(self, geovals, obss, nvars, nlocs, & write(message,'(A, A, 2I6)') trim(routine_name), 'after rttov_direct: error ', errorstatus, i_inst, & ' skipping profiles ', prof_start, ' -- ', prof_start + nprof_sim - 1 call fckit_log%info(message) - cycle end if endif ! jacobian_needed @@ -316,17 +314,20 @@ subroutine ufo_radiancerttov_simobs(self, geovals, obss, nvars, nlocs, & ! deallocate local chanprof so it can be re-allocated with a different number of channels if reqd. deallocate(chanprof) - if (jacobian_needed) call self % RTprof % zero_K() + if (jacobian_needed) call self % RTprof % zero_k() end do RTTOV_loop ! Deallocate structures for rttov_direct if(jacobian_needed) then call self % RTprof % alloc_k(errorstatus, self % conf, -1, -1, -1, asw=0) - call self % RTprof % alloc_profs_K(errorstatus, self % conf, -1, -1, asw=0) + call self % RTprof % alloc_profs_k(errorstatus, self % conf, size(self % RTprof % profiles_k), -1, asw=0) + ! deallocation of profiles_k isn't done by default in alloc_profs_k because it can contain the trajectory + ! which is currently used for the TL/AD but the 1dvar doesn't use it so it can be safely done here + deallocate (self % RTprof % profiles_k) endif call self % RTprof % alloc_direct(errorstatus, self % conf, -1, -1, -1, asw=0) - call self % RTprof % alloc_profs(errorstatus, self % conf, -1, -1, asw=0) + call self % RTprof % alloc_profs(errorstatus, self % conf, size(self % RTprof % profiles), -1, asw=0) deallocate(self % RTprof % chanprof) diff --git a/src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 b/src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 index ce3c4e306..15c0a24fc 100644 --- a/src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 +++ b/src/ufo/rttov/ufo_radiancerttov_utils_mod.F90 @@ -123,26 +123,26 @@ module ufo_radiancerttov_utils_mod integer :: iprof type, public :: ufo_rttov_io - logical, pointer :: calcemis(:) ! Flag to indicate calculation of emissivity within RTTOV - type(rttov_emissivity), pointer :: emissivity(:) ! Input/output surface emissivity - type(rttov_profile), pointer :: profiles(:) ! Input profiles - type(rttov_profile), pointer :: profiles_k(:) ! Input profiles - type(rttov_chanprof), pointer :: chanprof(:) ! Input profiles - type(rttov_transmission) :: transmission ! Output transmittances - type(rttov_radiance) :: radiance ! Output radiances + logical, pointer :: calcemis(:) ! Flag to indicate calculation of emissivity within RTTOV + type(rttov_emissivity), pointer :: emissivity(:) ! Input/output surface emissivity + type(rttov_profile), allocatable :: profiles(:) ! Input profiles + type(rttov_profile), allocatable :: profiles_k(:) ! Output jacobian profiles + type(rttov_chanprof), pointer :: chanprof(:) + type(rttov_transmission) :: transmission ! Output transmittances + type(rttov_radiance) :: radiance ! Output radiances - type(rttov_emissivity), pointer :: emissivity_k(:) !Input/output surface emissivity - type(rttov_transmission) :: transmission_k ! Output transmittances - type(rttov_radiance) :: radiance_k ! Output radiances + type(rttov_emissivity), pointer :: emissivity_k(:) ! Input/output surface emissivity + type(rttov_transmission) :: transmission_k ! Output transmittances + type(rttov_radiance) :: radiance_k ! Output radiances contains procedure :: alloc_direct => ufo_rttov_alloc_direct procedure :: alloc_k => ufo_rttov_alloc_k procedure :: alloc_profs => ufo_rttov_alloc_profiles - procedure :: alloc_profs_K => ufo_rttov_alloc_profiles_K - procedure :: zero_K => ufo_rttov_zero_K + procedure :: alloc_profs_k => ufo_rttov_alloc_profiles_k + procedure :: zero_k => ufo_rttov_zero_k procedure :: init_emissivity => ufo_rttov_init_emissivity procedure :: setup => ufo_rttov_setup_rtprof procedure :: check => ufo_rttov_check_rtprof @@ -331,7 +331,7 @@ subroutine rttov_conf_setup(conf, f_confOpts, f_confOper) endif if( .not. conf % rttov_is_setup) then - call conf % setup(f_confOpts, asw=1) + call conf % setup(f_confOpts) end if !DARFIX THIS ONLY WORKS FOR ONE INSTRUMENT @@ -393,6 +393,16 @@ subroutine rttov_conf_delete(conf) implicit none type(rttov_conf), intent(inout) :: conf + + integer :: i + + include 'rttov_dealloc_coefs.interface' + + do i = 1, size(conf % rttov_coef_array) + call rttov_dealloc_coefs(rttov_errorstatus, conf % rttov_coef_array(i)) + enddo + deallocate(conf % rttov_coef_array) + conf%rttov_is_setup =.false. deallocate(conf%SENSOR_ID) deallocate(conf%Absorbers) @@ -661,10 +671,9 @@ end subroutine set_options_rttov ! ------------------------------------------------------------------------------ - subroutine setup_rttov(self, f_confOpts, asw) + subroutine setup_rttov(self, f_confOpts) class(rttov_conf), intent(inout) :: self type(fckit_configuration), intent(in) :: f_confOpts ! RTcontrol - integer, intent(in) :: asw !allocate switch character(len=255) :: coef_filename character(len=4) :: coef_ext @@ -674,8 +683,7 @@ subroutine setup_rttov(self, f_confOpts, asw) coef_ext = '.dat' if (.not. self%rttov_is_setup ) then - if(asw == 1) then - + ! -------------------------------------------------------------------------- ! 1. Setup rttov options ! -------------------------------------------------------------------------- @@ -696,20 +704,16 @@ subroutine setup_rttov(self, f_confOpts, asw) file_coef = coef_filename) !in if (rttov_errorstatus /= errorstatus_success) then - write(message,*) 'fatal error reading coefficients' - call abor1_ftn(message) + write(message,*) 'fatal error reading coefficients' + call abor1_ftn(message) else - write(message,*) 'successfully read' // coef_filename - call fckit_log%info(message) + write(message,*) 'successfully read' // coef_filename + call fckit_log%info(message) end if end do self % rttov_is_setup =.true. - else !asw == 0 - deallocate(self % rttov_coef_array) - self%rttov_is_setup =.false. - end if endif end subroutine setup_rttov @@ -962,7 +966,7 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) deallocate(windsp) endif - if (.not. allocated(tmpvar)) allocate(TmpVar(nlocs_total)) + allocate(TmpVar(nlocs_total)) !Hard code watertype to ocean. Only used to determine BRDF in visible calculations profiles(1:nprofiles) % skin % watertype = watertype_ocean_water ! always assume ocean @@ -1217,11 +1221,11 @@ subroutine ufo_rttov_setup_rtprof(self,geovals,obss,conf,ob_info) call ufo_geovals_get_var(geovals, var_surf_type_rttov, geoval) profiles(1:nprofiles)%elevation = geoval%vals(1, 1:nprofiles) * m_to_km endif - - deallocate(TmpVar) end if + deallocate(TmpVar) + ! deallocate(profiles) ! nullify(profiles) ! deallocate(geoval) @@ -1467,7 +1471,6 @@ subroutine ufo_rttov_alloc_profiles_k(self, errorstatus, conf, nprofiles, nlevel allocate (self % profiles_k(nprofiles)) endif - ! Allocate structures for rttov_direct call rttov_alloc_prof( & errorstatus, & nprofiles, & @@ -1561,7 +1564,6 @@ subroutine ufo_rttov_init_emissivity(self, conf, prof_start) enddo endif - end subroutine ufo_rttov_init_emissivity subroutine set_defaults_rttov(self, default_opts_set) From 4c48590271ec2b853427a9ec3564b4080bfaf48b Mon Sep 17 00:00:00 2001 From: Stuart Newman <50145691+smnewman@users.noreply.github.com> Date: Sat, 17 Jul 2021 03:16:20 +0100 Subject: [PATCH 49/98] Extend variable transforms to enable renumbering of satellite scan position (#1308) * Commit working changes * Templates and changes responding to Mike Cooke's review Co-authored-by: Ryan Honeyager --- src/ufo/variabletransforms/CMakeLists.txt | 2 + .../Cal_RemapScanPosition.cc | 56 +++++++++++++++++++ .../Cal_RemapScanPosition.h | 45 +++++++++++++++ src/ufo/variabletransforms/Formulas.cc | 7 +++ src/ufo/variabletransforms/Formulas.h | 11 ++++ src/ufo/variabletransforms/TransformBase.cc | 37 ------------ src/ufo/variabletransforms/TransformBase.h | 45 ++++++++++++++- test/CMakeLists.txt | 8 +++ .../variabletransforms_scanposition.yaml | 34 +++++++++++ 9 files changed, 205 insertions(+), 40 deletions(-) create mode 100644 src/ufo/variabletransforms/Cal_RemapScanPosition.cc create mode 100644 src/ufo/variabletransforms/Cal_RemapScanPosition.h create mode 100644 test/testinput/unit_tests/variabletransforms_scanposition.yaml diff --git a/src/ufo/variabletransforms/CMakeLists.txt b/src/ufo/variabletransforms/CMakeLists.txt index 626760b99..b71331cd1 100644 --- a/src/ufo/variabletransforms/CMakeLists.txt +++ b/src/ufo/variabletransforms/CMakeLists.txt @@ -8,6 +8,8 @@ set ( variabletransforms_files Cal_PressureFromHeight.cc Cal_Humidity.h Cal_Humidity.cc + Cal_RemapScanPosition.h + Cal_RemapScanPosition.cc Cal_Wind.h Cal_Wind.cc LookupTable.h diff --git a/src/ufo/variabletransforms/Cal_RemapScanPosition.cc b/src/ufo/variabletransforms/Cal_RemapScanPosition.cc new file mode 100644 index 000000000..2b9eb208c --- /dev/null +++ b/src/ufo/variabletransforms/Cal_RemapScanPosition.cc @@ -0,0 +1,56 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/variabletransforms/Cal_RemapScanPosition.h" + +namespace ufo { + +/************************************************************************************/ +// Cal_RemapScanPosition +/************************************************************************************/ + +static TransformMaker + makerCal_RemapScanPosition_("RemapScanPosition"); + +Cal_RemapScanPosition::Cal_RemapScanPosition( + const VariableTransformsParameters &options, ioda::ObsSpace &os, + const std::shared_ptr> &flags, + const std::vector &apply) + : TransformBase(options, os, flags, apply) {} + +/************************************************************************************/ + +void Cal_RemapScanPosition::runTransform() { + oops::Log::trace() << " --> Renumber satellite scan position" + << std::endl; + oops::Log::trace() << " --> method: " << method() << std::endl; + oops::Log::trace() << " --> obsName: " << obsName() << std::endl; + + const size_t nlocs = obsdb_.nlocs(); + + std::vector original_scan_position; + getObservation("MetaData", "scan_position", original_scan_position, true); + + std::vector remapped_scan_position(nlocs); + remapped_scan_position.assign(nlocs, missingValueInt); + + // Loop over all obs + for (size_t jobs = 0; jobs < nlocs; ++jobs) { + // if the data have been excluded by the where statement + if (!apply_[jobs]) continue; + + // Calculate wind vector + if (original_scan_position[jobs] != missingValueInt) { + remapped_scan_position[jobs] = formulas::RenumberScanPosition(original_scan_position[jobs]); + } + } + // Overwrite variable at existing locations + obsdb_.put_db("MetaData", "scan_position", remapped_scan_position); +} + +} // namespace ufo + diff --git a/src/ufo/variabletransforms/Cal_RemapScanPosition.h b/src/ufo/variabletransforms/Cal_RemapScanPosition.h new file mode 100644 index 000000000..c8f16c9d8 --- /dev/null +++ b/src/ufo/variabletransforms/Cal_RemapScanPosition.h @@ -0,0 +1,45 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_VARIABLETRANSFORMS_CAL_REMAPSCANPOSITION_H_ +#define UFO_VARIABLETRANSFORMS_CAL_REMAPSCANPOSITION_H_ + +#include +#include +#include +#include + +#include "oops/util/ObjectCounter.h" +#include "ufo/variabletransforms/TransformBase.h" + +namespace ufo { + +/*! +* \brief Renumber satellite scan position +* +* \details Within the Variable Transforms filter, apply the transform "RemapScanPosition" +* in order to renumber satellite scan position. At the Met Office ATMS observations are +* spatially resampled, resulting in 32 fields of view per record sampled from the raw +* 96 FOVs. From the initial observation data the values of scan_position@MetaData are +* 2, 5, 8, ..., 92, 95 (integers). However, in the calculation of observation bias we +* require scan_position@MetaData to be renumbered as 1, 2, 3, ..., 32. +/// +* +* See VariableTransformsParameters for filter setup. +*/ +class Cal_RemapScanPosition : public TransformBase { + public: + Cal_RemapScanPosition(const VariableTransformsParameters &options, + ioda::ObsSpace &os, + const std::shared_ptr> &flags, + const std::vector &apply); + // Run variable conversion + void runTransform() override; +}; +} // namespace ufo + +#endif // UFO_VARIABLETRANSFORMS_CAL_REMAPSCANPOSITION_H_ diff --git a/src/ufo/variabletransforms/Formulas.cc b/src/ufo/variabletransforms/Formulas.cc index 8c68132fc..cad1509b3 100644 --- a/src/ufo/variabletransforms/Formulas.cc +++ b/src/ufo/variabletransforms/Formulas.cc @@ -313,6 +313,13 @@ float GetWind_V(float windSpeed, float windFromDirection) { return v; } +/* -------------------------------------------------------------------------------------*/ + +int RenumberScanPosition(int scanpos) { + // Renumber from 2,5,8,... to 1,2,3,... + int newpos = (scanpos + 1)/3; + return newpos; +} } // namespace formulas } // namespace ufo diff --git a/src/ufo/variabletransforms/Formulas.h b/src/ufo/variabletransforms/Formulas.h index 4c72e31fd..c0a52e007 100644 --- a/src/ufo/variabletransforms/Formulas.h +++ b/src/ufo/variabletransforms/Formulas.h @@ -232,6 +232,17 @@ float GetWind_U(float windSpeed, float windFromDirection); */ float GetWind_V(float windSpeed, float windFromDirection); +// ------------------------------------------------------------------------------------- +/*! +* \brief Get renumbered scan position 1,2,3,... for satellite instrument +* which has been spatially resampled and for which scan position is 2,5,8,... +* +* \param scanpos +* satellite instrument scan position +* \return newpos +*/ +int RenumberScanPosition(int scanpos); + } // namespace formulas } // namespace ufo diff --git a/src/ufo/variabletransforms/TransformBase.cc b/src/ufo/variabletransforms/TransformBase.cc index 66fd37080..d52309eac 100644 --- a/src/ufo/variabletransforms/TransformBase.cc +++ b/src/ufo/variabletransforms/TransformBase.cc @@ -25,43 +25,6 @@ TransformBase::TransformBase(const VariableTransformsParameters& options, obsName_ = os.obsname(); } -void TransformBase::filterObservation(const std::string &variableName, - std::vector &obsVector) const { - if (flags_.has(variableName)) { - const float missing = missingValueFloat; - const std::vector *varFlags = &flags_[variableName]; - - std::transform(obsVector.begin(), obsVector.end(), // Input range 1 - varFlags->begin(), // First element of input range vector 2 (must be same size) - obsVector.begin(), // First element of output range (must be same size) - [missing](float obsvalue, int flag) - { return flag == QCflags::missing || flag == QCflags::bounds - ? missing : obsvalue; }); - } -} - -void TransformBase::getObservation(const std::string &originalTag, const std::string &varName, - std::vector &obsVector, bool require) const { - const size_t nlocs = obsdb_.nlocs(); - - if (obsdb_.has("DerivedValue", varName)) { - obsVector = std::vector(nlocs); - obsdb_.get_db("DerivedValue", varName, obsVector); - // Set obsValue to missingValueFloat if flag is equal to QCflags::missing or QCflags::bounds - if (UseValidDataOnly()) filterObservation(varName, obsVector); - } else if (obsdb_.has(originalTag, varName)) { - obsVector = std::vector(nlocs); - obsdb_.get_db(originalTag, varName, obsVector); - // Set obsValue to missingValueFloat if flag is equal to QCflags::missing or QCflags::bounds - if (UseValidDataOnly()) filterObservation(varName, obsVector); - } - - if (require && obsVector.empty()) { - throw eckit::BadValue("The parameter `" + varName + "@" + originalTag + - "` does not exist in the ObsSpace ", Here()); - } -} - TransformFactory::TransformFactory(const std::string& name) { if (getMakers().find(name) != getMakers().end()) throw eckit::BadParameter( diff --git a/src/ufo/variabletransforms/TransformBase.h b/src/ufo/variabletransforms/TransformBase.h index 28919fec6..631de0b33 100644 --- a/src/ufo/variabletransforms/TransformBase.h +++ b/src/ufo/variabletransforms/TransformBase.h @@ -58,7 +58,6 @@ class TransformBase { virtual void runTransform() = 0; private: - void filterObservation(const std::string &variable, std::vector &obsVector) const; /// Method used for the calculation formulas::MethodFormulation method_; formulas::MethodFormulation formulation_; @@ -66,10 +65,25 @@ class TransformBase { bool AllowSuperSaturation_; /// The observation name std::string obsName_; + /// templated function for float, int data types + template + void filterObservation(const std::string &variableName, + std::vector &obsVector) const { + if (flags_.has(variableName)) { + constexpr T one = T(1.0); + const T missing = util::missingValue(one); + const std::vector *varFlags = &flags_[variableName]; + + std::transform(obsVector.begin(), obsVector.end(), // Input range 1 + varFlags->begin(), // 1st element of input range vector 2 (must be same size) + obsVector.begin(), // 1st element of output range (must be same size) + [missing](T obsvalue, int flag) + { return flag == QCflags::missing || flag == QCflags::bounds + ? missing : obsvalue; }); + } + } protected: // variables - void getObservation(const std::string &originalTag, const std::string &varName, - std::vector &obsVector, bool require = false) const; /// subclasses to access Method and formualtion used for the calculation formulas::MethodFormulation method() const { return method_; } formulas::MethodFormulation formulation() const { return formulation_; } @@ -84,10 +98,35 @@ class TransformBase { ioda::ObsSpace &obsdb_; const ioda::ObsDataVector &flags_; const std::vector &apply_; + /// Missing value (int) + const int missingValueInt = util::missingValue(1); /// Missing value (float) const float missingValueFloat = util::missingValue(1.0f); /// output tag for derived parameters const std::string outputTag = "DerivedValue"; + /// templated function for float, int data types + template + void getObservation(const std::string &originalTag, const std::string &varName, + std::vector &obsVector, bool require = false) const { + const size_t nlocs = obsdb_.nlocs(); + + if (obsdb_.has("DerivedValue", varName)) { + obsVector = std::vector(nlocs); + obsdb_.get_db("DerivedValue", varName, obsVector); + // Set obsValue to missingValueFloat if flag is equal to QCflags::missing or QCflags::bounds + if (UseValidDataOnly()) filterObservation(varName, obsVector); + } else if (obsdb_.has(originalTag, varName)) { + obsVector = std::vector(nlocs); + obsdb_.get_db(originalTag, varName, obsVector); + // Set obsValue to missingValueFloat if flag is equal to QCflags::missing or QCflags::bounds + if (UseValidDataOnly()) filterObservation(varName, obsVector); + } + + if (require && obsVector.empty()) { + throw eckit::BadValue("The parameter `" + varName + "@" + originalTag + + "` does not exist in the ObsSpace ", Here()); + } + } }; /// \brief Transform factory diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3fa23358c..0f3f179cf 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -295,6 +295,7 @@ list( APPEND ufo_test_input testinput/unit_tests/variabletransforms_rhumidity.yaml testinput/unit_tests/variabletransforms_rhumidity_part2.yaml testinput/unit_tests/variabletransforms_shumidity.yaml + testinput/unit_tests/variabletransforms_scanposition.yaml testinput/unit_tests/variabletransforms_SondePressureFromHeight.yaml testinput/unit_tests/variabletransforms_SondePressureFromHeightICAO.yaml testinput/unit_tests/variabletransforms_windcomponents.yaml @@ -2533,6 +2534,13 @@ ecbuild_add_test( TARGET test_ufo_variabletransforms_humidityrelative_part2 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ioda_test_data ) +ecbuild_add_test( TARGET test_ufo_variabletransforms_scanposition + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x + ARGS "testinput/unit_tests/variabletransforms_scanposition.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFilters.x + TEST_DEPENDS ufo_get_ufo_test_data ) + ecbuild_add_test( TARGET test_ufo_variabletransforms_humidityspecific COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x ARGS "testinput/unit_tests/variabletransforms_shumidity.yaml" diff --git a/test/testinput/unit_tests/variabletransforms_scanposition.yaml b/test/testinput/unit_tests/variabletransforms_scanposition.yaml new file mode 100644 index 000000000..5e0d51c60 --- /dev/null +++ b/test/testinput/unit_tests/variabletransforms_scanposition.yaml @@ -0,0 +1,34 @@ +window begin: 2019-12-29T21:00:00Z +window end: 2019-12-30T03:00:00Z + +observations: +- obs operator: + name: RTTOV + GeoVal_type: MetO + Absorbers: &rttov_absorbers [Water_vapour, CLW] + linear obs operator: + Absorbers: [Water_vapour] + obs options: &rttov_options + RTTOV_default_opts: UKMO_PS45 + SatRad_compatibility: true + Sensor_ID: noaa_20_atms + CoefficientPath: Data/ + RTTOV_GasUnitConv: true + UseRHwaterForQC: true + obs space: + name: atms_n20 + obsdatain: + obsfile: Data/ufo/testinput_tier_1/atms_n20_obs_20191230T0000_rttov.nc4 + simulated variables: [brightness_temperature] + channels: 1-22 + geovals: + filename: Data/ufo/testinput_tier_1/geovals_atms_20191230T0000Z_benchmark.nc4 + obs filters: + - filter: Variable Transforms + Transform: ["RemapScanPosition"] + UseValidDataOnly: false + compareVariables: + - reference: + name: scan_position@TestReference + test: + name: scan_position@MetaData From 8aadaf2a6e1ecb6f8c0bd0acdcf7b5ed6f2ee934 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Sun, 18 Jul 2021 20:03:14 -0400 Subject: [PATCH 50/98] Bug fix for averaging kernel unit conversions (#1353) * update nonlinear and TL; no change to AD yet * bugfix for 0 Pa top pressure; updated rms ref * Change to adjoint; updated test tolerance * Test with fixed averaging kernel * Update test reference YAML * revert constant avgkernel hack Co-authored-by: Ryan Honeyager --- src/ufo/avgkernel/ufo_avgkernel_mod.F90 | 2 + src/ufo/avgkernel/ufo_avgkernel_tlad_mod.F90 | 22 +++-- src/ufo/avgkernel/ufo_satcolumn_mod.F90 | 99 +++++++++++++++----- test/testinput/tropomi_no2.yaml | 8 +- 4 files changed, 92 insertions(+), 39 deletions(-) diff --git a/src/ufo/avgkernel/ufo_avgkernel_mod.F90 b/src/ufo/avgkernel/ufo_avgkernel_mod.F90 index 972ae5700..29a236d8f 100644 --- a/src/ufo/avgkernel/ufo_avgkernel_mod.F90 +++ b/src/ufo/avgkernel/ufo_avgkernel_mod.F90 @@ -197,12 +197,14 @@ subroutine ufo_avgkernel_simobs(self, geovals_in, obss, nvars, nlocs, hofx) if (avgkernel_obs(1,iobs) /= missing) then ! take care of missing obs if (self%troposphere) then call simulate_column_ob(self%nlayers_kernel, tracer%nval, avgkernel_obs(:,iobs), & + prsi_obs(:,iobs), prsi%vals(:,iobs), & prsl_obs(:,iobs), prsl%vals(:,iobs), temp%vals(:,iobs),& phii%vals(:,iobs), tracer%vals(:,iobs)*self%convert_factor_model, & hofx_tmp, troplev_obs(iobs), airmass_tot(iobs), airmass_trop(iobs)) hofx(ivar,iobs) = hofx_tmp * self%convert_factor_hofx else if (self%totalcolumn) then call simulate_column_ob(self%nlayers_kernel, tracer%nval, avgkernel_obs(:,iobs), & + prsi_obs(:,iobs), prsi%vals(:,iobs), & prsl_obs(:,iobs), prsl%vals(:,iobs), temp%vals(:,iobs),& phii%vals(:,iobs), tracer%vals(:,iobs)*self%convert_factor_model, & hofx_tmp) diff --git a/src/ufo/avgkernel/ufo_avgkernel_tlad_mod.F90 b/src/ufo/avgkernel/ufo_avgkernel_tlad_mod.F90 index 26929927a..a9e209bb3 100644 --- a/src/ufo/avgkernel/ufo_avgkernel_tlad_mod.F90 +++ b/src/ufo/avgkernel/ufo_avgkernel_tlad_mod.F90 @@ -29,8 +29,8 @@ module ufo_avgkernel_tlad_mod character(kind=c_char,len=:), allocatable :: obskernelvar, tracervars(:) logical :: troposphere, totalcolumn real(kind_real) :: convert_factor_model, convert_factor_hofx - real(kind_real), allocatable, dimension(:,:) :: avgkernel_obs, prsl_obs - real(kind_real), allocatable, dimension(:,:) :: prsl, temp, phii + real(kind_real), allocatable, dimension(:,:) :: avgkernel_obs, prsl_obs, prsi_obs + real(kind_real), allocatable, dimension(:,:) :: prsl, prsi, temp, phii real(kind_real), allocatable, dimension(:) :: airmass_tot, airmass_trop integer, allocatable, dimension(:) :: troplev_obs logical :: flip_it @@ -136,7 +136,6 @@ subroutine avgkernel_tlad_settraj_(self, geovals_in, obss) integer :: ivar, iobs, ilev character(len=MAXVARLEN) :: varstring character(len=4) :: levstr - real(kind_real), allocatable, dimension(:,:) :: prsi_obs type(ufo_geovals) :: geovals type(ufo_geoval), pointer :: prsl, temp, phii, p_temp @@ -165,8 +164,10 @@ subroutine avgkernel_tlad_settraj_(self, geovals_in, obss) allocate(self%prsl(self%nval, self%nlocs)) allocate(self%temp(self%nval, self%nlocs)) allocate(self%phii(self%nval+1, self%nlocs)) + allocate(self%prsi(self%nval+1, self%nlocs)) do iobs = 1, self%nlocs self%prsl(:,iobs) = prsl%vals(:,iobs) + self%prsi(:,iobs) = prsi%vals(:,iobs) self%temp(:,iobs) = temp%vals(:,iobs) self%phii(:,iobs) = phii%vals(:,iobs) end do @@ -184,14 +185,14 @@ subroutine avgkernel_tlad_settraj_(self, geovals_in, obss) ! compute prsl_obs/prsi_obs from ak/bk/psfc allocate(self%prsl_obs(self%nlayers_kernel, self%nlocs)) - allocate(prsi_obs(self%nlayers_kernel+1, self%nlocs)) + allocate(self%prsi_obs(self%nlayers_kernel+1, self%nlocs)) ! prsi_obs calculation do ilev = 1, self%nlayers_kernel+1 - prsi_obs(ilev,:) = self%ak_kernel(ilev) + self%bk_kernel(ilev) * psfc%vals(1,:) + self%prsi_obs(ilev,:) = self%ak_kernel(ilev) + self%bk_kernel(ilev) * psfc%vals(1,:) end do ! using simple averaging for now for prsl, can use more complex way later do ilev = 1, self%nlayers_kernel - self%prsl_obs(ilev,:) = (prsi_obs(ilev,:) + prsi_obs(ilev+1,:)) * half + self%prsl_obs(ilev,:) = (self%prsi_obs(ilev,:) + self%prsi_obs(ilev+1,:)) * half end do if (self%troposphere) then @@ -203,9 +204,6 @@ subroutine avgkernel_tlad_settraj_(self, geovals_in, obss) call obsspace_get_db(obss, "MetaData", "air_mass_factor_total", self%airmass_tot) end if - ! cleanup things we do not need now - deallocate(prsi_obs) - end subroutine avgkernel_tlad_settraj_ ! ------------------------------------------------------------------------------ @@ -240,12 +238,14 @@ subroutine avgkernel_simobs_tl_(self, geovals_in, obss, nvars, nlocs, hofx) if(self%flip_it) tracer%vals(1:tracer%nval,iobs) = tracer%vals(tracer%nval:1:-1,iobs) if (self%troposphere) then call simulate_column_ob_tl(self%nlayers_kernel, tracer%nval, self%avgkernel_obs(:,iobs), & + self%prsi_obs(:,iobs), self%prsi(:,iobs),& self%prsl_obs(:,iobs), self%prsl(:,iobs), self%temp(:,iobs),& self%phii(:,iobs), tracer%vals(:,iobs)*self%convert_factor_model, & hofx_tmp, self%troplev_obs(iobs), self%airmass_tot(iobs), self%airmass_trop(iobs)) hofx(ivar,iobs) = hofx_tmp * self%convert_factor_hofx else if (self%totalcolumn) then call simulate_column_ob_tl(self%nlayers_kernel, tracer%nval, self%avgkernel_obs(:,iobs), & + self%prsi_obs(:,iobs), self%prsi(:,iobs),& self%prsl_obs(:,iobs), self%prsl(:,iobs), self%temp(:,iobs),& self%phii(:,iobs), tracer%vals(:,iobs)*self%convert_factor_model, & hofx_tmp) @@ -293,6 +293,7 @@ subroutine avgkernel_simobs_ad_(self, geovals_in, obss, nvars, nlocs, hofx) if (self%troposphere) then hofx_tmp = hofx(ivar,iobs) * self%convert_factor_hofx call simulate_column_ob_ad(self%nlayers_kernel, tracer%nval, self%avgkernel_obs(:,iobs), & + self%prsi_obs(:,iobs), self%prsi(:,iobs),& self%prsl_obs(:,iobs), self%prsl(:,iobs), self%temp(:,iobs),& self%phii(:,iobs), tracer%vals(:,iobs), & hofx_tmp, self%troplev_obs(iobs), self%airmass_tot(iobs), self%airmass_trop(iobs)) @@ -300,6 +301,7 @@ subroutine avgkernel_simobs_ad_(self, geovals_in, obss, nvars, nlocs, hofx) else if (self%totalcolumn) then hofx_tmp = hofx(ivar,iobs) * self%convert_factor_hofx call simulate_column_ob_ad(self%nlayers_kernel, tracer%nval, self%avgkernel_obs(:,iobs), & + self%prsi_obs(:,iobs), self%prsi(:,iobs),& self%prsl_obs(:,iobs), self%prsl(:,iobs), self%temp(:,iobs),& self%phii(:,iobs), tracer%vals(:,iobs), hofx_tmp) tracer%vals(:,iobs) = tracer%vals(:,iobs) * self%convert_factor_model @@ -325,7 +327,9 @@ subroutine avgkernel_tlad_cleanup_(self) if (allocated(self%tracervars)) deallocate(self%tracervars) if (allocated(self%avgkernel_obs)) deallocate(self%avgkernel_obs) if (allocated(self%prsl_obs)) deallocate(self%prsl_obs) + if (allocated(self%prsi_obs)) deallocate(self%prsi_obs) if (allocated(self%prsl)) deallocate(self%prsl) + if (allocated(self%prsi)) deallocate(self%prsi) if (allocated(self%temp)) deallocate(self%temp) if (allocated(self%phii)) deallocate(self%phii) if (allocated(self%airmass_tot)) deallocate(self%airmass_tot) diff --git a/src/ufo/avgkernel/ufo_satcolumn_mod.F90 b/src/ufo/avgkernel/ufo_satcolumn_mod.F90 index 9a712d416..bee1d2451 100644 --- a/src/ufo/avgkernel/ufo_satcolumn_mod.F90 +++ b/src/ufo/avgkernel/ufo_satcolumn_mod.F90 @@ -2,6 +2,7 @@ module satcolumn_mod contains subroutine simulate_column_ob(nlayers_obs, nlayers_model, avgkernel_obs, & + prsi_obs, prsi_model, & prsl_obs, prsl_model, temp_model, zi_model, & profile_model, hofx, & troplev_obs, airmass_tot, airmass_trop) @@ -11,6 +12,8 @@ subroutine simulate_column_ob(nlayers_obs, nlayers_model, avgkernel_obs, & implicit none integer, intent(in ) :: nlayers_obs, nlayers_model real(kind_real), intent(in ), dimension(nlayers_obs) :: avgkernel_obs + real(kind_real), intent(in ), dimension(nlayers_obs+1) :: prsi_obs + real(kind_real), intent(in ), dimension(nlayers_model+1) :: prsi_model real(kind_real), intent(in ), dimension(nlayers_obs) :: prsl_obs real(kind_real), intent(in ), dimension(nlayers_model) :: prsl_model real(kind_real), intent(in ), dimension(nlayers_model) :: profile_model @@ -19,10 +22,12 @@ subroutine simulate_column_ob(nlayers_obs, nlayers_model, avgkernel_obs, & real(kind_real), intent( out) :: hofx real(kind_real), intent(in ), optional :: airmass_tot, airmass_trop integer, intent(in ), optional :: troplev_obs - real(kind_real) :: airmass_ratio, lnp_ob, wf, dz + real(kind_real) :: airmass_ratio, lnp_ob, lnpi_ob, wf, dz, ptmp real(kind_real), dimension(nlayers_obs) :: avgkernel_use real(kind_real), dimension(nlayers_model) :: lnp_model, profile_model_use + real(kind_real), dimension(nlayers_model+1) :: lnpi_model real(kind_real), dimension(nlayers_obs) :: profile_obslayers + real(kind_real), dimension(nlayers_obs+1) :: zi_obs integer :: k, wi logical :: troposphere @@ -46,22 +51,33 @@ subroutine simulate_column_ob(nlayers_obs, nlayers_model, avgkernel_obs, & ! need to convert from kg/kg to molec/m3 for each layer's T and P profile_model_use = profile_model * ((avogadro*prsl_model)/(temp_model*gas_constant)) - ! need to convert from molec/m3 to molec/cm2 to sum later - do k=1,nlayers_model - dz = (zi_model(k+1) - zi_model(k)) - profile_model_use(k) = profile_model_use(k) * dz ! convert to molec/m2 - profile_model_use(k) = profile_model_use(k) / 1.0e4_kind_real ! to molec/cm2 - end do - ! need to interpolate model profile layers to observation layers lnp_model = log(prsl_model) do k=1,nlayers_obs - lnp_ob = log(prsl_obs(k)) + ptmp = max(prsl_obs(k), 1.0_kind_real) + lnp_ob = log(ptmp) call vert_interp_weights(nlayers_model, lnp_ob, lnp_model, wi, wf) call vert_interp_apply(nlayers_model, profile_model_use, & profile_obslayers(k), wi, wf) end do + ! next, interpolate model z to observation interface levels + lnpi_model = log(prsi_model) + do k=1,nlayers_obs+1 + ptmp = max(prsi_obs(k), 1.0_kind_real) + lnpi_ob = log(ptmp) + call vert_interp_weights(nlayers_model+1, lnpi_ob, lnpi_model, wi, wf) + call vert_interp_apply(nlayers_model+1, zi_model, & + zi_obs(k), wi, wf) + end do + + ! now, convert from molec/m3 to molec/cm2 using dz and 1e4 + do k=1,nlayers_obs + dz = (zi_obs(k+1) - zi_obs(k)) + profile_obslayers(k) = profile_obslayers(k) * dz ! convert to molec/m2 + profile_obslayers(k) = profile_obslayers(k) / 1.0e4_kind_real ! to molec/cm2 + end do + ! compute hofx as column integral hofx = zero do k=1,nlayers_obs @@ -71,6 +87,7 @@ subroutine simulate_column_ob(nlayers_obs, nlayers_model, avgkernel_obs, & end subroutine simulate_column_ob subroutine simulate_column_ob_tl(nlayers_obs, nlayers_model, avgkernel_obs, & + prsi_obs, prsi_model, & prsl_obs, prsl_model, temp_model, zi_model, & profile_model, hofx, & troplev_obs, airmass_tot, airmass_trop) @@ -80,6 +97,8 @@ subroutine simulate_column_ob_tl(nlayers_obs, nlayers_model, avgkernel_obs, & implicit none integer, intent(in ) :: nlayers_obs, nlayers_model real(kind_real), intent(in ), dimension(nlayers_obs) :: avgkernel_obs + real(kind_real), intent(in ), dimension(nlayers_obs+1) :: prsi_obs + real(kind_real), intent(in ), dimension(nlayers_model+1) :: prsi_model real(kind_real), intent(in ), dimension(nlayers_obs) :: prsl_obs real(kind_real), intent(in ), dimension(nlayers_model) :: prsl_model real(kind_real), intent(in ), dimension(nlayers_model) :: profile_model @@ -88,10 +107,12 @@ subroutine simulate_column_ob_tl(nlayers_obs, nlayers_model, avgkernel_obs, & real(kind_real), intent( out) :: hofx real(kind_real), intent(in ), optional :: airmass_tot, airmass_trop integer, intent(in ), optional :: troplev_obs - real(kind_real) :: airmass_ratio, lnp_ob, wf, dz + real(kind_real) :: airmass_ratio, lnp_ob, lnpi_ob, wf, dz, ptmp real(kind_real), dimension(nlayers_obs) :: avgkernel_use real(kind_real), dimension(nlayers_model) :: lnp_model, profile_model_use + real(kind_real), dimension(nlayers_model+1) :: lnpi_model real(kind_real), dimension(nlayers_obs) :: profile_obslayers + real(kind_real), dimension(nlayers_obs+1) :: zi_obs integer :: k, wi logical :: troposphere @@ -115,22 +136,33 @@ subroutine simulate_column_ob_tl(nlayers_obs, nlayers_model, avgkernel_obs, & ! need to convert from kg/kg to molec/m3 for each layer's T and P profile_model_use = profile_model * ((avogadro*prsl_model)/(temp_model*gas_constant)) - ! need to convert from molec/m3 to molec/cm2 to sum later - do k=1,nlayers_model - dz = (zi_model(k+1) - zi_model(k)) - profile_model_use(k) = profile_model_use(k) * dz ! convert to molec/m2 - profile_model_use(k) = profile_model_use(k) / 1.0e4_kind_real ! to molec/cm2 - end do - ! need to interpolate model profile layers to observation layers lnp_model = log(prsl_model) do k=1,nlayers_obs - lnp_ob = log(prsl_obs(k)) + ptmp = max(prsl_obs(k), 1.0_kind_real) + lnp_ob = log(ptmp) call vert_interp_weights(nlayers_model, lnp_ob, lnp_model, wi, wf) call vert_interp_apply_tl(nlayers_model, profile_model_use, & profile_obslayers(k), wi, wf) end do + ! next, interpolate model z to observation interface levels + lnpi_model = log(prsi_model) + do k=1,nlayers_obs+1 + ptmp = max(prsi_obs(k), 1.0_kind_real) + lnpi_ob = log(ptmp) + call vert_interp_weights(nlayers_model+1, lnpi_ob, lnpi_model, wi, wf) + call vert_interp_apply_tl(nlayers_model+1, zi_model, & + zi_obs(k), wi, wf) + end do + + ! now, convert from molec/m3 to molec/cm2 using dz and 1e4 + do k=1,nlayers_obs + dz = (zi_obs(k+1) - zi_obs(k)) + profile_obslayers(k) = profile_obslayers(k) * dz ! convert to molec/m2 + profile_obslayers(k) = profile_obslayers(k) / 1.0e4_kind_real ! to molec/cm2 + end do + ! compute hofx as column integral hofx = zero do k=1,nlayers_obs @@ -140,6 +172,7 @@ subroutine simulate_column_ob_tl(nlayers_obs, nlayers_model, avgkernel_obs, & end subroutine simulate_column_ob_tl subroutine simulate_column_ob_ad(nlayers_obs, nlayers_model, avgkernel_obs, & + prsi_obs, prsi_model, & prsl_obs, prsl_model, temp_model, zi_model, & profile_model_ad, hofx_ad, & troplev_obs, airmass_tot, airmass_trop) @@ -149,6 +182,8 @@ subroutine simulate_column_ob_ad(nlayers_obs, nlayers_model, avgkernel_obs, & implicit none integer, intent(in ) :: nlayers_obs, nlayers_model real(kind_real), intent(in ), dimension(nlayers_obs) :: avgkernel_obs + real(kind_real), intent(in ), dimension(nlayers_obs+1) :: prsi_obs + real(kind_real), intent(in ), dimension(nlayers_model+1) :: prsi_model real(kind_real), intent(in ), dimension(nlayers_obs) :: prsl_obs real(kind_real), intent(in ), dimension(nlayers_model) :: prsl_model real(kind_real), intent(inout), dimension(nlayers_model) :: profile_model_ad @@ -157,10 +192,12 @@ subroutine simulate_column_ob_ad(nlayers_obs, nlayers_model, avgkernel_obs, & real(kind_real), intent(in ) :: hofx_ad real(kind_real), intent(in ), optional :: airmass_tot, airmass_trop integer, intent(in ), optional :: troplev_obs - real(kind_real) :: airmass_ratio, lnp_ob, wf, dz + real(kind_real) :: airmass_ratio, lnp_ob, lnpi_ob, wf, dz, ptmp real(kind_real), dimension(nlayers_obs) :: avgkernel_use real(kind_real), dimension(nlayers_model) :: lnp_model, profile_model_use_ad + real(kind_real), dimension(nlayers_model+1) :: lnpi_model real(kind_real), dimension(nlayers_obs) :: profile_obslayers_ad + real(kind_real), dimension(nlayers_obs+1) :: zi_obs integer :: k, wi logical :: troposphere integer :: nlevs @@ -187,6 +224,23 @@ subroutine simulate_column_ob_ad(nlayers_obs, nlayers_model, avgkernel_obs, & profile_obslayers_ad(k) = profile_obslayers_ad(k) + avgkernel_use(k) * hofx_ad end do + ! interpolate model z to observation interface levels + lnpi_model = log(prsi_model) + do k=nlayers_obs+1,1,-1 + ptmp = max(prsi_obs(k), 1.0_kind_real) + lnpi_ob = log(ptmp) + call vert_interp_weights(nlayers_model+1, lnpi_ob, lnpi_model, wi, wf) + call vert_interp_apply_tl(nlayers_model+1, zi_model, & + zi_obs(k), wi, wf) + end do + + ! now, convert from molec/m3 to molec/m2 using dz and 1e4 + do k=nlayers_obs,1,-1 + dz = (zi_obs(k+1) - zi_obs(k)) + profile_obslayers_ad(k) = profile_obslayers_ad(k) * dz ! to molec/m2 + profile_obslayers_ad(k) = profile_obslayers_ad(k) / 1.0e4_kind_real ! to molec/cm2 + end do + profile_model_use_ad = zero ! need to interpolate model profile layers to observation layers lnp_model = log(prsl_model) @@ -197,13 +251,6 @@ subroutine simulate_column_ob_ad(nlayers_obs, nlayers_model, avgkernel_obs, & profile_obslayers_ad(k), wi, wf) end do - ! adjoint of unit conversions - do k=nlayers_model,1,-1 - dz = (zi_model(k+1) - zi_model(k)) - profile_model_use_ad(k) = profile_model_use_ad(k) / 1.0e4_kind_real ! to molec/m2 - profile_model_use_ad(k) = profile_model_use_ad(k) * dz ! convert to molec/m3 - end do - profile_model_ad = profile_model_use_ad * ((avogadro*prsl_model)/(temp_model*gas_constant)) end subroutine simulate_column_ob_ad diff --git a/test/testinput/tropomi_no2.yaml b/test/testinput/tropomi_no2.yaml index e124afabf..7947ce55b 100644 --- a/test/testinput/tropomi_no2.yaml +++ b/test/testinput/tropomi_no2.yaml @@ -32,10 +32,10 @@ observations: linear obs operator test: coef TL: 1.0e-5 tolerance TL: 1.0e-12 - tolerance AD: 1.0e-14 + tolerance AD: 1.0e-13 geovals: filename: Data/ufo/testinput_tier_1/tropomi_no2_geoval_2020090318_m.nc4 - rms ref: 2.1272229395313009e-05 + rms ref: 5.9775801600415118e-05 tolerance: 1.0e-7 - obs space: name: NO2 @@ -68,8 +68,8 @@ observations: linear obs operator test: coef TL: 1.0e-5 tolerance TL: 1.0e-12 - tolerance AD: 1.0e-14 + tolerance AD: 1.0e-13 geovals: filename: Data/ufo/testinput_tier_1/tropomi_no2_geoval_2020090318_m.nc4 - rms ref: 1.4128255400842815e-05 + rms ref: 3.9691718286217631e-05 tolerance: 1.0e-7 From 64f9981713e8179443ba84fdad18dde1b144f4cb Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Tue, 20 Jul 2021 11:13:04 -0600 Subject: [PATCH 51/98] change ObsLocalization to specify values outside of loc as missing (#1366) --- src/ufo/obslocalization/ObsLocGC99.h | 13 ++++------ src/ufo/obslocalization/ObsLocSOAR.h | 13 ++++------ src/ufo/obslocalization/ObsLocalization.h | 29 ++++++++++++----------- 3 files changed, 25 insertions(+), 30 deletions(-) diff --git a/src/ufo/obslocalization/ObsLocGC99.h b/src/ufo/obslocalization/ObsLocGC99.h index 2504d506e..382b66aed 100644 --- a/src/ufo/obslocalization/ObsLocGC99.h +++ b/src/ufo/obslocalization/ObsLocGC99.h @@ -13,7 +13,6 @@ #include "eckit/config/Configuration.h" -#include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" @@ -32,11 +31,10 @@ class ObsLocGC99: public ufo::ObsLocalization { public: ObsLocGC99(const eckit::Configuration &, const ioda::ObsSpace &); - /// compute localization and save localization values in \p obsvector and - /// localization flags (1: outside of localization; 0: inside localization area) - /// in \p outside - void computeLocalization(const GeometryIterator_ &, ioda::ObsDataVector & outside, - ioda::ObsVector & obsvector) const override; + /// compute localization and save localization values in \p locfactor + /// (missing values indicate that observation is outside of localization) + void computeLocalization(const GeometryIterator_ &, + ioda::ObsVector & locfactor) const override; private: void print(std::ostream &) const override; @@ -56,12 +54,11 @@ ObsLocGC99::ObsLocGC99(const eckit::Configuration & config, template void ObsLocGC99::computeLocalization(const GeometryIterator_ & i, - ioda::ObsDataVector & outside, ioda::ObsVector & locvector) const { oops::Log::trace() << "ObsLocGC99::computeLocalization" << std::endl; // do distance search and compute box-car locvector - ObsLocalization::computeLocalization(i, outside, locvector); + ObsLocalization::computeLocalization(i, locvector); // return refs to internals of ObsLocalization const std::vector & localobs = ObsLocalization::localobs(); diff --git a/src/ufo/obslocalization/ObsLocSOAR.h b/src/ufo/obslocalization/ObsLocSOAR.h index 58326e4bb..a90bb4908 100644 --- a/src/ufo/obslocalization/ObsLocSOAR.h +++ b/src/ufo/obslocalization/ObsLocSOAR.h @@ -13,7 +13,6 @@ #include "eckit/config/Configuration.h" -#include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" @@ -32,11 +31,10 @@ class ObsLocSOAR: public ufo::ObsLocalization { public: ObsLocSOAR(const eckit::Configuration &, const ioda::ObsSpace &); - /// compute localization and save localization values in \p obsvector and - /// localization flags (1: outside of localization; 0: inside localization area) - /// in \p outside - void computeLocalization(const GeometryIterator_ &, ioda::ObsDataVector & outside, - ioda::ObsVector & obsvector) const override; + /// compute localization and save localization values in \p locvector + /// (missing values indicate that observation is outside of localization) + void computeLocalization(const GeometryIterator_ &, + ioda::ObsVector & locvector) const override; private: void print(std::ostream &) const override; @@ -57,11 +55,10 @@ ObsLocSOAR::ObsLocSOAR(const eckit::Configuration & config, template void ObsLocSOAR::computeLocalization(const GeometryIterator_ & i, - ioda::ObsDataVector & outside, ioda::ObsVector & locvector) const { oops::Log::trace() << "ObsLocSOAR::computeLocalization" << std::endl; // do distance search and compute box-car locvector - ObsLocalization::computeLocalization(i, outside, locvector); + ObsLocalization::computeLocalization(i, locvector); // return refs to internals of ObsLocalization const std::vector & localobs = ObsLocalization::localobs(); diff --git a/src/ufo/obslocalization/ObsLocalization.h b/src/ufo/obslocalization/ObsLocalization.h index 9fff9b85d..2fbb7548a 100644 --- a/src/ufo/obslocalization/ObsLocalization.h +++ b/src/ufo/obslocalization/ObsLocalization.h @@ -23,11 +23,11 @@ #include "eckit/geometry/Point3.h" #include "eckit/geometry/UnitSphere.h" -#include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" #include "oops/base/ObsLocalizationBase.h" +#include "oops/util/missingValues.h" #include "ufo/obslocalization/ObsLocParameters.h" #include "ufo/ObsTraits.h" @@ -47,11 +47,10 @@ class ObsLocalization: public oops::ObsLocalizationBase { typedef eckit::KDTreeMemory KDTree; ObsLocalization(const eckit::Configuration &, const ioda::ObsSpace &); - /// compute localization and save localization values in \p obsvector and - /// localization flags (1: outside of localization; 0: inside localization area) - /// in \p outside - void computeLocalization(const GeometryIterator_ &, ioda::ObsDataVector & outside, - ioda::ObsVector & obsvector) const override; + /// compute localization and save localization values in \p locvector + /// (missing values indicate that observation is outside of localization) + void computeLocalization(const GeometryIterator_ &, + ioda::ObsVector & locvector) const override; const std::vector & localobs() const {return localobs_;} const std::vector & horizontalObsdist() const {return obsdist_;} @@ -118,8 +117,7 @@ ObsLocalization::ObsLocalization(const eckit::Configuration & config, template void ObsLocalization::computeLocalization(const GeometryIterator_ & i, - ioda::ObsDataVector & outside, - ioda::ObsVector & locvector) const { + ioda::ObsVector & locvector) const { oops::Log::trace() << "ObsLocalization::computeLocalization" << std::endl; // check that this distribution supports local obs space @@ -207,16 +205,19 @@ void ObsLocalization::computeLocalization(const GeometryIterator_ & i, obsdist_.resize(*maxnobs); } } - for (size_t jloc = 0; jloc < outside.nlocs(); ++jloc) { - for (size_t jvar = 0; jvar < outside.nvars(); ++jvar) { - outside[jvar][jloc] = 1; - } + + // set all to missing (outside of localization distance) + const double missing = util::missingValue(double()); + for (size_t jj = 0; jj < locvector.size(); ++jj) { + locvector[jj] = missing; } + + // set localization for the obs inside localization distance to 1.0 const size_t nvars = locvector.nvars(); - for (size_t jlocal = 0; jlocal < localobs_.size(); ++jlocal) { + const size_t nlocal = localobs_.size(); + for (size_t jlocal = 0; jlocal < nlocal; ++jlocal) { // obsdist is calculated at each location; need to update R for each variable for (size_t jvar = 0; jvar < nvars; ++jvar) { - outside[jvar][localobs_[jlocal]] = 0; locvector[jvar + localobs_[jlocal] * nvars] = 1.0; } } From a638b5cfc529a5ceccf8b03c064acad630bd583d Mon Sep 17 00:00:00 2001 From: Stuart Newman <50145691+smnewman@users.noreply.github.com> Date: Wed, 21 Jul 2021 23:24:16 +0100 Subject: [PATCH 52/98] Move test_ufo_variabletransforms_scanposition behind rttov check (#1363) Co-authored-by: Stephen Herbener <32968781+srherbener@users.noreply.github.com> --- test/CMakeLists.txt | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0f3f179cf..051d31a7d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -947,6 +947,14 @@ if( ${rttov_FOUND} ) ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsOperatorTLAD.x TEST_DEPENDS ufo_get_ufo_test_data ) + + ecbuild_add_test( TARGET test_ufo_variabletransforms_scanposition + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x + ARGS "testinput/unit_tests/variabletransforms_scanposition.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFilters.x + TEST_DEPENDS ufo_get_ufo_test_data ) + endif( ${rttov_FOUND} ) @@ -2534,13 +2542,6 @@ ecbuild_add_test( TARGET test_ufo_variabletransforms_humidityrelative_part2 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ioda_test_data ) -ecbuild_add_test( TARGET test_ufo_variabletransforms_scanposition - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/unit_tests/variabletransforms_scanposition.yaml" - ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ObsFilters.x - TEST_DEPENDS ufo_get_ufo_test_data ) - ecbuild_add_test( TARGET test_ufo_variabletransforms_humidityspecific COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x ARGS "testinput/unit_tests/variabletransforms_shumidity.yaml" From 9f93a130d5eac6bbb28861a8c789bf2da5b851ca Mon Sep 17 00:00:00 2001 From: Stephen Herbener <32968781+srherbener@users.noreply.github.com> Date: Thu, 22 Jul 2021 11:42:58 -0600 Subject: [PATCH 53/98] Add handling of ioda v2 syntax to the utility function splitVarGroup (#1377) * Added handling of ioda v2 syntax to the utility funciton splitVarGroup. * Fixed typo in comments --- src/ufo/utils/StringUtils.cc | 25 ++++++- test/testinput/qc_boundscheck.yaml | 110 ++++++++++++++--------------- 2 files changed, 77 insertions(+), 58 deletions(-) diff --git a/src/ufo/utils/StringUtils.cc b/src/ufo/utils/StringUtils.cc index c30c928d3..d0dcaf727 100644 --- a/src/ufo/utils/StringUtils.cc +++ b/src/ufo/utils/StringUtils.cc @@ -15,15 +15,34 @@ namespace ufo { // ----------------------------------------------------------------------------- - +// For now support both ioda v1 an ioda v2 syntax for specifying the combined +// variable and group name: +// ioda v1: variable@group +// ioda v2: group/variable +// +// We will eventually be obsoleting the ioda v1 syntax. For ioda v1 syntax, only +// allow for one "@" separator. For ioda v2 syntax, since it allows for nested groups, +// allow for multiple "/" separators, and the variable name is after the last "/". +// void splitVarGroup(const std::string & vargrp, std::string & var, std::string & grp) { const size_t at = vargrp.find("@"); - var = vargrp.substr(0, at); - grp = ""; + const size_t slash = vargrp.find_last_of("/"); + if (at != std::string::npos) { + // ioda v1 syntax + var = vargrp.substr(0, at); grp = vargrp.substr(at + 1, std::string::npos); const size_t no_at = grp.find("@"); ASSERT(no_at == std::string::npos); + } else if (slash != std::string::npos) { + // ioda v2 syntax + grp = vargrp.substr(0, slash); + var = vargrp.substr(slash + 1, std::string::npos); + } else { + // vargrp has no "@" nor "/" then assume that vargrp is a variable + // name (with no group specified). + var = vargrp; + grp = ""; } } diff --git a/test/testinput/qc_boundscheck.yaml b/test/testinput/qc_boundscheck.yaml index c2849ad53..30768e795 100644 --- a/test/testinput/qc_boundscheck.yaml +++ b/test/testinput/qc_boundscheck.yaml @@ -16,9 +16,9 @@ observations: minvalue: 14.0 maxvalue: 19.0 # Compare variables with minvalue/maxvalue -# variable1@ObsValue = 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 -# variable2@ObsValue = 10, 12, 14, 16, 18, 20, 22, 24, 26, 28 -# variable3@ObsValue = 25, 24, 23, 22, 21, 20, 19, 18, 17, 16 +# ObsValue/variable1 = 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 +# ObsValue/variable2 = 10, 12, 14, 16, 18, 20, 22, 24, 26, 28 +# ObsValue/variable3 = 25, 24, 23, 22, 21, 20, 19, 18, 17, 16 passedBenchmark: 13 - obs space: name: test data @@ -29,9 +29,9 @@ observations: - filter: Bounds Check # test min/max value with all variables and a where statement where: - variable: - name: var1@MetaData + name: MetaData/var1 maxvalue: 8 -# var1@MetaData = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 +# MetaData/var1 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 filter variables: - name: variable1 - name: variable2 @@ -39,9 +39,9 @@ observations: minvalue: 14.0 maxvalue: 19.0 # Compare variables with minvalue/maxvalue -# variable1@ObsValue = 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 -# variable2@ObsValue = 10, 12, 14, 16, 18, 20, 22, 24, 26, 28 -# variable3@ObsValue = 25, 24, 23, 22, 21, 20, 19, 18, 17, 16 +# ObsValue/variable1 = 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 +# ObsValue/variable2 = 10, 12, 14, 16, 18, 20, 22, 24, 26, 28 +# ObsValue/variable3 = 25, 24, 23, 22, 21, 20, 19, 18, 17, 16 passedBenchmark: 15 - obs space: name: test data @@ -57,8 +57,8 @@ observations: maxvalue: 19.0 # Compare variables with minvalue/maxvalue # Note: variable1 is not specified in filtered variables, all obs for variable1 will pass -# variable2@ObsValue = 10, 12, 14, 16, 18, 20, 22, 24, 26, 28 -# variable3@ObsValue = 25, 24, 23, 22, 21, 20, 19, 18, 17, 16 +# ObsValue/variable2 = 10, 12, 14, 16, 18, 20, 22, 24, 26, 28 +# ObsValue/variable3 = 25, 24, 23, 22, 21, 20, 19, 18, 17, 16 passedBenchmark: 17 - obs space: name: test data @@ -74,8 +74,8 @@ observations: maxvalue: 19.0 # Compare variables with minvalue/maxvalue # Note: variable2 is not specified in filtered variables, all obs for variable2 will pass -# variable3@ObsValue = 25, 24, 23, 22, 21, 20, 19, 18, 17, 16 -# variable1@ObsValue = 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 +# ObsValue/variable3 = 25, 24, 23, 22, 21, 20, 19, 18, 17, 16 +# ObsValue/variable1 = 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 passedBenchmark: 20 - obs space: name: test data @@ -89,15 +89,15 @@ observations: - name: variable2 - name: variable3 test variables: - - name: var1@MetaData - - name: var2@MetaData - - name: var3@MetaData + - name: MetaData/var1 + - name: MetaData/var2 + - name: MetaData/var3 minvalue: 14.0 maxvalue: 19.0 -# Filter variable1, variable2, variable3 based on var1@MetaData, var2@MetaData, var3@MetaData values -# var1@MetaData = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 -# var2@MetaData = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 -# var3@MetaData = 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 +# Filter variable1, variable2, variable3 based on MetaData/var1, MetaData/var2, MetaData/var3 values +# MetaData/var1 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 +# MetaData/var2 = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +# MetaData/var3 = 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 passedBenchmark: 0 - obs space: name: test data @@ -111,15 +111,15 @@ observations: - name: variable2 - name: variable3 test variables: - - name: var1@MetaData - - name: var2@MetaData - - name: var3@MetaData + - name: MetaData/var1 + - name: MetaData/var2 + - name: MetaData/var3 minvalue: 1 maxvalue: 3 -# Filter variable1, variable2, variable3 based on var1@MetaData, var2@MetaData, var3@MetaData values -# var1@MetaData = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 -# var2@MetaData = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 -# var3@MetaData = 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 +# Filter variable1, variable2, variable3 based on MetaData/var1, MetaData/var2, MetaData/var3 values +# MetaData/var1 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 +# MetaData/var2 = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +# MetaData/var3 = 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 passedBenchmark: 11 - obs space: name: test data @@ -132,12 +132,12 @@ observations: - name: variable2 - name: variable3 test variables: - - name: var1@MetaData - - name: var2@MetaData -# Filter variable2, variable3 based on var1@MetaData, var2@MetaData values + - name: MetaData/var1 + - name: MetaData/var2 +# Filter variable2, variable3 based on MetaData/var1, MetaData/var2 values # Note: variable1 is not specified in filtered variables, all obs for variable1 will pass -# var1@MetaData = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 -# var2@MetaData = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +# MetaData/var1 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 +# MetaData/var2 = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 minvalue: 14.0 maxvalue: 19.0 passedBenchmark: 10 @@ -152,14 +152,14 @@ observations: - name: variable2 - name: variable3 test variables: - - name: var1@MetaData - - name: var2@MetaData + - name: MetaData/var1 + - name: MetaData/var2 minvalue: 1 maxvalue: 3 -# Filter variable2, variable3 based on var1@MetaData, var2@MetaData values +# Filter variable2, variable3 based on MetaData/var1, MetaData/var2 values # Note: variable1 is not specified in filtered variables, all obs for variable1 will pass -# var1@MetaData = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 -# var2@MetaData = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +# MetaData/var1 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 +# MetaData/var2 = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 passedBenchmark: 16 - obs space: name: test data @@ -172,12 +172,12 @@ observations: - name: variable2 - name: variable3 test variables: - - name: var1@MetaData + - name: MetaData/var1 minvalue: 4 -# Filter variable2 AND variable3 based only on single test var: var1@MetaData values +# Filter variable2 AND variable3 based only on single test var: MetaData/var1 values # Note: variable1 is not specified in filtered variables, all obs for variable1 will pass -# var1@MetaData = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 -# var2@MetaData = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +# MetaData/var1 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 +# MetaData/var2 = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 passedBenchmark: 24 - obs space: name: test data @@ -190,11 +190,11 @@ observations: - name: variable2 - name: variable3 test variables: - - name: variable1@ObsValue + - name: ObsValue/variable1 maxvalue: 16 -# Filter variable2 AND variable3 based only on single test var of variable1 (@ObsValue) +# Filter variable2 AND variable3 based only on single test var of (ObsValue/) variable1 # Note: variable1 is not specified in filtered variables, all obs for variable1 will pass -# variable1@ObsValue = 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 +# ObsValue/variable1 = 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 passedBenchmark: 24 - obs space: name: test data @@ -207,14 +207,14 @@ observations: - name: variable2 - name: variable3 test variables: - - name: var1@MetaData - - name: var2@MetaData + - name: MetaData/var1 + - name: MetaData/var2 flag all filter variables if any test variable is out of bounds: true minvalue: 3 -# Filter variable2 AND variable3 based the values of var1 and var2@MetaData +# Filter variable2 AND variable3 based the values of var1 and MetaData/var2 # Note: variable1 is not specified in filtered variables, all obs for variable1 will pass -# var1@MetaData = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 -# var2@MetaData = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +# MetaData/var1 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 +# MetaData/var2 = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 passedBenchmark: 22 # variable2 and variable3 should be rejected at locations 0, 1, 8 and 9 - obs space: name: test data @@ -227,12 +227,12 @@ observations: - name: variable2 - name: variable3 test variables: - - name: var5@MetaData + - name: MetaData/var5 treat missing as out of bounds: false maxvalue: 3 -# Filter variable2 AND variable3 based the values of var1 and var2@MetaData +# Filter variable2 AND variable3 based the values of var1 and MetaData/var2 # Note: variable1 is not specified in filtered variables, all obs for variable1 will pass -# var5@MetaData = 1, _, 2, _, 3, _, 4, _, 5, 0 ; +# MetaData/var5 = 1, _, 2, _, 3, _, 4, _, 5, 0 ; passedBenchmark: 26 # variable2 and variable3 should be rejected at locations 6 and 8 - obs space: name: test data @@ -245,12 +245,12 @@ observations: - name: variable2 - name: variable3 test variables: - - name: var5@MetaData + - name: MetaData/var5 treat missing as out of bounds: true maxvalue: 3 -# Filter variable2 AND variable3 based the values of var1 and var2@MetaData +# Filter variable2 AND variable3 based the values of var1 and MetaData/var2 # Note: variable1 is not specified in filtered variables, all obs for variable1 will pass -# var5@MetaData = 1, _, 2, _, 3, _, 4, _, 5, 0 ; +# MetaData/var5 = 1, _, 2, _, 3, _, 4, _, 5, 0 ; passedBenchmark: 18 # variable2 and variable3 should be rejected at locations 1, 3, 5, 6, 7 and 8 - obs space: name: Where using datetimes @@ -262,7 +262,7 @@ observations: - filter: BlackList where: - variable: - name: datetime@MetaData + name: MetaData/datetime minvalue: 2018-04-14T21:00:00Z maxvalue: 2018-04-14T22:00:00Z failedBenchmark: 17 From d87e0b6077bdbf1ea67e618c85d67becadbb6b1d Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Fri, 23 Jul 2021 16:45:12 +0100 Subject: [PATCH 54/98] Rename Profile Consistency Checks (#1227) * rename profile QC routines * change name of one of the tests * update nc4 filenames * remove comments * QC -> processing * renamed files * indentation and comments * fix indentation Co-authored-by: Ryan Honeyager --- src/ufo/filters/CMakeLists.txt | 6 +- ...ks.cc => ConventionalProfileProcessing.cc} | 23 +- ...ecks.h => ConventionalProfileProcessing.h} | 31 +- ...ConventionalProfileProcessingParameters.h} | 12 +- src/ufo/filters/ProfileBackgroundCheck.cc | 2 +- src/ufo/instantiateObsFilterFactory.h | 6 +- src/ufo/profile/ProfileAveragePressure.cc | 2 +- src/ufo/profile/ProfileAveragePressure.h | 4 +- .../profile/ProfileAverageRelativeHumidity.cc | 2 +- .../profile/ProfileAverageRelativeHumidity.h | 4 +- src/ufo/profile/ProfileAverageTemperature.cc | 2 +- src/ufo/profile/ProfileAverageTemperature.h | 4 +- src/ufo/profile/ProfileAverageWindSpeed.cc | 2 +- src/ufo/profile/ProfileAverageWindSpeed.h | 4 +- ...rofileCheckBackgroundGeopotentialHeight.cc | 2 +- ...ProfileCheckBackgroundGeopotentialHeight.h | 4 +- .../ProfileCheckBackgroundRelativeHumidity.cc | 2 +- .../ProfileCheckBackgroundRelativeHumidity.h | 4 +- .../ProfileCheckBackgroundTemperature.cc | 2 +- .../ProfileCheckBackgroundTemperature.h | 5 +- .../ProfileCheckBackgroundWindSpeed.cc | 2 +- .../profile/ProfileCheckBackgroundWindSpeed.h | 5 +- src/ufo/profile/ProfileCheckBase.cc | 4 +- src/ufo/profile/ProfileCheckBase.h | 13 +- src/ufo/profile/ProfileCheckBasic.cc | 2 +- src/ufo/profile/ProfileCheckBasic.h | 2 +- src/ufo/profile/ProfileCheckHistory.cc | 2 +- src/ufo/profile/ProfileCheckHistory.h | 4 +- src/ufo/profile/ProfileCheckHydrostatic.cc | 3 +- src/ufo/profile/ProfileCheckHydrostatic.h | 4 +- src/ufo/profile/ProfileCheckInterpolation.cc | 2 +- src/ufo/profile/ProfileCheckInterpolation.h | 4 +- .../profile/ProfileCheckPermanentReject.cc | 2 +- src/ufo/profile/ProfileCheckPermanentReject.h | 4 +- src/ufo/profile/ProfileCheckRH.cc | 2 +- src/ufo/profile/ProfileCheckRH.h | 4 +- src/ufo/profile/ProfileCheckSamePDiffT.cc | 3 +- src/ufo/profile/ProfileCheckSamePDiffT.h | 4 +- src/ufo/profile/ProfileCheckSign.cc | 2 +- src/ufo/profile/ProfileCheckSign.h | 4 +- src/ufo/profile/ProfileCheckTime.cc | 2 +- src/ufo/profile/ProfileCheckTime.h | 4 +- src/ufo/profile/ProfileCheckUInterp.cc | 2 +- src/ufo/profile/ProfileCheckUInterp.h | 4 +- .../profile/ProfileCheckUInterpAlternative.cc | 2 +- .../profile/ProfileCheckUInterpAlternative.h | 5 +- src/ufo/profile/ProfileCheckUnstableLayer.cc | 2 +- src/ufo/profile/ProfileCheckUnstableLayer.h | 4 +- src/ufo/profile/ProfileCheckValidator.cc | 3 +- src/ufo/profile/ProfileCheckValidator.h | 6 +- src/ufo/profile/ProfileChecker.cc | 2 +- src/ufo/profile/ProfileChecker.h | 6 +- src/ufo/profile/ProfilePressure.cc | 2 +- src/ufo/profile/ProfilePressure.h | 4 +- src/ufo/profile/ProfileSondeFlags.cc | 2 +- src/ufo/profile/ProfileSondeFlags.h | 4 +- src/ufo/profile/ProfileStandardLevels.cc | 3 +- src/ufo/profile/ProfileStandardLevels.h | 6 +- src/ufo/profile/ProfileWindProfilerFlags.cc | 2 +- src/ufo/profile/ProfileWindProfilerFlags.h | 4 +- src/ufo/variabletransforms/Cal_Humidity.cc | 1 - .../Cal_PressureFromHeight.cc | 1 - src/ufo/variabletransforms/Cal_Wind.cc | 1 - test/CMakeLists.txt | 460 +++++++++--------- ...c => TestConventionalProfileProcessing.cc} | 4 +- ...ionalprofileprocessing_OPScomparison.yaml} | 8 +- ...alprofileprocessing_RH_OPScomparison.yaml} | 6 +- ...tionalprofileprocessing_RH_obsfilter.yaml} | 6 +- ...ing_UInterpAlternative_OPScomparison.yaml} | 6 +- ...cessing_UInterpAlternative_obsfilter.yaml} | 6 +- ...fileprocessing_UInterp_OPScomparison.yaml} | 6 +- ...lprofileprocessing_UInterp_obsfilter.yaml} | 6 +- ...ssing_average_pressure_OPScomparison.yaml} | 6 +- ...rocessing_average_pressure_obsfilter.yaml} | 6 +- ...erage_relativehumidity_OPScomparison.yaml} | 6 +- ...g_average_relativehumidity_obsfilter.yaml} | 6 +- ...ng_average_temperature_OPScomparison.yaml} | 6 +- ...essing_average_temperature_obsfilter.yaml} | 6 +- ...sing_average_windspeed_OPScomparison.yaml} | 6 +- ...ocessing_average_windspeed_obsfilter.yaml} | 6 +- ...rocessing_bkgqc_modobs_OPScomparison.yaml} | 8 +- ...ileprocessing_bkgqc_modobs_obsfilter.yaml} | 8 +- ...gqc_modobs_separatefilters_obsfilter.yaml} | 18 +- ...rocessing_bkgqc_repobs_OPScomparison.yaml} | 8 +- ...ileprocessing_bkgqc_repobs_obsfilter.yaml} | 8 +- ...gqc_repobs_separatefilters_obsfilter.yaml} | 18 +- ...alprofileprocessing_monolithicfilter.yaml} | 8 +- ...ntionalprofileprocessing_opr_average.yaml} | 0 ...ileprocessing_pressure_OPScomparison.yaml} | 6 +- ...profileprocessing_pressure_obsfilter.yaml} | 6 +- ...nalprofileprocessing_separatefilters.yaml} | 18 +- ...eprocessing_sondeflags_OPScomparison.yaml} | 4 +- ...ofileprocessing_sondeflags_obsfilter.yaml} | 4 +- ...ileprocessing_unittest_oneprofileMPI.yaml} | 8 +- ...ventionalprofileprocessing_unittests.yaml} | 52 +- ...ocessing_unittests_profiledataholder.yaml} | 8 +- ...fileprocessing_unittests_vertaverage.yaml} | 6 +- ...sing_unittests_vertaverage_ascending.yaml} | 6 +- ...ofileprocessing_unittests_vertinterp.yaml} | 8 +- ...processing_winproflags_OPScomparison.yaml} | 4 +- ...fileprocessing_winproflags_obsfilter.yaml} | 4 +- ...profileprocessing_wrongOPScomparison.yaml} | 136 +++--- ...ecks.h => ConventionalProfileProcessing.h} | 42 +- 103 files changed, 610 insertions(+), 601 deletions(-) rename src/ufo/filters/{ProfileConsistencyChecks.cc => ConventionalProfileProcessing.cc} (90%) rename src/ufo/filters/{ProfileConsistencyChecks.h => ConventionalProfileProcessing.h} (81%) rename src/ufo/filters/{ProfileConsistencyCheckParameters.h => ConventionalProfileProcessingParameters.h} (97%) rename test/mains/{TestProfileConsistencyChecks.cc => TestConventionalProfileProcessing.cc} (76%) rename test/testinput/{profileconsistencychecks_OPScomparison.yaml => conventionalprofileprocessing_OPScomparison.yaml} (67%) rename test/testinput/{profileconsistencychecks_RH_OPScomparison.yaml => conventionalprofileprocessing_RH_OPScomparison.yaml} (74%) rename test/testinput/{profileconsistencychecks_RH_obsfilter.yaml => conventionalprofileprocessing_RH_obsfilter.yaml} (77%) rename test/testinput/{profileconsistencychecks_UInterpAlternative_OPScomparison.yaml => conventionalprofileprocessing_UInterpAlternative_OPScomparison.yaml} (66%) rename test/testinput/{profileconsistencychecks_UInterpAlternative_obsfilter.yaml => conventionalprofileprocessing_UInterpAlternative_obsfilter.yaml} (72%) rename test/testinput/{profileconsistencychecks_UInterp_OPScomparison.yaml => conventionalprofileprocessing_UInterp_OPScomparison.yaml} (69%) rename test/testinput/{profileconsistencychecks_UInterp_obsfilter.yaml => conventionalprofileprocessing_UInterp_obsfilter.yaml} (76%) rename test/testinput/{profileconsistencychecks_average_pressure_OPScomparison.yaml => conventionalprofileprocessing_average_pressure_OPScomparison.yaml} (69%) rename test/testinput/{profileconsistencychecks_average_pressure_obsfilter.yaml => conventionalprofileprocessing_average_pressure_obsfilter.yaml} (71%) rename test/testinput/{profileconsistencychecks_average_relativehumidity_OPScomparison.yaml => conventionalprofileprocessing_average_relativehumidity_OPScomparison.yaml} (71%) rename test/testinput/{profileconsistencychecks_average_relativehumidity_obsfilter.yaml => conventionalprofileprocessing_average_relativehumidity_obsfilter.yaml} (73%) rename test/testinput/{profileconsistencychecks_average_temperature_OPScomparison.yaml => conventionalprofileprocessing_average_temperature_OPScomparison.yaml} (71%) rename test/testinput/{profileconsistencychecks_average_temperature_obsfilter.yaml => conventionalprofileprocessing_average_temperature_obsfilter.yaml} (72%) rename test/testinput/{profileconsistencychecks_average_windspeed_OPScomparison.yaml => conventionalprofileprocessing_average_windspeed_OPScomparison.yaml} (71%) rename test/testinput/{profileconsistencychecks_average_windspeed_obsfilter.yaml => conventionalprofileprocessing_average_windspeed_obsfilter.yaml} (73%) rename test/testinput/{profileconsistencychecks_bkgqc_modobs_OPScomparison.yaml => conventionalprofileprocessing_bkgqc_modobs_OPScomparison.yaml} (74%) rename test/testinput/{profileconsistencychecks_bkgqc_modobs_obsfilter.yaml => conventionalprofileprocessing_bkgqc_modobs_obsfilter.yaml} (79%) rename test/testinput/{profileconsistencychecks_bkgqc_modobs_separatefilters_obsfilter.yaml => conventionalprofileprocessing_bkgqc_modobs_separatefilters_obsfilter.yaml} (82%) rename test/testinput/{profileconsistencychecks_bkgqc_repobs_OPScomparison.yaml => conventionalprofileprocessing_bkgqc_repobs_OPScomparison.yaml} (74%) rename test/testinput/{profileconsistencychecks_bkgqc_repobs_obsfilter.yaml => conventionalprofileprocessing_bkgqc_repobs_obsfilter.yaml} (79%) rename test/testinput/{profileconsistencychecks_bkgqc_repobs_separatefilters_obsfilter.yaml => conventionalprofileprocessing_bkgqc_repobs_separatefilters_obsfilter.yaml} (82%) rename test/testinput/{profileconsistencychecks_monolithicfilter.yaml => conventionalprofileprocessing_monolithicfilter.yaml} (73%) rename test/testinput/{profileconsistencychecks_opr_average.yaml => conventionalprofileprocessing_opr_average.yaml} (100%) rename test/testinput/{profileconsistencychecks_pressure_OPScomparison.yaml => conventionalprofileprocessing_pressure_OPScomparison.yaml} (70%) rename test/testinput/{profileconsistencychecks_pressure_obsfilter.yaml => conventionalprofileprocessing_pressure_obsfilter.yaml} (68%) rename test/testinput/{profileconsistencychecks_separatefilters.yaml => conventionalprofileprocessing_separatefilters.yaml} (72%) rename test/testinput/{profileconsistencychecks_sondeflags_OPScomparison.yaml => conventionalprofileprocessing_sondeflags_OPScomparison.yaml} (80%) rename test/testinput/{profileconsistencychecks_sondeflags_obsfilter.yaml => conventionalprofileprocessing_sondeflags_obsfilter.yaml} (81%) rename test/testinput/{profileconsistencychecks_unittest_oneprofileMPI.yaml => conventionalprofileprocessing_unittest_oneprofileMPI.yaml} (70%) rename test/testinput/{profileconsistencychecks_unittests.yaml => conventionalprofileprocessing_unittests.yaml} (83%) rename test/testinput/{profileconsistencychecks_unittests_profiledataholder.yaml => conventionalprofileprocessing_unittests_profiledataholder.yaml} (56%) rename test/testinput/{profileconsistencychecks_unittests_vertaverage.yaml => conventionalprofileprocessing_unittests_vertaverage.yaml} (64%) rename test/testinput/{profileconsistencychecks_unittests_vertaverage_ascending.yaml => conventionalprofileprocessing_unittests_vertaverage_ascending.yaml} (60%) rename test/testinput/{profileconsistencychecks_unittests_vertinterp.yaml => conventionalprofileprocessing_unittests_vertinterp.yaml} (76%) rename test/testinput/{profileconsistencychecks_winproflags_OPScomparison.yaml => conventionalprofileprocessing_winproflags_OPScomparison.yaml} (80%) rename test/testinput/{profileconsistencychecks_winproflags_obsfilter.yaml => conventionalprofileprocessing_winproflags_obsfilter.yaml} (81%) rename test/testinput/{profileconsistencychecks_wrongOPScomparison.yaml => conventionalprofileprocessing_wrongOPScomparison.yaml} (77%) rename test/ufo/{ProfileConsistencyChecks.h => ConventionalProfileProcessing.h} (93%) diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index fc043567e..0e76c8e48 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -12,6 +12,9 @@ set ( filters_files BayesianBackgroundCheck.h BlackList.cc BlackList.h + ConventionalProfileProcessing.cc + ConventionalProfileProcessing.h + ConventionalProfileProcessingParameters.h DifferenceCheck.cc DifferenceCheck.h FilterBase.cc @@ -70,9 +73,6 @@ set ( filters_files PerformAction.h ProfileBackgroundCheck.cc ProfileBackgroundCheck.h - ProfileConsistencyChecks.cc - ProfileConsistencyChecks.h - ProfileConsistencyCheckParameters.h ProfileFewObsCheck.cc ProfileFewObsCheck.h SatName.h diff --git a/src/ufo/filters/ProfileConsistencyChecks.cc b/src/ufo/filters/ConventionalProfileProcessing.cc similarity index 90% rename from src/ufo/filters/ProfileConsistencyChecks.cc rename to src/ufo/filters/ConventionalProfileProcessing.cc index 6dcb6a0b7..c1b4b02d9 100644 --- a/src/ufo/filters/ProfileConsistencyChecks.cc +++ b/src/ufo/filters/ConventionalProfileProcessing.cc @@ -21,8 +21,8 @@ #include "oops/util/abor1_cpp.h" #include "oops/util/Logger.h" -#include "ufo/filters/ProfileConsistencyCheckParameters.h" -#include "ufo/filters/ProfileConsistencyChecks.h" +#include "ufo/filters/ConventionalProfileProcessing.h" +#include "ufo/filters/ConventionalProfileProcessingParameters.h" #include "ufo/GeoVaLs.h" @@ -30,7 +30,7 @@ namespace ufo { // ----------------------------------------------------------------------------- - ProfileConsistencyChecks::ProfileConsistencyChecks + ConventionalProfileProcessing::ConventionalProfileProcessing (ioda::ObsSpace & obsdb, const Parameters_ & parameters, std::shared_ptr > flags, @@ -77,11 +77,11 @@ namespace ufo { // ----------------------------------------------------------------------------- - ProfileConsistencyChecks::~ProfileConsistencyChecks() {} + ConventionalProfileProcessing::~ConventionalProfileProcessing() {} // ----------------------------------------------------------------------------- - void ProfileConsistencyChecks::individualProfileChecks + void ConventionalProfileProcessing::individualProfileChecks (ProfileDataHandler &profileDataHandler, ProfileCheckValidator &profileCheckValidator, ProfileChecker &profileChecker, @@ -126,7 +126,7 @@ namespace ufo { // ----------------------------------------------------------------------------- - void ProfileConsistencyChecks::entireSampleChecks + void ConventionalProfileProcessing::entireSampleChecks (ProfileDataHandler &profileDataHandler, ProfileCheckValidator &profileCheckValidator, ProfileChecker &profileChecker, @@ -147,9 +147,10 @@ namespace ufo { // ----------------------------------------------------------------------------- - void ProfileConsistencyChecks::applyFilter(const std::vector & apply, - const Variables & filtervars, - std::vector> & flagged) const + void ConventionalProfileProcessing::applyFilter + (const std::vector & apply, + const Variables & filtervars, + std::vector> & flagged) const { print(oops::Log::trace()); @@ -197,8 +198,8 @@ namespace ufo { // ----------------------------------------------------------------------------- - void ProfileConsistencyChecks::print(std::ostream & os) const { - os << "ProfileConsistencyChecks: config = " << options_ << std::endl; + void ConventionalProfileProcessing::print(std::ostream & os) const { + os << "ConventionalProfileProcessing: config = " << options_ << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/ProfileConsistencyChecks.h b/src/ufo/filters/ConventionalProfileProcessing.h similarity index 81% rename from src/ufo/filters/ProfileConsistencyChecks.h rename to src/ufo/filters/ConventionalProfileProcessing.h index 84d6837d0..4b8280318 100644 --- a/src/ufo/filters/ProfileConsistencyChecks.h +++ b/src/ufo/filters/ConventionalProfileProcessing.h @@ -5,8 +5,8 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef UFO_FILTERS_PROFILECONSISTENCYCHECKS_H_ -#define UFO_FILTERS_PROFILECONSISTENCYCHECKS_H_ +#ifndef UFO_FILTERS_CONVENTIONALPROFILEPROCESSING_H_ +#define UFO_FILTERS_CONVENTIONALPROFILEPROCESSING_H_ #include #include @@ -16,8 +16,8 @@ #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" +#include "ufo/filters/ConventionalProfileProcessingParameters.h" #include "ufo/filters/FilterBase.h" -#include "ufo/filters/ProfileConsistencyCheckParameters.h" #include "ufo/filters/QCflags.h" #include "ufo/profile/EntireSampleDataHandler.h" @@ -39,7 +39,10 @@ namespace ioda { namespace ufo { - /// \brief Profile QC checks. + /// \brief Conventional profile processing. + /// + /// This filter applies a variety of QC checks to profile data. + /// The filter also averages profiles onto model levels prior to their use in data assimilation. /// /// The temperature consistency checks available are as follows: /// -# Basic checks of pressure @@ -80,17 +83,17 @@ namespace ufo { /// - corrections to sign of temperature /// - corrections to height (multiples of 100 m) - class ProfileConsistencyChecks : public FilterBase, - private util::ObjectCounter { + class ConventionalProfileProcessing : public FilterBase, + private util::ObjectCounter { public: - typedef ProfileConsistencyCheckParameters Parameters_; + typedef ConventionalProfileProcessingParameters Parameters_; - static const std::string classname() {return "ufo::ProfileConsistencyChecks";} + static const std::string classname() {return "ufo::ConventionalProfileProcessing";} - ProfileConsistencyChecks(ioda::ObsSpace &, const Parameters_ &, - std::shared_ptr >, - std::shared_ptr >); - ~ProfileConsistencyChecks(); + ConventionalProfileProcessing(ioda::ObsSpace &, const Parameters_ &, + std::shared_ptr >, + std::shared_ptr >); + ~ConventionalProfileProcessing(); /// Return the number of mismatches between values produced by the checking routines /// and the equivalents produced in the OPS code. @@ -119,7 +122,7 @@ namespace ufo { int qcFlag() const override {return QCflags::profile;} /// Configurable options - ProfileConsistencyCheckParameters options_; + ConventionalProfileProcessingParameters options_; /// Number of mismatches between values produced in this code /// and their OPS equivalents (used for validation) @@ -128,4 +131,4 @@ namespace ufo { } // namespace ufo -#endif // UFO_FILTERS_PROFILECONSISTENCYCHECKS_H_ +#endif // UFO_FILTERS_CONVENTIONALPROFILEPROCESSING_H_ diff --git a/src/ufo/filters/ProfileConsistencyCheckParameters.h b/src/ufo/filters/ConventionalProfileProcessingParameters.h similarity index 97% rename from src/ufo/filters/ProfileConsistencyCheckParameters.h rename to src/ufo/filters/ConventionalProfileProcessingParameters.h index 4e06c7ba2..51dd949ce 100644 --- a/src/ufo/filters/ProfileConsistencyCheckParameters.h +++ b/src/ufo/filters/ConventionalProfileProcessingParameters.h @@ -5,8 +5,8 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef UFO_FILTERS_PROFILECONSISTENCYCHECKPARAMETERS_H_ -#define UFO_FILTERS_PROFILECONSISTENCYCHECKPARAMETERS_H_ +#ifndef UFO_FILTERS_CONVENTIONALPROFILEPROCESSINGPARAMETERS_H_ +#define UFO_FILTERS_CONVENTIONALPROFILEPROCESSINGPARAMETERS_H_ #include #include @@ -32,9 +32,9 @@ namespace eckit { namespace ufo { - /// \brief Options controlling the operation of the ProfileConsistencyChecks filter. - class ProfileConsistencyCheckParameters : public FilterParametersBase { - OOPS_CONCRETE_PARAMETERS(ProfileConsistencyCheckParameters, FilterParametersBase) + /// \brief Options controlling the operation of the ConventionalProfileProcessing filter. + class ConventionalProfileProcessingParameters : public FilterParametersBase { + OOPS_CONCRETE_PARAMETERS(ConventionalProfileProcessingParameters, FilterParametersBase) public: // variables /// @name Generic parameters @@ -410,5 +410,5 @@ namespace ufo { }; } // namespace ufo -#endif // UFO_FILTERS_PROFILECONSISTENCYCHECKPARAMETERS_H_ +#endif // UFO_FILTERS_CONVENTIONALPROFILEPROCESSINGPARAMETERS_H_ diff --git a/src/ufo/filters/ProfileBackgroundCheck.cc b/src/ufo/filters/ProfileBackgroundCheck.cc index 2ba0859e5..d13d642e6 100644 --- a/src/ufo/filters/ProfileBackgroundCheck.cc +++ b/src/ufo/filters/ProfileBackgroundCheck.cc @@ -40,7 +40,7 @@ namespace ufo { /// taken from the threshold given for the first observation in the profile. /// /// This is related to BackgroundCheck, which checks each observation against a threshold. -/// There is also a group of other profile checks (ProfileConsistencyChecks) which are +/// There is also a group of other profile checks (ConventionalProfileProcessing) which are /// mostly aimed to processing radiosondes. ProfileBackgroundCheck::ProfileBackgroundCheck( diff --git a/src/ufo/instantiateObsFilterFactory.h b/src/ufo/instantiateObsFilterFactory.h index 51a43b4db..53f6e6c92 100644 --- a/src/ufo/instantiateObsFilterFactory.h +++ b/src/ufo/instantiateObsFilterFactory.h @@ -15,6 +15,7 @@ #include "ufo/filters/BayesianBackgroundCheck.h" #include "ufo/filters/BayesianBackgroundQCFlags.h" #include "ufo/filters/BlackList.h" +#include "ufo/filters/ConventionalProfileProcessing.h" #include "ufo/filters/DifferenceCheck.h" #include "ufo/filters/FinalCheck.h" #include "ufo/filters/Gaussian_Thinning.h" @@ -35,7 +36,6 @@ #include "ufo/filters/PreQC.h" #include "ufo/filters/ProbabilityGrossErrorWholeReport.h" #include "ufo/filters/ProfileBackgroundCheck.h" -#include "ufo/filters/ProfileConsistencyChecks.h" #include "ufo/filters/ProfileFewObsCheck.h" #include "ufo/filters/QCmanager.h" #include "ufo/filters/SatName.h" @@ -95,8 +95,8 @@ template void instantiateObsFilterFactory() { MWCLWCheckMaker("MWCLW Check"); static oops::FilterMaker > domainErrCheckMaker("DomainErr Check"); - static oops::FilterMaker > - profileConsistencyChecksMaker("Profile Consistency Checks"); + static oops::FilterMaker > + conventionalProfileProcessingMaker("Conventional Profile Processing"); static oops::FilterMaker > backgroundCheckRONBAMMaker("Background Check RONBAM"); static oops::FilterMaker > diff --git a/src/ufo/profile/ProfileAveragePressure.cc b/src/ufo/profile/ProfileAveragePressure.cc index bab9e8dad..6a63c1687 100644 --- a/src/ufo/profile/ProfileAveragePressure.cc +++ b/src/ufo/profile/ProfileAveragePressure.cc @@ -23,7 +23,7 @@ namespace ufo { makerProfileAveragePressure_("AveragePressure"); ProfileAveragePressure::ProfileAveragePressure - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileAveragePressure.h b/src/ufo/profile/ProfileAveragePressure.h index 403116791..b223af7af 100644 --- a/src/ufo/profile/ProfileAveragePressure.h +++ b/src/ufo/profile/ProfileAveragePressure.h @@ -19,7 +19,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; class ProfileDataHandler; class ProfileDataHolder; } @@ -30,7 +30,7 @@ namespace ufo { /// The transformed pressures are used in subsequent profile averaging routines. class ProfileAveragePressure : public ProfileCheckBase { public: - explicit ProfileAveragePressure(const ProfileConsistencyCheckParameters &options); + explicit ProfileAveragePressure(const ConventionalProfileProcessingParameters &options); /// Run check on all profiles. void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileAverageRelativeHumidity.cc b/src/ufo/profile/ProfileAverageRelativeHumidity.cc index da9b450d9..1d8cf78cc 100644 --- a/src/ufo/profile/ProfileAverageRelativeHumidity.cc +++ b/src/ufo/profile/ProfileAverageRelativeHumidity.cc @@ -23,7 +23,7 @@ namespace ufo { makerProfileAverageRelativeHumidity_("AverageRelativeHumidity"); ProfileAverageRelativeHumidity::ProfileAverageRelativeHumidity - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileAverageRelativeHumidity.h b/src/ufo/profile/ProfileAverageRelativeHumidity.h index 3b12a46cc..b7273b072 100644 --- a/src/ufo/profile/ProfileAverageRelativeHumidity.h +++ b/src/ufo/profile/ProfileAverageRelativeHumidity.h @@ -21,7 +21,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -42,7 +42,7 @@ namespace ufo { /// and the associated temperature thresholds. class ProfileAverageRelativeHumidity : public ProfileCheckBase { public: - explicit ProfileAverageRelativeHumidity(const ProfileConsistencyCheckParameters &options); + explicit ProfileAverageRelativeHumidity(const ConventionalProfileProcessingParameters &options); /// Average relative humidity observations onto model levels and store the results. /// \throws eckit::BadValue if vectors produced by the AveragePressure routine diff --git a/src/ufo/profile/ProfileAverageTemperature.cc b/src/ufo/profile/ProfileAverageTemperature.cc index a551d066a..9bf316c1a 100644 --- a/src/ufo/profile/ProfileAverageTemperature.cc +++ b/src/ufo/profile/ProfileAverageTemperature.cc @@ -23,7 +23,7 @@ namespace ufo { makerProfileAverageTemperature_("AverageTemperature"); ProfileAverageTemperature::ProfileAverageTemperature - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileAverageTemperature.h b/src/ufo/profile/ProfileAverageTemperature.h index 3728415f5..e40f74084 100644 --- a/src/ufo/profile/ProfileAverageTemperature.h +++ b/src/ufo/profile/ProfileAverageTemperature.h @@ -21,7 +21,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -43,7 +43,7 @@ namespace ufo { /// the averaged observation value. class ProfileAverageTemperature : public ProfileCheckBase { public: - explicit ProfileAverageTemperature(const ProfileConsistencyCheckParameters &options); + explicit ProfileAverageTemperature(const ConventionalProfileProcessingParameters &options); /// Average temperature observations onto model levels and store the results. /// \throws eckit::BadValue if vectors produced by the AveragePressure routine diff --git a/src/ufo/profile/ProfileAverageWindSpeed.cc b/src/ufo/profile/ProfileAverageWindSpeed.cc index 4b7ce3818..026e9ba79 100644 --- a/src/ufo/profile/ProfileAverageWindSpeed.cc +++ b/src/ufo/profile/ProfileAverageWindSpeed.cc @@ -23,7 +23,7 @@ namespace ufo { makerProfileAverageWindSpeed_("AverageWindSpeed"); ProfileAverageWindSpeed::ProfileAverageWindSpeed - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileAverageWindSpeed.h b/src/ufo/profile/ProfileAverageWindSpeed.h index 6ca1914bf..f83bc51f5 100644 --- a/src/ufo/profile/ProfileAverageWindSpeed.h +++ b/src/ufo/profile/ProfileAverageWindSpeed.h @@ -21,7 +21,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -35,7 +35,7 @@ namespace ufo { /// over model layers defined by adjacent pressure levels, including the surface pressure. class ProfileAverageWindSpeed : public ProfileCheckBase { public: - explicit ProfileAverageWindSpeed(const ProfileConsistencyCheckParameters &options); + explicit ProfileAverageWindSpeed(const ConventionalProfileProcessingParameters &options); /// Average wind speed observations onto model levels and store the results. /// \throws eckit::BadValue if vectors produced by the AveragePressure routine diff --git a/src/ufo/profile/ProfileCheckBackgroundGeopotentialHeight.cc b/src/ufo/profile/ProfileCheckBackgroundGeopotentialHeight.cc index adc7f2942..a49093a49 100644 --- a/src/ufo/profile/ProfileCheckBackgroundGeopotentialHeight.cc +++ b/src/ufo/profile/ProfileCheckBackgroundGeopotentialHeight.cc @@ -13,7 +13,7 @@ namespace ufo { makerProfileCheckBackgroundGeopotentialHeight_("BackgroundGeopotentialHeight"); ProfileCheckBackgroundGeopotentialHeight::ProfileCheckBackgroundGeopotentialHeight - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileCheckBackgroundGeopotentialHeight.h b/src/ufo/profile/ProfileCheckBackgroundGeopotentialHeight.h index 571c8e6d2..457c730fe 100644 --- a/src/ufo/profile/ProfileCheckBackgroundGeopotentialHeight.h +++ b/src/ufo/profile/ProfileCheckBackgroundGeopotentialHeight.h @@ -22,7 +22,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -36,7 +36,7 @@ namespace ufo { class ProfileCheckBackgroundGeopotentialHeight : public ProfileCheckBase { public: explicit ProfileCheckBackgroundGeopotentialHeight - (const ProfileConsistencyCheckParameters &options); + (const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckBackgroundRelativeHumidity.cc b/src/ufo/profile/ProfileCheckBackgroundRelativeHumidity.cc index d1f3d0ef6..b1256f122 100644 --- a/src/ufo/profile/ProfileCheckBackgroundRelativeHumidity.cc +++ b/src/ufo/profile/ProfileCheckBackgroundRelativeHumidity.cc @@ -13,7 +13,7 @@ namespace ufo { makerProfileCheckBackgroundRelativeHumidity_("BackgroundRelativeHumidity"); ProfileCheckBackgroundRelativeHumidity::ProfileCheckBackgroundRelativeHumidity - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileCheckBackgroundRelativeHumidity.h b/src/ufo/profile/ProfileCheckBackgroundRelativeHumidity.h index 549d37566..c0380bc99 100644 --- a/src/ufo/profile/ProfileCheckBackgroundRelativeHumidity.h +++ b/src/ufo/profile/ProfileCheckBackgroundRelativeHumidity.h @@ -22,7 +22,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -36,7 +36,7 @@ namespace ufo { class ProfileCheckBackgroundRelativeHumidity : public ProfileCheckBase { public: explicit ProfileCheckBackgroundRelativeHumidity - (const ProfileConsistencyCheckParameters &options); + (const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckBackgroundTemperature.cc b/src/ufo/profile/ProfileCheckBackgroundTemperature.cc index a37c8622d..6941b39c6 100644 --- a/src/ufo/profile/ProfileCheckBackgroundTemperature.cc +++ b/src/ufo/profile/ProfileCheckBackgroundTemperature.cc @@ -13,7 +13,7 @@ namespace ufo { makerProfileCheckBackgroundTemperature_("BackgroundTemperature"); ProfileCheckBackgroundTemperature::ProfileCheckBackgroundTemperature - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileCheckBackgroundTemperature.h b/src/ufo/profile/ProfileCheckBackgroundTemperature.h index ebb656781..7197c55cb 100644 --- a/src/ufo/profile/ProfileCheckBackgroundTemperature.h +++ b/src/ufo/profile/ProfileCheckBackgroundTemperature.h @@ -22,7 +22,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -35,7 +35,8 @@ namespace ufo { /// (except those with PGE > 0.999) will be used in vertical averaging. class ProfileCheckBackgroundTemperature : public ProfileCheckBase { public: - explicit ProfileCheckBackgroundTemperature(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckBackgroundTemperature + (const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckBackgroundWindSpeed.cc b/src/ufo/profile/ProfileCheckBackgroundWindSpeed.cc index eeffe04ba..1cc8b6650 100644 --- a/src/ufo/profile/ProfileCheckBackgroundWindSpeed.cc +++ b/src/ufo/profile/ProfileCheckBackgroundWindSpeed.cc @@ -13,7 +13,7 @@ namespace ufo { makerProfileCheckBackgroundWindSpeed_("BackgroundWindSpeed"); ProfileCheckBackgroundWindSpeed::ProfileCheckBackgroundWindSpeed - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileCheckBackgroundWindSpeed.h b/src/ufo/profile/ProfileCheckBackgroundWindSpeed.h index 1aa105a26..5d11cb361 100644 --- a/src/ufo/profile/ProfileCheckBackgroundWindSpeed.h +++ b/src/ufo/profile/ProfileCheckBackgroundWindSpeed.h @@ -22,7 +22,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -35,7 +35,8 @@ namespace ufo { /// (except those with PGE > 0.999) will be used in vertical averaging. class ProfileCheckBackgroundWindSpeed : public ProfileCheckBase { public: - explicit ProfileCheckBackgroundWindSpeed(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckBackgroundWindSpeed + (const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckBase.cc b/src/ufo/profile/ProfileCheckBase.cc index 1a5a6e69b..379f74b9b 100644 --- a/src/ufo/profile/ProfileCheckBase.cc +++ b/src/ufo/profile/ProfileCheckBase.cc @@ -13,7 +13,7 @@ #include "oops/util/Logger.h" namespace ufo { - ProfileCheckBase::ProfileCheckBase(const ProfileConsistencyCheckParameters &options) + ProfileCheckBase::ProfileCheckBase(const ConventionalProfileProcessingParameters &options) : options_(options) {} @@ -26,7 +26,7 @@ namespace ufo { std::unique_ptr ProfileCheckFactory::create(const std::string& name, - const ProfileConsistencyCheckParameters &options) + const ConventionalProfileProcessingParameters &options) { oops::Log::trace() << "ProfileCheckBase::create starting" << std::endl; typename std::map::iterator jloc = diff --git a/src/ufo/profile/ProfileCheckBase.h b/src/ufo/profile/ProfileCheckBase.h index ddc26b113..45fea3fbf 100644 --- a/src/ufo/profile/ProfileCheckBase.h +++ b/src/ufo/profile/ProfileCheckBase.h @@ -25,7 +25,7 @@ #include "oops/util/missingValues.h" #include "oops/util/PropertiesOfNVectors.h" -#include "ufo/filters/ProfileConsistencyCheckParameters.h" +#include "ufo/filters/ConventionalProfileProcessingParameters.h" #include "ufo/profile/ProfileDataHandler.h" #include "ufo/profile/VariableNames.h" @@ -37,7 +37,7 @@ namespace ufo { /// \brief Profile QC checker base class class ProfileCheckBase { public: - explicit ProfileCheckBase(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckBase(const ConventionalProfileProcessingParameters &options); virtual ~ProfileCheckBase() {} @@ -104,7 +104,7 @@ namespace ufo { protected: // variables /// Configurable parameters - const ProfileConsistencyCheckParameters &options_; + const ConventionalProfileProcessingParameters &options_; /// Missing value (int) const int missingValueInt = util::missingValue(1); @@ -118,12 +118,13 @@ namespace ufo { { public: static std::unique_ptr create(const std::string&, - const ProfileConsistencyCheckParameters&); + const ConventionalProfileProcessingParameters&); virtual ~ProfileCheckFactory() = default; protected: explicit ProfileCheckFactory(const std::string &); private: - virtual std::unique_ptr make(const ProfileConsistencyCheckParameters&) = 0; + virtual std::unique_ptr make + (const ConventionalProfileProcessingParameters&) = 0; static std::map & getMakers() { @@ -136,7 +137,7 @@ namespace ufo { class ProfileCheckMaker : public ProfileCheckFactory { virtual std::unique_ptr - make(const ProfileConsistencyCheckParameters &options) + make(const ConventionalProfileProcessingParameters &options) { return std::unique_ptr(new T(options)); } diff --git a/src/ufo/profile/ProfileCheckBasic.cc b/src/ufo/profile/ProfileCheckBasic.cc index 5a04a587f..8f2d95013 100644 --- a/src/ufo/profile/ProfileCheckBasic.cc +++ b/src/ufo/profile/ProfileCheckBasic.cc @@ -12,7 +12,7 @@ namespace ufo { static ProfileCheckMaker makerProfileCheckBasic_("Basic"); - ProfileCheckBasic::ProfileCheckBasic(const ProfileConsistencyCheckParameters &options) + ProfileCheckBasic::ProfileCheckBasic(const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileCheckBasic.h b/src/ufo/profile/ProfileCheckBasic.h index d105b900c..d4e69af3b 100644 --- a/src/ufo/profile/ProfileCheckBasic.h +++ b/src/ufo/profile/ProfileCheckBasic.h @@ -18,7 +18,7 @@ namespace ufo { /// \brief Profile QC: basic checks on pressure class ProfileCheckBasic : public ProfileCheckBase { public: - explicit ProfileCheckBasic(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckBasic(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckHistory.cc b/src/ufo/profile/ProfileCheckHistory.cc index b306b1a07..b037d2264 100644 --- a/src/ufo/profile/ProfileCheckHistory.cc +++ b/src/ufo/profile/ProfileCheckHistory.cc @@ -13,7 +13,7 @@ namespace ufo { makerProfileCheckHistory_("History"); ProfileCheckHistory::ProfileCheckHistory - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileCheckHistory.h b/src/ufo/profile/ProfileCheckHistory.h index 01f05449b..30048748d 100644 --- a/src/ufo/profile/ProfileCheckHistory.h +++ b/src/ufo/profile/ProfileCheckHistory.h @@ -23,7 +23,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -31,7 +31,7 @@ namespace ufo { /// \brief Profile QC: history check. class ProfileCheckHistory : public ProfileCheckBase { public: - explicit ProfileCheckHistory(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckHistory(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckHydrostatic.cc b/src/ufo/profile/ProfileCheckHydrostatic.cc index f3e2aea6b..a12c51656 100644 --- a/src/ufo/profile/ProfileCheckHydrostatic.cc +++ b/src/ufo/profile/ProfileCheckHydrostatic.cc @@ -12,7 +12,8 @@ namespace ufo { static ProfileCheckMaker makerProfileCheckHydrostatic_("Hydrostatic"); - ProfileCheckHydrostatic::ProfileCheckHydrostatic(const ProfileConsistencyCheckParameters &options) + ProfileCheckHydrostatic::ProfileCheckHydrostatic + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options), ProfileStandardLevels(options) {} diff --git a/src/ufo/profile/ProfileCheckHydrostatic.h b/src/ufo/profile/ProfileCheckHydrostatic.h index c7fd6add4..4cd2debc2 100644 --- a/src/ufo/profile/ProfileCheckHydrostatic.h +++ b/src/ufo/profile/ProfileCheckHydrostatic.h @@ -18,7 +18,7 @@ #include "ufo/profile/ProfileStandardLevels.h" namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -27,7 +27,7 @@ namespace ufo { class ProfileCheckHydrostatic : public ProfileCheckBase, private ProfileStandardLevels { public: - explicit ProfileCheckHydrostatic(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckHydrostatic(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckInterpolation.cc b/src/ufo/profile/ProfileCheckInterpolation.cc index c783e262f..123ab65dc 100644 --- a/src/ufo/profile/ProfileCheckInterpolation.cc +++ b/src/ufo/profile/ProfileCheckInterpolation.cc @@ -14,7 +14,7 @@ namespace ufo { makerProfileCheckInterpolation_("Interpolation"); ProfileCheckInterpolation::ProfileCheckInterpolation - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options), ProfileStandardLevels(options) {} diff --git a/src/ufo/profile/ProfileCheckInterpolation.h b/src/ufo/profile/ProfileCheckInterpolation.h index dc676b5e1..bb5e3fa88 100644 --- a/src/ufo/profile/ProfileCheckInterpolation.h +++ b/src/ufo/profile/ProfileCheckInterpolation.h @@ -17,7 +17,7 @@ #include "ufo/profile/ProfileStandardLevels.h" namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -26,7 +26,7 @@ namespace ufo { class ProfileCheckInterpolation : public ProfileCheckBase, private ProfileStandardLevels { public: - explicit ProfileCheckInterpolation(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckInterpolation(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckPermanentReject.cc b/src/ufo/profile/ProfileCheckPermanentReject.cc index 165beb7da..8e642c4eb 100644 --- a/src/ufo/profile/ProfileCheckPermanentReject.cc +++ b/src/ufo/profile/ProfileCheckPermanentReject.cc @@ -13,7 +13,7 @@ namespace ufo { makerProfileCheckPermanentReject_("PermanentReject"); ProfileCheckPermanentReject::ProfileCheckPermanentReject - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileCheckPermanentReject.h b/src/ufo/profile/ProfileCheckPermanentReject.h index 61eea4e8d..44cc8a104 100644 --- a/src/ufo/profile/ProfileCheckPermanentReject.h +++ b/src/ufo/profile/ProfileCheckPermanentReject.h @@ -20,7 +20,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -28,7 +28,7 @@ namespace ufo { /// \brief Profile QC: reject observations which are flagged to be permanently rejected. class ProfileCheckPermanentReject : public ProfileCheckBase { public: - explicit ProfileCheckPermanentReject(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckPermanentReject(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckRH.cc b/src/ufo/profile/ProfileCheckRH.cc index 22e938a0d..d6d4c7b99 100644 --- a/src/ufo/profile/ProfileCheckRH.cc +++ b/src/ufo/profile/ProfileCheckRH.cc @@ -13,7 +13,7 @@ namespace ufo { static ProfileCheckMaker makerProfileCheckRH_("RH"); ProfileCheckRH::ProfileCheckRH - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileCheckRH.h b/src/ufo/profile/ProfileCheckRH.h index dd0fe44ab..b9ce814e3 100644 --- a/src/ufo/profile/ProfileCheckRH.h +++ b/src/ufo/profile/ProfileCheckRH.h @@ -16,7 +16,7 @@ #include "ufo/profile/ProfileDataHandler.h" namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -27,7 +27,7 @@ namespace ufo { class ProfileCheckRH : public ProfileCheckBase { public: - explicit ProfileCheckRH(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckRH(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckSamePDiffT.cc b/src/ufo/profile/ProfileCheckSamePDiffT.cc index 06592caaf..2a41951aa 100644 --- a/src/ufo/profile/ProfileCheckSamePDiffT.cc +++ b/src/ufo/profile/ProfileCheckSamePDiffT.cc @@ -12,7 +12,8 @@ namespace ufo { static ProfileCheckMaker makerProfileCheckSamePDiffT_("SamePDiffT"); - ProfileCheckSamePDiffT::ProfileCheckSamePDiffT(const ProfileConsistencyCheckParameters &options) + ProfileCheckSamePDiffT::ProfileCheckSamePDiffT + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileCheckSamePDiffT.h b/src/ufo/profile/ProfileCheckSamePDiffT.h index 7f7555477..70f5c51bb 100644 --- a/src/ufo/profile/ProfileCheckSamePDiffT.h +++ b/src/ufo/profile/ProfileCheckSamePDiffT.h @@ -14,7 +14,7 @@ #include "ufo/profile/ProfileDataHandler.h" namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -22,7 +22,7 @@ namespace ufo { /// \brief Profile QC: same P/different T check class ProfileCheckSamePDiffT : public ProfileCheckBase { public: - explicit ProfileCheckSamePDiffT(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckSamePDiffT(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckSign.cc b/src/ufo/profile/ProfileCheckSign.cc index 617501939..8d4e58a5b 100644 --- a/src/ufo/profile/ProfileCheckSign.cc +++ b/src/ufo/profile/ProfileCheckSign.cc @@ -12,7 +12,7 @@ namespace ufo { static ProfileCheckMaker makerProfileCheckSign_("Sign"); - ProfileCheckSign::ProfileCheckSign(const ProfileConsistencyCheckParameters &options) + ProfileCheckSign::ProfileCheckSign(const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileCheckSign.h b/src/ufo/profile/ProfileCheckSign.h index c0f1df76f..8e8ead0d3 100644 --- a/src/ufo/profile/ProfileCheckSign.h +++ b/src/ufo/profile/ProfileCheckSign.h @@ -14,7 +14,7 @@ #include "ufo/profile/ProfileDataHandler.h" namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -22,7 +22,7 @@ namespace ufo { /// \brief Profile QC: sign check class ProfileCheckSign : public ProfileCheckBase { public: - explicit ProfileCheckSign(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckSign(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckTime.cc b/src/ufo/profile/ProfileCheckTime.cc index 1b8dce7ee..afcc9fdc9 100644 --- a/src/ufo/profile/ProfileCheckTime.cc +++ b/src/ufo/profile/ProfileCheckTime.cc @@ -16,7 +16,7 @@ namespace ufo { makerProfileCheckTime_("Time"); ProfileCheckTime::ProfileCheckTime - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileCheckTime.h b/src/ufo/profile/ProfileCheckTime.h index ea00a2756..8a97b0ffb 100644 --- a/src/ufo/profile/ProfileCheckTime.h +++ b/src/ufo/profile/ProfileCheckTime.h @@ -23,7 +23,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -32,7 +32,7 @@ namespace ufo { /// Also, if requested, reject data taken a short period after the sonde launch. class ProfileCheckTime : public ProfileCheckBase { public: - explicit ProfileCheckTime(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckTime(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckUInterp.cc b/src/ufo/profile/ProfileCheckUInterp.cc index 49f212735..851f039bd 100644 --- a/src/ufo/profile/ProfileCheckUInterp.cc +++ b/src/ufo/profile/ProfileCheckUInterp.cc @@ -13,7 +13,7 @@ namespace ufo { static ProfileCheckMaker makerProfileCheckUInterp_("UInterp"); ProfileCheckUInterp::ProfileCheckUInterp - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options), ProfileStandardLevels(options) {} diff --git a/src/ufo/profile/ProfileCheckUInterp.h b/src/ufo/profile/ProfileCheckUInterp.h index a919b8100..06661c0c9 100644 --- a/src/ufo/profile/ProfileCheckUInterp.h +++ b/src/ufo/profile/ProfileCheckUInterp.h @@ -17,7 +17,7 @@ #include "ufo/profile/ProfileStandardLevels.h" namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -26,7 +26,7 @@ namespace ufo { class ProfileCheckUInterp : public ProfileCheckBase, private ProfileStandardLevels { public: - explicit ProfileCheckUInterp(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckUInterp(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckUInterpAlternative.cc b/src/ufo/profile/ProfileCheckUInterpAlternative.cc index 682e9b3f3..1abe23a06 100644 --- a/src/ufo/profile/ProfileCheckUInterpAlternative.cc +++ b/src/ufo/profile/ProfileCheckUInterpAlternative.cc @@ -17,7 +17,7 @@ namespace ufo { makerProfileCheckUInterpAlternative_("UInterpAlternative"); ProfileCheckUInterpAlternative::ProfileCheckUInterpAlternative - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options), ProfileStandardLevels(options) {} diff --git a/src/ufo/profile/ProfileCheckUInterpAlternative.h b/src/ufo/profile/ProfileCheckUInterpAlternative.h index e1389c7f5..2332c24bf 100644 --- a/src/ufo/profile/ProfileCheckUInterpAlternative.h +++ b/src/ufo/profile/ProfileCheckUInterpAlternative.h @@ -18,7 +18,7 @@ #include "ufo/profile/ProfileStandardLevels.h" namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -29,7 +29,8 @@ namespace ufo { class ProfileCheckUInterpAlternative : public ProfileCheckBase, private ProfileStandardLevels { public: - explicit ProfileCheckUInterpAlternative(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckUInterpAlternative + (const ConventionalProfileProcessingParameters &options); /// Run check on all profiles. void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckUnstableLayer.cc b/src/ufo/profile/ProfileCheckUnstableLayer.cc index a8e768cd3..aa68ae8a8 100644 --- a/src/ufo/profile/ProfileCheckUnstableLayer.cc +++ b/src/ufo/profile/ProfileCheckUnstableLayer.cc @@ -14,7 +14,7 @@ namespace ufo { makerProfileCheckUnstableLayer_("UnstableLayer"); ProfileCheckUnstableLayer::ProfileCheckUnstableLayer - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileCheckUnstableLayer.h b/src/ufo/profile/ProfileCheckUnstableLayer.h index 2f40a1451..9a406de3d 100644 --- a/src/ufo/profile/ProfileCheckUnstableLayer.h +++ b/src/ufo/profile/ProfileCheckUnstableLayer.h @@ -16,7 +16,7 @@ #include "ufo/profile/ProfileDataHandler.h" namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -24,7 +24,7 @@ namespace ufo { /// \brief Profile QC: unstable layer check class ProfileCheckUnstableLayer : public ProfileCheckBase { public: - explicit ProfileCheckUnstableLayer(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckUnstableLayer(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileCheckValidator.cc b/src/ufo/profile/ProfileCheckValidator.cc index 54bc9ab9a..cdba4922f 100644 --- a/src/ufo/profile/ProfileCheckValidator.cc +++ b/src/ufo/profile/ProfileCheckValidator.cc @@ -20,7 +20,8 @@ #include "ufo/utils/StringUtils.h" namespace ufo { - ProfileCheckValidator::ProfileCheckValidator(const ProfileConsistencyCheckParameters &options) + ProfileCheckValidator::ProfileCheckValidator + (const ConventionalProfileProcessingParameters &options) : options_(options) { // Set offsets due to C++ and Fortran array index starting values diff --git a/src/ufo/profile/ProfileCheckValidator.h b/src/ufo/profile/ProfileCheckValidator.h index 99caaf26a..4c56f62d5 100644 --- a/src/ufo/profile/ProfileCheckValidator.h +++ b/src/ufo/profile/ProfileCheckValidator.h @@ -13,7 +13,7 @@ #include #include -#include "ufo/filters/ProfileConsistencyCheckParameters.h" +#include "ufo/filters/ConventionalProfileProcessingParameters.h" #include "ufo/profile/ProfileDataHandler.h" #include "ufo/profile/VariableNames.h" @@ -26,7 +26,7 @@ namespace ufo { /// the equivalent values produced in the OPS code. class ProfileCheckValidator { public: - explicit ProfileCheckValidator(const ProfileConsistencyCheckParameters &options); + explicit ProfileCheckValidator(const ConventionalProfileProcessingParameters &options); /// Validate check results against OPS values. void validate(ProfileDataHandler &profileDataHandler, @@ -61,7 +61,7 @@ namespace ufo { private: // variables /// Configurable parameters. - const ProfileConsistencyCheckParameters &options_; + const ConventionalProfileProcessingParameters &options_; /// Counters that are accumulated across profiles. std::map cumulativeCounters_; diff --git a/src/ufo/profile/ProfileChecker.cc b/src/ufo/profile/ProfileChecker.cc index 8fee0072b..4584dbf66 100644 --- a/src/ufo/profile/ProfileChecker.cc +++ b/src/ufo/profile/ProfileChecker.cc @@ -27,7 +27,7 @@ #include "ufo/utils/metoffice/MetOfficeQCFlags.h" namespace ufo { - ProfileChecker::ProfileChecker(const ProfileConsistencyCheckParameters &options) + ProfileChecker::ProfileChecker(const ConventionalProfileProcessingParameters &options) : options_(options), checks_(options.Checks.value()) { diff --git a/src/ufo/profile/ProfileChecker.h b/src/ufo/profile/ProfileChecker.h index 6574f2415..c307807ac 100644 --- a/src/ufo/profile/ProfileChecker.h +++ b/src/ufo/profile/ProfileChecker.h @@ -14,7 +14,7 @@ #include #include -#include "ufo/filters/ProfileConsistencyCheckParameters.h" +#include "ufo/filters/ConventionalProfileProcessingParameters.h" namespace ufo { class ProfileDataHandler; @@ -36,7 +36,7 @@ namespace ufo { /// Runs the various QC checks on individual profiles and modifies flags accordingly. class ProfileChecker { public: - explicit ProfileChecker(const ProfileConsistencyCheckParameters &options); + explicit ProfileChecker(const ConventionalProfileProcessingParameters &options); /// Type for container of check subgroups. typedef std::vector CheckSubgroupList; @@ -65,7 +65,7 @@ namespace ufo { private: /// Configurable parameters - const ProfileConsistencyCheckParameters &options_; + const ConventionalProfileProcessingParameters &options_; /// Checks to perform std::vector checks_; diff --git a/src/ufo/profile/ProfilePressure.cc b/src/ufo/profile/ProfilePressure.cc index b10a7a1a5..8da3faac3 100644 --- a/src/ufo/profile/ProfilePressure.cc +++ b/src/ufo/profile/ProfilePressure.cc @@ -13,7 +13,7 @@ namespace ufo { makerProfilePressure_("Pressure"); ProfilePressure::ProfilePressure - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfilePressure.h b/src/ufo/profile/ProfilePressure.h index 13611c052..23f0307eb 100644 --- a/src/ufo/profile/ProfilePressure.h +++ b/src/ufo/profile/ProfilePressure.h @@ -26,7 +26,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -39,7 +39,7 @@ namespace ufo { /// Profiles without a pressure sensor are flagged. class ProfilePressure : public ProfileCheckBase { public: - explicit ProfilePressure(const ProfileConsistencyCheckParameters &options); + explicit ProfilePressure(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileSondeFlags.cc b/src/ufo/profile/ProfileSondeFlags.cc index f4a75741a..88d5c9e20 100644 --- a/src/ufo/profile/ProfileSondeFlags.cc +++ b/src/ufo/profile/ProfileSondeFlags.cc @@ -13,7 +13,7 @@ namespace ufo { makerProfileSondeFlags_("SondeFlags"); ProfileSondeFlags::ProfileSondeFlags - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileSondeFlags.h b/src/ufo/profile/ProfileSondeFlags.h index ac37e443b..257c34168 100644 --- a/src/ufo/profile/ProfileSondeFlags.h +++ b/src/ufo/profile/ProfileSondeFlags.h @@ -24,7 +24,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -40,7 +40,7 @@ namespace ufo { /// 11 800.0 missing 50.0 knots, 270 degs class ProfileSondeFlags : public ProfileCheckBase { public: - explicit ProfileSondeFlags(const ProfileConsistencyCheckParameters &options); + explicit ProfileSondeFlags(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/profile/ProfileStandardLevels.cc b/src/ufo/profile/ProfileStandardLevels.cc index 4893e45dd..017a32e97 100644 --- a/src/ufo/profile/ProfileStandardLevels.cc +++ b/src/ufo/profile/ProfileStandardLevels.cc @@ -8,7 +8,8 @@ #include "ufo/profile/ProfileStandardLevels.h" namespace ufo { - ProfileStandardLevels::ProfileStandardLevels(const ProfileConsistencyCheckParameters &options) + ProfileStandardLevels::ProfileStandardLevels + (const ConventionalProfileProcessingParameters &options) : optionsSL_(options) { StandardLevels_ = optionsSL_.StandardLevels.value(); diff --git a/src/ufo/profile/ProfileStandardLevels.h b/src/ufo/profile/ProfileStandardLevels.h index 706a1408b..38871599d 100644 --- a/src/ufo/profile/ProfileStandardLevels.h +++ b/src/ufo/profile/ProfileStandardLevels.h @@ -17,7 +17,7 @@ #include "oops/util/Logger.h" #include "oops/util/missingValues.h" -#include "ufo/filters/ProfileConsistencyCheckParameters.h" +#include "ufo/filters/ConventionalProfileProcessingParameters.h" #include "ufo/utils/metoffice/MetOfficeQCFlags.h" @@ -30,7 +30,7 @@ namespace ufo { /// \brief Calculate standard levels class ProfileStandardLevels { public: - explicit ProfileStandardLevels(const ProfileConsistencyCheckParameters &options); + explicit ProfileStandardLevels(const ConventionalProfileProcessingParameters &options); virtual ~ProfileStandardLevels() {} protected: // functions @@ -58,7 +58,7 @@ namespace ufo { std::vector BigGaps_; /// Configurable parameters - const ProfileConsistencyCheckParameters &optionsSL_; + const ConventionalProfileProcessingParameters &optionsSL_; /// Number of significant levels int NumSig_; diff --git a/src/ufo/profile/ProfileWindProfilerFlags.cc b/src/ufo/profile/ProfileWindProfilerFlags.cc index 50a3887cc..3407afaea 100644 --- a/src/ufo/profile/ProfileWindProfilerFlags.cc +++ b/src/ufo/profile/ProfileWindProfilerFlags.cc @@ -13,7 +13,7 @@ namespace ufo { static ProfileCheckMaker makerProfileWindProfilerFlags_("WinProFlags"); ProfileWindProfilerFlags::ProfileWindProfilerFlags - (const ProfileConsistencyCheckParameters &options) + (const ConventionalProfileProcessingParameters &options) : ProfileCheckBase(options) {} diff --git a/src/ufo/profile/ProfileWindProfilerFlags.h b/src/ufo/profile/ProfileWindProfilerFlags.h index 434082b81..2699d5216 100644 --- a/src/ufo/profile/ProfileWindProfilerFlags.h +++ b/src/ufo/profile/ProfileWindProfilerFlags.h @@ -22,7 +22,7 @@ namespace ioda { } namespace ufo { - class ProfileConsistencyCheckParameters; + class ConventionalProfileProcessingParameters; } namespace ufo { @@ -31,7 +31,7 @@ namespace ufo { /// Rejects levels of wind-profiler observations for which reported QC flags indicate bad obs. class ProfileWindProfilerFlags : public ProfileCheckBase { public: - explicit ProfileWindProfilerFlags(const ProfileConsistencyCheckParameters &options); + explicit ProfileWindProfilerFlags(const ConventionalProfileProcessingParameters &options); /// Run check void runCheck(ProfileDataHandler &profileDataHandler) override; diff --git a/src/ufo/variabletransforms/Cal_Humidity.cc b/src/ufo/variabletransforms/Cal_Humidity.cc index 4d885b9df..d83e3e582 100644 --- a/src/ufo/variabletransforms/Cal_Humidity.cc +++ b/src/ufo/variabletransforms/Cal_Humidity.cc @@ -5,7 +5,6 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#include "ufo/filters/ProfileConsistencyCheckParameters.h" #include "ufo/utils/Constants.h" #include "ufo/variabletransforms/Cal_Humidity.h" diff --git a/src/ufo/variabletransforms/Cal_PressureFromHeight.cc b/src/ufo/variabletransforms/Cal_PressureFromHeight.cc index 664010b5c..c874ec120 100644 --- a/src/ufo/variabletransforms/Cal_PressureFromHeight.cc +++ b/src/ufo/variabletransforms/Cal_PressureFromHeight.cc @@ -6,7 +6,6 @@ */ #include "ufo/variabletransforms/Cal_PressureFromHeight.h" -#include "ufo/filters/ProfileConsistencyCheckParameters.h" #include "ufo/utils/Constants.h" namespace ufo { diff --git a/src/ufo/variabletransforms/Cal_Wind.cc b/src/ufo/variabletransforms/Cal_Wind.cc index de471ac82..1fb8626f7 100644 --- a/src/ufo/variabletransforms/Cal_Wind.cc +++ b/src/ufo/variabletransforms/Cal_Wind.cc @@ -6,7 +6,6 @@ */ #include "ufo/variabletransforms/Cal_Wind.h" -#include "ufo/filters/ProfileConsistencyCheckParameters.h" #include "ufo/utils/Constants.h" namespace ufo { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 051d31a7d..297debfbb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -232,43 +232,43 @@ list( APPEND ufo_test_input testinput/sndrd1-4_crtm.yaml testinput/sfcpcorrected.yaml testinput/thickness_predictor.yaml - testinput/profileconsistencychecks_monolithicfilter.yaml - testinput/profileconsistencychecks_OPScomparison.yaml - testinput/profileconsistencychecks_wrongOPScomparison.yaml - testinput/profileconsistencychecks_separatefilters.yaml - testinput/profileconsistencychecks_unittests.yaml - testinput/profileconsistencychecks_unittest_oneprofileMPI.yaml - testinput/profileconsistencychecks_unittests_vertaverage.yaml - testinput/profileconsistencychecks_unittests_vertaverage_ascending.yaml - testinput/profileconsistencychecks_unittests_vertinterp.yaml - testinput/profileconsistencychecks_unittests_profiledataholder.yaml - testinput/profileconsistencychecks_RH_obsfilter.yaml - testinput/profileconsistencychecks_RH_OPScomparison.yaml - testinput/profileconsistencychecks_UInterp_obsfilter.yaml - testinput/profileconsistencychecks_UInterp_OPScomparison.yaml - testinput/profileconsistencychecks_UInterpAlternative_obsfilter.yaml - testinput/profileconsistencychecks_UInterpAlternative_OPScomparison.yaml - testinput/profileconsistencychecks_bkgqc_modobs_obsfilter.yaml - testinput/profileconsistencychecks_bkgqc_modobs_separatefilters_obsfilter.yaml - testinput/profileconsistencychecks_bkgqc_repobs_obsfilter.yaml - testinput/profileconsistencychecks_bkgqc_repobs_separatefilters_obsfilter.yaml - testinput/profileconsistencychecks_bkgqc_modobs_OPScomparison.yaml - testinput/profileconsistencychecks_bkgqc_repobs_OPScomparison.yaml - testinput/profileconsistencychecks_sondeflags_obsfilter.yaml - testinput/profileconsistencychecks_sondeflags_OPScomparison.yaml - testinput/profileconsistencychecks_winproflags_obsfilter.yaml - testinput/profileconsistencychecks_winproflags_OPScomparison.yaml - testinput/profileconsistencychecks_pressure_obsfilter.yaml - testinput/profileconsistencychecks_pressure_OPScomparison.yaml - testinput/profileconsistencychecks_average_pressure_obsfilter.yaml - testinput/profileconsistencychecks_average_pressure_OPScomparison.yaml - testinput/profileconsistencychecks_average_temperature_obsfilter.yaml - testinput/profileconsistencychecks_average_temperature_OPScomparison.yaml - testinput/profileconsistencychecks_average_windspeed_obsfilter.yaml - testinput/profileconsistencychecks_average_windspeed_OPScomparison.yaml - testinput/profileconsistencychecks_average_relativehumidity_obsfilter.yaml - testinput/profileconsistencychecks_average_relativehumidity_OPScomparison.yaml - testinput/profileconsistencychecks_opr_average.yaml + testinput/conventionalprofileprocessing_monolithicfilter.yaml + testinput/conventionalprofileprocessing_OPScomparison.yaml + testinput/conventionalprofileprocessing_wrongOPScomparison.yaml + testinput/conventionalprofileprocessing_separatefilters.yaml + testinput/conventionalprofileprocessing_unittests.yaml + testinput/conventionalprofileprocessing_unittest_oneprofileMPI.yaml + testinput/conventionalprofileprocessing_unittests_vertaverage.yaml + testinput/conventionalprofileprocessing_unittests_vertaverage_ascending.yaml + testinput/conventionalprofileprocessing_unittests_vertinterp.yaml + testinput/conventionalprofileprocessing_unittests_profiledataholder.yaml + testinput/conventionalprofileprocessing_RH_obsfilter.yaml + testinput/conventionalprofileprocessing_RH_OPScomparison.yaml + testinput/conventionalprofileprocessing_UInterp_obsfilter.yaml + testinput/conventionalprofileprocessing_UInterp_OPScomparison.yaml + testinput/conventionalprofileprocessing_UInterpAlternative_obsfilter.yaml + testinput/conventionalprofileprocessing_UInterpAlternative_OPScomparison.yaml + testinput/conventionalprofileprocessing_bkgqc_modobs_obsfilter.yaml + testinput/conventionalprofileprocessing_bkgqc_modobs_separatefilters_obsfilter.yaml + testinput/conventionalprofileprocessing_bkgqc_repobs_obsfilter.yaml + testinput/conventionalprofileprocessing_bkgqc_repobs_separatefilters_obsfilter.yaml + testinput/conventionalprofileprocessing_bkgqc_modobs_OPScomparison.yaml + testinput/conventionalprofileprocessing_bkgqc_repobs_OPScomparison.yaml + testinput/conventionalprofileprocessing_sondeflags_obsfilter.yaml + testinput/conventionalprofileprocessing_sondeflags_OPScomparison.yaml + testinput/conventionalprofileprocessing_winproflags_obsfilter.yaml + testinput/conventionalprofileprocessing_winproflags_OPScomparison.yaml + testinput/conventionalprofileprocessing_pressure_obsfilter.yaml + testinput/conventionalprofileprocessing_pressure_OPScomparison.yaml + testinput/conventionalprofileprocessing_average_pressure_obsfilter.yaml + testinput/conventionalprofileprocessing_average_pressure_OPScomparison.yaml + testinput/conventionalprofileprocessing_average_temperature_obsfilter.yaml + testinput/conventionalprofileprocessing_average_temperature_OPScomparison.yaml + testinput/conventionalprofileprocessing_average_windspeed_obsfilter.yaml + testinput/conventionalprofileprocessing_average_windspeed_OPScomparison.yaml + testinput/conventionalprofileprocessing_average_relativehumidity_obsfilter.yaml + testinput/conventionalprofileprocessing_average_relativehumidity_OPScomparison.yaml + testinput/conventionalprofileprocessing_opr_average.yaml testinput/ssmis_f17_gfs_backgroundcheck_bc.yaml testinput/ssmis_f17_gfs_backgroundcheck_nbc.yaml testinput/timeoper.yaml @@ -494,8 +494,8 @@ ecbuild_add_executable( TARGET test_ObsFunction.x LIBS ufo ) -ecbuild_add_executable( TARGET test_ProfileConsistencyChecks.x - SOURCES mains/TestProfileConsistencyChecks.cc +ecbuild_add_executable( TARGET test_ConventionalProfileProcessing.x + SOURCES mains/TestConventionalProfileProcessing.cc LIBS ufo ) @@ -1610,7 +1610,7 @@ ecbuild_add_test( TARGET test_ufo_linopr_categorical ecbuild_add_test( TARGET test_ufo_opr_profile_average COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x - ARGS "testinput/profileconsistencychecks_opr_average.yaml" + ARGS "testinput/conventionalprofileprocessing_opr_average.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsOperator.x TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) @@ -1875,101 +1875,101 @@ ecbuild_add_test( TARGET test_ufo_qc_trackcheckship ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_monolithicfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_monolithicfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_monolithicfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_monolithicfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_separatefilters +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_separatefilters COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_separatefilters.yaml" + ARGS "testinput/conventionalprofileprocessing_separatefilters.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_unittests +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_unittests COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_unittests.yaml" + ARGS "testinput/conventionalprofileprocessing_unittests.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_unittest_oneprofileMPI +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_unittest_oneprofileMPI MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_unittest_oneprofileMPI.yaml" + ARGS "testinput/conventionalprofileprocessing_unittest_oneprofileMPI.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_RH_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_RH_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_RH_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_RH_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_UInterp_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_UInterp_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_UInterp_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_UInterp_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_UInterpAlternative_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_UInterpAlternative_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_UInterpAlternative_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_UInterpAlternative_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_bkgqc_repobs_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_bkgqc_repobs_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_bkgqc_repobs_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_bkgqc_repobs_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_bkgqc_repobs_separatefilters_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_bkgqc_repobs_separatefilters_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_bkgqc_repobs_separatefilters_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_bkgqc_repobs_separatefilters_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_bkgqc_modobs_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_bkgqc_modobs_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_bkgqc_modobs_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_bkgqc_modobs_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_bkgqc_modobs_separatefilters_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_bkgqc_modobs_separatefilters_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_bkgqc_modobs_separatefilters_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_bkgqc_modobs_separatefilters_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_sondeflags_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_sondeflags_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_sondeflags_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_sondeflags_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_winproflags_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_winproflags_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_winproflags_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_winproflags_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_pressure_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_pressure_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_pressure_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_pressure_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) @@ -2009,30 +2009,30 @@ ecbuild_add_test( TARGET test_ufo_variabletransforms_SondePressureFromHeightICA DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_average_pressure_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_average_pressure_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_average_pressure_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_average_pressure_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_average_temperature_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_average_temperature_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_average_temperature_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_average_temperature_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_average_windspeed_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_average_windspeed_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_average_windspeed_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_average_windspeed_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_average_relativehumidity_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_average_relativehumidity_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_average_relativehumidity_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_average_relativehumidity_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) @@ -2044,138 +2044,138 @@ ecbuild_add_test( TARGET test_ufo_qc_acceptlist DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_monolithicfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_monolithicfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_monolithicfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_monolithicfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_separatefilters +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_separatefilters MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_separatefilters.yaml" + ARGS "testinput/conventionalprofileprocessing_separatefilters.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_unittests +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_unittests MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_unittests.yaml" + ARGS "testinput/conventionalprofileprocessing_unittests.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_RH_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_RH_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_RH_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_RH_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_UInterp_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_UInterp_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_UInterp_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_UInterp_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_UInterpAlternative_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_UInterpAlternative_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_UInterpAlternative_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_UInterpAlternative_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_bkgqc_repobs_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_bkgqc_repobs_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_bkgqc_repobs_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_bkgqc_repobs_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_bkgqc_repobs_separatefilters_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_bkgqc_repobs_separatefilters_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_bkgqc_repobs_separatefilters_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_bkgqc_repobs_separatefilters_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_bkgqc_modobs_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_bkgqc_modobs_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_bkgqc_modobs_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_bkgqc_modobs_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_bkgqc_modobs_separatefilters_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_bkgqc_modobs_separatefilters_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_bkgqc_modobs_separatefilters_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_bkgqc_modobs_separatefilters_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_sondeflags_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_sondeflags_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_sondeflags_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_sondeflags_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_winproflags_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_winproflags_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_winproflags_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_winproflags_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_pressure_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_pressure_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_pressure_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_pressure_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_average_pressure_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_average_pressure_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_average_pressure_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_average_pressure_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_average_temperature_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_average_temperature_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_average_temperature_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_average_temperature_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_average_windspeed_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_average_windspeed_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_average_windspeed_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_average_windspeed_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_average_relativehumidity_obsfilter +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_average_relativehumidity_obsfilter MPI 2 COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "testinput/profileconsistencychecks_average_relativehumidity_obsfilter.yaml" + ARGS "testinput/conventionalprofileprocessing_average_relativehumidity_obsfilter.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) @@ -2276,102 +2276,102 @@ ecbuild_add_test( TARGET test_ufo_qc_poisson_disk_thinning_parallel LIBS ufo TEST_DEPENDS ufo_get_ufo_test_data) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_OPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_OPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_OPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_bkgqc_modobs_OPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_bkgqc_modobs_OPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_bkgqc_modobs_OPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_bkgqc_modobs_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_bkgqc_repobs_OPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_bkgqc_repobs_OPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_bkgqc_repobs_OPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_bkgqc_repobs_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_RH_OPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_RH_OPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_RH_OPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_RH_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_wrongOPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_wrongOPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_wrongOPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_wrongOPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_UInterp_OPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_UInterp_OPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_UInterp_OPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_UInterp_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_UInterpAlternative_OPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_UInterpAlternative_OPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_UInterpAlternative_OPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_UInterpAlternative_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_sondeflags_OPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_sondeflags_OPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_sondeflags_OPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_sondeflags_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_winproflags_OPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_winproflags_OPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_winproflags_OPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_winproflags_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_pressure_OPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_pressure_OPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_pressure_OPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_pressure_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_unittests_vertaverage - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_unittests_vertaverage.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_unittests_vertaverage + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_unittests_vertaverage.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_unittests_vertaverage_ascending - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_unittests_vertaverage_ascending.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_unittests_vertaverage_ascending + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_unittests_vertaverage_ascending.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_unittests_vertinterp - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_unittests_vertinterp.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_unittests_vertinterp + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_unittests_vertinterp.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_unittests_profiledataholder - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_unittests_profiledataholder.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_unittests_profiledataholder + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_unittests_profiledataholder.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) ecbuild_add_test( TARGET test_ufo_qc_met_office_buddy_check_unittests @@ -2388,128 +2388,128 @@ ecbuild_add_test( TARGET test_ufo_qc_met_office_buddy_pair_finder LIBS ufo TEST_DEPENDS ufo_get_ufo_test_data) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_average_pressure_OPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_average_pressure_OPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_average_pressure_OPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_average_pressure_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_average_temperature_OPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_average_temperature_OPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_average_temperature_OPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_average_temperature_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_average_windspeed_OPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_average_windspeed_OPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_average_windspeed_OPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_average_windspeed_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_average_relativehumidity_OPScomparison - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_average_relativehumidity_OPScomparison.yaml" +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_average_relativehumidity_OPScomparison + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_average_relativehumidity_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_OPScomparison +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_OPScomparison MPI 2 - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_OPScomparison.yaml" + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_bkgqc_modobs_OPScomparison +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_bkgqc_modobs_OPScomparison MPI 2 - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_bkgqc_modobs_OPScomparison.yaml" + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_bkgqc_modobs_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_bkgqc_repobs_OPScomparison +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_bkgqc_repobs_OPScomparison MPI 2 - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_bkgqc_repobs_OPScomparison.yaml" + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_bkgqc_repobs_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_RH_OPScomparison +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_RH_OPScomparison MPI 2 - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_RH_OPScomparison.yaml" + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_RH_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_UInterp_OPScomparison +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_UInterp_OPScomparison MPI 2 - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_UInterp_OPScomparison.yaml" + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_UInterp_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_sondeflags_OPScomparison +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_sondeflags_OPScomparison MPI 2 - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_sondeflags_OPScomparison.yaml" + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_sondeflags_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_winproflags_OPScomparison +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_winproflags_OPScomparison MPI 2 - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_winproflags_OPScomparison.yaml" + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_winproflags_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_pressure_OPScomparison +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_pressure_OPScomparison MPI 2 - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_pressure_OPScomparison.yaml" + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_pressure_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_average_pressure_OPScomparison +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_average_pressure_OPScomparison MPI 2 - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_average_pressure_OPScomparison.yaml" + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_average_pressure_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_average_temperature_OPScomparison +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_average_temperature_OPScomparison MPI 2 - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_average_temperature_OPScomparison.yaml" + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_average_temperature_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_average_windspeed_OPScomparison +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_average_windspeed_OPScomparison MPI 2 - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_average_windspeed_OPScomparison.yaml" + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_average_windspeed_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) -ecbuild_add_test( TARGET test_ufo_profileconsistencychecks_MPI_average_relativehumidity_OPScomparison +ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_MPI_average_relativehumidity_OPScomparison MPI 2 - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ProfileConsistencyChecks.x - ARGS "testinput/profileconsistencychecks_average_relativehumidity_OPScomparison.yaml" + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ConventionalProfileProcessing.x + ARGS "testinput/conventionalprofileprocessing_average_relativehumidity_OPScomparison.yaml" ENVIRONMENT OOPS_TRAPFPE=1 - DEPENDS test_ProfileConsistencyChecks.x + DEPENDS test_ConventionalProfileProcessing.x TEST_DEPENDS ufo_get_ufo_test_data ) # Test Variable Transforms diff --git a/test/mains/TestProfileConsistencyChecks.cc b/test/mains/TestConventionalProfileProcessing.cc similarity index 76% rename from test/mains/TestProfileConsistencyChecks.cc rename to test/mains/TestConventionalProfileProcessing.cc index 315d6b194..536fa4d0a 100644 --- a/test/mains/TestProfileConsistencyChecks.cc +++ b/test/mains/TestConventionalProfileProcessing.cc @@ -5,11 +5,11 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#include "../ufo/ProfileConsistencyChecks.h" +#include "../ufo/ConventionalProfileProcessing.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); - ufo::test::ProfileConsistencyChecks tests; + ufo::test::ConventionalProfileProcessing tests; return run.execute(tests); } diff --git a/test/testinput/profileconsistencychecks_OPScomparison.yaml b/test/testinput/conventionalprofileprocessing_OPScomparison.yaml similarity index 67% rename from test/testinput/profileconsistencychecks_OPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_OPScomparison.yaml index 45ef6d563..b7ae04bca 100644 --- a/test/testinput/profileconsistencychecks_OPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_OPScomparison.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks: comparison with OPS values ===# +#=== Profile processing: comparison with OPS values ===# # Sondes: @@ -8,15 +8,15 @@ Sondes: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing.nc4 obsgrouping: group variables: [ "station_id" ] #Sorting is not performed for this test in order to ensure exact correspondence with OPS simulated variables: [air_temperature, geopotential_height] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_geovals.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "SamePDiffT", "Sign", "UnstableLayer", "Interpolation", "Hydrostatic"] compareWithOPS: true flagBasicChecksFail: true diff --git a/test/testinput/profileconsistencychecks_RH_OPScomparison.yaml b/test/testinput/conventionalprofileprocessing_RH_OPScomparison.yaml similarity index 74% rename from test/testinput/profileconsistencychecks_RH_OPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_RH_OPScomparison.yaml index 28f31c14c..430d7e1cc 100644 --- a/test/testinput/profileconsistencychecks_RH_OPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_RH_OPScomparison.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks for RH: comparison with OPS values ===# +#=== Profile processing for RH: comparison with OPS values ===# # Sondes: @@ -8,13 +8,13 @@ Sondes: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_rh.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_rh.nc4 obsgrouping: group variables: [ "station_id" ] #Sorting is not performed for this test in order to ensure exact correspondence with OPS simulated variables: [air_temperature, relative_humidity, dew_point_temperature] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "RH"] compareWithOPS: true flagBasicChecksFail: true diff --git a/test/testinput/profileconsistencychecks_RH_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_RH_obsfilter.yaml similarity index 77% rename from test/testinput/profileconsistencychecks_RH_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_RH_obsfilter.yaml index c6a17cdbd..685c9232a 100644 --- a/test/testinput/profileconsistencychecks_RH_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_RH_obsfilter.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks for wind speed interpolation ===# +#=== Profile processing for wind speed interpolation ===# # window begin: 2018-04-14T20:30:00Z @@ -9,14 +9,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_rh.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_rh.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, relative_humidity] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity diff --git a/test/testinput/profileconsistencychecks_UInterpAlternative_OPScomparison.yaml b/test/testinput/conventionalprofileprocessing_UInterpAlternative_OPScomparison.yaml similarity index 66% rename from test/testinput/profileconsistencychecks_UInterpAlternative_OPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_UInterpAlternative_OPScomparison.yaml index 5746fb191..e90c10aaa 100644 --- a/test/testinput/profileconsistencychecks_UInterpAlternative_OPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_UInterpAlternative_OPScomparison.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks for wind speed interpolation: comparison with OPS values, alternative implementation of UInterp check ===# +#=== Profile processing for wind speed interpolation: comparison with OPS values, alternative implementation of UInterp check ===# # Sondes: @@ -8,13 +8,13 @@ Sondes: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_uinterp.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_uinterp.nc4 obsgrouping: group variables: [ "station_id" ] #Sorting is not performed for this test in order to ensure exact correspondence with OPS simulated variables: [eastward_wind, northward_wind] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "UInterpAlternative"] compareWithOPS: true flagBasicChecksFail: true diff --git a/test/testinput/profileconsistencychecks_UInterpAlternative_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_UInterpAlternative_obsfilter.yaml similarity index 72% rename from test/testinput/profileconsistencychecks_UInterpAlternative_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_UInterpAlternative_obsfilter.yaml index 00e862d7f..aa59fc8d7 100644 --- a/test/testinput/profileconsistencychecks_UInterpAlternative_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_UInterpAlternative_obsfilter.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks for wind speed interpolation: alternative implementation of UInterp check ===# +#=== Profile processing for wind speed interpolation: alternative implementation of UInterp check ===# # window begin: 2018-04-14T20:30:00Z @@ -9,14 +9,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_uinterp.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_uinterp.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [eastward_wind, northward_wind] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: eastward_wind - name: northward_wind diff --git a/test/testinput/profileconsistencychecks_UInterp_OPScomparison.yaml b/test/testinput/conventionalprofileprocessing_UInterp_OPScomparison.yaml similarity index 69% rename from test/testinput/profileconsistencychecks_UInterp_OPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_UInterp_OPScomparison.yaml index e1eeffbd5..eb849f087 100644 --- a/test/testinput/profileconsistencychecks_UInterp_OPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_UInterp_OPScomparison.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks for wind speed interpolation: comparison with OPS values ===# +#=== Profile processing for wind speed interpolation: comparison with OPS values ===# # Sondes: @@ -8,13 +8,13 @@ Sondes: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_uinterp.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_uinterp.nc4 obsgrouping: group variables: [ "station_id" ] #Sorting is not performed for this test in order to ensure exact correspondence with OPS simulated variables: [eastward_wind, northward_wind] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "UInterp"] compareWithOPS: true flagBasicChecksFail: true diff --git a/test/testinput/profileconsistencychecks_UInterp_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_UInterp_obsfilter.yaml similarity index 76% rename from test/testinput/profileconsistencychecks_UInterp_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_UInterp_obsfilter.yaml index 3e7426ad4..04905f6d0 100644 --- a/test/testinput/profileconsistencychecks_UInterp_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_UInterp_obsfilter.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks for wind speed interpolation ===# +#=== Profile processing for wind speed interpolation ===# # window begin: 2018-04-14T20:30:00Z @@ -9,14 +9,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_uinterp.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_uinterp.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [eastward_wind, northward_wind] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: eastward_wind - name: northward_wind diff --git a/test/testinput/profileconsistencychecks_average_pressure_OPScomparison.yaml b/test/testinput/conventionalprofileprocessing_average_pressure_OPScomparison.yaml similarity index 69% rename from test/testinput/profileconsistencychecks_average_pressure_OPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_average_pressure_OPScomparison.yaml index ed4100d6c..32ae15b19 100644 --- a/test/testinput/profileconsistencychecks_average_pressure_OPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_average_pressure_OPScomparison.yaml @@ -8,7 +8,7 @@ Sondes: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_pressure_obs_extended.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_pressure_obs_extended.nc4 obsgrouping: group variables: [ "station_id" ] #Sorting is not performed for this test in order to ensure exact correspondence with OPS @@ -16,9 +16,9 @@ Sondes: extension: average profiles onto model levels: 71 geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_pressure_geovals_extended.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_pressure_geovals_extended.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AveragePressure"] compareWithOPS: true flagBasicChecksFail: true diff --git a/test/testinput/profileconsistencychecks_average_pressure_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_average_pressure_obsfilter.yaml similarity index 71% rename from test/testinput/profileconsistencychecks_average_pressure_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_average_pressure_obsfilter.yaml index 9dae043f9..6e3cf8d75 100644 --- a/test/testinput/profileconsistencychecks_average_pressure_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_average_pressure_obsfilter.yaml @@ -9,7 +9,7 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_pressure_obs_extended.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_pressure_obs_extended.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" @@ -18,9 +18,9 @@ observations: extension: average profiles onto model levels: 71 geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_pressure_geovals_extended.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_pressure_geovals_extended.nc4 obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature Checks: ["Basic", "AveragePressure"] diff --git a/test/testinput/profileconsistencychecks_average_relativehumidity_OPScomparison.yaml b/test/testinput/conventionalprofileprocessing_average_relativehumidity_OPScomparison.yaml similarity index 71% rename from test/testinput/profileconsistencychecks_average_relativehumidity_OPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_average_relativehumidity_OPScomparison.yaml index 27b5e65d0..c2be2279c 100644 --- a/test/testinput/profileconsistencychecks_average_relativehumidity_OPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_average_relativehumidity_OPScomparison.yaml @@ -8,7 +8,7 @@ Sondes: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_relativehumidity_obs_extended.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_relativehumidity_obs_extended.nc4 obsgrouping: group variables: [ "station_id" ] #Sorting is not performed for this test in order to ensure exact correspondence with OPS @@ -16,9 +16,9 @@ Sondes: extension: average profiles onto model levels: 71 geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_geovals_extended.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_geovals_extended.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AveragePressure", "AverageTemperature", "AverageRelativeHumidity"] compareWithOPS: true flagBasicChecksFail: true diff --git a/test/testinput/profileconsistencychecks_average_relativehumidity_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_average_relativehumidity_obsfilter.yaml similarity index 73% rename from test/testinput/profileconsistencychecks_average_relativehumidity_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_average_relativehumidity_obsfilter.yaml index bc5ef7f46..8f95814e0 100644 --- a/test/testinput/profileconsistencychecks_average_relativehumidity_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_average_relativehumidity_obsfilter.yaml @@ -9,7 +9,7 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_relativehumidity_obs_extended.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_relativehumidity_obs_extended.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" @@ -18,9 +18,9 @@ observations: extension: average profiles onto model levels: 71 geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_geovals_extended.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_geovals_extended.nc4 obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: relative_humidity Checks: ["Basic", "AveragePressure", "AverageTemperature", "AverageRelativeHumidity"] diff --git a/test/testinput/profileconsistencychecks_average_temperature_OPScomparison.yaml b/test/testinput/conventionalprofileprocessing_average_temperature_OPScomparison.yaml similarity index 71% rename from test/testinput/profileconsistencychecks_average_temperature_OPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_average_temperature_OPScomparison.yaml index c98252840..332dae7b7 100644 --- a/test/testinput/profileconsistencychecks_average_temperature_OPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_average_temperature_OPScomparison.yaml @@ -8,7 +8,7 @@ Sondes: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_temperature_obs_extended.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_temperature_obs_extended.nc4 obsgrouping: group variables: [ "station_id" ] #Sorting is not performed for this test in order to ensure exact correspondence with OPS @@ -16,9 +16,9 @@ Sondes: extension: average profiles onto model levels: 71 geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_geovals_extended.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_geovals_extended.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AveragePressure", "AverageTemperature"] compareWithOPS: true flagBasicChecksFail: true diff --git a/test/testinput/profileconsistencychecks_average_temperature_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_average_temperature_obsfilter.yaml similarity index 72% rename from test/testinput/profileconsistencychecks_average_temperature_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_average_temperature_obsfilter.yaml index ea7540199..14a6b65a3 100644 --- a/test/testinput/profileconsistencychecks_average_temperature_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_average_temperature_obsfilter.yaml @@ -9,7 +9,7 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_temperature_obs_extended.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_temperature_obs_extended.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" @@ -18,9 +18,9 @@ observations: extension: average profiles onto model levels: 71 geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_geovals_extended.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_geovals_extended.nc4 obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature Checks: ["Basic", "AveragePressure", "AverageTemperature"] diff --git a/test/testinput/profileconsistencychecks_average_windspeed_OPScomparison.yaml b/test/testinput/conventionalprofileprocessing_average_windspeed_OPScomparison.yaml similarity index 71% rename from test/testinput/profileconsistencychecks_average_windspeed_OPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_average_windspeed_OPScomparison.yaml index 27c63bf04..d14b03bfa 100644 --- a/test/testinput/profileconsistencychecks_average_windspeed_OPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_average_windspeed_OPScomparison.yaml @@ -8,7 +8,7 @@ Sondes: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_windspeed_obs_extended.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_windspeed_obs_extended.nc4 obsgrouping: group variables: [ "station_id" ] #Sorting is not performed for this test in order to ensure exact correspondence with OPS @@ -16,9 +16,9 @@ Sondes: extension: average profiles onto model levels: 71 geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_geovals_extended.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_geovals_extended.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AveragePressure", "AverageWindSpeed"] compareWithOPS: true flagBasicChecksFail: true diff --git a/test/testinput/profileconsistencychecks_average_windspeed_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_average_windspeed_obsfilter.yaml similarity index 73% rename from test/testinput/profileconsistencychecks_average_windspeed_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_average_windspeed_obsfilter.yaml index a907d7770..7ab16d02e 100644 --- a/test/testinput/profileconsistencychecks_average_windspeed_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_average_windspeed_obsfilter.yaml @@ -9,7 +9,7 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_windspeed_obs_extended.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_windspeed_obs_extended.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" @@ -18,9 +18,9 @@ observations: extension: average profiles onto model levels: 71 geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_geovals_extended.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_geovals_extended.nc4 obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: eastward_wind - name: northward_wind diff --git a/test/testinput/profileconsistencychecks_bkgqc_modobs_OPScomparison.yaml b/test/testinput/conventionalprofileprocessing_bkgqc_modobs_OPScomparison.yaml similarity index 74% rename from test/testinput/profileconsistencychecks_bkgqc_modobs_OPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_bkgqc_modobs_OPScomparison.yaml index 5e4850f44..62943d9bd 100644 --- a/test/testinput/profileconsistencychecks_bkgqc_modobs_OPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_bkgqc_modobs_OPScomparison.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks for background QC: comparison with OPS values on model levels ===# +#=== Profile processing for background QC: comparison with OPS values on model levels ===# # Sondes: @@ -8,21 +8,21 @@ Sondes: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_modobs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_modobs.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: ["air_temperature", "relative_humidity", "eastward_wind", "northward_wind", "geopotential_height"] obs diagnostics: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_obsdiagnostics_modobs.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_obsdiagnostics_modobs.nc4 variables: - name: air_temperature_background_error@ObsDiag - name: relative_humidity_background_error@ObsDiag - name: eastward_wind_background_error@ObsDiag - name: northward_wind_background_error@ObsDiag - name: geopotential_height_background_error@ObsDiag - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "Time", "PermanentReject", "BackgroundTemperature", "BackgroundRelativeHumidity", "BackgroundWindSpeed", "BackgroundGeopotentialHeight"] compareWithOPS: true flagBasicChecksFail: true diff --git a/test/testinput/profileconsistencychecks_bkgqc_modobs_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_bkgqc_modobs_obsfilter.yaml similarity index 79% rename from test/testinput/profileconsistencychecks_bkgqc_modobs_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_bkgqc_modobs_obsfilter.yaml index 984cd4d2d..2d02a476d 100644 --- a/test/testinput/profileconsistencychecks_bkgqc_modobs_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_bkgqc_modobs_obsfilter.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks for background QC on model levels ==# +#=== Profile processing for background QC on model levels ==# # window begin: 2019-06-14T21:00:00Z @@ -9,14 +9,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_modobs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_modobs.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, relative_humidity, eastward_wind, northward_wind, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -31,7 +31,7 @@ observations: ModelLevels: true HofX: HofX obs diagnostics: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_obsdiagnostics_modobs.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_obsdiagnostics_modobs.nc4 variables: - name: air_temperature_background_error@ObsDiag - name: relative_humidity_background_error@ObsDiag diff --git a/test/testinput/profileconsistencychecks_bkgqc_modobs_separatefilters_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_bkgqc_modobs_separatefilters_obsfilter.yaml similarity index 82% rename from test/testinput/profileconsistencychecks_bkgqc_modobs_separatefilters_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_bkgqc_modobs_separatefilters_obsfilter.yaml index 871b996ba..81e9678b4 100644 --- a/test/testinput/profileconsistencychecks_bkgqc_modobs_separatefilters_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_bkgqc_modobs_separatefilters_obsfilter.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks for background QC on model levels using sequential separate filters ==# +#=== Profile processing for background QC on model levels using sequential separate filters ==# # window begin: 2019-06-14T21:00:00Z @@ -9,14 +9,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_modobs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_modobs.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, relative_humidity, eastward_wind, northward_wind, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -29,7 +29,7 @@ observations: PrintStationID: true BChecks_Skip: true ModelLevels: true - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -42,7 +42,7 @@ observations: PrintStationID: true BChecks_Skip: true ModelLevels: true - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -55,7 +55,7 @@ observations: PrintStationID: true BChecks_Skip: true ModelLevels: true - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -68,7 +68,7 @@ observations: PrintStationID: true BChecks_Skip: true ModelLevels: true - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -81,7 +81,7 @@ observations: PrintStationID: true BChecks_Skip: true ModelLevels: true - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -96,7 +96,7 @@ observations: ModelLevels: true HofX: HofX obs diagnostics: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_obsdiagnostics_modobs.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_obsdiagnostics_modobs.nc4 variables: - name: air_temperature_background_error@ObsDiag - name: relative_humidity_background_error@ObsDiag diff --git a/test/testinput/profileconsistencychecks_bkgqc_repobs_OPScomparison.yaml b/test/testinput/conventionalprofileprocessing_bkgqc_repobs_OPScomparison.yaml similarity index 74% rename from test/testinput/profileconsistencychecks_bkgqc_repobs_OPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_bkgqc_repobs_OPScomparison.yaml index 98d7fbfe0..3deb69243 100644 --- a/test/testinput/profileconsistencychecks_bkgqc_repobs_OPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_bkgqc_repobs_OPScomparison.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks for background QC: comparison with OPS values on reported levels ===# +#=== Profile processing for background QC: comparison with OPS values on reported levels ===# # Sondes: @@ -8,21 +8,21 @@ Sondes: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_repobs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_repobs.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: ["air_temperature", "relative_humidity", "eastward_wind", "northward_wind", "geopotential_height"] obs diagnostics: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_obsdiagnostics_repobs.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_obsdiagnostics_repobs.nc4 variables: - name: air_temperature_background_error@ObsDiag - name: relative_humidity_background_error@ObsDiag - name: eastward_wind_background_error@ObsDiag - name: northward_wind_background_error@ObsDiag - name: geopotential_height_background_error@ObsDiag - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "Time", "PermanentReject", "BackgroundTemperature", "BackgroundRelativeHumidity", "BackgroundWindSpeed", "BackgroundGeopotentialHeight"] compareWithOPS: true flagBasicChecksFail: true diff --git a/test/testinput/profileconsistencychecks_bkgqc_repobs_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_bkgqc_repobs_obsfilter.yaml similarity index 79% rename from test/testinput/profileconsistencychecks_bkgqc_repobs_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_bkgqc_repobs_obsfilter.yaml index c256ba88c..e176770a7 100644 --- a/test/testinput/profileconsistencychecks_bkgqc_repobs_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_bkgqc_repobs_obsfilter.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks for background QC on reported levels ==# +#=== Profile processing for background QC on reported levels ==# # window begin: 2019-06-14T21:00:00Z @@ -9,14 +9,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_repobs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_repobs.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, relative_humidity, eastward_wind, northward_wind, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -31,7 +31,7 @@ observations: ModelLevels: false HofX: HofX obs diagnostics: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_obsdiagnostics_repobs.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_obsdiagnostics_repobs.nc4 variables: - name: air_temperature_background_error@ObsDiag - name: relative_humidity_background_error@ObsDiag diff --git a/test/testinput/profileconsistencychecks_bkgqc_repobs_separatefilters_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_bkgqc_repobs_separatefilters_obsfilter.yaml similarity index 82% rename from test/testinput/profileconsistencychecks_bkgqc_repobs_separatefilters_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_bkgqc_repobs_separatefilters_obsfilter.yaml index b2ddc67a2..31f271a0f 100644 --- a/test/testinput/profileconsistencychecks_bkgqc_repobs_separatefilters_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_bkgqc_repobs_separatefilters_obsfilter.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks for background QC on reported levels using sequential separate filters ==# +#=== Profile processing for background QC on reported levels using sequential separate filters ==# # window begin: 2019-06-14T21:00:00Z @@ -9,14 +9,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_repobs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_repobs.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, relative_humidity, eastward_wind, northward_wind, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -29,7 +29,7 @@ observations: PrintStationID: true BChecks_Skip: true ModelLevels: false - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -42,7 +42,7 @@ observations: PrintStationID: true BChecks_Skip: true ModelLevels: false - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -55,7 +55,7 @@ observations: PrintStationID: true BChecks_Skip: true ModelLevels: false - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -68,7 +68,7 @@ observations: PrintStationID: true BChecks_Skip: true ModelLevels: false - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -81,7 +81,7 @@ observations: PrintStationID: true BChecks_Skip: true ModelLevels: false - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: relative_humidity @@ -96,7 +96,7 @@ observations: ModelLevels: false HofX: HofX obs diagnostics: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_obsdiagnostics_repobs.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_obsdiagnostics_repobs.nc4 variables: - name: air_temperature_background_error@ObsDiag - name: relative_humidity_background_error@ObsDiag diff --git a/test/testinput/profileconsistencychecks_monolithicfilter.yaml b/test/testinput/conventionalprofileprocessing_monolithicfilter.yaml similarity index 73% rename from test/testinput/profileconsistencychecks_monolithicfilter.yaml rename to test/testinput/conventionalprofileprocessing_monolithicfilter.yaml index e2927e39d..4ca1ed36e 100644 --- a/test/testinput/profileconsistencychecks_monolithicfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_monolithicfilter.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks using one monolithic filter ===# +#=== Profile processing using one monolithic filter ===# # window begin: 2018-04-14T20:30:00Z @@ -9,16 +9,16 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, geopotential_height] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_geovals.nc4 obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height diff --git a/test/testinput/profileconsistencychecks_opr_average.yaml b/test/testinput/conventionalprofileprocessing_opr_average.yaml similarity index 100% rename from test/testinput/profileconsistencychecks_opr_average.yaml rename to test/testinput/conventionalprofileprocessing_opr_average.yaml diff --git a/test/testinput/profileconsistencychecks_pressure_OPScomparison.yaml b/test/testinput/conventionalprofileprocessing_pressure_OPScomparison.yaml similarity index 70% rename from test/testinput/profileconsistencychecks_pressure_OPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_pressure_OPScomparison.yaml index fc115f367..aa1ba6471 100644 --- a/test/testinput/profileconsistencychecks_pressure_OPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_pressure_OPScomparison.yaml @@ -8,15 +8,15 @@ Sondes: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_pressure_obs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_pressure_obs.nc4 obsgrouping: group variables: [ "station_id" ] #Sorting is not performed for this test in order to ensure exact correspondence with OPS simulated variables: [geopotential_height] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_pressure_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_pressure_geovals.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "Pressure"] compareWithOPS: true BChecks_Skip: true diff --git a/test/testinput/profileconsistencychecks_pressure_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_pressure_obsfilter.yaml similarity index 68% rename from test/testinput/profileconsistencychecks_pressure_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_pressure_obsfilter.yaml index 680a5be24..4e09526d3 100644 --- a/test/testinput/profileconsistencychecks_pressure_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_pressure_obsfilter.yaml @@ -9,14 +9,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests_vertinterp_obs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests_vertinterp_obs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [geopotential_height] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests_vertinterp_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests_vertinterp_geovals.nc4 obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: geopotential_height Checks: ["Basic", "Pressure"] diff --git a/test/testinput/profileconsistencychecks_separatefilters.yaml b/test/testinput/conventionalprofileprocessing_separatefilters.yaml similarity index 72% rename from test/testinput/profileconsistencychecks_separatefilters.yaml rename to test/testinput/conventionalprofileprocessing_separatefilters.yaml index cf2f96612..61d10f43d 100644 --- a/test/testinput/profileconsistencychecks_separatefilters.yaml +++ b/test/testinput/conventionalprofileprocessing_separatefilters.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks using sequential separate filters ===# +#=== Profile processing using sequential separate filters ===# # window begin: 2018-04-14T20:30:00Z @@ -9,43 +9,43 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, geopotential_height] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_geovals.nc4 obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height Checks: ["Basic"] flagBasicChecksFail: true - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height Checks: ["SamePDiffT"] - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height Checks: ["Sign"] SCheck_CorrectT: true - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height Checks: ["UnstableLayer"] - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height Checks: ["Interpolation"] - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height diff --git a/test/testinput/profileconsistencychecks_sondeflags_OPScomparison.yaml b/test/testinput/conventionalprofileprocessing_sondeflags_OPScomparison.yaml similarity index 80% rename from test/testinput/profileconsistencychecks_sondeflags_OPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_sondeflags_OPScomparison.yaml index 11b02cc27..ce1b1a9f5 100644 --- a/test/testinput/profileconsistencychecks_sondeflags_OPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_sondeflags_OPScomparison.yaml @@ -8,13 +8,13 @@ Sondes: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_sondeflags.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_sondeflags.nc4 obsgrouping: group variables: [ "station_id" ] #Sorting is not performed for this test in order to ensure exact correspondence with OPS simulated variables: [eastward_wind] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "SondeFlags"] compareWithOPS: true flagBasicChecksFail: true diff --git a/test/testinput/profileconsistencychecks_sondeflags_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_sondeflags_obsfilter.yaml similarity index 81% rename from test/testinput/profileconsistencychecks_sondeflags_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_sondeflags_obsfilter.yaml index 1da3c02f7..1c918e485 100644 --- a/test/testinput/profileconsistencychecks_sondeflags_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_sondeflags_obsfilter.yaml @@ -9,14 +9,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_sondeflags.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_sondeflags.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [eastward_wind] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: eastward_wind Checks: ["Basic", "SondeFlags"] diff --git a/test/testinput/profileconsistencychecks_unittest_oneprofileMPI.yaml b/test/testinput/conventionalprofileprocessing_unittest_oneprofileMPI.yaml similarity index 70% rename from test/testinput/profileconsistencychecks_unittest_oneprofileMPI.yaml rename to test/testinput/conventionalprofileprocessing_unittest_oneprofileMPI.yaml index 71199eb12..d5a38446b 100644 --- a/test/testinput/profileconsistencychecks_unittest_oneprofileMPI.yaml +++ b/test/testinput/conventionalprofileprocessing_unittest_oneprofileMPI.yaml @@ -1,5 +1,5 @@ # -#=== Unit test with one profile for the profile consistency checks ===# +#=== Unit test with one profile for the profile QC ===# # window begin: 2018-04-14T20:30:00Z @@ -9,16 +9,16 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_oneprofile.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_oneprofile.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, geopotential_height] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_oneprofile_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_oneprofile_geovals.nc4 obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height diff --git a/test/testinput/profileconsistencychecks_unittests.yaml b/test/testinput/conventionalprofileprocessing_unittests.yaml similarity index 83% rename from test/testinput/profileconsistencychecks_unittests.yaml rename to test/testinput/conventionalprofileprocessing_unittests.yaml index d5d82c3a6..89b66cb24 100644 --- a/test/testinput/profileconsistencychecks_unittests.yaml +++ b/test/testinput/conventionalprofileprocessing_unittests.yaml @@ -1,5 +1,5 @@ # -#=== Unit tests for the profile consistency checks ===# +#=== Unit tests for the profile QC ===# # #These tests focus on the basic checks. #The same sonde profile has been used multiple times (with minor tweaks) to generate a test sample. @@ -23,14 +23,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height @@ -50,12 +50,12 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height @@ -75,14 +75,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height @@ -102,14 +102,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height @@ -131,14 +131,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height @@ -158,14 +158,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height @@ -186,14 +186,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height @@ -213,14 +213,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height @@ -240,12 +240,12 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height @@ -265,12 +265,12 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height @@ -292,12 +292,12 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, geopotential_height] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height @@ -318,14 +318,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, geopotential_height] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_geovals.nc4 obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: air_temperature - name: geopotential_height diff --git a/test/testinput/profileconsistencychecks_unittests_profiledataholder.yaml b/test/testinput/conventionalprofileprocessing_unittests_profiledataholder.yaml similarity index 56% rename from test/testinput/profileconsistencychecks_unittests_profiledataholder.yaml rename to test/testinput/conventionalprofileprocessing_unittests_profiledataholder.yaml index 41610719c..ae8858603 100644 --- a/test/testinput/profileconsistencychecks_unittests_profiledataholder.yaml +++ b/test/testinput/conventionalprofileprocessing_unittests_profiledataholder.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks: unit tests of ProfileDataHolder ===# +#=== Profile processing: unit tests of ProfileDataHolder ===# # ProfileDataHolder: @@ -8,15 +8,15 @@ ProfileDataHolder: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_oneprofile.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_oneprofile.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature] extension: average profiles onto model levels: 71 geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_oneprofile_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_oneprofile_geovals.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic"] testProfileDataHolder: true diff --git a/test/testinput/profileconsistencychecks_unittests_vertaverage.yaml b/test/testinput/conventionalprofileprocessing_unittests_vertaverage.yaml similarity index 64% rename from test/testinput/profileconsistencychecks_unittests_vertaverage.yaml rename to test/testinput/conventionalprofileprocessing_unittests_vertaverage.yaml index 7f2b2ae6b..714987310 100644 --- a/test/testinput/profileconsistencychecks_unittests_vertaverage.yaml +++ b/test/testinput/conventionalprofileprocessing_unittests_vertaverage.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks: unit tests of vertical averaging routine ===# +#=== Profile processing: unit tests of vertical averaging routine ===# # Vertical averaging: @@ -8,13 +8,13 @@ Vertical averaging: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests_vertaverage.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests_vertaverage.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [eastward_wind] HofX: HofX obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic"] compareWithOPS: true BypassMismatchComparison: true diff --git a/test/testinput/profileconsistencychecks_unittests_vertaverage_ascending.yaml b/test/testinput/conventionalprofileprocessing_unittests_vertaverage_ascending.yaml similarity index 60% rename from test/testinput/profileconsistencychecks_unittests_vertaverage_ascending.yaml rename to test/testinput/conventionalprofileprocessing_unittests_vertaverage_ascending.yaml index b039a6cdb..1ffe3cf16 100644 --- a/test/testinput/profileconsistencychecks_unittests_vertaverage_ascending.yaml +++ b/test/testinput/conventionalprofileprocessing_unittests_vertaverage_ascending.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks: unit tests of vertical averaging routine with ascending vertical coordinate ===# +#=== Profile processing: unit tests of vertical averaging routine with ascending vertical coordinate ===# # Vertical averaging: @@ -8,13 +8,13 @@ Vertical averaging: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests_vertaverage_ascending.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests_vertaverage_ascending.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [eastward_wind] HofX: HofX obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic"] compareWithOPS: true BypassMismatchComparison: true diff --git a/test/testinput/profileconsistencychecks_unittests_vertinterp.yaml b/test/testinput/conventionalprofileprocessing_unittests_vertinterp.yaml similarity index 76% rename from test/testinput/profileconsistencychecks_unittests_vertinterp.yaml rename to test/testinput/conventionalprofileprocessing_unittests_vertinterp.yaml index 472370222..791d0c5f3 100644 --- a/test/testinput/profileconsistencychecks_unittests_vertinterp.yaml +++ b/test/testinput/conventionalprofileprocessing_unittests_vertinterp.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks: unit tests of vertical interpolation routine ===# +#=== Profile processing: unit tests of vertical interpolation routine ===# # Vertical interpolation: @@ -8,15 +8,15 @@ Vertical interpolation: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests_vertinterp_obs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests_vertinterp_obs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [geopotential_height] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_unittests_vertinterp_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_unittests_vertinterp_geovals.nc4 HofX: HofX obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "Pressure"] compareWithOPS: true BypassMismatchComparison: true diff --git a/test/testinput/profileconsistencychecks_winproflags_OPScomparison.yaml b/test/testinput/conventionalprofileprocessing_winproflags_OPScomparison.yaml similarity index 80% rename from test/testinput/profileconsistencychecks_winproflags_OPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_winproflags_OPScomparison.yaml index 1c80f08f8..3260c3b39 100644 --- a/test/testinput/profileconsistencychecks_winproflags_OPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_winproflags_OPScomparison.yaml @@ -8,13 +8,13 @@ Sondes: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_winproflags.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_winproflags.nc4 obsgrouping: group variables: [ "station_id" ] #Sorting is not performed for this test in order to ensure exact correspondence with OPS simulated variables: [eastward_wind] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "WinProFlags"] compareWithOPS: true flagBasicChecksFail: true diff --git a/test/testinput/profileconsistencychecks_winproflags_obsfilter.yaml b/test/testinput/conventionalprofileprocessing_winproflags_obsfilter.yaml similarity index 81% rename from test/testinput/profileconsistencychecks_winproflags_obsfilter.yaml rename to test/testinput/conventionalprofileprocessing_winproflags_obsfilter.yaml index 0f177b2df..a8037e617 100644 --- a/test/testinput/profileconsistencychecks_winproflags_obsfilter.yaml +++ b/test/testinput/conventionalprofileprocessing_winproflags_obsfilter.yaml @@ -9,14 +9,14 @@ observations: - obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_winproflags.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_winproflags.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [eastward_wind] obs filters: - - filter: Profile Consistency Checks + - filter: Conventional Profile Processing filter variables: - name: eastward_wind Checks: ["Basic", "WinProFlags"] diff --git a/test/testinput/profileconsistencychecks_wrongOPScomparison.yaml b/test/testinput/conventionalprofileprocessing_wrongOPScomparison.yaml similarity index 77% rename from test/testinput/profileconsistencychecks_wrongOPScomparison.yaml rename to test/testinput/conventionalprofileprocessing_wrongOPScomparison.yaml index 6876bb664..c0db014dc 100644 --- a/test/testinput/profileconsistencychecks_wrongOPScomparison.yaml +++ b/test/testinput/conventionalprofileprocessing_wrongOPScomparison.yaml @@ -1,5 +1,5 @@ # -#=== Profile consistency checks: OPS comparison with deliberately wrong values ===# +#=== Profile processing: OPS comparison with deliberately wrong values ===# # #This is used to cover code paths that would otherwise be missed. #It is recommended to run this test with OOPS_DEBUG=0 due to the large amount of output generated. @@ -14,12 +14,12 @@ Incorrect and missing values, RH: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_rh_wrong.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_rh_wrong.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, relative_humidity, dew_point_temperature] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "SamePDiffT", "Sign", "UnstableLayer", "Interpolation", "Hydrostatic", "RH", "UInterp", "Time", "PermanentReject", "BackgroundTemperature", "BackgroundRelativeHumidity", "BackgroundWindSpeed", "BackgroundGeopotentialHeight", "Pressure"] compareWithOPS: true BChecks_Skip: true @@ -35,12 +35,12 @@ Incorrect and missing values: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_oneprofile_wrong.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_oneprofile_wrong.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, geopotential_height] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "SamePDiffT", "Sign", "UnstableLayer", "Interpolation", "Hydrostatic", "RH", "UInterp", "Time", "PermanentReject", "BackgroundTemperature", "BackgroundRelativeHumidity", "BackgroundWindSpeed", "BackgroundGeopotentialHeight", "Pressure", "AverageTemperature", "AverageWindSpeed", "AverageRelativeHumidity"] compareWithOPS: true BChecks_Skip: true @@ -56,12 +56,12 @@ Incorrect and missing values, Basic: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_oneprofile_wrong.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_oneprofile_wrong.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, geopotential_height] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic"] compareWithOPS: true flagBasicChecksFail: true @@ -76,14 +76,14 @@ Wrong number of entries per profile: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_oneprofile.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_oneprofile.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, geopotential_height] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_oneprofile_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_oneprofile_geovals.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["SamePDiffT", "Basic", "Sign", "UnstableLayer", "Interpolation", "Hydrostatic"] compareWithOPS: true BChecks_Skip: true @@ -100,12 +100,12 @@ Wrong number of entries per profile, UInterp: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_uinterp.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_uinterp.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [eastward_wind, northward_wind] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["UInterp"] compareWithOPS: true BChecks_Skip: true @@ -122,12 +122,12 @@ Wrong number of entries per profile, UInterpAlternative: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_uinterp.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_uinterp.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [eastward_wind, northward_wind] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["UInterpAlternative"] BChecks_Skip: true flagBasicChecksFail: true @@ -142,12 +142,12 @@ Wrong number of entries per profile, RH: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_rh.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_rh.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, relative_humidity, dew_point_temperature] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["RH"] compareWithOPS: true BChecks_Skip: true @@ -164,12 +164,12 @@ Wrong number of entries per profile, background: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_repobs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_repobs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, relative_humidity, eastward_wind, northward_wind, geopotential_height] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Time", "PermanentReject", "BackgroundTemperature", "BackgroundRelativeHumidity", "BackgroundWindSpeed", "BackgroundGeopotentialHeight"] compareWithOPS: true BChecks_Skip: true @@ -186,14 +186,14 @@ Wrong number of entries per profile, pressure transformation: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_temperature_obs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_temperature_obs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_geovals.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AveragePressure"] compareWithOPS: true BChecks_Skip: true @@ -211,14 +211,14 @@ Wrong number of entries per profile, temperature averaging: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_temperature_obs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_temperature_obs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_geovals.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AveragePressure", "AverageTemperature"] compareWithOPS: true BChecks_Skip: true @@ -235,14 +235,14 @@ Wrong number of entries per profile, wind speed averaging: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_windspeed_obs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_windspeed_obs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [eastward_wind, northward_wind] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_geovals.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AveragePressure", "AverageWindSpeed"] compareWithOPS: true BChecks_Skip: true @@ -259,14 +259,14 @@ Wrong number of entries per profile, wind speed averaging: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_relativehumidity_obs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_relativehumidity_obs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [relative_humidity] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_geovals.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AveragePressure", "AverageRelativeHumidity"] compareWithOPS: true BChecks_Skip: true @@ -282,10 +282,10 @@ No group variable chosen: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_rh.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_rh.nc4 simulated variables: [air_temperature, relative_humidity, dew_point_temperature] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["RH"] compareWithOPS: true BChecks_Skip: true @@ -301,14 +301,14 @@ Ascending sort order: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_rh.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_rh.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "ascending" simulated variables: [air_temperature, relative_humidity, dew_point_temperature] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["RH"] compareWithOPS: true BChecks_Skip: true @@ -324,12 +324,12 @@ Invalid check requested: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_rh.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_rh.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, relative_humidity, dew_point_temperature] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["WrongCheck"] compareWithOPS: true BChecks_Skip: true @@ -345,14 +345,14 @@ Duplicate check requested: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_oneprofile.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_oneprofile.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, geopotential_height] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_oneprofile_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_oneprofile_geovals.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Sign"] compareWithOPS: true BChecks_Skip: true @@ -368,14 +368,14 @@ Incorrect type in get: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_oneprofile.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_oneprofile.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, geopotential_height] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_oneprofile_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_oneprofile_geovals.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Sign"] compareWithOPS: true BChecks_Skip: true @@ -391,14 +391,14 @@ Standard levels out of order: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_oneprofile.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_oneprofile.nc4 obsgrouping: group variables: [ "station_id" ] sort variable: "air_pressure" sort order: "descending" simulated variables: [air_temperature, geopotential_height] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Hydrostatic"] compareWithOPS: true BChecks_Skip: true @@ -415,12 +415,12 @@ Background SondeLaunchWindRej: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_repobs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_repobs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [eastward_wind, northward_wind] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Time"] compareWithOPS: true BChecks_Skip: true @@ -436,12 +436,12 @@ Background manual QC flags: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_bkgqc_repobs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_bkgqc_repobs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature, relative_humidity, eastward_wind, northward_wind, geopotential_height] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Time"] compareWithOPS: true BChecks_Skip: true @@ -457,14 +457,14 @@ Profile pressure not run before temperature averaging: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_temperature_obs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_temperature_obs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_geovals.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AverageTemperature"] compareWithOPS: true flagBasicChecksFail: true @@ -480,14 +480,14 @@ Missing GeoVaLs for temperature averaging: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_temperature_obs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_temperature_obs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_temperature.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_temperature.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AverageTemperature"] compareWithOPS: true flagBasicChecksFail: true @@ -504,14 +504,14 @@ Profile pressure not run before wind speed averaging: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_windspeed_obs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_windspeed_obs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [eastward_wind, northward_wind] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_geovals.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AverageWindSpeed"] compareWithOPS: true flagBasicChecksFail: true @@ -527,14 +527,14 @@ Missing GeoVaLs for wind speed averaging: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_windspeed_obs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_windspeed_obs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [eastward_wind, northward_wind] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_windspeed.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_windspeed.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AverageWindSpeed"] compareWithOPS: true flagBasicChecksFail: true @@ -551,14 +551,14 @@ Profile pressure not run before relative humidity averaging: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_relativehumidity_obs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_relativehumidity_obs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [relative_humidity] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_geovals.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_geovals.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AverageRelativeHumidity"] compareWithOPS: true flagBasicChecksFail: true @@ -574,14 +574,14 @@ Missing GeoVaLs for relative humidity averaging: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_relativehumidity_obs.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_relativehumidity_obs.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [relative_humidity] geovals: - filename: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks_average_relativehumidity.nc4 + filename: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing_average_relativehumidity.nc4 obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "AverageRelativeHumidity"] compareWithOPS: true flagBasicChecksFail: true @@ -598,12 +598,12 @@ Missing GeoVaLs for sign check: obs space: name: Radiosonde obsdatain: - obsfile: Data/ufo/testinput_tier_1/met_office_profile_consistency_checks.nc4 + obsfile: Data/ufo/testinput_tier_1/met_office_conventional_profile_processing.nc4 obsgrouping: group variables: [ "station_id" ] simulated variables: [air_temperature] obs diagnostics: - ProfileConsistencyChecks: + Conventional Profile Processing: Checks: ["Basic", "Sign"] compareWithOPS: true flagBasicChecksFail: true diff --git a/test/ufo/ProfileConsistencyChecks.h b/test/ufo/ConventionalProfileProcessing.h similarity index 93% rename from test/ufo/ProfileConsistencyChecks.h rename to test/ufo/ConventionalProfileProcessing.h index f78ca7bee..e7181bb96 100644 --- a/test/ufo/ProfileConsistencyChecks.h +++ b/test/ufo/ConventionalProfileProcessing.h @@ -5,8 +5,8 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef TEST_UFO_PROFILECONSISTENCYCHECKS_H_ -#define TEST_UFO_PROFILECONSISTENCYCHECKS_H_ +#ifndef TEST_UFO_CONVENTIONALPROFILEPROCESSING_H_ +#define TEST_UFO_CONVENTIONALPROFILEPROCESSING_H_ #include #include @@ -26,9 +26,9 @@ #include "oops/util/Expect.h" #include "oops/util/FloatCompare.h" #include "test/TestEnvironment.h" +#include "ufo/filters/ConventionalProfileProcessing.h" +#include "ufo/filters/ConventionalProfileProcessingParameters.h" #include "ufo/filters/ObsFilterData.h" -#include "ufo/filters/ProfileConsistencyCheckParameters.h" -#include "ufo/filters/ProfileConsistencyChecks.h" #include "ufo/filters/Variables.h" #include "ufo/GeoVaLs.h" #include "ufo/ObsDiagnostics.h" @@ -55,7 +55,7 @@ namespace ufo { namespace test { -void testProfileConsistencyChecks(const eckit::LocalConfiguration &conf) { +void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { util::DateTime bgn(conf.getString("window begin")); util::DateTime end(conf.getString("window end")); @@ -80,8 +80,8 @@ void testProfileConsistencyChecks(const eckit::LocalConfiguration &conf) { std::shared_ptr> qcflags(new ioda::ObsDataVector( obsspace, obsspace.obsvariables())); - const eckit::LocalConfiguration filterConf(conf, "ProfileConsistencyChecks"); - ufo::ProfileConsistencyCheckParameters filterParameters; + const eckit::LocalConfiguration filterConf(conf, "Conventional Profile Processing"); + ufo::ConventionalProfileProcessingParameters filterParameters; filterParameters.validateAndDeserialize(filterConf); // Determine whether an exception is expected to be thrown. @@ -92,14 +92,14 @@ void testProfileConsistencyChecks(const eckit::LocalConfiguration &conf) { bool expectThrowDuringOperation = conf.getBool("ExpectThrowDuringOperation", false); if (expectThrowOnInstantiation) { - EXPECT_THROWS(ufo::ProfileConsistencyChecks filterThrow(obsspace, filterParameters, - qcflags, obserr)); + EXPECT_THROWS(ufo::ConventionalProfileProcessing filterThrow(obsspace, filterParameters, + qcflags, obserr)); // Do not proceed further in this case. return; } // Instantiate filter. - ufo::ProfileConsistencyChecks filter(obsspace, filterParameters, qcflags, obserr); + ufo::ConventionalProfileProcessing filter(obsspace, filterParameters, qcflags, obserr); // Obtain GeoVaLs. const bool ignoreGeoVaLs = conf.getBool("IgnoreGeoVaLs", false); @@ -142,7 +142,7 @@ void testProfileConsistencyChecks(const eckit::LocalConfiguration &conf) { // Test whether using the get function with the wrong type throws an exception. const bool getWrongType = conf.getBool("GetWrongType", false); if (getWrongType) { - ProfileConsistencyCheckParameters options; + ConventionalProfileProcessingParameters options; options.deserialize(conf); EntireSampleDataHandler entireSampleDataHandler(obsspace, options.DHParameters); @@ -163,10 +163,10 @@ void testProfileConsistencyChecks(const eckit::LocalConfiguration &conf) { EXPECT_THROWS(profileDataHandler.get(ufo::VariableNames::obs_air_pressure)); } - // Manually modify QC flags in order to cover rare code paths. + // Manually modify Processing flags in order to cover rare code paths. const bool ManualFlagModification = conf.getBool("ManualFlagModification", false); if (ManualFlagModification) { - ProfileConsistencyCheckParameters options; + ConventionalProfileProcessingParameters options; options.deserialize(conf); EntireSampleDataHandler entireSampleDataHandler(obsspace, options.DHParameters); @@ -234,7 +234,7 @@ void testProfileConsistencyChecks(const eckit::LocalConfiguration &conf) { const bool testProfileVerticalInterpolation = conf.getBool("testProfileVerticalInterpolation", false); if (testProfileVerticalInterpolation) { - ProfileConsistencyCheckParameters options; + ConventionalProfileProcessingParameters options; options.deserialize(conf); std::vector apply(obsspace.nlocs(), true); @@ -318,7 +318,7 @@ void testProfileConsistencyChecks(const eckit::LocalConfiguration &conf) { const bool testProfileVerticalAveraging = conf.getBool("testProfileVerticalAveraging", false); if (testProfileVerticalAveraging) { - ProfileConsistencyCheckParameters options; + ConventionalProfileProcessingParameters options; options.deserialize(conf); std::vector apply(obsspace.nlocs(), true); @@ -386,7 +386,7 @@ void testProfileConsistencyChecks(const eckit::LocalConfiguration &conf) { const bool testProfileDataHolder = conf.getBool("testProfileDataHolder", false); if (testProfileDataHolder) { - ProfileConsistencyCheckParameters options; + ConventionalProfileProcessingParameters options; options.deserialize(conf); std::vector apply(obsspace.nlocs(), true); @@ -437,9 +437,9 @@ void testProfileConsistencyChecks(const eckit::LocalConfiguration &conf) { } } -class ProfileConsistencyChecks : public oops::Test { +class ConventionalProfileProcessing : public oops::Test { private: - std::string testid() const override {return "ufo::test::ProfileConsistencyChecks";} + std::string testid() const override {return "ufo::test::ConventionalProfileProcessing";} void register_tests() const override { std::vector& ts = eckit::testing::specification(); @@ -448,9 +448,9 @@ class ProfileConsistencyChecks : public oops::Test { for (const std::string & testCaseName : conf.keys()) { const eckit::LocalConfiguration testCaseConf(::test::TestEnvironment::config(), testCaseName); - ts.emplace_back(CASE("ufo/ProfileConsistencyChecks/" + testCaseName, testCaseConf) + ts.emplace_back(CASE("ufo/ConventionalProfileProcessing/" + testCaseName, testCaseConf) { - testProfileConsistencyChecks(testCaseConf); + testConventionalProfileProcessing(testCaseConf); }); } } @@ -461,4 +461,4 @@ class ProfileConsistencyChecks : public oops::Test { } // namespace test } // namespace ufo -#endif // TEST_UFO_PROFILECONSISTENCYCHECKS_H_ +#endif // TEST_UFO_CONVENTIONALPROFILEPROCESSING_H_ From e60d6685d69c01df8b6ddbd940699fb0e4d53f56 Mon Sep 17 00:00:00 2001 From: JoanneAWaller <52206323+JoanneAWaller@users.noreply.github.com> Date: Fri, 23 Jul 2021 17:54:17 +0100 Subject: [PATCH 55/98] Add obs function to calculate adjusted marine wind vector (#1352) * Begin adding surface adjustment for marine winds * Adding marine surface wind adjustment obs function. * Modfying function to appy to u and v. * Coding norms * Tidy * Address Chris' comments * Corrected typo * Accounting for 0m obs * Coding norms * Dealing with 0m winds * Added a todo for flags * Restart test system Co-authored-by: Ryan Honeyager --- src/ufo/filters/CMakeLists.txt | 2 + .../ModelHeightAdjustedMarineWind.cc | 70 +++++++++++++++++++ .../ModelHeightAdjustedMarineWind.h | 63 +++++++++++++++++ test/CMakeLists.txt | 9 +++ ...n_ModelSurfaceHeightAdjustMarineWinds.yaml | 25 +++++++ 5 files changed, 169 insertions(+) create mode 100644 src/ufo/filters/obsfunctions/ModelHeightAdjustedMarineWind.cc create mode 100644 src/ufo/filters/obsfunctions/ModelHeightAdjustedMarineWind.h create mode 100644 test/testinput/unit_tests/function_ModelSurfaceHeightAdjustMarineWinds.yaml diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index 0e76c8e48..a5fc2a4b3 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -191,6 +191,8 @@ set ( filters_files obsfunctions/ModelHeightAdjustedAirTemperature.h obsfunctions/ModelHeightAdjustedRelativeHumidity.cc obsfunctions/ModelHeightAdjustedRelativeHumidity.h + obsfunctions/ModelHeightAdjustedMarineWind.cc + obsfunctions/ModelHeightAdjustedMarineWind.h obsfunctions/ObsErrorBoundIR.cc obsfunctions/ObsErrorBoundIR.h obsfunctions/ObsErrorBoundMW.cc diff --git a/src/ufo/filters/obsfunctions/ModelHeightAdjustedMarineWind.cc b/src/ufo/filters/obsfunctions/ModelHeightAdjustedMarineWind.cc new file mode 100644 index 000000000..966b9b52d --- /dev/null +++ b/src/ufo/filters/obsfunctions/ModelHeightAdjustedMarineWind.cc @@ -0,0 +1,70 @@ +/* ----------------------------------------------------------------------------- + * (C) British Crown Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * ----------------------------------------------------------------------------- + */ + +#include + +#include "ioda/ObsDataVector.h" +#include "oops/util/missingValues.h" +#include "ufo/filters/obsfunctions/ModelHeightAdjustedMarineWind.h" +#include "ufo/filters/Variable.h" +#include "ufo/utils/Constants.h" + +namespace ufo { + +static ObsFunctionMaker + eastwardMaker_("ModelHeightAdjustedEastwardMarineWind"); +static ObsFunctionMaker + northwardMaker_("ModelHeightAdjustedNorthwardMarineWind"); + +// ----------------------------------------------------------------------------- + +ModelHeightAdjustedMarineWindComponent::ModelHeightAdjustedMarineWindComponent( + const eckit::LocalConfiguration & conf, const Variable &windComponent) + : invars_(), wind_(windComponent) { + // Required observation station height + invars_ += Variable("anemometer_height@MetaData"); + // Required wind component + invars_ += wind_; +} + +// ----------------------------------------------------------------------------- + +void ModelHeightAdjustedMarineWindComponent::compute(const ObsFilterData & in, + ioda::ObsDataVector & out) const { + const size_t nlocs = in.nlocs(); + std::vector WindComponent(nlocs); + std::vector StationHeight(nlocs); + + in.get(Variable(wind_), WindComponent); + in.get(Variable("anemometer_height@MetaData"), StationHeight); + + const float missing = util::missingValue(missing); + const float a = 1.0/0.0016; + const float ref_height = std::log(10.0*a); + // TODO(jwaller): When QC flaging is availiable add flags indicate corrected values + // Compute adjusted marine wind. + for (size_t jj = 0; jj < nlocs; ++jj) { + if (StationHeight[jj] == missing || WindComponent[jj] == missing) { + out[0][jj] = missing; + } else { + float ScaleFactor = ref_height/std::log(std::abs(StationHeight[jj])*a); + out[0][jj] = WindComponent[jj]*ScaleFactor; + } + } +} + +// ----------------------------------------------------------------------------- + +const ufo::Variables & ModelHeightAdjustedMarineWindComponent::requiredVariables() const { + return invars_; +} + + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/filters/obsfunctions/ModelHeightAdjustedMarineWind.h b/src/ufo/filters/obsfunctions/ModelHeightAdjustedMarineWind.h new file mode 100644 index 000000000..2d7c8f17d --- /dev/null +++ b/src/ufo/filters/obsfunctions/ModelHeightAdjustedMarineWind.h @@ -0,0 +1,63 @@ +/* ----------------------------------------------------------------------------- + * (C) British Crown Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * ----------------------------------------------------------------------------- + */ + +#ifndef UFO_FILTERS_OBSFUNCTIONS_MODELHEIGHTADJUSTEDMARINEWIND_H_ +#define UFO_FILTERS_OBSFUNCTIONS_MODELHEIGHTADJUSTEDMARINEWIND_H_ + +#include + +#include "ufo/filters/ObsFilterData.h" +#include "ufo/filters/obsfunctions/ObsFunctionBase.h" +#include "ufo/filters/Variables.h" + +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" +#include "ufo/utils/parameters/ParameterTraitsVariable.h" + +namespace ufo { + +/// \brief Function to calculate surface marine wind value adjusted from +/// anemometer height to reference height of 10m. Outputs a derived eastward +/// or northward wind adjusted to the reference height. Observed winds are +/// adjusted to the reference height by multiplying by a scaling factor (S) +/// based on a simple stability-independent adjustment based on one of the +/// TurboWin options (Eqn 1 of Thomas et al, 2005; doi.org/10.1002/joc.1176). +/// Observations where the anemometer height is 0m are NOT corrected, +/// instead a missing data indicator is assigned. + +class ModelHeightAdjustedMarineWindComponent : public ObsFunctionBase { + protected: + ModelHeightAdjustedMarineWindComponent(const eckit::LocalConfiguration &conf, + const Variable & windComponent); + public: + void compute(const ObsFilterData &, + ioda::ObsDataVector &) const; + const ufo::Variables & requiredVariables() const; + + private: + ufo::Variables invars_; + ufo::Variable wind_; +}; + +class ModelHeightAdjustedEastwardMarineWind : public ModelHeightAdjustedMarineWindComponent { + public: + explicit ModelHeightAdjustedEastwardMarineWind(const eckit::LocalConfiguration &conf) + : ModelHeightAdjustedMarineWindComponent(conf, Variable("eastward_wind@ObsValue")) + {} +}; + +class ModelHeightAdjustedNorthwardMarineWind : public ModelHeightAdjustedMarineWindComponent { + public: + explicit ModelHeightAdjustedNorthwardMarineWind(const eckit::LocalConfiguration &conf) + : ModelHeightAdjustedMarineWindComponent(conf, Variable("northward_wind@ObsValue")) + {} +}; + +} // namespace ufo + +#endif // UFO_FILTERS_OBSFUNCTIONS_MODELHEIGHTADJUSTEDMARINEWIND_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 297debfbb..552d19dcb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -284,6 +284,7 @@ list( APPEND ufo_test_input testinput/unit_tests/function_tropo_estimate.yaml testinput/unit_tests/function_lam_domaincheck.yaml testinput/unit_tests/function_lam_domaincheck_circle.yaml + testinput/unit_tests/function_ModelSurfaceHeightAdjustMarineWinds.yaml testinput/unit_tests/function_ModelSurfaceHeightAdjustRelativeHumidity.yaml testinput/unit_tests/function_ModelSurfaceHeightAdjustWindVector.yaml testinput/unit_tests/function_ModelSurfaceHeightAdjustTemperature.yaml @@ -2874,6 +2875,14 @@ ecbuild_add_test( TARGET test_ufo_function_modelsurfaceheightadjust_relativehum DEPENDS test_ObsFunction.x TEST_DEPENDS ufo_get_ufo_test_data) + +ecbuild_add_test( TARGET test_ufo_function_modelsurfaceheightadjust_marinewind + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x + ARGS "testinput/unit_tests/function_ModelSurfaceHeightAdjustMarineWinds.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFunction.x + TEST_DEPENDS ufo_get_ufo_test_data) + # Test Diagnostics ecbuild_add_test( TARGET test_ufo_obsdiag_background_error_vert_interp_air_pressure diff --git a/test/testinput/unit_tests/function_ModelSurfaceHeightAdjustMarineWinds.yaml b/test/testinput/unit_tests/function_ModelSurfaceHeightAdjustMarineWinds.yaml new file mode 100644 index 000000000..85264d9d4 --- /dev/null +++ b/test/testinput/unit_tests/function_ModelSurfaceHeightAdjustMarineWinds.yaml @@ -0,0 +1,25 @@ +window begin: 2021-05-21T00:00:00Z +window end: 2021-05-21T23:00:00Z + +observations: +- obs space: + name: Surface + obsdatain: + obsfile: Data/ufo/testinput_tier_1/surface_marine_obs_20210521T1200Z.nc4 + simulated variables: [eastward_wind,northward_wind] + + obs function: + name: ModelHeightAdjustedEastwardMarineWind@ObsFunction + variables: [eastward_wind] + tolerance: 1.0e-6 + +- obs space: + name: Surface + obsdatain: + obsfile: Data/ufo/testinput_tier_1/surface_marine_obs_20210521T1200Z.nc4 + simulated variables: [eastward_wind,northward_wind] + + obs function: + name: ModelHeightAdjustedNorthwardMarineWind@ObsFunction + variables: [northward_wind] + tolerance: 1.0e-6 From d8d6c965c4a61ec6930e67b94e81db482a713b51 Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan <2984440+yaswant@users.noreply.github.com> Date: Fri, 23 Jul 2021 20:44:01 +0100 Subject: [PATCH 56/98] exclude new_obsop new_qc templates if ufo source dir is not writeable (#1371) Co-authored-by: Stephen Herbener <32968781+srherbener@users.noreply.github.com> Co-authored-by: Ryan Honeyager --- tools/CMakeLists.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 8d0af37a6..1ac618cab 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -9,5 +9,11 @@ foreach(FILENAME ${test_files}) ${CMAKE_BINARY_DIR}/bin/${PROJECT_NAME}_${FILENAME} ) endforeach(FILENAME) -add_subdirectory(new_obsop) -add_subdirectory(new_qc) +execute_process( COMMAND test -w @CMAKE_CURRENT_SOURCE_DIR RESULT_VARIABLE NO_WRITE ) +if ( NO_WRITE ) + message( WARNING "Source dir ${CMAKE_CURRENT_SOURCE_DIR} is not writeable - autogenerated templates for obsoperator and filters - new_obsop and new_qc - wont be produced in ufo build." ) +else() + add_subdirectory(new_obsop) + add_subdirectory(new_qc) +endif() + From 1ac4499778dc7a362153d68351d677578acf30b1 Mon Sep 17 00:00:00 2001 From: Greg Thompson <63315545+gthompsnJCSDA@users.noreply.github.com> Date: Fri, 23 Jul 2021 14:43:58 -0600 Subject: [PATCH 57/98] Change surface-pressure obsError inflation by delta of surface obs height and model height (when dz>1km) (#1375) * scale the obsError inflation by delta of surface obs height and model height * ensure max limit above 5km DZ, even though it is rather unlikely this large * small indent change Co-authored-by: gthompsnJCSDA Co-authored-by: Ryan Honeyager --- src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.cc index bd2f02c7c..a53a36b35 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorSfcPressure.cc @@ -135,9 +135,11 @@ void ObsErrorFactorSfcPressure::compute(const ObsFilterData & data, } else { rdelz = ob_elevation[iloc]-model_elevation[iloc]; - // If more than a km between model and real elevation, set error factor very high. - if (std::abs(rdelz) > 1000.0f) { - obserr[iv][iloc] = 49.9f; + // If more than a km between model and real elevation, set error factor linearly higher. + if (std::abs(rdelz) > 5000.0f) { + obserr[iv][iloc] = 50.0f; + } else if (std::abs(rdelz) > 1000.0f) { + obserr[iv][iloc] = 3.0f + 47.0f*(std::abs(rdelz)-1000.0f)/4000.0f; } else { pgesorig = model_pres_sfc[iloc]*0.001; // Converting Pascals to cb psges = log(pgesorig); From d78d87600459f5cd854296a43d5d258033222ef6 Mon Sep 17 00:00:00 2001 From: Greg Thompson <63315545+gthompsnJCSDA@users.noreply.github.com> Date: Fri, 23 Jul 2021 14:47:03 -0600 Subject: [PATCH 58/98] Bugfix, FPE, divide by zero error when windspeed is basically zero (#1384) * FPE, divide by zero error when windspeed is basically zero * forgot semicolon at end Co-authored-by: gthompsnJCSDA Co-authored-by: Ryan Honeyager --- src/ufo/filters/obsfunctions/SatWindsLNVDCheck.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/ufo/filters/obsfunctions/SatWindsLNVDCheck.cc b/src/ufo/filters/obsfunctions/SatWindsLNVDCheck.cc index f3c3203a0..ebe40aa66 100644 --- a/src/ufo/filters/obsfunctions/SatWindsLNVDCheck.cc +++ b/src/ufo/filters/obsfunctions/SatWindsLNVDCheck.cc @@ -64,11 +64,12 @@ void SatWindsLNVDCheck::compute(const ObsFilterData & in, for (size_t jj = 0; jj < nlocs; ++jj) { if (u[jj] != missing && v[jj] != missing) { - out[0][jj] = sqrt((u[jj]-um[jj])*(u[jj]-um[jj]) + (v[jj]-vm[jj])*(v[jj]-vm[jj])) + if ((u[jj]*u[jj] + v[jj]*v[jj]) > 1.01f) { + out[0][jj] = sqrt((u[jj]-um[jj])*(u[jj]-um[jj]) + (v[jj]-vm[jj])*(v[jj]-vm[jj])) / log(sqrt(u[jj]*u[jj] + v[jj]*v[jj])); - oops::Log::debug() << "u, v: " << u[jj] << ", " << v[jj] - << " um, vm: " << um[jj] << ", " << vm[jj] - << " LNVD: " << out[0][jj] << std::endl; + } else { + out[0][jj] = sqrt((u[jj]-um[jj])*(u[jj]-um[jj]) + (v[jj]-vm[jj])*(v[jj]-vm[jj])); + } } else { out[0][jj] = missing; } From d885156343e591768d5e433851d25192a678d7d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Wed, 28 Jul 2021 19:38:52 +0100 Subject: [PATCH 59/98] Make ObsError implementations subclasses of oops::interface::ObsErrorBase. (#1396) * Make ObsError implementations subclasses of oops::interface::ObsErrorBase. * Make obserrorsImpl() and inverseVarianceImpl() return unique_ptrs to OBS::ObsVector. * Rename obserrorsImpl to getObsErrors and inverseVarianceImpl to getInverseVariance. --- src/ufo/errors/ObsErrorCrossVarCov.cc | 18 +++++++------ src/ufo/errors/ObsErrorCrossVarCov.h | 27 ++++++++++--------- src/ufo/errors/ObsErrorDiagonal.cc | 13 ++++----- src/ufo/errors/ObsErrorDiagonal.h | 38 ++++++++++++++++----------- src/ufo/instantiateObsErrorFactory.h | 6 ++--- 5 files changed, 58 insertions(+), 44 deletions(-) diff --git a/src/ufo/errors/ObsErrorCrossVarCov.cc b/src/ufo/errors/ObsErrorCrossVarCov.cc index 014ebc6f8..738fd6c0e 100644 --- a/src/ufo/errors/ObsErrorCrossVarCov.cc +++ b/src/ufo/errors/ObsErrorCrossVarCov.cc @@ -19,8 +19,10 @@ namespace ufo { // ----------------------------------------------------------------------------- ObsErrorCrossVarCov::ObsErrorCrossVarCov(const eckit::Configuration & conf, - ioda::ObsSpace & obspace) - : stddev_(obspace, "ObsError"), + ioda::ObsSpace & obspace, + const eckit::mpi::Comm &timeComm) + : ObsErrorBase(timeComm), + stddev_(obspace, "ObsError"), varcorrelations_(Eigen::MatrixXd::Identity(stddev_.nvars(), stddev_.nvars())) { ObsErrorCrossVarCovParameters options; @@ -167,16 +169,16 @@ void ObsErrorCrossVarCov::save(const std::string & name) const { // ----------------------------------------------------------------------------- -ioda::ObsVector ObsErrorCrossVarCov::obserrors() const { - return stddev_; +std::unique_ptr ObsErrorCrossVarCov::getObsErrors() const { + return std::make_unique(stddev_); } // ----------------------------------------------------------------------------- -ioda::ObsVector ObsErrorCrossVarCov::inverseVariance() const { - ioda::ObsVector inverseVariance = stddev_; - inverseVariance *= stddev_; - inverseVariance.invert(); +std::unique_ptr ObsErrorCrossVarCov::getInverseVariance() const { + std::unique_ptr inverseVariance = std::make_unique(stddev_); + *inverseVariance *= stddev_; + inverseVariance->invert(); return inverseVariance; } diff --git a/src/ufo/errors/ObsErrorCrossVarCov.h b/src/ufo/errors/ObsErrorCrossVarCov.h index 51cf278bc..32c48e8ae 100644 --- a/src/ufo/errors/ObsErrorCrossVarCov.h +++ b/src/ufo/errors/ObsErrorCrossVarCov.h @@ -10,14 +10,16 @@ #include -#include +#include #include #include "ioda/ObsVector.h" +#include "oops/interface/ObsErrorBase.h" #include "oops/util/parameters/Parameters.h" #include "oops/util/parameters/RequiredParameter.h" -#include "oops/util/Printable.h" + +#include "ufo/ObsTraits.h" namespace eckit { class Configuration; @@ -46,40 +48,41 @@ class ObsErrorCrossVarCovParameters : public oops::Parameters { /// Full observation error covariance matrix is R = D^{1/2} * C * D^{1/2} /// where D^{1/2} is a diagonal matrix with stddev_ (ObsError group) /// on the diagonal, and C is the correlation matrix. -class ObsErrorCrossVarCov : public util::Printable { +class ObsErrorCrossVarCov : public oops::interface::ObsErrorBase { public: /// Initialize observation errors - ObsErrorCrossVarCov(const eckit::Configuration &, ioda::ObsSpace &); + ObsErrorCrossVarCov(const eckit::Configuration &, ioda::ObsSpace &, + const eckit::mpi::Comm &timeComm); /// Update obs error standard deviations to be equal to \p stddev - void update(const ioda::ObsVector & stddev); + void update(const ioda::ObsVector & stddev) override; /// Multiply \p y by this observation error covariance /// Computed as R * dy = D^{1/2} * C * D^{1/2} * dy /// where D^{1/2} - diagonal matrix with stddev_ on the diagonal /// C - correlations - void multiply(ioda::ObsVector & y) const; + void multiply(ioda::ObsVector & y) const override; /// Multiply \p y by inverse of this observation error covariance /// Computed as R^{-1} * dy = D^{-1/2} * C^{-1] * D^{-1/2} * dy /// where D^{1/2} - diagonal matrix with stddev_ on the diagonal /// C - correlations - void inverseMultiply(ioda::ObsVector & y) const; + void inverseMultiply(ioda::ObsVector & y) const override; /// Generate \p y as a random perturbation - void randomize(ioda::ObsVector & y) const; + void randomize(ioda::ObsVector & y) const override; /// Save obs error standard deviations under \p name group name - void save(const std::string & name) const; + void save(const std::string & name) const override; /// Return RMS of obs error standard deviations - double getRMSE() const {return stddev_.rms();} + double getRMSE() const override {return stddev_.rms();} /// Return obs errors std deviation - ioda::ObsVector obserrors() const; + std::unique_ptr getObsErrors() const override; /// Return inverse of obs error variance - ioda::ObsVector inverseVariance() const; + std::unique_ptr getInverseVariance() const override; private: /// Print covariance details (for logging) diff --git a/src/ufo/errors/ObsErrorDiagonal.cc b/src/ufo/errors/ObsErrorDiagonal.cc index 9dcee1247..eb8cb9416 100644 --- a/src/ufo/errors/ObsErrorDiagonal.cc +++ b/src/ufo/errors/ObsErrorDiagonal.cc @@ -16,8 +16,9 @@ namespace ufo { // ----------------------------------------------------------------------------- -ObsErrorDiagonal::ObsErrorDiagonal(const eckit::Configuration & conf, ioda::ObsSpace & obsgeom) - : stddev_(obsgeom, "ObsError"), inverseVariance_(obsgeom) +ObsErrorDiagonal::ObsErrorDiagonal(const eckit::Configuration & conf, ioda::ObsSpace & obsgeom, + const eckit::mpi::Comm &timeComm) + : ObsErrorBase(timeComm), stddev_(obsgeom, "ObsError"), inverseVariance_(obsgeom) { options_.deserialize(conf); inverseVariance_ = stddev_; @@ -65,14 +66,14 @@ void ObsErrorDiagonal::save(const std::string & name) const { // ----------------------------------------------------------------------------- -ioda::ObsVector ObsErrorDiagonal::obserrors() const { - return stddev_; +std::unique_ptr ObsErrorDiagonal::getObsErrors() const { + return std::make_unique(stddev_); } // ----------------------------------------------------------------------------- -ioda::ObsVector ObsErrorDiagonal::inverseVariance() const { - return inverseVariance_; +std::unique_ptr ObsErrorDiagonal::getInverseVariance() const { + return std::make_unique(inverseVariance_); } // ----------------------------------------------------------------------------- diff --git a/src/ufo/errors/ObsErrorDiagonal.h b/src/ufo/errors/ObsErrorDiagonal.h index 77b20a2c3..521c83210 100644 --- a/src/ufo/errors/ObsErrorDiagonal.h +++ b/src/ufo/errors/ObsErrorDiagonal.h @@ -8,19 +8,26 @@ #ifndef UFO_ERRORS_OBSERRORDIAGONAL_H_ #define UFO_ERRORS_OBSERRORDIAGONAL_H_ -#include +#include #include #include "eckit/config/Configuration.h" -#include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" -#include "oops/util/Logger.h" +#include "oops/interface/ObsErrorBase.h" #include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/Parameters.h" -#include "oops/util/Printable.h" +#include "ufo/ObsTraits.h" + +namespace eckit { + class Configuration; +} + +namespace ioda { + class ObsSpace; +} namespace ufo { @@ -34,38 +41,39 @@ class ObsErrorDiagonalParameters : public oops::Parameters { // ----------------------------------------------------------------------------- /// \brief Diagonal observation error covariance matrix. -class ObsErrorDiagonal : public util::Printable { +class ObsErrorDiagonal : public oops::interface::ObsErrorBase { public: static const std::string classname() {return "ufo::ObsErrorDiagonal";} - ObsErrorDiagonal(const eckit::Configuration &, ioda::ObsSpace &); + ObsErrorDiagonal(const eckit::Configuration &, ioda::ObsSpace &, + const eckit::mpi::Comm &timeComm); /// Update after obs errors potentially changed - void update(const ioda::ObsVector &); + void update(const ioda::ObsVector &) override; /// Multiply a Departure by \f$R\f$ - void multiply(ioda::ObsVector &) const; + void multiply(ioda::ObsVector &) const override; /// Multiply a Departure by \f$R^{-1}\f$ - void inverseMultiply(ioda::ObsVector &) const; + void inverseMultiply(ioda::ObsVector &) const override; /// Generate random perturbation - void randomize(ioda::ObsVector &) const; + void randomize(ioda::ObsVector &) const override; /// Save obs errors - void save(const std::string &) const; + void save(const std::string &) const override; /// Get mean error for Jo table - double getRMSE() const {return stddev_.rms();} + double getRMSE() const override {return stddev_.rms();} /// Get obs errors std deviation - ioda::ObsVector obserrors() const; + std::unique_ptr getObsErrors() const override; /// Return inverseVariance - ioda::ObsVector inverseVariance() const; + std::unique_ptr getInverseVariance() const override; private: - void print(std::ostream &) const; + void print(std::ostream &) const override; ioda::ObsVector stddev_; ioda::ObsVector inverseVariance_; ObsErrorDiagonalParameters options_; diff --git a/src/ufo/instantiateObsErrorFactory.h b/src/ufo/instantiateObsErrorFactory.h index 32928f430..274435ccd 100644 --- a/src/ufo/instantiateObsErrorFactory.h +++ b/src/ufo/instantiateObsErrorFactory.h @@ -9,7 +9,7 @@ #define UFO_INSTANTIATEOBSERRORFACTORY_H_ #include "oops/generic/instantiateObsErrorFactory.h" -#include "oops/interface/ObsErrorCovariance.h" +#include "oops/interface/ObsErrorBase.h" #include "ufo/errors/ObsErrorCrossVarCov.h" #include "ufo/errors/ObsErrorDiagonal.h" @@ -17,9 +17,9 @@ namespace ufo { template void instantiateObsErrorFactory() { oops::instantiateObsErrorFactory(); - static oops::ObsErrorMaker > + static oops::interface::ObsErrorMaker makerDiagUFO("diagonal ufo"); - static oops::ObsErrorMaker > + static oops::interface::ObsErrorMaker makerCrossVarCov("cross variable covariances"); } From de293145872ce709efb5cf8cfa6d363e3a58032c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Fri, 30 Jul 2021 13:44:03 +0100 Subject: [PATCH 60/98] Refactor ObsFilterBase and related classes (#1400) * Removed unnecessary includes. * Make all filters subclasses of oops::interface::ObsFilterBase. --- src/ufo/filters/BackgroundCheck.cc | 1 - src/ufo/filters/BayesianBackgroundCheck.cc | 1 - .../filters/ConventionalProfileProcessing.cc | 1 - src/ufo/filters/FilterBase.cc | 1 - src/ufo/filters/FilterParametersBase.h | 2 +- src/ufo/filters/FinalCheck.h | 2 +- src/ufo/filters/ModelObThreshold.cc | 2 - src/ufo/filters/ObsDiagnosticsWriter.cc | 1 - src/ufo/filters/ObsDiagnosticsWriter.h | 19 ++-- src/ufo/filters/ObsProcessorBase.cc | 1 - src/ufo/filters/ObsProcessorBase.h | 16 ++-- src/ufo/filters/ProfileBackgroundCheck.cc | 1 - src/ufo/filters/QCmanager.cc | 3 +- src/ufo/filters/QCmanager.h | 16 ++-- src/ufo/filters/SatwindInversionCorrection.cc | 2 - src/ufo/filters/VariableAssignment.h | 2 +- src/ufo/filters/VariableTransforms.cc | 1 - src/ufo/gnssro/QC/BackgroundCheckRONBAM.cc | 1 - src/ufo/gnssro/QC/ROobserror.cc | 2 - src/ufo/instantiateObsFilterFactory.h | 88 +++++++++---------- 20 files changed, 74 insertions(+), 89 deletions(-) diff --git a/src/ufo/filters/BackgroundCheck.cc b/src/ufo/filters/BackgroundCheck.cc index 0002014c9..a835b5885 100644 --- a/src/ufo/filters/BackgroundCheck.cc +++ b/src/ufo/filters/BackgroundCheck.cc @@ -20,7 +20,6 @@ #include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/abor1_cpp.h" #include "oops/util/IntSetParser.h" #include "oops/util/Logger.h" diff --git a/src/ufo/filters/BayesianBackgroundCheck.cc b/src/ufo/filters/BayesianBackgroundCheck.cc index a367485bd..83c47885e 100644 --- a/src/ufo/filters/BayesianBackgroundCheck.cc +++ b/src/ufo/filters/BayesianBackgroundCheck.cc @@ -18,7 +18,6 @@ #include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/Logger.h" #include "ufo/filters/getScalarOrFilterData.h" diff --git a/src/ufo/filters/ConventionalProfileProcessing.cc b/src/ufo/filters/ConventionalProfileProcessing.cc index c1b4b02d9..9d93bba91 100644 --- a/src/ufo/filters/ConventionalProfileProcessing.cc +++ b/src/ufo/filters/ConventionalProfileProcessing.cc @@ -17,7 +17,6 @@ #include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/abor1_cpp.h" #include "oops/util/Logger.h" diff --git a/src/ufo/filters/FilterBase.cc b/src/ufo/filters/FilterBase.cc index 097319738..916ecb96b 100644 --- a/src/ufo/filters/FilterBase.cc +++ b/src/ufo/filters/FilterBase.cc @@ -16,7 +16,6 @@ #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/Logger.h" #include "ufo/filters/actions/FilterAction.h" diff --git a/src/ufo/filters/FilterParametersBase.h b/src/ufo/filters/FilterParametersBase.h index 59cfccef2..01076b202 100644 --- a/src/ufo/filters/FilterParametersBase.h +++ b/src/ufo/filters/FilterParametersBase.h @@ -11,7 +11,7 @@ #include #include "eckit/config/LocalConfiguration.h" -#include "oops/base/ObsFilterParametersBase.h" +#include "oops/generic/ObsFilterParametersBase.h" #include "oops/util/parameters/OptionalParameter.h" #include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/PolymorphicParameter.h" diff --git a/src/ufo/filters/FinalCheck.h b/src/ufo/filters/FinalCheck.h index 945c5ef1b..308d9931d 100644 --- a/src/ufo/filters/FinalCheck.h +++ b/src/ufo/filters/FinalCheck.h @@ -12,7 +12,7 @@ #include #include -#include "oops/base/ObsFilterParametersBase.h" +#include "oops/generic/ObsFilterParametersBase.h" #include "oops/util/ObjectCounter.h" #include "ufo/filters/ObsProcessorBase.h" diff --git a/src/ufo/filters/ModelObThreshold.cc b/src/ufo/filters/ModelObThreshold.cc index c1c1ccd4f..e5b620591 100644 --- a/src/ufo/filters/ModelObThreshold.cc +++ b/src/ufo/filters/ModelObThreshold.cc @@ -12,8 +12,6 @@ #include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" -#include "oops/base/ObsFilterBase.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/Logger.h" #include "oops/util/PropertiesOfNVectors.h" diff --git a/src/ufo/filters/ObsDiagnosticsWriter.cc b/src/ufo/filters/ObsDiagnosticsWriter.cc index 55ecb8d59..70916f3c1 100644 --- a/src/ufo/filters/ObsDiagnosticsWriter.cc +++ b/src/ufo/filters/ObsDiagnosticsWriter.cc @@ -13,7 +13,6 @@ #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/Logger.h" namespace ufo { diff --git a/src/ufo/filters/ObsDiagnosticsWriter.h b/src/ufo/filters/ObsDiagnosticsWriter.h index 016574d38..92c60f134 100644 --- a/src/ufo/filters/ObsDiagnosticsWriter.h +++ b/src/ufo/filters/ObsDiagnosticsWriter.h @@ -14,9 +14,11 @@ #include "ioda/ObsDataVector.h" #include "oops/base/Variables.h" +#include "oops/interface/ObsFilterBase.h" #include "oops/util/Printable.h" #include "ufo/filters/Variables.h" #include "ufo/ObsDiagnostics.h" +#include "ufo/ObsTraits.h" namespace eckit { class Configuration; @@ -32,27 +34,24 @@ namespace ioda { namespace ufo { class GeoVaLs; -class ObsDiagnosticsWriter : public util::Printable, - private util::ObjectCounter { +class ObsDiagnosticsWriter : public oops::interface::ObsFilterBase { public: - static const std::string classname() {return "ufo::ObsDiagnosticsWriter";} - ObsDiagnosticsWriter(ioda::ObsSpace &, const eckit::Configuration &, std::shared_ptr >, std::shared_ptr >); ~ObsDiagnosticsWriter() {} - void preProcess() const {} - void priorFilter(const GeoVaLs &) const {} - void postFilter(const ioda::ObsVector &, const ObsDiagnostics & diags) { + void preProcess() override {} + void priorFilter(const GeoVaLs &) override {} + void postFilter(const ioda::ObsVector &, const ObsDiagnostics & diags) override { diags.write(config_); } - const oops::Variables & requiredVars() const {return nogeovals_;} - const oops::Variables & requiredHdiagnostics() const {return extradiagvars_;} + oops::Variables requiredVars() const override {return nogeovals_;} + oops::Variables requiredHdiagnostics() const override {return extradiagvars_;} private: - void print(std::ostream &) const; + void print(std::ostream &) const override; const eckit::LocalConfiguration config_; const oops::Variables nogeovals_; oops::Variables extradiagvars_; diff --git a/src/ufo/filters/ObsProcessorBase.cc b/src/ufo/filters/ObsProcessorBase.cc index cb5f93f03..7e4943d0e 100644 --- a/src/ufo/filters/ObsProcessorBase.cc +++ b/src/ufo/filters/ObsProcessorBase.cc @@ -16,7 +16,6 @@ #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/Logger.h" #include "ufo/filters/actions/FilterAction.h" diff --git a/src/ufo/filters/ObsProcessorBase.h b/src/ufo/filters/ObsProcessorBase.h index 954f467ff..cdf824f78 100644 --- a/src/ufo/filters/ObsProcessorBase.h +++ b/src/ufo/filters/ObsProcessorBase.h @@ -12,10 +12,10 @@ #include "ioda/ObsDataVector.h" #include "oops/base/Variables.h" -#include "oops/util/ObjectCounter.h" -#include "oops/util/Printable.h" +#include "oops/interface/ObsFilterBase.h" #include "ufo/filters/ObsFilterData.h" #include "ufo/filters/Variables.h" +#include "ufo/ObsTraits.h" namespace eckit { class Configuration; @@ -36,20 +36,20 @@ namespace ufo { /// Observation processors only need to implement the constructor and the doFilter method; /// the base class takes care of applying the processor at the pre, prior or post stage. -class ObsProcessorBase : public util::Printable { +class ObsProcessorBase : public oops::interface::ObsFilterBase { public: ObsProcessorBase(ioda::ObsSpace &, bool deferToPost, std::shared_ptr >, std::shared_ptr >); ~ObsProcessorBase(); - void preProcess(); - void priorFilter(const GeoVaLs &); - void postFilter(const ioda::ObsVector &, const ObsDiagnostics &); + void preProcess() override; + void priorFilter(const GeoVaLs &) override; + void postFilter(const ioda::ObsVector &, const ObsDiagnostics &) override; - oops::Variables requiredVars() const { + oops::Variables requiredVars() const override { return allvars_.allFromGroup("GeoVaLs").toOopsVariables();} - oops::Variables requiredHdiagnostics() const { + oops::Variables requiredHdiagnostics() const override { return allvars_.allFromGroup("ObsDiag").toOopsVariables();} protected: diff --git a/src/ufo/filters/ProfileBackgroundCheck.cc b/src/ufo/filters/ProfileBackgroundCheck.cc index d13d642e6..042e95290 100644 --- a/src/ufo/filters/ProfileBackgroundCheck.cc +++ b/src/ufo/filters/ProfileBackgroundCheck.cc @@ -18,7 +18,6 @@ #include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/Logger.h" #include "ufo/filters/getScalarOrFilterData.h" diff --git a/src/ufo/filters/QCmanager.cc b/src/ufo/filters/QCmanager.cc index c9e46059e..6ee70757f 100644 --- a/src/ufo/filters/QCmanager.cc +++ b/src/ufo/filters/QCmanager.cc @@ -19,7 +19,6 @@ #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" #include "oops/base/Variables.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/Logger.h" #include "oops/util/missingValues.h" #include "ufo/filters/QCflags.h" @@ -95,7 +94,7 @@ QCmanager::QCmanager(ioda::ObsSpace & obsdb, const eckit::Configuration & /*conf // ----------------------------------------------------------------------------- -void QCmanager::postFilter(const ioda::ObsVector & hofx, const ObsDiagnostics &) const { +void QCmanager::postFilter(const ioda::ObsVector & hofx, const ObsDiagnostics &) { oops::Log::trace() << "QCmanager postFilter" << std::endl; const double missing = util::missingValue(missing); diff --git a/src/ufo/filters/QCmanager.h b/src/ufo/filters/QCmanager.h index b6f6852a5..4d3f887d8 100644 --- a/src/ufo/filters/QCmanager.h +++ b/src/ufo/filters/QCmanager.h @@ -15,7 +15,9 @@ #include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" #include "oops/base/Variables.h" +#include "oops/interface/ObsFilterBase.h" #include "oops/util/Printable.h" +#include "ufo/ObsTraits.h" namespace ioda { template class ObsDataVector; @@ -31,22 +33,22 @@ class ObsDiagnostics; /// The constructor sets the QC flag to `missing` at all locations with missing obs values of QC /// flags. The postFilter() function sets the QC flag to `Hfailed` if it was previously set to /// `pass`, but the obs operator failed to produce a valid value. -class QCmanager : public util::Printable { +class QCmanager : public oops::interface::ObsFilterBase { public: QCmanager(ioda::ObsSpace &, const eckit::Configuration &, std::shared_ptr >, std::shared_ptr >); ~QCmanager(); - void preProcess() const {} - void priorFilter(const GeoVaLs &) const {} - void postFilter(const ioda::ObsVector &, const ObsDiagnostics &) const; + void preProcess() override {} + void priorFilter(const GeoVaLs &) override {} + void postFilter(const ioda::ObsVector &, const ObsDiagnostics &) override; - const oops::Variables & requiredVars() const {return nogeovals_;} - const oops::Variables & requiredHdiagnostics() const {return nodiags_;} + oops::Variables requiredVars() const override {return nogeovals_;} + oops::Variables requiredHdiagnostics() const override {return nodiags_;} private: - void print(std::ostream &) const; + void print(std::ostream &) const override; ioda::ObsSpace & obsdb_; const oops::Variables nogeovals_; diff --git a/src/ufo/filters/SatwindInversionCorrection.cc b/src/ufo/filters/SatwindInversionCorrection.cc index 9ccd9d20e..501b60712 100644 --- a/src/ufo/filters/SatwindInversionCorrection.cc +++ b/src/ufo/filters/SatwindInversionCorrection.cc @@ -15,8 +15,6 @@ #include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" -#include "oops/base/ObsFilterBase.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" diff --git a/src/ufo/filters/VariableAssignment.h b/src/ufo/filters/VariableAssignment.h index badeb2ec4..4fded1ae4 100644 --- a/src/ufo/filters/VariableAssignment.h +++ b/src/ufo/filters/VariableAssignment.h @@ -15,7 +15,7 @@ #include "eckit/config/LocalConfiguration.h" #include "ioda/core/ParameterTraitsObsDtype.h" -#include "oops/base/ObsFilterParametersBase.h" +#include "oops/generic/ObsFilterParametersBase.h" #include "oops/util/ObjectCounter.h" #include "oops/util/parameters/OptionalParameter.h" #include "oops/util/parameters/Parameter.h" diff --git a/src/ufo/filters/VariableTransforms.cc b/src/ufo/filters/VariableTransforms.cc index 06d3b044b..f0f542976 100644 --- a/src/ufo/filters/VariableTransforms.cc +++ b/src/ufo/filters/VariableTransforms.cc @@ -17,7 +17,6 @@ #include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/abor1_cpp.h" #include "oops/util/Logger.h" diff --git a/src/ufo/gnssro/QC/BackgroundCheckRONBAM.cc b/src/ufo/gnssro/QC/BackgroundCheckRONBAM.cc index 82f871677..0adceb9fb 100644 --- a/src/ufo/gnssro/QC/BackgroundCheckRONBAM.cc +++ b/src/ufo/gnssro/QC/BackgroundCheckRONBAM.cc @@ -18,7 +18,6 @@ #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/Logger.h" #include "ufo/filters/QCflags.h" diff --git a/src/ufo/gnssro/QC/ROobserror.cc b/src/ufo/gnssro/QC/ROobserror.cc index 8f573519b..5c0158b95 100644 --- a/src/ufo/gnssro/QC/ROobserror.cc +++ b/src/ufo/gnssro/QC/ROobserror.cc @@ -12,8 +12,6 @@ #include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" -#include "oops/base/ObsFilterBase.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" diff --git a/src/ufo/instantiateObsFilterFactory.h b/src/ufo/instantiateObsFilterFactory.h index 53f6e6c92..8f9854a2d 100644 --- a/src/ufo/instantiateObsFilterFactory.h +++ b/src/ufo/instantiateObsFilterFactory.h @@ -9,7 +9,7 @@ #define UFO_INSTANTIATEOBSFILTERFACTORY_H_ #include "oops/base/instantiateObsFilterFactory.h" -#include "oops/interface/ObsFilter.h" +#include "oops/interface/ObsFilterBase.h" #include "ufo/filters/AcceptList.h" #include "ufo/filters/BackgroundCheck.h" #include "ufo/filters/BayesianBackgroundCheck.h" @@ -57,97 +57,97 @@ namespace ufo { template void instantiateObsFilterFactory() { oops::instantiateObsFilterFactory(); - static oops::FilterMaker > + static oops::interface::FilterMaker qcManagerMaker("QCmanager"); - static oops::FilterMaker > + static oops::interface::FilterMaker finalCheckMaker("Final Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker preQCMaker("PreQC"); - static oops::FilterMaker > + static oops::interface::FilterMaker domainCheckMaker("Domain Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker satnameCheckMaker("satname"); - static oops::FilterMaker > + static oops::interface::FilterMaker boundsCheckMaker("Bounds Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker blackListMaker("BlackList"); - static oops::FilterMaker > + static oops::interface::FilterMaker rejectListMaker("RejectList"); // alternative name - static oops::FilterMaker > + static oops::interface::FilterMaker backgroundCheckMaker("Background Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker BayesianBackgroundCheckMaker("Bayesian Background Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker differenceCheckMaker("Difference Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker historyCheckMaker("History Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker ModelBestFitPressureMaker("Model Best Fit Pressure"); - static oops::FilterMaker > + static oops::interface::FilterMaker ModelObThresholdMaker("ModelOb Threshold"); - static oops::FilterMaker > + static oops::interface::FilterMaker ROobserrorMaker("ROobserror"); - static oops::FilterMaker > + static oops::interface::FilterMaker thinningMaker("Thinning"); - static oops::FilterMaker > + static oops::interface::FilterMaker gaussianThinningMaker("Gaussian Thinning"); - static oops::FilterMaker > + static oops::interface::FilterMaker MWCLWCheckMaker("MWCLW Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker domainErrCheckMaker("DomainErr Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker conventionalProfileProcessingMaker("Conventional Profile Processing"); - static oops::FilterMaker > + static oops::interface::FilterMaker backgroundCheckRONBAMMaker("Background Check RONBAM"); - static oops::FilterMaker > + static oops::interface::FilterMaker temporalThinningMaker("Temporal Thinning"); - static oops::FilterMaker > + static oops::interface::FilterMaker poissonDiskThinningMaker("Poisson Disk Thinning"); - static oops::FilterMaker > + static oops::interface::FilterMaker YDIAGsaverMaker("YDIAGsaver"); - static oops::FilterMaker > + static oops::interface::FilterMaker TrackCheckMaker("Track Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker MetOfficeBuddyCheckMaker("Met Office Buddy Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker DerivativeCheckMaker("Derivative Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker ShipTrackCheckMaker("Ship Track Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker StuckCheckMaker("Stuck Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker GNSSROOneDVarCheckMaker("GNSS-RO 1DVar Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker variableAssignmentMaker("Variable Assignment"); - static oops::FilterMaker > + static oops::interface::FilterMaker VariableTransformsMaker("Variable Transforms"); - static oops::FilterMaker > + static oops::interface::FilterMaker ProfileBackgroundCheckMaker("Profile Background Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker ProfileFewObsCheckMaker("Profile Few Observations Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker acceptListMaker("AcceptList"); - static oops::FilterMaker > + static oops::interface::FilterMaker performActionMaker("Perform Action"); - static oops::FilterMaker > + static oops::interface::FilterMaker BayesianBackgroundQCFlagsMaker("Bayesian Background QC Flags"); - static oops::FilterMaker > + static oops::interface::FilterMaker ProbabilityGrossErrorWholeReportMaker("Bayesian Whole Report"); - static oops::FilterMaker > + static oops::interface::FilterMaker ImpactHeightCheckMaker("GNSSRO Impact Height Check"); - static oops::FilterMaker > + static oops::interface::FilterMaker SatwindInversionCorrectionMaker("Satwind Inversion Correction"); // Only include this filter if rttov is present #if defined(RTTOV_FOUND) - static oops::FilterMaker > + static oops::interface::FilterMaker RTTOVOneDVarCheckMaker("RTTOV OneDVar Check"); #endif // For backward compatibility, register some filters under legacy names used in the past - static oops::FilterMaker > + static oops::interface::FilterMaker legacyGaussianThinningMaker("Gaussian_Thinning"); - static oops::FilterMaker > + static oops::interface::FilterMaker legacyTemporalThinningMaker("TemporalThinning"); } From 8fb93f6965d49144fd9358eb17c39f7b31afe800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Mon, 2 Aug 2021 15:42:21 +0100 Subject: [PATCH 61/98] Use of derived variables in variable transforms (#1337) * VariableTransforms: Output to the DerivedObsValue group. Update QC flags when an obsvalue changes from missing to non-missing and vice versa. * Add a test case for the VariableTransforms filter creating a derived simulated variable. * Correct ObsSpace name. * Restart CI Co-authored-by: Ryan Honeyager --- src/ufo/filters/VariableTransforms.h | 6 +- src/ufo/variabletransforms/Cal_Humidity.cc | 10 +- .../Cal_PressureFromHeight.cc | 8 +- src/ufo/variabletransforms/Cal_Wind.cc | 8 +- src/ufo/variabletransforms/TransformBase.h | 91 ++++++++++++------- ...bletransforms_SondePressureFromHeight.yaml | 2 +- ...ransforms_SondePressureFromHeightICAO.yaml | 2 +- .../variabletransforms_rhumidity.yaml | 8 +- .../variabletransforms_rhumidity_part2.yaml | 2 +- .../variabletransforms_shumidity.yaml | 10 +- .../variabletransforms_windcomponents.yaml | 4 +- ...iabletransforms_windspeedanddirection.yaml | 21 ++++- 12 files changed, 105 insertions(+), 67 deletions(-) diff --git a/src/ufo/filters/VariableTransforms.h b/src/ufo/filters/VariableTransforms.h index 957d0582d..223ee0917 100644 --- a/src/ufo/filters/VariableTransforms.h +++ b/src/ufo/filters/VariableTransforms.h @@ -33,14 +33,14 @@ class VariableTransformsParameters; namespace ufo { -/// \brief Main filter to apply some variable convertion. +/// \brief Main filter to apply some variable conversion. /// -/// See variabletransformsParameters for the documentation of the available +/// See VariableTransformsParameters for the documentation of the available /// parameters and options. /// /// \par Important: /// Any new variable created is assigned to the observation space with the -/// "@DerivedValue" tag. +/// "@DerivedObsValue" tag. /// class VariableTransforms : public FilterBase, private util::ObjectCounter { diff --git a/src/ufo/variabletransforms/Cal_Humidity.cc b/src/ufo/variabletransforms/Cal_Humidity.cc index d83e3e582..e792b9119 100644 --- a/src/ufo/variabletransforms/Cal_Humidity.cc +++ b/src/ufo/variabletransforms/Cal_Humidity.cc @@ -214,12 +214,12 @@ void Cal_RelativeHumidity::methodUKMO() { } } - // assign the derived relative humidity as DerivedValue + // assign the derived relative humidity as DerivedObsValue if (hasBeenUpdated) { if (surfaceData) { - obsdb_.put_db(outputTag, "relative_humidity_surface", relativeHumidity); + putObservation("relative_humidity_surface", relativeHumidity); } else { - obsdb_.put_db(outputTag, "relative_humidity", relativeHumidity); + putObservation("relative_humidity", relativeHumidity); } } } @@ -282,7 +282,7 @@ void Cal_RelativeHumidity::methodDEFAULT() { } } - obsdb_.put_db("DerivedValue", "relative_humidity", relativeHumidity); + putObservation("relative_humidity", relativeHumidity); } /************************************************************************************/ @@ -368,7 +368,7 @@ void Cal_SpecificHumidity::methodDEFAULT() { specificHumidity[jobs] = std::max(1.0e-12f, qv/(1.0f+qv)); } } - obsdb_.put_db("DerivedValue", "specific_humidity", specificHumidity); + putObservation("specific_humidity", specificHumidity); } diff --git a/src/ufo/variabletransforms/Cal_PressureFromHeight.cc b/src/ufo/variabletransforms/Cal_PressureFromHeight.cc index c874ec120..3f7e5cdea 100644 --- a/src/ufo/variabletransforms/Cal_PressureFromHeight.cc +++ b/src/ufo/variabletransforms/Cal_PressureFromHeight.cc @@ -233,8 +233,8 @@ void Cal_PressureFromHeightForProfile::methodUKMO() { if (hasBeenUpdated) { // if updated the airPressure - // assign the derived air pressure as DerivedValue - obsdb_.put_db(outputTag, "air_pressure", airPressure); + // assign the derived air pressure as DerivedObsValue + putObservation("air_pressure", airPressure); } } @@ -326,8 +326,8 @@ void Cal_PressureFromHeightForICAO::methodUKMO() { if (hasBeenUpdated) { // if updated the airPressure - // assign the derived air pressure as DerivedValue - obsdb_.put_db(outputTag, "air_pressure", airPressure); + // assign the derived air pressure as DerivedObsValue + putObservation("air_pressure", airPressure); } } diff --git a/src/ufo/variabletransforms/Cal_Wind.cc b/src/ufo/variabletransforms/Cal_Wind.cc index 1fb8626f7..e6b84795c 100644 --- a/src/ufo/variabletransforms/Cal_Wind.cc +++ b/src/ufo/variabletransforms/Cal_Wind.cc @@ -63,8 +63,8 @@ void Cal_WindSpeedAndDirection::runTransform() { } } // put new variable at existing locations - obsdb_.put_db(outputTag, "wind_speed", windSpeed); - obsdb_.put_db(outputTag, "wind_from_direction", windFromDirection); + putObservation("wind_speed", windSpeed); + putObservation("wind_from_direction", windFromDirection); } /************************************************************************************/ @@ -121,8 +121,8 @@ void Cal_WindComponents::runTransform() { } // put new variable at existing locations - obsdb_.put_db(outputTag, "eastward_wind", u); - obsdb_.put_db(outputTag, "northward_wind", v); + putObservation("eastward_wind", u); + putObservation("northward_wind", v); } } // namespace ufo diff --git a/src/ufo/variabletransforms/TransformBase.h b/src/ufo/variabletransforms/TransformBase.h index 631de0b33..2bcb5b85f 100644 --- a/src/ufo/variabletransforms/TransformBase.h +++ b/src/ufo/variabletransforms/TransformBase.h @@ -58,20 +58,12 @@ class TransformBase { virtual void runTransform() = 0; private: - /// Method used for the calculation - formulas::MethodFormulation method_; - formulas::MethodFormulation formulation_; - bool UseValidDataOnly_; - bool AllowSuperSaturation_; - /// The observation name - std::string obsName_; /// templated function for float, int data types template void filterObservation(const std::string &variableName, std::vector &obsVector) const { if (flags_.has(variableName)) { - constexpr T one = T(1.0); - const T missing = util::missingValue(one); + const T missing = util::missingValue(T()); const std::vector *varFlags = &flags_[variableName]; std::transform(obsVector.begin(), obsVector.end(), // Input range 1 @@ -83,7 +75,59 @@ class TransformBase { } } - protected: // variables + /// Method used for the calculation + formulas::MethodFormulation method_; + formulas::MethodFormulation formulation_; + bool UseValidDataOnly_; + bool AllowSuperSaturation_; + /// The observation name + std::string obsName_; + + protected: + /// templated function for float, int data types + template + void getObservation(const std::string &originalTag, const std::string &varName, + std::vector &obsVector, bool require = false) const { + if (!obsdb_.has(originalTag, varName)) { + if (require) + throw eckit::BadValue("The parameter `" + varName + "@" + originalTag + + "` does not exist in the ObsSpace ", Here()); + else + return; + } + + obsVector.resize(obsdb_.nlocs()); + obsdb_.get_db(originalTag, varName, obsVector); + // Set obsValue to missingValue if flag is equal to QCflags::missing or QCflags::bounds + if (UseValidDataOnly()) filterObservation(varName, obsVector); + } + + /// \brief Save a transformed variable to the `DerivedObsValue` group of the obs space. + /// + /// If the saved variable is a simulated variable, QC flags previously set to `missing` are reset + /// to `pass` at locations where a valid obs value has been assigned. Conversely, QC flags + /// previously set to `pass` are reset to `missing` at locations where the variable is set to a + /// missing value. + /// + /// \param varName Variable name. + /// \param obsVactor Variable values. + template + void putObservation(const std::string &varName, const std::vector &obsVector) { + obsdb_.put_db(outputTag, varName, obsVector); + if (flags_.has(varName)) { + std::vector &varFlags = flags_[varName]; + ASSERT(varFlags.size() == obsVector.size()); + + const T missing = util::missingValue(T()); + for (size_t iloc = 0; iloc < obsVector.size(); ++iloc) { + if (varFlags[iloc] == QCflags::missing && obsVector[iloc] != missing) + varFlags[iloc] = QCflags::pass; + else if (varFlags[iloc] == QCflags::pass && obsVector[iloc] == missing) + varFlags[iloc] = QCflags::missing; + } + } + } + /// subclasses to access Method and formualtion used for the calculation formulas::MethodFormulation method() const { return method_; } formulas::MethodFormulation formulation() const { return formulation_; } @@ -96,37 +140,14 @@ class TransformBase { const VariableTransformsParameters &options_; /// Observation space ioda::ObsSpace &obsdb_; - const ioda::ObsDataVector &flags_; + ioda::ObsDataVector &flags_; const std::vector &apply_; /// Missing value (int) const int missingValueInt = util::missingValue(1); /// Missing value (float) const float missingValueFloat = util::missingValue(1.0f); /// output tag for derived parameters - const std::string outputTag = "DerivedValue"; - /// templated function for float, int data types - template - void getObservation(const std::string &originalTag, const std::string &varName, - std::vector &obsVector, bool require = false) const { - const size_t nlocs = obsdb_.nlocs(); - - if (obsdb_.has("DerivedValue", varName)) { - obsVector = std::vector(nlocs); - obsdb_.get_db("DerivedValue", varName, obsVector); - // Set obsValue to missingValueFloat if flag is equal to QCflags::missing or QCflags::bounds - if (UseValidDataOnly()) filterObservation(varName, obsVector); - } else if (obsdb_.has(originalTag, varName)) { - obsVector = std::vector(nlocs); - obsdb_.get_db(originalTag, varName, obsVector); - // Set obsValue to missingValueFloat if flag is equal to QCflags::missing or QCflags::bounds - if (UseValidDataOnly()) filterObservation(varName, obsVector); - } - - if (require && obsVector.empty()) { - throw eckit::BadValue("The parameter `" + varName + "@" + originalTag + - "` does not exist in the ObsSpace ", Here()); - } - } + const std::string outputTag = "DerivedObsValue"; }; /// \brief Transform factory diff --git a/test/testinput/unit_tests/variabletransforms_SondePressureFromHeight.yaml b/test/testinput/unit_tests/variabletransforms_SondePressureFromHeight.yaml index 9ce3d7dd4..ee3c409ff 100644 --- a/test/testinput/unit_tests/variabletransforms_SondePressureFromHeight.yaml +++ b/test/testinput/unit_tests/variabletransforms_SondePressureFromHeight.yaml @@ -21,7 +21,7 @@ observations: Method: UKMO compareVariables: - test: - name: air_pressure@DerivedValue + name: air_pressure@DerivedObsValue reference: name: air_pressure_reference@ObsValue relTol: 1.0e-5 diff --git a/test/testinput/unit_tests/variabletransforms_SondePressureFromHeightICAO.yaml b/test/testinput/unit_tests/variabletransforms_SondePressureFromHeightICAO.yaml index 73bc1af48..620fb71da 100644 --- a/test/testinput/unit_tests/variabletransforms_SondePressureFromHeightICAO.yaml +++ b/test/testinput/unit_tests/variabletransforms_SondePressureFromHeightICAO.yaml @@ -21,7 +21,7 @@ observations: Method: UKMO compareVariables: - test: - name: air_pressure@DerivedValue + name: air_pressure@DerivedObsValue reference: name: air_pressure_ICAO_reference@ObsValue relTol: 1.0e-5 diff --git a/test/testinput/unit_tests/variabletransforms_rhumidity.yaml b/test/testinput/unit_tests/variabletransforms_rhumidity.yaml index 4747a43ad..4c5eeca7a 100644 --- a/test/testinput/unit_tests/variabletransforms_rhumidity.yaml +++ b/test/testinput/unit_tests/variabletransforms_rhumidity.yaml @@ -15,7 +15,7 @@ observations: - reference: name: relative_humidity_NOAA@TestReference test: - name: relative_humidity@DerivedValue + name: relative_humidity@DerivedObsValue absTol: 1.0e-5 passedBenchmark: 99 # (33 sites with 3 vars listed in simulated variables) @@ -33,7 +33,7 @@ observations: - reference: name: relative_humidity_UKMO@TestReference test: - name: relative_humidity@DerivedValue + name: relative_humidity@DerivedObsValue absTol: 1.0e-5 passedBenchmark: 99 @@ -51,7 +51,7 @@ observations: - reference: name: relative_humidity_Walko@TestReference test: - name: relative_humidity@DerivedValue + name: relative_humidity@DerivedObsValue absTol: 1.0e-5 passedBenchmark: 99 @@ -69,6 +69,6 @@ observations: - reference: name: relative_humidity_Murphy@TestReference test: - name: relative_humidity@DerivedValue + name: relative_humidity@DerivedObsValue absTol: 1.0e-5 passedBenchmark: 99 diff --git a/test/testinput/unit_tests/variabletransforms_rhumidity_part2.yaml b/test/testinput/unit_tests/variabletransforms_rhumidity_part2.yaml index 0a2aa51af..48911c482 100644 --- a/test/testinput/unit_tests/variabletransforms_rhumidity_part2.yaml +++ b/test/testinput/unit_tests/variabletransforms_rhumidity_part2.yaml @@ -20,5 +20,5 @@ observations: - reference: name: relative_humidity_reference@ObsValue test: - name: relative_humidity_surface@DerivedValue + name: relative_humidity_surface@DerivedObsValue relTol: 1.0e-5 diff --git a/test/testinput/unit_tests/variabletransforms_shumidity.yaml b/test/testinput/unit_tests/variabletransforms_shumidity.yaml index bf8a631be..e9a467c85 100644 --- a/test/testinput/unit_tests/variabletransforms_shumidity.yaml +++ b/test/testinput/unit_tests/variabletransforms_shumidity.yaml @@ -15,7 +15,7 @@ observations: - reference: name: specific_humidity_NOAA@TestReference test: - name: specific_humidity@DerivedValue + name: specific_humidity@DerivedObsValue absTol: 1.0e-7 passedBenchmark: 99 @@ -32,7 +32,7 @@ observations: - reference: name: specific_humidity_UKMO@TestReference test: - name: specific_humidity@DerivedValue + name: specific_humidity@DerivedObsValue absTol: 1.0e-7 passedBenchmark: 99 @@ -50,7 +50,7 @@ observations: - reference: name: specific_humidity_UKMO@TestReference test: - name: specific_humidity@DerivedValue + name: specific_humidity@DerivedObsValue absTol: 1.0e-7 passedBenchmark: 99 @@ -67,7 +67,7 @@ observations: - reference: name: specific_humidity_Walko@TestReference test: - name: specific_humidity@DerivedValue + name: specific_humidity@DerivedObsValue absTol: 1.0e-7 passedBenchmark: 99 @@ -84,6 +84,6 @@ observations: - reference: name: specific_humidity_Murphy@TestReference test: - name: specific_humidity@DerivedValue + name: specific_humidity@DerivedObsValue absTol: 1.0e-7 passedBenchmark: 99 diff --git a/test/testinput/unit_tests/variabletransforms_windcomponents.yaml b/test/testinput/unit_tests/variabletransforms_windcomponents.yaml index 80ebfc15a..f36df6c44 100644 --- a/test/testinput/unit_tests/variabletransforms_windcomponents.yaml +++ b/test/testinput/unit_tests/variabletransforms_windcomponents.yaml @@ -18,10 +18,10 @@ observations: - reference: name: eastward_wind@TestReference test: - name: eastward_wind@DerivedValue + name: eastward_wind@DerivedObsValue absTol: 5.0e-5 - reference: name: northward_wind@TestReference test: - name: northward_wind@DerivedValue + name: northward_wind@DerivedObsValue absTol: 5.0e-5 diff --git a/test/testinput/unit_tests/variabletransforms_windspeedanddirection.yaml b/test/testinput/unit_tests/variabletransforms_windspeedanddirection.yaml index 0c3a48a1c..e418d700c 100644 --- a/test/testinput/unit_tests/variabletransforms_windspeedanddirection.yaml +++ b/test/testinput/unit_tests/variabletransforms_windspeedanddirection.yaml @@ -19,10 +19,27 @@ observations: - reference: name: wind_speed@TestReference test: - name: wind_speed@DerivedValue + name: wind_speed@DerivedObsValue absTol: 5.0e-6 - reference: name: wind_from_direction@TestReference test: - name: wind_from_direction@DerivedValue + name: wind_from_direction@DerivedObsValue absTol: 5.0e-5 + +# Check the filter can create derived simulated variables and sets correct QC flags +- obs space: + name: Sonde + # Ioda file we want to apply the filter to + obsdatain: + obsfile: Data/ufo/testinput_tier_1/sondes_obs_2018041500_s.nc4 + simulated variables: [] + derived simulated variables: [wind_speed, wind_from_direction] + obs filters: + - filter: Variable Transforms + Transform: ["WindSpeedAndDirection"] + - filter: Perform Action + action: + name: assign error + error parameter: 1 + failedBenchmark: 50 # there should be 25 missing values in each of the two simulated variables From cf91dfd57d248173bdd0289d4ed15575d6768efd Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Mon, 2 Aug 2021 16:34:07 +0100 Subject: [PATCH 62/98] TL/AD code for profile average observation operator (#1343) * modifications to algorithm * Modifications to algorithm; fill H(x) * refactor to use data handling class * improve documentation * coding norms * more unit tests * more error checking; improved error messages * coding norms * coding norms * add extra tests to trigger exceptions * Trigger CI * clear stringstreams correctly * improve comments; change function name * no longer clear stringstream * allow for different vertical coordinate; test for T * add test with RH * add variables option * coding norms * Trigger CI * correct include path * update doxygen comments * change 'location' to 'level' in a couple of places * correct treatment of operator variable indices * profile average TL/AD code * add expected simulateObsTL exceptions * move GeoVaL caching to data object * Trigger CI * coding norms * rename test Co-authored-by: Ryan Honeyager --- src/ufo/profile/CMakeLists.txt | 2 + src/ufo/profile/ObsProfileAverage.cc | 7 +- src/ufo/profile/ObsProfileAverageData.cc | 12 +- src/ufo/profile/ObsProfileAverageData.h | 13 +- src/ufo/profile/ObsProfileAverageTLAD.cc | 135 ++++++++++++++++++ src/ufo/profile/ObsProfileAverageTLAD.h | 69 +++++++++ test/CMakeLists.txt | 9 +- ...entionalprofileprocessing_opr_average.yaml | 2 + 8 files changed, 239 insertions(+), 10 deletions(-) create mode 100644 src/ufo/profile/ObsProfileAverageTLAD.cc create mode 100644 src/ufo/profile/ObsProfileAverageTLAD.h diff --git a/src/ufo/profile/CMakeLists.txt b/src/ufo/profile/CMakeLists.txt index 7ea79bfa8..0570c7e04 100644 --- a/src/ufo/profile/CMakeLists.txt +++ b/src/ufo/profile/CMakeLists.txt @@ -16,6 +16,8 @@ set ( profile_files ObsProfileAverageParameters.h ObsProfileAverageData.cc ObsProfileAverageData.h + ObsProfileAverageTLAD.cc + ObsProfileAverageTLAD.h ProfileAveragePressure.cc ProfileAveragePressure.h ProfileAverageRelativeHumidity.cc diff --git a/src/ufo/profile/ObsProfileAverage.cc b/src/ufo/profile/ObsProfileAverage.cc index b7aa6eff7..369d270ea 100644 --- a/src/ufo/profile/ObsProfileAverage.cc +++ b/src/ufo/profile/ObsProfileAverage.cc @@ -46,6 +46,9 @@ void ObsProfileAverage::simulateObs(const GeoVaLs & gv, ioda::ObsVector & ovec, ObsDiagnostics & ydiags) const { oops::Log::trace() << "ObsProfileAverage: simulateObs started" << std::endl; + // Cache the GeoVaLs the first time this function is called. + data_.cacheGeoVaLs(gv); + // Get correspondence between record numbers and indices in the total sample. const std::vector &recnums = odb_.recidx_all_recnums(); @@ -67,7 +70,7 @@ void ObsProfileAverage::simulateObs(const GeoVaLs & gv, ioda::ObsVector & ovec, // Retrieve slant path locations. const std::vector& slant_path_location = - data_.getSlantPathLocations(locsOriginal, locsExtended, gv); + data_.getSlantPathLocations(locsOriginal, locsExtended); // Fill H(x) vector for each variable. for (int jvar : data_.operatorVarIndices()) { @@ -75,7 +78,7 @@ void ObsProfileAverage::simulateObs(const GeoVaLs & gv, ioda::ObsVector & ovec, // Number of levels for this variable. const std::size_t nlevs_var = gv.nlevs(variable); // GeoVaL vector for this variable. - std::vector var_gv(nlevs_var); + std::vector var_gv(nlevs_var); // For each level: // - get the relevant slant path location, // - retrieve the GeoVaL at that location, diff --git a/src/ufo/profile/ObsProfileAverageData.cc b/src/ufo/profile/ObsProfileAverageData.cc index 784334c12..7a8a4b711 100644 --- a/src/ufo/profile/ObsProfileAverageData.cc +++ b/src/ufo/profile/ObsProfileAverageData.cc @@ -67,10 +67,14 @@ namespace ufo { return operatorVarIndices_; } + void ObsProfileAverageData::cacheGeoVaLs(const GeoVaLs & gv) const { + // Only perform the caching once. + if (!cachedGeoVaLs_) cachedGeoVaLs_.reset(new GeoVaLs(gv)); + } + std::vector ObsProfileAverageData::getSlantPathLocations (const std::vector & locsOriginal, - const std::vector & locsExtended, - const GeoVaLs & gv) const + const std::vector & locsExtended) const { const float missing = util::missingValue(missing); @@ -80,7 +84,7 @@ namespace ufo { // Set up GeoVaLs and H(x) vectors. // Number of levels for model pressure. - const std::size_t nlevs_p = gv.nlevs(modelVerticalCoord_); + const std::size_t nlevs_p = cachedGeoVaLs_->nlevs(modelVerticalCoord_); // Vector storing location for each level along the slant path. // Initially the first location in the profile is used everywhere. std::vector slant_path_location(nlevs_p, locsOriginal.front()); @@ -101,7 +105,7 @@ namespace ufo { for (std::size_t mlev = 0; mlev < nlevs_p; ++mlev) { for (int iter = 0; iter <= itermax; ++iter) { // Get the GeoVaL that corresponds to the current slanted profile location. - gv.getAtLocation(pressure_gv, modelVerticalCoord_, jlocslant); + cachedGeoVaLs_->getAtLocation(pressure_gv, modelVerticalCoord_, jlocslant); // Define an iteration-specific location that is initialised to the // current slanted profile location. std::size_t jlociter = jlocslant; diff --git a/src/ufo/profile/ObsProfileAverageData.h b/src/ufo/profile/ObsProfileAverageData.h index b38820b8b..a3fa56864 100644 --- a/src/ufo/profile/ObsProfileAverageData.h +++ b/src/ufo/profile/ObsProfileAverageData.h @@ -8,6 +8,7 @@ #ifndef UFO_PROFILE_OBSPROFILEAVERAGEDATA_H_ #define UFO_PROFILE_OBSPROFILEAVERAGEDATA_H_ +#include #include #include #include @@ -40,11 +41,14 @@ namespace ufo { /// Return operator variable indices for the operator. const std::vector & operatorVarIndices() const; + /// Cache the initial values of the GeoVaLs. + void cacheGeoVaLs(const GeoVaLs & gv) const; + /// Get slant path locations. This determines, for each model level, the location that /// corresponds to the intersection of the observed profile with that level. - std::vector getSlantPathLocations(const std::vector & locsOriginal, - const std::vector & locsExtended, - const GeoVaLs & gv) const; + std::vector getSlantPathLocations + (const std::vector & locsOriginal, + const std::vector & locsExtended) const; /// Print operator configuration options. void print(std::ostream & os) const; @@ -80,6 +84,9 @@ namespace ufo { /// Indices of operator variables. std::vector operatorVarIndices_; + /// Cached GeoVaLs. + mutable std::unique_ptr cachedGeoVaLs_; + /// Reference values of slant path locations. std::vector slant_path_location_ref_; diff --git a/src/ufo/profile/ObsProfileAverageTLAD.cc b/src/ufo/profile/ObsProfileAverageTLAD.cc new file mode 100644 index 000000000..07722069e --- /dev/null +++ b/src/ufo/profile/ObsProfileAverageTLAD.cc @@ -0,0 +1,135 @@ +/* + * (C) Copyright 2021 UK Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/profile/ObsProfileAverageTLAD.h" + +#include +#include +#include + +#include "ioda/ObsVector.h" + +#include "oops/base/Variables.h" +#include "oops/util/Logger.h" +#include "oops/util/missingValues.h" + +#include "ufo/GeoVaLs.h" +#include "ufo/ObsDiagnostics.h" + +namespace ufo { + +// ----------------------------------------------------------------------------- +static LinearObsOperatorMaker obsProfileAverageMaker_("ProfileAverage"); +// ----------------------------------------------------------------------------- + +ObsProfileAverageTLAD::ObsProfileAverageTLAD(const ioda::ObsSpace & odb, + const eckit::Configuration & config) + : LinearObsOperatorBase(odb), odb_(odb), data_(odb, config) +{ + oops::Log::trace() << "ObsProfileAverageTLAD constructed" << std::endl; +} + +// ----------------------------------------------------------------------------- + +ObsProfileAverageTLAD::~ObsProfileAverageTLAD() { + oops::Log::trace() << "ObsProfileAverageTLAD destructed" << std::endl; +} + +// ----------------------------------------------------------------------------- + +void ObsProfileAverageTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics &) { + // The initial trajectory is used to determine the slant path locations. + data_.cacheGeoVaLs(geovals); + oops::Log::trace() << "ObsProfileAverageTLAD: trajectory set" << std::endl; +} + +// ----------------------------------------------------------------------------- + +void ObsProfileAverageTLAD::simulateObsTL(const GeoVaLs & dx, ioda::ObsVector & dy) const { + oops::Log::trace() << "ObsProfileAverageTLAD: simulateObsTL started" << std::endl; + + // Get correspondence between record numbers and indices in the total sample. + const std::vector &recnums = odb_.recidx_all_recnums(); + + // Number of profiles in the original ObsSpace. + const std::size_t nprofs = recnums.size() / 2; + + // Loop over profiles. + for (std::size_t jprof = 0; jprof < nprofs; ++jprof) { + const std::vector &locsOriginal = odb_.recidx_vector(recnums[jprof]); + const std::vector &locsExtended = odb_.recidx_vector(recnums[jprof + nprofs]); + + // Retrieve slant path locations. + const std::vector& slant_path_location = + data_.getSlantPathLocations(locsOriginal, locsExtended); + + for (int jvar : data_.operatorVarIndices()) { + const auto& variable = dy.varnames().variables()[jvar]; + const std::size_t nlevs_var = dx.nlevs(variable); + std::vector var_gv(nlevs_var); + for (std::size_t mlev = 0; mlev < nlevs_var; ++mlev) { + const std::size_t jloc = slant_path_location[mlev]; + dx.getAtLocation(var_gv, variable, jloc); + dy[locsExtended[mlev] * dy.nvars() + jvar] = var_gv[mlev]; + } + } + } + + oops::Log::trace() << "ObsProfileAverageTLAD: simulateObsTL finished" << std::endl; +} + +// ----------------------------------------------------------------------------- + +void ObsProfileAverageTLAD::simulateObsAD(GeoVaLs & dx, const ioda::ObsVector & dy) const { + oops::Log::trace() << "ObsProfileAverageTLAD: simulateObsAD started" << std::endl; + + const double missing = util::missingValue(missing); + + // Get correspondence between record numbers and indices in the total sample. + const std::vector &recnums = odb_.recidx_all_recnums(); + + // Number of profiles in the original ObsSpace. + const std::size_t nprofs = recnums.size() / 2; + + // Loop over profiles. + for (std::size_t jprof = 0; jprof < nprofs; ++jprof) { + const std::vector &locsOriginal = odb_.recidx_vector(recnums[jprof]); + const std::vector &locsExtended = odb_.recidx_vector(recnums[jprof + nprofs]); + + // Retrieve slant path locations. + const std::vector& slant_path_location = + data_.getSlantPathLocations(locsOriginal, locsExtended); + + for (int jvar : data_.operatorVarIndices()) { + const auto& variable = dy.varnames().variables()[jvar]; + const std::size_t nlevs_var = dx.nlevs(variable); + std::vector var_gv(nlevs_var); + for (std::size_t mlev = 0; mlev < nlevs_var; ++mlev) { + const std::size_t jloc = slant_path_location[mlev]; + // Get the current value of dx. + dx.getAtLocation(var_gv, variable, jloc); + const std::size_t idx = locsExtended[mlev] * dy.nvars() + jvar; + if (dy[idx] != missing) + var_gv[mlev] += dy[idx]; + // Store the new value of dx. + dx.putAtLocation(var_gv, variable, jloc); + } + } + } + + oops::Log::trace() << "ObsProfileAverageTLAD: simulateObsAD finished" << std::endl; +} + +// ----------------------------------------------------------------------------- + +void ObsProfileAverageTLAD::print(std::ostream & os) const { + os << "ObsProfileAverageTLAD operator" << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/profile/ObsProfileAverageTLAD.h b/src/ufo/profile/ObsProfileAverageTLAD.h new file mode 100644 index 000000000..ff5305f69 --- /dev/null +++ b/src/ufo/profile/ObsProfileAverageTLAD.h @@ -0,0 +1,69 @@ +/* + * (C) Copyright 2021 UK Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_PROFILE_OBSPROFILEAVERAGETLAD_H_ +#define UFO_PROFILE_OBSPROFILEAVERAGETLAD_H_ + +#include +#include +#include +#include + +#include "oops/base/Variables.h" +#include "oops/util/ObjectCounter.h" + +#include "ufo/LinearObsOperatorBase.h" + +#include "ufo/profile/ObsProfileAverageData.h" +#include "ufo/profile/ObsProfileAverageParameters.h" + +/// Forward declarations +namespace eckit { + class Configuration; +} + +namespace ioda { + class ObsSpace; + class ObsVector; +} + +namespace ufo { + class GeoVaLs; + class ObsDiagnostics; + +/// \brief TL/AD code for the ProfileAverage observation operator. +class ObsProfileAverageTLAD : public LinearObsOperatorBase, + private util::ObjectCounter { + public: + static const std::string classname() {return "ufo::ObsProfileAverageTLAD";} + + ObsProfileAverageTLAD(const ioda::ObsSpace &, const eckit::Configuration &); + virtual ~ObsProfileAverageTLAD(); + + void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; + void simulateObsTL(const GeoVaLs &, ioda::ObsVector &) const override; + void simulateObsAD(GeoVaLs &, const ioda::ObsVector &) const override; + + const oops::Variables & requiredVars() const override { return data_.requiredVars(); } + + oops::Variables simulatedVars() const override { return data_.simulatedVars(); } + + private: + void print(std::ostream &) const override; + + private: + /// ObsSpace. + const ioda::ObsSpace& odb_; + + /// Data handler for the ProfileAverage operator and TL/AD code. + ObsProfileAverageData data_; +}; + +// ----------------------------------------------------------------------------- + +} // namespace ufo +#endif // UFO_PROFILE_OBSPROFILEAVERAGETLAD_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 552d19dcb..d11cf8f15 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1614,7 +1614,14 @@ ecbuild_add_test( TARGET test_ufo_opr_profile_average ARGS "testinput/conventionalprofileprocessing_opr_average.yaml" ENVIRONMENT OOPS_TRAPFPE=1 DEPENDS test_ObsOperator.x - TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) + TEST_DEPENDS ufo_get_ufo_test_data ) + +ecbuild_add_test( TARGET test_ufo_linopr_profile_average + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperatorTLAD.x + ARGS "testinput/conventionalprofileprocessing_opr_average.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsOperatorTLAD.x + TEST_DEPENDS ufo_get_ufo_test_data ) ecbuild_add_test( TARGET test_ufo_opr_identity COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x diff --git a/test/testinput/conventionalprofileprocessing_opr_average.yaml b/test/testinput/conventionalprofileprocessing_opr_average.yaml index 936b2886c..841403cfb 100644 --- a/test/testinput/conventionalprofileprocessing_opr_average.yaml +++ b/test/testinput/conventionalprofileprocessing_opr_average.yaml @@ -254,6 +254,7 @@ observations: number of intersection iterations: 3 compare with OPS: true expect simulateObs to throw exception with message: "Mismatch for slant_path_location, level = 0 (this code, OPS): 2659, 0" + expect simulateObsTL to throw exception with message: "Mismatch for slant_path_location, level = 0 (this code, OPS): 2659, 0" geovals: filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 linear obs operator test: @@ -281,6 +282,7 @@ observations: number of intersection iterations: 3 compare with OPS: true expect simulateObs to throw exception with message: "Mismatch for slant_pressure, level = 0 (this code, OPS): 100958, 0" + expect simulateObsTL to throw exception with message: "Mismatch for slant_pressure, level = 0 (this code, OPS): 100958, 0" geovals: filename: Data/ufo/testinput_tier_1/met_office_profile_cxinterpolation_geovals.nc4 linear obs operator test: From 59d4a115ca262a6d4f3f405f611d5cb4599b74e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Mon, 2 Aug 2021 17:30:59 +0100 Subject: [PATCH 63/98] Port the sorting algorithm used in OPS to JEDI. (#1368) Co-authored-by: Ryan Honeyager --- src/ufo/utils/CMakeLists.txt | 1 + src/ufo/utils/metoffice/MetOfficeSort.h | 42 +++++++++++++ test/CMakeLists.txt | 8 +++ test/mains/TestMetOfficeSort.cc | 15 +++++ .../unit_tests/utils_metoffice_sort.yaml | 30 +++++++++ test/ufo/MetOfficeSort.h | 61 +++++++++++++++++++ 6 files changed, 157 insertions(+) create mode 100644 src/ufo/utils/metoffice/MetOfficeSort.h create mode 100644 test/mains/TestMetOfficeSort.cc create mode 100644 test/testinput/unit_tests/utils_metoffice_sort.yaml create mode 100644 test/ufo/MetOfficeSort.h diff --git a/src/ufo/utils/CMakeLists.txt b/src/ufo/utils/CMakeLists.txt index 6980e7b13..fc664951a 100644 --- a/src/ufo/utils/CMakeLists.txt +++ b/src/ufo/utils/CMakeLists.txt @@ -30,6 +30,7 @@ set ( utils_files metoffice/MetOfficeRMatrixRadiance.h metoffice/MetOfficeRMatrixRadiance.interface.h metoffice/MetOfficeRMatrixRadiance.interface.F90 + metoffice/MetOfficeSort.h metoffice/MetOfficeObservationIDs.h metoffice/ufo_metoffice_bmatrixstatic_mod.f90 metoffice/ufo_metoffice_rmatrixradiance_mod.f90 diff --git a/src/ufo/utils/metoffice/MetOfficeSort.h b/src/ufo/utils/metoffice/MetOfficeSort.h new file mode 100644 index 000000000..827b26b7a --- /dev/null +++ b/src/ufo/utils/metoffice/MetOfficeSort.h @@ -0,0 +1,42 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_UTILS_METOFFICE_METOFFICESORT_H_ +#define UFO_UTILS_METOFFICE_METOFFICESORT_H_ + +#include + +namespace ufo { + +/// \brief Sort the range `[first, last)` in the order of ascending keys using the same algorithm as +/// the `Ops_Integer/Char/RealSort` subroutines from the Met Office OPS system. +/// +/// \param first, last +/// The range of elements to sort. +/// \param key +/// An unary functor taking an element of the range (a dereferenced iterator `RandomIt`) +/// and returning a key used as the sorting criterion. +template +void metOfficeSort(RandomIt first, RandomIt last, const UnaryOperation &key) { + auto compare = [&key] (const auto &a, const auto &b) { return key(a) <= key(b); }; + std::make_heap(first, last, compare); + std::sort_heap(first, last, compare); +} + +/// \brief Sort the range `[first, last)` in ascending order using the same algorithm as +/// the `Ops_Integer/Char/RealSort` subroutines from the Met Office OPS system. +/// +/// \param first, last +/// The range of elements to sort. +template +void metOfficeSort(RandomIt first, RandomIt last) { + metOfficeSort(first, last, [](const auto &x) { return x; }); +} + +} // namespace ufo + +#endif // UFO_UTILS_METOFFICE_METOFFICESORT_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d11cf8f15..15f8623ae 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -293,6 +293,7 @@ list( APPEND ufo_test_input testinput/unit_tests/function_satwindsLNVDcheck.yaml testinput/unit_tests/function_satwindsSPDBcheck.yaml testinput/unit_tests/function_wdirdiff.yaml + testinput/unit_tests/utils_metoffice_sort.yaml testinput/unit_tests/variabletransforms_rhumidity.yaml testinput/unit_tests/variabletransforms_rhumidity_part2.yaml testinput/unit_tests/variabletransforms_shumidity.yaml @@ -3036,6 +3037,13 @@ ecbuild_add_test( TARGET test_ufo_metoffice_error_matrices ENVIRONMENT OOPS_TRAPFPE=1 LIBS ufo) +# Test the sorting algorithm used for compatibility with the Met Office OPS system +ecbuild_add_test( TARGET test_ufo_metoffice_sort + SOURCES mains/TestMetOfficeSort.cc + ARGS "testinput/unit_tests/utils_metoffice_sort.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + LIBS ufo) + # Test QC for specific instruments if( crtm_FOUND ) diff --git a/test/mains/TestMetOfficeSort.cc b/test/mains/TestMetOfficeSort.cc new file mode 100644 index 000000000..0138f537e --- /dev/null +++ b/test/mains/TestMetOfficeSort.cc @@ -0,0 +1,15 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "../ufo/MetOfficeSort.h" +#include "oops/runs/Run.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + ufo::test::MetOfficeSort tests; + return run.execute(tests); +} diff --git a/test/testinput/unit_tests/utils_metoffice_sort.yaml b/test/testinput/unit_tests/utils_metoffice_sort.yaml new file mode 100644 index 000000000..9c5d0f178 --- /dev/null +++ b/test/testinput/unit_tests/utils_metoffice_sort.yaml @@ -0,0 +1,30 @@ +no key: +# Here the expected output is just the sorted input +- input: [] + output: [] +- input: [1] + output: [1] +- input: [5, -3, 6, 2, 7, 2] + output: [-3, 2, 2, 5, 6, 7] +- input: [0, 7, 1, 4, 1] + output: [0, 1, 1, 4, 7] +with key: +# Here the expected output is effectively argsort(keys) +- keys: [] + output: [] +- keys: [a] + output: [0] +- keys: [b, a, c] + output: [1, 0, 2] +# A sequence with duplicate keys +- keys: [4, 3, 1, 8, 9, 7, 1, 8, 7, 3, 0, 5, 6, 8, 4, 6, 0, 9, 5, 0, 4, 9, 2, 0, 0, & + 4, 7, 8, 6, 2, 5, 9, 0, 9, 7, 6, 7, 8, 0, 1, 0, 7, 0, 2, 9, 1, 9, 5, 6, 9, & + 4, 3, 9, 0, 0, 7, 2, 0, 2, 7, 0, 4, 4, 7, 2, 2, 8, 4, 3, 3, 7, 5, 9, 3, 6, & + 2, 5, 6, 0, 7, 8, 9, 6, 2, 3, 2, 9, 0, 4, 9, 7, 9, 0, 0, 3, 7, 4, 9, 6, 1] +# The reference output was generated using Ops_IntegerSort. + output: [93, 40, 78, 24, 57, 54, 10, 19, 32, 42, 38, 87, 60, 53, 23, 92, 16, 39, + 6, 99, 2, 45, 58, 56, 43, 29, 83, 22, 64, 75, 65, 85, 51, 9, 94, 73, + 84, 69, 68, 1, 61, 14, 25, 50, 88, 62, 20, 96, 0, 67, 30, 11, 47, 18, + 76, 71, 28, 12, 98, 48, 82, 77, 74, 35, 15, 59, 55, 26, 5, 95, 90, 41, + 79, 36, 8, 70, 34, 63, 27, 13, 80, 37, 3, 66, 7, 52, 49, 97, 46, 91, + 89, 44, 21, 86, 81, 4, 72, 17, 33, 31] diff --git a/test/ufo/MetOfficeSort.h b/test/ufo/MetOfficeSort.h new file mode 100644 index 000000000..e4e817f25 --- /dev/null +++ b/test/ufo/MetOfficeSort.h @@ -0,0 +1,61 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef TEST_UFO_METOFFICESORT_H_ +#define TEST_UFO_METOFFICESORT_H_ + +#include +#include +#include +#include + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/testing/Test.h" +#include "oops/runs/Test.h" +#include "oops/util/Expect.h" +#include "oops/util/Logger.h" +#include "test/TestEnvironment.h" +#include "ufo/utils/metoffice/MetOfficeSort.h" + +namespace ufo { +namespace test { + +CASE("ufo/MetOfficeSort/noKey") { + const eckit::Configuration &topLevelConf = ::test::TestEnvironment::config(); + for (const eckit::LocalConfiguration &conf : topLevelConf.getSubConfigurations("no key")) { + std::vector input = conf.getIntVector("input"); + const std::vector expectedOutput = conf.getIntVector("output"); + metOfficeSort(input.begin(), input.end()); + EXPECT_EQUAL(input, expectedOutput); + } +} + +CASE("ufo/MetOfficeSort/withKey") { + const eckit::Configuration &topLevelConf = ::test::TestEnvironment::config(); + for (const eckit::LocalConfiguration &conf : topLevelConf.getSubConfigurations("with key")) { + const std::vector keys = conf.getStringVector("keys"); + std::vector index(keys.size()); + std::iota(index.begin(), index.end(), 0); + const std::vector expectedOutput = conf.getUnsignedVector("output"); + metOfficeSort(index.begin(), index.end(), [&keys] (size_t i) { return keys[i]; } ); + EXPECT_EQUAL(index, expectedOutput); + } +} + +class MetOfficeSort : public oops::Test { + private: + std::string testid() const override {return "ufo::test::MetOfficeSort";} + + void register_tests() const override {} + + void clear() const override {} +}; + +} // namespace test +} // namespace ufo + +#endif // TEST_UFO_METOFFICESORT_H_ From 938b6483dcfae6c9b85dce5624d8fb9145fa79a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Mon, 2 Aug 2021 20:21:10 +0100 Subject: [PATCH 64/98] OPS compatibility mode for the Gaussian Thinning filter (#1369) * Port the sorting algorithm used in OPS to JEDI. * RecursiveSplitter: Allow using the same sorting algorithm as in OPS. * Introduce an "OPS compatibility mode" to Gaussian Thinning. In this mode the filter is expected to retain the same observations as the Ops_Thinning subroutine from the Met Office OPS system when run serially, even if there are multiple equally good candidates for retaining. * Fix copyright year. * Detect incompatible options. * Fix observation type description (GNSSRO -> GNSS). * Trigger re-run of the test suite Co-authored-by: Ryan Honeyager --- src/ufo/filters/GaussianThinningParameters.cc | 19 + src/ufo/filters/GaussianThinningParameters.h | 43 +- src/ufo/filters/Gaussian_Thinning.cc | 146 +- src/ufo/filters/Gaussian_Thinning.h | 11 +- src/ufo/filters/MetOfficeBuddyPairFinder.cc | 15 +- src/ufo/filters/ObsAccessor.cc | 4 +- src/ufo/filters/ObsAccessor.h | 7 +- src/ufo/filters/TemporalThinning.cc | 4 +- src/ufo/filters/TrackCheckUtils.cc | 4 +- src/ufo/utils/CMakeLists.txt | 4 +- src/ufo/utils/EquispacedBinSelectorBase.h | 46 + src/ufo/utils/RecursiveSplitter.cc | 17 +- src/ufo/utils/RecursiveSplitter.h | 27 +- src/ufo/utils/RoundingEquispacedBinSelector.h | 70 + src/ufo/utils/SpatialBinSelector.cc | 43 +- src/ufo/utils/SpatialBinSelector.h | 29 +- ...or.h => TruncatingEquispacedBinSelector.h} | 33 +- src/ufo/utils/dataextractor/DataExtractor.cc | 5 +- test/CMakeLists.txt | 8 + test/testinput/qc_gauss_thinning_ops.yaml | 4492 +++++++++++++++++ .../qc_gauss_thinning_unittests.yaml | 38 + test/ufo/GaussianThinning.h | 9 +- test/ufo/RecursiveSplitter.h | 5 +- 23 files changed, 4943 insertions(+), 136 deletions(-) create mode 100644 src/ufo/utils/EquispacedBinSelectorBase.h create mode 100644 src/ufo/utils/RoundingEquispacedBinSelector.h rename src/ufo/utils/{EquispacedBinSelector.h => TruncatingEquispacedBinSelector.h} (60%) create mode 100644 test/testinput/qc_gauss_thinning_ops.yaml diff --git a/src/ufo/filters/GaussianThinningParameters.cc b/src/ufo/filters/GaussianThinningParameters.cc index 7b7e32a5d..4a358870c 100644 --- a/src/ufo/filters/GaussianThinningParameters.cc +++ b/src/ufo/filters/GaussianThinningParameters.cc @@ -12,4 +12,23 @@ namespace ufo { constexpr char DistanceNormParameterTraitsHelper::enumTypeName[]; constexpr util::NamedEnumerator DistanceNormParameterTraitsHelper::namedValues[]; + +void GaussianThinningParameters::deserialize(util::CompositePath &path, + const eckit::Configuration &config) { + oops::Parameters::deserialize(path, config); + + if (opsCompatibilityMode) { + if (roundHorizontalBinCountToNearest.value() != boost::none && + *roundHorizontalBinCountToNearest.value() == false) + throw eckit::UserError( + path.path() + ": round_horizontal_bin_count_to_nearest must not be set to false when " + "ops_compatibility_mode is set to true", Here()); + if (distanceNorm.value() != boost::none && + *distanceNorm.value() == DistanceNorm::GEODESIC) + throw eckit::UserError( + path.path() + ": distance_norm must not be set to 'geodesic' when " + "ops_compatibility_mode is set to true", Here()); + } +} + } // namespace ufo diff --git a/src/ufo/filters/GaussianThinningParameters.h b/src/ufo/filters/GaussianThinningParameters.h index c423a42f7..6b8e2c685 100644 --- a/src/ufo/filters/GaussianThinningParameters.h +++ b/src/ufo/filters/GaussianThinningParameters.h @@ -56,6 +56,9 @@ class GaussianThinningParameters : public FilterParametersBase { OOPS_CONCRETE_PARAMETERS(GaussianThinningParameters, FilterParametersBase) public: + /// Reimplemented to detect incompatible options. + void deserialize(util::CompositePath &path, const eckit::Configuration &config) override; + // Horizontal grid /// Cell size (in km) along the meridians. Thinning in the horizontal direction is disabled if @@ -74,8 +77,11 @@ class GaussianThinningParameters : public FilterParametersBase { /// False to set the number of zonal bands so that the band width is as small as possible, but /// no smaller than \c horizontal_mesh, and the bin width in the zonal direction is as small as /// possible, but no smaller than in the meridional direction. - oops::Parameter roundHorizontalBinCountToNearest{ - "round_horizontal_bin_count_to_nearest", false, this}; + /// + /// Defaults to \c false unless the \c ops_compatibility_mode option is enabled, in which case + /// it's set to \c true. + oops::OptionalParameter roundHorizontalBinCountToNearest{ + "round_horizontal_bin_count_to_nearest", this}; // Vertical grid @@ -124,13 +130,44 @@ class GaussianThinningParameters : public FilterParametersBase { oops::OptionalParameter priorityVariable{"priority_variable", this}; /// Determines which of the highest-priority observations lying in a cell is retained. + /// /// Allowed values: + /// /// - \c geodesic: retain the observation closest to the cell centre in the horizontal direction /// (air pressure and time are ignored) /// - \c maximum: retain the observation lying furthest from the cell's bounding box in the /// system of coordinates in which the cell is a unit cube (all dimensions along which thinning /// is enabled are taken into account). - oops::Parameter distanceNorm{"distance_norm", DistanceNorm::GEODESIC, this}; + /// + /// Defaults to \c geodesic unless the \c ops_compatibility_mode option is enabled, in which case + /// it's set to \c maximum. + oops::OptionalParameter distanceNorm{"distance_norm", this}; + + /// Set this option to \c true to make the filter produce identical results as the Ops_Thinning + /// subroutine from the Met Office OPS system when both are run serially (on a single process). + /// + /// The filter behavior is modified in several ways: + /// + /// - The \c round_horizontal_bin_count_to_nearest option is set to \c true. + /// + /// - The \c distance_norm option is set to \c maximum. + /// + /// - Bin indices are calculated by rounding values away from rather towards zero. This can alter + /// the bin indices assigned to observations lying at bin boundaries. + /// + /// - The bin lattice is assumed to cover the whole real axis (for times and pressures) or the + /// [-360, 720] degrees interval (for longitudes) rather than just the intervals [\c time_min, + /// \c time_max], [\c pressure_min, \c pressure_max] and [0, 360] degrees, respectively. This + /// may cause observations lying at the boundaries of the latter intervals to be put in bins of + /// their own, which is normally undesirable. + /// + /// - A different (non-stable) sorting algorithm is used to order observations before inspection. + /// This can alter the set of retained observations if some bins contain multiple equally good + /// observations (with the same priority and distance to the cell center measured with the + /// selected norm). If this happens for a significant fraction of bins, it may be a sign the + /// criteria used to rank observations (the priority and the distance norm) are not specific + /// enough. + oops::Parameter opsCompatibilityMode{"ops_compatibility_mode", false, this}; private: static float defaultHorizontalMesh() { diff --git a/src/ufo/filters/Gaussian_Thinning.cc b/src/ufo/filters/Gaussian_Thinning.cc index b1c6bcbc9..7073e858f 100644 --- a/src/ufo/filters/Gaussian_Thinning.cc +++ b/src/ufo/filters/Gaussian_Thinning.cc @@ -24,11 +24,14 @@ #include "ufo/filters/GaussianThinningParameters.h" #include "ufo/filters/ObsAccessor.h" #include "ufo/utils/Constants.h" -#include "ufo/utils/EquispacedBinSelector.h" +#include "ufo/utils/EquispacedBinSelectorBase.h" #include "ufo/utils/GeodesicDistanceCalculator.h" #include "ufo/utils/MaxNormDistanceCalculator.h" +#include "ufo/utils/metoffice/MetOfficeSort.h" #include "ufo/utils/RecursiveSplitter.h" +#include "ufo/utils/RoundingEquispacedBinSelector.h" #include "ufo/utils/SpatialBinSelector.h" +#include "ufo/utils/TruncatingEquispacedBinSelector.h" namespace ufo { @@ -50,12 +53,19 @@ void Gaussian_Thinning::applyFilter(const std::vector & apply, std::vector> & flagged) const { ObsAccessor obsAccessor = createObsAccessor(); - const std::vector validObsIds = obsAccessor.getValidObservationIds(apply, *flags_); + std::vector validObsIds = obsAccessor.getValidObservationIds(apply, *flags_); + + if (options_.opsCompatibilityMode) { + // Sort observations by latitude + const std::vector lat = obsAccessor.getFloatVariableFromObsSpace("MetaData", "latitude"); + metOfficeSort(validObsIds.begin(), validObsIds.end(), [&lat] (size_t id) { return lat[id]; }); + } std::vector distancesToBinCenter(validObsIds.size(), 0.f); std::unique_ptr distanceCalculator = makeDistanceCalculator(options_); - RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups(validObsIds); + RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups( + validObsIds, options_.opsCompatibilityMode); groupObservationsByPressure(validObsIds, *distanceCalculator, obsAccessor, splitter, distancesToBinCenter); groupObservationsByTime(validObsIds, *distanceCalculator, obsAccessor, @@ -87,7 +97,10 @@ ObsAccessor Gaussian_Thinning::createObsAccessor() const { std::unique_ptr Gaussian_Thinning::makeDistanceCalculator( const GaussianThinningParameters &options) { - switch (options.distanceNorm.value()) { + DistanceNorm distanceNorm = options.distanceNorm.value().value_or(DistanceNorm::GEODESIC); + if (options.opsCompatibilityMode) + distanceNorm = DistanceNorm::MAXIMUM; + switch (distanceNorm) { case DistanceNorm::GEODESIC: return std::unique_ptr(new GeodesicDistanceCalculator()); case DistanceNorm::MAXIMUM: @@ -115,15 +128,18 @@ void Gaussian_Thinning::groupObservationsBySpatialLocation( std::vector lat = obsAccessor.getFloatVariableFromObsSpace("MetaData", "latitude"); std::vector lon = obsAccessor.getFloatVariableFromObsSpace("MetaData", "longitude"); - // Longitudes will typically be either in the [-180, 180] degree range or in the [0, 360] - // degree range. The spatial bin selector is written with the latter convention in mind, - // so let's shift any negative longitudes up by 360 degrees. - for (float &longitude : lon) - if (longitude < 0) - longitude += 360; - - std::vector latBins; - std::vector lonBins; + if (!options_.opsCompatibilityMode) { + // Longitudes will typically be either in the [-180, 180] degree range or in the [0, 360] + // degree range. When the OPS compatibility mode is off, the spatial bin selector is constructed + // with the latter convention in mind, so we need to shift any negative longitudes up by 360 + // degrees. + for (float &longitude : lon) + if (longitude < 0) + longitude += 360; + } + + std::vector latBins; + std::vector lonBins; latBins.reserve(validObsIds.size()); lonBins.reserve(validObsIds.size()); for (size_t obsId : validObsIds) { @@ -162,7 +178,11 @@ boost::optional Gaussian_Thinning::makeSpatialBinSelector( oops::Log::debug() << "Gaussian_Thinning: requested horizontal bin size (km) = " << options.horizontalMesh << std::endl; - SpatialBinCountRoundingMode roundingMode = options.roundHorizontalBinCountToNearest ? + bool roundHorizontalBinCountToNearest = + options.roundHorizontalBinCountToNearest.value().value_or(false); + if (options.opsCompatibilityMode) + roundHorizontalBinCountToNearest = true; + SpatialBinCountRoundingMode roundingMode = roundHorizontalBinCountToNearest ? SpatialBinCountRoundingMode::NEAREST : SpatialBinCountRoundingMode::DOWN; const float earthRadius = Constants::mean_earth_rad; // km @@ -172,11 +192,12 @@ boost::optional Gaussian_Thinning::makeSpatialBinSelector( if (options.useReducedHorizontalGrid) { // Use fewer bins at high latitudes - return SpatialBinSelector(numLatBins, roundingMode); + return SpatialBinSelector(numLatBins, roundingMode, options.opsCompatibilityMode); } else { // Use the same number of bins at all latitudes const int equatorToMeridianLengthRatio = 2; - return SpatialBinSelector(numLatBins, equatorToMeridianLengthRatio * numLatBins); + return SpatialBinSelector(numLatBins, equatorToMeridianLengthRatio * numLatBins, + options.opsCompatibilityMode); } } @@ -188,16 +209,17 @@ void Gaussian_Thinning::groupObservationsByPressure( const ObsAccessor &obsAccessor, RecursiveSplitter &splitter, std::vector &distancesToBinCenter) const { - boost::optional binSelector = makePressureBinSelector(options_); - if (binSelector == boost::none) + std::unique_ptr binSelector = makePressureBinSelector(options_); + if (!binSelector) return; - oops::Log::debug() << "Gaussian_Thinning: number of vertical bins = " - << binSelector->numBins() << std::endl; + if (binSelector->numBins() != boost::none) + oops::Log::debug() << "Gaussian_Thinning: number of vertical bins = " + << *binSelector->numBins() << std::endl; std::vector pres = obsAccessor.getFloatVariableFromObsSpace("MetaData", "air_pressure"); - std::vector bins; + std::vector bins; bins.reserve(validObsIds.size()); for (size_t obsId : validObsIds) { @@ -220,22 +242,28 @@ void Gaussian_Thinning::groupObservationsByPressure( // ----------------------------------------------------------------------------- -boost::optional Gaussian_Thinning::makePressureBinSelector( +std::unique_ptr Gaussian_Thinning::makePressureBinSelector( const GaussianThinningParameters &options) { if (options.verticalMesh <= 0) - return boost::none; - - const int numVerticalBins = std::max( - 1, - static_cast(std::ceil((options.verticalMax - options.verticalMin) / - options.verticalMesh))); - // Adjust verticalMax upwards to make the range of pressures - // evenly divisible into bins of width verticalMesh. - const float adjustedVerticalMax = options.verticalMin + numVerticalBins * options.verticalMesh; + return nullptr; - oops::Log::debug() << "Gaussian_Thinning: number of vertical bins = " + if (options.opsCompatibilityMode) { + return std::make_unique( + options.verticalMesh, options.verticalMin + options.verticalMesh / 2); + } else { + const int numVerticalBins = std::max( + 1, + static_cast(std::ceil((options.verticalMax - options.verticalMin) / + options.verticalMesh))); + // Adjust verticalMax upwards to make the range of pressures + // evenly divisible into bins of width verticalMesh. + const float adjustedVerticalMax = options.verticalMin + numVerticalBins * options.verticalMesh; + + oops::Log::debug() << "Gaussian_Thinning: number of vertical bins = " << numVerticalBins << std::endl; - return EquispacedBinSelector(options.verticalMin, adjustedVerticalMax, numVerticalBins); + return std::make_unique( + options.verticalMin, adjustedVerticalMax, numVerticalBins); + } } // ----------------------------------------------------------------------------- @@ -247,17 +275,19 @@ void Gaussian_Thinning::groupObservationsByTime( RecursiveSplitter &splitter, std::vector &distancesToBinCenter) const { util::DateTime timeOffset; - boost::optional binSelector = makeTimeBinSelector(options_, timeOffset); - if (binSelector == boost::none) + std::unique_ptr binSelector = + makeTimeBinSelector(options_, obsdb_.windowStart(), obsdb_.windowEnd(), timeOffset); + if (!binSelector) return; - oops::Log::debug() << "Gaussian_Thinning: number of time bins = " - << binSelector->numBins() << std::endl; + if (binSelector->numBins() != boost::none) + oops::Log::debug() << "Gaussian_Thinning: number of time bins = " + << *binSelector->numBins() << std::endl; std::vector times = obsAccessor.getDateTimeVariableFromObsSpace( "MetaData", "datetime"); - std::vector bins; + std::vector bins; bins.reserve(validObsIds.size()); for (size_t obsId : validObsIds) { @@ -283,16 +313,19 @@ void Gaussian_Thinning::groupObservationsByTime( // ----------------------------------------------------------------------------- -boost::optional Gaussian_Thinning::makeTimeBinSelector( - const GaussianThinningParameters &options, util::DateTime &timeOffset) { +std::unique_ptr Gaussian_Thinning::makeTimeBinSelector( + const GaussianThinningParameters &options, + const util::DateTime &windowStart, + const util::DateTime &windowEnd, + util::DateTime &timeOffset) { if (options.timeMesh.value() == boost::none || options.timeMin.value() == boost::none || options.timeMax.value() == boost::none) - return boost::none; + return nullptr; const util::Duration timeMesh = options.timeMesh.value().get(); if (timeMesh.toSeconds() == 0) - return boost::none; + return nullptr; const util::DateTime timeMin = options.timeMin.value().get(); const util::DateTime timeMax = options.timeMax.value().get(); @@ -301,16 +334,27 @@ boost::optional Gaussian_Thinning::makeTimeBinSelector( << ((timeMax - timeMin).toSeconds()) << std::endl; oops::Log::debug() << "timeMesh.toSeconds() = " << timeMesh.toSeconds() << std::endl; - const int numTimeBins = std::max( - 1, - static_cast(std::ceil((timeMax - timeMin).toSeconds() / - static_cast(timeMesh.toSeconds())))); - - // NOTE: the upper bound of the time interval is effectively adjusted upwards - // to make space for an integral number of bins of specified width. - timeOffset = timeMin; - return EquispacedBinSelector(0.0f, numTimeBins * timeMesh.toSeconds(), numTimeBins); + + if (options.opsCompatibilityMode) { + // Put bin 0 at the center of the assimilation window. + const util::Duration windowLength = windowEnd - windowStart; + const int numFullBinsLeftOfWindowCenter = (windowLength / 2).toSeconds() / timeMesh.toSeconds(); + const util::DateTime bin0Center = timeMin + numFullBinsLeftOfWindowCenter * timeMesh + + timeMesh / 2; + return std::make_unique( + timeMesh.toSeconds(), (bin0Center - timeOffset).toSeconds()); + } else { + // The upper bound of the time interval is effectively adjusted upwards + // to make space for an integral number of bins of specified width. + const int numTimeBins = std::max( + 1, + static_cast(std::ceil((timeMax - timeMin).toSeconds() / + static_cast(timeMesh.toSeconds())))); + + return std::make_unique( + 0.0f, numTimeBins * timeMesh.toSeconds(), numTimeBins); + } } // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/Gaussian_Thinning.h b/src/ufo/filters/Gaussian_Thinning.h index 2bbfcd48e..9f2b3f0ea 100644 --- a/src/ufo/filters/Gaussian_Thinning.h +++ b/src/ufo/filters/Gaussian_Thinning.h @@ -37,7 +37,7 @@ namespace util { namespace ufo { class DistanceCalculator; -class EquispacedBinSelector; +class EquispacedBinSelectorBase; class GaussianThinningParameters; class ObsAccessor; class RecursiveSplitter; @@ -109,11 +109,14 @@ class Gaussian_Thinning : public FilterBase, static boost::optional makeSpatialBinSelector( const GaussianThinningParameters &options); - static boost::optional makePressureBinSelector( + static std::unique_ptr makePressureBinSelector( const GaussianThinningParameters &options); - static boost::optional makeTimeBinSelector( - const GaussianThinningParameters &options, util::DateTime &timeOffset); + static std::unique_ptr makeTimeBinSelector( + const GaussianThinningParameters &options, + const util::DateTime &windowStart, + const util::DateTime &windowEnd, + util::DateTime &timeOffset); static std::unique_ptr makeDistanceCalculator( const GaussianThinningParameters &options); diff --git a/src/ufo/filters/MetOfficeBuddyPairFinder.cc b/src/ufo/filters/MetOfficeBuddyPairFinder.cc index df04b5e39..841ea197a 100644 --- a/src/ufo/filters/MetOfficeBuddyPairFinder.cc +++ b/src/ufo/filters/MetOfficeBuddyPairFinder.cc @@ -72,18 +72,11 @@ void MetOfficeBuddyPairFinder::sortObservations(const std::vector & vali RecursiveSplitter splitter(validObsIds.size()); splitter.groupBy(bandIndices); splitter.sortGroupsBy( - [this, &validObsIds](size_t obsIndexA, size_t obsIndexB) + [this, &validObsIds](size_t obsIndex) { - size_t obsIdA = validObsIds[obsIndexA]; - size_t obsIdB = validObsIds[obsIndexB]; - if (pressures_ != nullptr) - return std::make_tuple(longitudes_[obsIdA], -latitudes_[obsIdA], - (*pressures_)[obsIdA], datetimes_[obsIdA]) < - std::make_tuple(longitudes_[obsIdB], -latitudes_[obsIdB], - (*pressures_)[obsIdB], datetimes_[obsIdB]); - else - return std::make_tuple(longitudes_[obsIdA], -latitudes_[obsIdA], datetimes_[obsIdA]) < - std::make_tuple(longitudes_[obsIdB], -latitudes_[obsIdB], datetimes_[obsIdB]); + size_t obsId = validObsIds[obsIndex]; + return std::make_tuple(longitudes_[obsId], -latitudes_[obsId], + pressures_ ? (*pressures_)[obsId] : 0.0f, datetimes_[obsId]); }); // Fill the validObsIdsInSortOrder and bandLbounds vectors diff --git a/src/ufo/filters/ObsAccessor.cc b/src/ufo/filters/ObsAccessor.cc index 83dedcfb7..0577c96e3 100644 --- a/src/ufo/filters/ObsAccessor.cc +++ b/src/ufo/filters/ObsAccessor.cc @@ -163,8 +163,8 @@ size_t ObsAccessor::totalNumObservations() const { } RecursiveSplitter ObsAccessor::splitObservationsIntoIndependentGroups( - const std::vector &validObsIds) const { - RecursiveSplitter splitter(validObsIds.size()); + const std::vector &validObsIds, bool opsCompatibilityMode) const { + RecursiveSplitter splitter(validObsIds.size(), opsCompatibilityMode); switch (groupBy_) { case GroupBy::NOTHING: // Nothing to do diff --git a/src/ufo/filters/ObsAccessor.h b/src/ufo/filters/ObsAccessor.h index f9e07b1d4..1c3de6d8a 100644 --- a/src/ufo/filters/ObsAccessor.h +++ b/src/ufo/filters/ObsAccessor.h @@ -155,8 +155,13 @@ class ObsAccessor { /// Construct a RecursiveSplitter object whose groups() method will return groups of observations /// that can be processed independently from each other (according to the criterion specified when /// the ObsAccessor was constructed). + /// + /// \param validObsIds + /// Indices of valid observations. + /// \param opsCompatibilityMode + /// Parameter to pass to the RecursiveSplitter's constructor. RecursiveSplitter splitObservationsIntoIndependentGroups( - const std::vector &validObsIds) const; + const std::vector &validObsIds, bool opsCompatibilityMode = false) const; /// \brief Update flags of observations held on the current MPI rank. /// diff --git a/src/ufo/filters/TemporalThinning.cc b/src/ufo/filters/TemporalThinning.cc index 84c0dcd11..84a1701ca 100644 --- a/src/ufo/filters/TemporalThinning.cc +++ b/src/ufo/filters/TemporalThinning.cc @@ -351,8 +351,8 @@ std::vector TemporalThinning::identifyThinnedObservations( std::vector times = obsAccessor.getDateTimeVariableFromObsSpace( "MetaData", "datetime"); - splitter.sortGroupsBy([×, &validObsIds](size_t obsIndexA, size_t obsIndexB) - { return times[validObsIds[obsIndexA]] < times[validObsIds[obsIndexB]]; }); + splitter.sortGroupsBy([×, &validObsIds](size_t obsIndex) + { return times[validObsIds[obsIndex]]; }); boost::optional> priorities = getObservationPriorities(obsAccessor); diff --git a/src/ufo/filters/TrackCheckUtils.cc b/src/ufo/filters/TrackCheckUtils.cc index 5d985561b..bede377ba 100644 --- a/src/ufo/filters/TrackCheckUtils.cc +++ b/src/ufo/filters/TrackCheckUtils.cc @@ -72,8 +72,8 @@ void TrackCheckUtils::sortTracksChronologically(const std::vector &valid RecursiveSplitter &splitter) { const std::vector times = obsAccessor.getDateTimeVariableFromObsSpace( "MetaData", "datetime"); - splitter.sortGroupsBy([×, &validObsIds](size_t obsIndexA, size_t obsIndexB) - { return times[validObsIds[obsIndexA]] < times[validObsIds[obsIndexB]]; }); + splitter.sortGroupsBy([×, &validObsIds](size_t obsIndex) + { return times[validObsIds[obsIndex]]; }); } TrackCheckUtils::ObsGroupLocationTimes diff --git a/src/ufo/utils/CMakeLists.txt b/src/ufo/utils/CMakeLists.txt index fc664951a..7a768bcf1 100644 --- a/src/ufo/utils/CMakeLists.txt +++ b/src/ufo/utils/CMakeLists.txt @@ -16,7 +16,7 @@ set ( utils_files dataextractor/DataExtractorNetCDFBackend.h dataextractor/DataExtractorNetCDFBackend.cc DistanceCalculator.h - EquispacedBinSelector.h + EquispacedBinSelectorBase.h GeodesicDistanceCalculator.h IodaGroupIndices.cc IodaGroupIndices.h @@ -48,11 +48,13 @@ set ( utils_files RecursiveSplitter.cc RecursiveSplitter.h RefractivityCalculator.F90 + RoundingEquispacedBinSelector.h SpatialBinSelector.h SpatialBinSelector.cc StringUtils.cc StringUtils.h SurfaceReportConstants.h + TruncatingEquispacedBinSelector.h ufo_utils_mod.F90 ufo_utils.interface.F90 ufo_utils.interface.h diff --git a/src/ufo/utils/EquispacedBinSelectorBase.h b/src/ufo/utils/EquispacedBinSelectorBase.h new file mode 100644 index 000000000..b87c9ccb7 --- /dev/null +++ b/src/ufo/utils/EquispacedBinSelectorBase.h @@ -0,0 +1,46 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_UTILS_EQUISPACEDBINSELECTORBASE_H_ +#define UFO_UTILS_EQUISPACEDBINSELECTORBASE_H_ + +#include + +namespace ufo +{ + +/// \brief A finite or infinite collection of non-overlapping intervals (_bins_) of the same width. +/// +/// Call the bin() function to find the bin containing a particular value. +class EquispacedBinSelectorBase { + public: + // If necessary, these could be made template parameters. + typedef float ValueType; + typedef int IndexType; + + virtual ~EquispacedBinSelectorBase() {} + + /// \brief Return the index of the bin containing \p value, or the nearest bin + /// if \p value lies outside all bins. + virtual IndexType bin(ValueType value) const = 0; + + /// \brief Return the number of bins or boost::none if the bin collection is infinite. + virtual boost::optional numBins() const = 0; + + /// \brief Return the width of each bin. + virtual ValueType binWidth() const = 0; + + /// \brief Return the inverse of the width of each bin. + virtual ValueType inverseBinWidth() const = 0; + + /// \brief Return the value lying at the center of the bin with index \p bin. + virtual ValueType binCenter(IndexType bin) const = 0; +}; + +} // namespace ufo + +#endif // UFO_UTILS_EQUISPACEDBINSELECTORBASE_H_ diff --git a/src/ufo/utils/RecursiveSplitter.cc b/src/ufo/utils/RecursiveSplitter.cc index a364ef910..417482dbc 100644 --- a/src/ufo/utils/RecursiveSplitter.cc +++ b/src/ufo/utils/RecursiveSplitter.cc @@ -15,7 +15,8 @@ namespace ufo { -RecursiveSplitter::RecursiveSplitter(size_t numIds) { +RecursiveSplitter::RecursiveSplitter(size_t numIds, bool opsCompatibilityMode) : + opsCompatibilityMode_(opsCompatibilityMode) { orderedIds_.resize(numIds); std::iota(orderedIds_.begin(), orderedIds_.end(), 0); initializeEncodedGroups(); @@ -46,10 +47,16 @@ void RecursiveSplitter::groupByImpl(const std::vector &categories) { break; lastIndexInGroup = encodedGroups_[firstIndexInGroup + 1]; - std::stable_sort(orderedIds_.begin() + firstIndexInGroup, - orderedIds_.begin() + lastIndexInGroup + 1, - [&categories](size_t idA, size_t idB) - { return categories[idA] < categories[idB];}); + if (opsCompatibilityMode_) { + metOfficeSort(orderedIds_.begin() + firstIndexInGroup, + orderedIds_.begin() + lastIndexInGroup + 1, + [&categories](size_t id) { return categories[id]; }); + } else { + std::stable_sort(orderedIds_.begin() + firstIndexInGroup, + orderedIds_.begin() + lastIndexInGroup + 1, + [&categories](size_t idA, size_t idB) + { return categories[idA] < categories[idB]; }); + } // Now update the group size_t newFirstIndex = firstIndexInGroup; diff --git a/src/ufo/utils/RecursiveSplitter.h b/src/ufo/utils/RecursiveSplitter.h index 1b18a6431..d97b853e9 100644 --- a/src/ufo/utils/RecursiveSplitter.h +++ b/src/ufo/utils/RecursiveSplitter.h @@ -15,6 +15,7 @@ #include #include "ufo/utils/ArrowProxy.h" +#include "ufo/utils/metoffice/MetOfficeSort.h" namespace ufo { @@ -260,7 +261,12 @@ class RecursiveSplitter /// \brief Initialize partitioning of an array of \p numIds elements. /// /// Initially, all elements are assumed to belong to the same equivalence class. - explicit RecursiveSplitter(size_t numIds); + /// + /// By default, all sorting operations performed by this class are done using a stable sorting + /// algorithm, but if \p opsCompatibilityMode is true, the same algorithm as in the Met Office + /// OPS system (heap sort) is used instead. This has an effect on the order of elements in each + /// equivalence class. + explicit RecursiveSplitter(size_t numIds, bool opsCompatibilityMode = false); /// \brief Split existing equivalence classes according to a new criterion. /// @@ -294,10 +300,10 @@ class RecursiveSplitter /// \brief Sort the elements in each equivalence class in ascending order. /// - /// The elements are compared using the binary comparison function \p comp. This function needs - /// to satisfy the same requirements as the \c comp argument of std::sort(). - template - void sortGroupsBy(Compare comp); + /// The elements are ranked by keys produced by the unary function \p key taking the index + /// of an element of the partitioned array. + template + void sortGroupsBy(const UnaryOperation &key); /// \brief Randomly shuffle the elements of each equivalence class. void shuffleGroups(); @@ -318,20 +324,25 @@ class RecursiveSplitter template void groupByImpl(const std::vector &categories); + bool opsCompatibilityMode_; /// Indices of elements of the partitioned array ordered by equivalence class. std::vector orderedIds_; /// Encoded locations of multi-element equivalence classes in orderedIds_. std::vector encodedGroups_; }; -template -void RecursiveSplitter::sortGroupsBy(Compare comp) { +template +void RecursiveSplitter::sortGroupsBy(const UnaryOperation &key) { for (Group group : multiElementGroups()) { std::vector::iterator nonConstGroupBegin = orderedIds_.begin() + (group.begin() - orderedIds_.cbegin()); std::vector::iterator nonConstGroupEnd = orderedIds_.begin() + (group.end() - orderedIds_.cbegin()); - std::sort(nonConstGroupBegin, nonConstGroupEnd, comp); + if (opsCompatibilityMode_) + metOfficeSort(nonConstGroupBegin, nonConstGroupEnd, key); + else + std::stable_sort(nonConstGroupBegin, nonConstGroupEnd, + [&key] (size_t a, size_t b) { return key(a) < key(b); }); } } diff --git a/src/ufo/utils/RoundingEquispacedBinSelector.h b/src/ufo/utils/RoundingEquispacedBinSelector.h new file mode 100644 index 000000000..6774b856a --- /dev/null +++ b/src/ufo/utils/RoundingEquispacedBinSelector.h @@ -0,0 +1,70 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_UTILS_ROUNDINGEQUISPACEDBINSELECTOR_H_ +#define UFO_UTILS_ROUNDINGEQUISPACEDBINSELECTOR_H_ + +#include +#include + +#include "eckit/exception/Exceptions.h" +#include "ufo/utils/EquispacedBinSelectorBase.h" + +namespace ufo +{ + +/// \brief Represents an infinite set of consecutive intervals (_bins_) of the same width. +/// +/// Bin 0 is open; bins 1, 2 etc. are right-open; bins -1, -2 etc. are left-open. +/// +/// Call the bin() function to find the bin containing a particular value. +class RoundingEquispacedBinSelector : public EquispacedBinSelectorBase { + public: + // If necessary, these could be made template parameters. + typedef float ValueType; + typedef int IndexType; + + /// \brief Partition the real axis into bins of width \p binWidth, with the center of bin 0 + /// located at \p bin0Center. + explicit RoundingEquispacedBinSelector(ValueType binWidth, ValueType bin0Center = 0) + : binWidth_(binWidth), + inverseBinWidth_(1 / binWidth_), + bin0Center_(bin0Center) + { + ASSERT_MSG(binWidth > 0, "Bin width must be positive"); + } + + IndexType bin(ValueType value) const override { + IndexType binIndex = std::lround((value - bin0Center_) * inverseBinWidth_); + return binIndex; + } + + boost::optional numBins() const override { + return boost::none; + } + + ValueType binWidth() const override { + return binWidth_; + } + + ValueType inverseBinWidth() const override { + return inverseBinWidth_; + } + + ValueType binCenter(IndexType bin) const override { + return bin0Center_ + binWidth_ * bin; + } + + private: + ValueType binWidth_; + ValueType inverseBinWidth_; + ValueType bin0Center_; +}; + +} // namespace ufo + +#endif // UFO_UTILS_ROUNDINGEQUISPACEDBINSELECTOR_H_ diff --git a/src/ufo/utils/SpatialBinSelector.cc b/src/ufo/utils/SpatialBinSelector.cc index 5a244a29a..41009e362 100644 --- a/src/ufo/utils/SpatialBinSelector.cc +++ b/src/ufo/utils/SpatialBinSelector.cc @@ -10,14 +10,16 @@ #include #include "ufo/utils/Constants.h" -#include "ufo/utils/EquispacedBinSelector.h" #include "ufo/utils/SpatialBinSelector.h" +#include "ufo/utils/TruncatingEquispacedBinSelector.h" namespace ufo { SpatialBinSelector::SpatialBinSelector(IndexType numLatitudeBins, - SpatialBinCountRoundingMode roundingMode) - : latitudeBinSelector_(latitudeLowerBound_, latitudeUpperBound_, numLatitudeBins) { + SpatialBinCountRoundingMode roundingMode, + bool metOfficeOpsCompatibilityMode) + : metOfficeOpsCompatibilityMode_(metOfficeOpsCompatibilityMode), + latitudeBinSelector_(latitudeLowerBound_, latitudeUpperBound_, numLatitudeBins) { longitudeBinSelectors_.reserve(numLatitudeBins); for (IndexType latBin = 0; latBin < numLatitudeBins; ++latBin) { ValueType latBinCenter = latitudeBinCenter(latBin); @@ -28,23 +30,40 @@ SpatialBinSelector::SpatialBinSelector(IndexType numLatitudeBins, std::cos(latBinCenter * static_cast(Constants::deg2rad)); const IndexType numLonBins = roundNumBins(tentativeNumLongitudeBins, roundingMode); - longitudeBinSelectors_.emplace_back( - static_cast(longitudeLowerBound_), - static_cast(longitudeUpperBound_), numLonBins); + if (metOfficeOpsCompatibilityMode) + longitudeBinSelectors_.emplace_back( + static_cast(opsCompatibilityModeLongitudeLowerBound_), + static_cast(opsCompatibilityModeLongitudeUpperBound_), + opsCompatibilityModeRelativeLongitudeRange_ * numLonBins); + else + longitudeBinSelectors_.emplace_back( + static_cast(longitudeLowerBound_), + static_cast(longitudeUpperBound_), numLonBins); } } -SpatialBinSelector::SpatialBinSelector(IndexType numLatitudeBins, IndexType numLongitudeBins) - : latitudeBinSelector_(latitudeLowerBound_, latitudeUpperBound_, numLatitudeBins), +SpatialBinSelector::SpatialBinSelector(IndexType numLatitudeBins, IndexType numLongitudeBins, + bool metOfficeOpsCompatibilityMode) + : metOfficeOpsCompatibilityMode_(metOfficeOpsCompatibilityMode), + latitudeBinSelector_(latitudeLowerBound_, latitudeUpperBound_, numLatitudeBins), longitudeBinSelectors_(numLatitudeBins, - EquispacedBinSelector(longitudeLowerBound_, longitudeUpperBound_, - numLongitudeBins)) + metOfficeOpsCompatibilityMode ? + TruncatingEquispacedBinSelector( + opsCompatibilityModeLongitudeLowerBound_, + opsCompatibilityModeLongitudeUpperBound_, + opsCompatibilityModeRelativeLongitudeRange_ * numLongitudeBins) : + TruncatingEquispacedBinSelector( + longitudeLowerBound_, + longitudeUpperBound_, + numLongitudeBins)) {} SpatialBinSelector::IndexType SpatialBinSelector::totalNumBins() const { size_t n = 0; - for (const EquispacedBinSelector & selector : longitudeBinSelectors_) - n += selector.numBins(); + for (const TruncatingEquispacedBinSelector & selector : longitudeBinSelectors_) + n += *selector.numBins(); + if (metOfficeOpsCompatibilityMode_) + n /= opsCompatibilityModeRelativeLongitudeRange_; return n; } diff --git a/src/ufo/utils/SpatialBinSelector.h b/src/ufo/utils/SpatialBinSelector.h index 598bc6149..eb2c3b5f9 100644 --- a/src/ufo/utils/SpatialBinSelector.h +++ b/src/ufo/utils/SpatialBinSelector.h @@ -12,7 +12,7 @@ #include #include "ufo/utils/Constants.h" -#include "ufo/utils/EquispacedBinSelector.h" +#include "ufo/utils/TruncatingEquispacedBinSelector.h" namespace ufo { @@ -33,6 +33,9 @@ class SpatialBinSelector { static constexpr ValueType latitudeUpperBound_ = 90; static constexpr ValueType longitudeLowerBound_ = 0; static constexpr ValueType longitudeUpperBound_ = 360; + static constexpr ValueType opsCompatibilityModeLongitudeLowerBound_ = -360; + static constexpr ValueType opsCompatibilityModeLongitudeUpperBound_ = 720; + static constexpr int opsCompatibilityModeRelativeLongitudeRange_ = 3; public: /// \brief Partition a sphere into bins whose centers lie on a reduced Gaussian grid. @@ -44,7 +47,16 @@ class SpatialBinSelector { /// in the zonal direction is as close as possible to that in the meridional direction. /// - If set to DOWN, the number of bins is chosen so that the bin width in the zonal direction /// is as small as possible, but no smaller than in the meridional direction. - SpatialBinSelector(IndexType numLatitudeBins, SpatialBinCountRoundingMode roundingMode); + /// \param metOfficeOpsCompatibilityMode + /// If true, the compatibility mode with the Met Office OPS system will be activated. + /// This will stop longitudeBin() from clamping longitudes to the interval [0, 360] deg. + /// + /// Activation of this mode in new code is discouraged; even if input longitudes are e.g. in + /// the range [-180, 180] deg, it is normally better to wrap them to the [0, 360] deg range + /// before passing them to longitudeBin(). Otherwise there's a risk that points lying exactly + /// at -180 or 180 deg will be put into a bin of their own. + SpatialBinSelector(IndexType numLatitudeBins, SpatialBinCountRoundingMode roundingMode, + bool metOfficeOpsCompatibilityMode = false); /// \brief Partition a sphere into bins whose centers lie on a regular Gaussian grid. /// @@ -52,7 +64,8 @@ class SpatialBinSelector { /// The number of zonal bands of bins into which the sphere is split. /// \param numLongitudeBins /// The number of meridional bands of bins into which the sphere is split. - SpatialBinSelector(IndexType numLatitudeBins, IndexType numLongitudeBins); + SpatialBinSelector(IndexType numLatitudeBins, IndexType numLongitudeBins, + bool metOfficeOpsCompatibilityMode = false); /// \brief Return the index of the zonal band of bins containing points with a given latitude /// (in degrees, assumed to lie in the interval [-90, 90]). @@ -61,7 +74,10 @@ class SpatialBinSelector { } /// \brief Return the index of the bin within the zonal band of index \p latitudeBin - /// containing points with a given latitude (in degrees, assumed to lie in the interval [0, 360)). + /// containing points with a given longitude (in degrees). + /// + /// The longitude is assumed to lie in the interval [0, 360] unless the compatibility mode with + /// the Met Office OPS system is in effect. IndexType longitudeBin(IndexType latitudeBin, ValueType longitude) const { return longitudeBinSelectors_[latitudeBin].bin(longitude); } @@ -106,8 +122,9 @@ class SpatialBinSelector { static IndexType roundNumBins(float idealNumBins, SpatialBinCountRoundingMode roundingMode); private: - EquispacedBinSelector latitudeBinSelector_; - std::vector longitudeBinSelectors_; + bool metOfficeOpsCompatibilityMode_; + TruncatingEquispacedBinSelector latitudeBinSelector_; + std::vector longitudeBinSelectors_; }; } // namespace ufo diff --git a/src/ufo/utils/EquispacedBinSelector.h b/src/ufo/utils/TruncatingEquispacedBinSelector.h similarity index 60% rename from src/ufo/utils/EquispacedBinSelector.h rename to src/ufo/utils/TruncatingEquispacedBinSelector.h index cb5222a98..73d3ef008 100644 --- a/src/ufo/utils/EquispacedBinSelector.h +++ b/src/ufo/utils/TruncatingEquispacedBinSelector.h @@ -5,29 +5,30 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef UFO_UTILS_EQUISPACEDBINSELECTOR_H_ -#define UFO_UTILS_EQUISPACEDBINSELECTOR_H_ +#ifndef UFO_UTILS_TRUNCATINGEQUISPACEDBINSELECTOR_H_ +#define UFO_UTILS_TRUNCATINGEQUISPACEDBINSELECTOR_H_ #include #include "eckit/exception/Exceptions.h" +#include "ufo/utils/EquispacedBinSelectorBase.h" namespace ufo { -/// \brief Represents a set of consecutive intervals (_bins_) of the same width. +/// \brief Represents a finite set of consecutive intervals (_bins_) of the same width, each closed +/// from the left and open from the right. /// /// Call the bin() function to find the bin containing a particular value. -class EquispacedBinSelector -{ +class TruncatingEquispacedBinSelector : public EquispacedBinSelectorBase { public: // If necessary, these could be made template parameters. typedef float ValueType; typedef int IndexType; - /// \brief Partition the interval [\p lowerBound, \p upperBound] into \p numBins + /// \brief Partition the interval [\p lowerBound, \p upperBound) into \p numBins /// bins of the same width. - EquispacedBinSelector(ValueType lowerBound, ValueType upperBound, IndexType numBins) + TruncatingEquispacedBinSelector(ValueType lowerBound, ValueType upperBound, IndexType numBins) : lowerBound_(lowerBound), binWidth_((upperBound - lowerBound) / numBins), inverseBinWidth_(1 / binWidth_), @@ -37,32 +38,26 @@ class EquispacedBinSelector ASSERT_MSG(numBins > 0, "Number of bins must be positive"); } - /// \brief Return the (0-based) index of the bin containing \p value, or the nearest bin - /// if \p value lies outside all bins. - IndexType bin(ValueType value) const { + IndexType bin(ValueType value) const override { IndexType binIndex = static_cast((value - lowerBound_) * inverseBinWidth_); binIndex = std::max(IndexType(0), binIndex); binIndex = std::min(numBins_ - 1, binIndex); return binIndex; } - /// \brief Return the number of bins. - IndexType numBins() const { + boost::optional numBins() const override { return numBins_; } - /// \brief Return the width of each bin. - ValueType binWidth() const { + ValueType binWidth() const override { return binWidth_; } - /// \brief Return the inverse of the width of each bin. - ValueType inverseBinWidth() const { + ValueType inverseBinWidth() const override { return inverseBinWidth_; } - /// \brief Return the value lying at the center of the bin with index \p bin. - ValueType binCenter(IndexType bin) const { + ValueType binCenter(IndexType bin) const override { return lowerBound_ + binWidth_ * (bin + ValueType(0.5)); } @@ -75,4 +70,4 @@ class EquispacedBinSelector } // namespace ufo -#endif // UFO_UTILS_EQUISPACEDBINSELECTOR_H_ +#endif // UFO_UTILS_TRUNCATINGEQUISPACEDBINSELECTOR_H_ diff --git a/src/ufo/utils/dataextractor/DataExtractor.cc b/src/ufo/utils/dataextractor/DataExtractor.cc index 2243aef83..7fab07eb8 100644 --- a/src/ufo/utils/dataextractor/DataExtractor.cc +++ b/src/ufo/utils/dataextractor/DataExtractor.cc @@ -42,10 +42,7 @@ class SortUpdateVisitor : public boost::static_visitor { } void operator()(const std::vector &coord) { - splitter.sortGroupsBy( - [&coord](int indexA, int indexB) { - return coord[static_cast(indexA)] < coord[static_cast(indexB)]; - }); + splitter.sortGroupsBy([&coord](int index) { return coord[static_cast(index)]; }); } ufo::RecursiveSplitter &splitter; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 15f8623ae..4bc8a38c4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -181,6 +181,7 @@ list( APPEND ufo_test_input testinput/qc_differencecheck.yaml testinput/qc_finalcheck.yaml testinput/qc_gauss_thinning.yaml + testinput/qc_gauss_thinning_ops.yaml testinput/qc_gauss_thinning_unittests.yaml testinput/qc_grosserrorwholereport.yaml testinput/qc_historycheck_unittests.yaml @@ -1802,6 +1803,13 @@ ecbuild_add_test( TARGET test_ufo_qc_gauss_thinning DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_qc_gauss_thinning_ops + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x + ARGS "testinput/qc_gauss_thinning_ops.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFilters.x + TEST_DEPENDS ufo_get_ufo_test_data ) + ecbuild_add_test( TARGET test_ufo_qc_met_office_buddy_check COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x ARGS "testinput/qc_met_office_buddy_check.yaml" diff --git a/test/testinput/qc_gauss_thinning_ops.yaml b/test/testinput/qc_gauss_thinning_ops.yaml new file mode 100644 index 000000000..bcf82a5f9 --- /dev/null +++ b/test/testinput/qc_gauss_thinning_ops.yaml @@ -0,0 +1,4492 @@ +# The purpose of this test is to verify that the ops_compatibility_mode option of the Gaussian +# Thinning filter makes it retain the same observations as the Ops_Thinning function from the +# Met Office OPS system when run in serial. + +window begin: 2020-05-11T02:59:59Z # Shifted back by 1 s to include obs taken at 03:00:00 +window end: 2020-05-11T09:00:00Z + +# The first two tests operate on test data containing series of observations that differ by only +# one coordinate (latitude, longitude, time or pressure); many of these have the same distance from +# the bin center in the sense of the maximum norm, which makes the choice of the observations to +# retain sensitive to the order in which they are processed. In addition, some observations lie on +# bin boundaries, making their classification sensitive to the rounding method in use. + +observations: +- obs space: + name: Longitudes from -180 to 180 + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_gauss_thinning_lon_-180_to_180_obs.nc4 + simulated variables: [air_temperature] + obs filters: + - filter: Gaussian Thinning + ops_compatibility_mode: true + horizontal_mesh: 5000 + vertical_mesh: 2500 + vertical_min: -1250 + vertical_max: 11250 + time_mesh: PT02H00M00S + time_min: 2020-05-11T03:00:00Z + time_max: 2020-05-11T09:00:00Z + passedObservationsBenchmark: + - 17 + - 39 + - 46 + - 58 + - 62 + - 80 + - 100 + - 137 + - 150 + - 153 + - 163 + - 176 + - 182 + passedBenchmark: 13 + +- obs space: + name: Longitudes from 0 to 360 # Same obs, but with negative longitudes shifted upwards by 360 deg + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_gauss_thinning_lon_0_to_360_obs.nc4 + simulated variables: [air_temperature] + obs filters: + - filter: Gaussian Thinning + ops_compatibility_mode: true + horizontal_mesh: 5000 + vertical_mesh: 2500 + vertical_min: -1250 + vertical_max: 11250 + time_mesh: PT02H00M00S + time_min: 2020-05-11T03:00:00Z + time_max: 2020-05-11T09:00:00Z + passedObservationsBenchmark: + - 17 + - 23 + - 39 + - 46 + - 58 + - 80 + - 100 + - 153 + - 156 + - 163 + - 174 + - 176 + - 182 + passedBenchmark: 13 + +# The following two tests operate on a realistic distribution of ground-GNSS obs. + +- obs space: + name: Ground GNSS, no thinning scores + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_gauss_thinning_groundgnss_obs.nc4 + simulated variables: [ZTD] + obs filters: + - filter: Gaussian Thinning + ops_compatibility_mode: true + horizontal_mesh: 100 + time_mesh: PT02H00M00S + time_min: 2020-05-11T03:00:00Z + time_max: 2020-05-11T09:00:00Z + passedObservationsBenchmark: + - 0 + - 5 + - 20 + - 25 + - 29 + - 30 + - 34 + - 35 + - 50 + - 60 + - 69 + - 70 + - 75 + - 80 + - 83 + - 95 + - 110 + - 114 + - 155 + - 165 + - 175 + - 185 + - 189 + - 204 + - 205 + - 210 + - 215 + - 219 + - 235 + - 245 + - 260 + - 275 + - 277 + - 280 + - 285 + - 299 + - 300 + - 305 + - 322 + - 325 + - 354 + - 365 + - 370 + - 375 + - 380 + - 383 + - 397 + - 415 + - 420 + - 435 + - 445 + - 450 + - 455 + - 465 + - 481 + - 490 + - 503 + - 505 + - 515 + - 520 + - 523 + - 530 + - 540 + - 555 + - 556 + - 560 + - 569 + - 593 + - 595 + - 600 + - 604 + - 605 + - 615 + - 616 + - 620 + - 635 + - 662 + - 685 + - 695 + - 698 + - 700 + - 709 + - 715 + - 720 + - 750 + - 753 + - 755 + - 763 + - 771 + - 785 + - 795 + - 796 + - 805 + - 810 + - 814 + - 823 + - 850 + - 852 + - 865 + - 869 + - 872 + - 890 + - 900 + - 905 + - 910 + - 920 + - 925 + - 935 + - 940 + - 945 + - 950 + - 955 + - 959 + - 960 + - 964 + - 975 + - 977 + - 985 + - 990 + - 1000 + - 1004 + - 1005 + - 1007 + - 1040 + - 1045 + - 1050 + - 1064 + - 1065 + - 1068 + - 1080 + - 1090 + - 1095 + - 1098 + - 1105 + - 1115 + - 1134 + - 1135 + - 1140 + - 1150 + - 1155 + - 1159 + - 1160 + - 1165 + - 1170 + - 1175 + - 1177 + - 1180 + - 1182 + - 1190 + - 1195 + - 1200 + - 1204 + - 1209 + - 1215 + - 1220 + - 1225 + - 1235 + - 1240 + - 1245 + - 1249 + - 1255 + - 1260 + - 1265 + - 1269 + - 1270 + - 1275 + - 1285 + - 1295 + - 1300 + - 1302 + - 1310 + - 1314 + - 1315 + - 1337 + - 1340 + - 1345 + - 1354 + - 1355 + - 1359 + - 1365 + - 1370 + - 1373 + - 1379 + - 1380 + - 1385 + - 1390 + - 1405 + - 1415 + - 1420 + - 1430 + - 1434 + - 1460 + - 1465 + - 1470 + - 1480 + - 1485 + - 1490 + - 1495 + - 1500 + - 1502 + - 1505 + - 1510 + - 1515 + - 1525 + - 1530 + - 1534 + - 1535 + - 1539 + - 1545 + - 1550 + - 1559 + - 1570 + - 1572 + - 1580 + - 1585 + - 1590 + - 1595 + - 1598 + - 1600 + - 1605 + - 1607 + - 1610 + - 1615 + - 1625 + - 1630 + - 1631 + - 1635 + - 1650 + - 1653 + - 1660 + - 1665 + - 1669 + - 1675 + - 1678 + - 1689 + - 1690 + - 1695 + - 1700 + - 1710 + - 1713 + - 1716 + - 1720 + - 1723 + - 1725 + - 1729 + - 1740 + - 1743 + - 1750 + - 1754 + - 1760 + - 1765 + - 1773 + - 1780 + - 1785 + - 1788 + - 1795 + - 1800 + - 1820 + - 1822 + - 1835 + - 1840 + - 1844 + - 1850 + - 1852 + - 1855 + - 1864 + - 1865 + - 1868 + - 1875 + - 1880 + - 1890 + - 1894 + - 1895 + - 1928 + - 1930 + - 1937 + - 1940 + - 1945 + - 1950 + - 1965 + - 1970 + - 1973 + - 1975 + - 1990 + - 1994 + - 1995 + - 1998 + - 2005 + - 2010 + - 2025 + - 2027 + - 2030 + - 2032 + - 2035 + - 2069 + - 2085 + - 2099 + - 2120 + - 2154 + - 2158 + - 2185 + - 2188 + - 2190 + - 2195 + - 2214 + - 2215 + - 2225 + - 2238 + - 2250 + - 2252 + - 2339 + - 2345 + - 2359 + - 2390 + - 2392 + - 2408 + - 2420 + - 2423 + - 2445 + - 2449 + - 2465 + - 2470 + - 2485 + - 2510 + - 2533 + - 2540 + - 2543 + - 2555 + - 2559 + - 2560 + - 2585 + - 2625 + - 2630 + - 2640 + - 2644 + - 2655 + - 2660 + - 2664 + - 2670 + - 2710 + - 2720 + - 2723 + - 2730 + - 2755 + - 2756 + - 2774 + - 2784 + - 2785 + - 2790 + - 2800 + - 2820 + - 2830 + - 2840 + - 2845 + - 2850 + - 2855 + - 2873 + - 2880 + - 2884 + - 2900 + - 2965 + - 2970 + - 2971 + - 2994 + - 3010 + - 3025 + - 3029 + - 3034 + - 3109 + - 3145 + - 3150 + - 3164 + - 3200 + - 3210 + - 3214 + - 3250 + - 3260 + - 3273 + - 3275 + - 3280 + - 3285 + - 3300 + - 3305 + - 3315 + - 3340 + - 3355 + - 3393 + - 3420 + - 3424 + - 3429 + - 3440 + - 3445 + - 3475 + - 3480 + - 3485 + - 3490 + - 3495 + - 3509 + - 3515 + - 3550 + - 3562 + - 3565 + - 3583 + - 3625 + - 3642 + - 3675 + - 3690 + - 3704 + - 3709 + - 3760 + - 3770 + - 3774 + - 3778 + - 3780 + - 3795 + - 3800 + - 3810 + - 3814 + - 3835 + - 3840 + - 3855 + - 3865 + - 3890 + - 3895 + - 3905 + - 3909 + - 3910 + - 3915 + - 3916 + - 3919 + - 3920 + - 3923 + - 3926 + - 3927 + - 3928 + - 3933 + - 3942 + - 3950 + - 3955 + - 3960 + - 3962 + - 3964 + - 3966 + - 3970 + - 3972 + - 3976 + - 3977 + - 3982 + - 3983 + - 3984 + - 3985 + - 3986 + - 3992 + - 4011 + - 4013 + - 4015 + - 4017 + - 4018 + - 4021 + - 4029 + - 4035 + - 4041 + - 4047 + - 4049 + - 4050 + - 4053 + - 4055 + - 4059 + - 4063 + - 4066 + - 4073 + - 4075 + - 4087 + - 4089 + - 4092 + - 4095 + - 4097 + - 4102 + - 4109 + - 4111 + - 4113 + - 4115 + - 4119 + - 4121 + - 4127 + - 4131 + - 4135 + - 4137 + - 4143 + - 4145 + - 4167 + - 4169 + - 4171 + - 4183 + - 4187 + - 4195 + - 4201 + - 4203 + - 4205 + - 4213 + - 4215 + - 4222 + - 4225 + - 4227 + - 4229 + - 4233 + - 4237 + - 4241 + - 4243 + - 4244 + - 4246 + - 4249 + - 4251 + - 4259 + - 4262 + - 4267 + - 4268 + - 4271 + - 4273 + - 4279 + - 4281 + - 4284 + - 4286 + - 4291 + - 4297 + - 4299 + - 4301 + - 4311 + - 4316 + - 4319 + - 4321 + - 4327 + - 4329 + - 4335 + - 4337 + - 4339 + - 4351 + - 4353 + - 4359 + - 4365 + - 4368 + - 4370 + - 4374 + - 4377 + - 4381 + - 4387 + - 4389 + - 4391 + - 4395 + - 4397 + - 4403 + - 4413 + - 4415 + - 4417 + - 4431 + - 4445 + - 4449 + - 4450 + - 4453 + - 4458 + - 4460 + - 4463 + - 4465 + - 4469 + - 4471 + - 4473 + - 4475 + - 4477 + - 4478 + - 4481 + - 4493 + - 4499 + - 4513 + - 4515 + - 4517 + - 4519 + - 4523 + - 4524 + - 4533 + - 4535 + - 4553 + - 4555 + - 4569 + - 4577 + - 4579 + - 4581 + - 4585 + - 4586 + - 4589 + - 4591 + - 4597 + - 4609 + - 4628 + - 4635 + - 4637 + - 4638 + - 4640 + - 4643 + - 4649 + - 4653 + - 4655 + - 4657 + - 4658 + - 4661 + - 4665 + - 4673 + - 4675 + - 4679 + - 4681 + - 4689 + - 4695 + - 4699 + - 4701 + - 4702 + - 4707 + - 4709 + - 4711 + - 4717 + - 4718 + - 4722 + - 4727 + - 4733 + - 4739 + - 4745 + - 4789 + - 4798 + - 4808 + - 4830 + - 4848 + - 4874 + - 4934 + - 5013 + - 5019 + - 5023 + - 5039 + - 5044 + - 5089 + - 5103 + - 5125 + - 5149 + - 5158 + - 5180 + - 5193 + - 5208 + - 5214 + - 5224 + - 5230 + - 5249 + - 5255 + - 5258 + - 5273 + - 5301 + - 5324 + - 5334 + - 5386 + - 5393 + - 5413 + - 5449 + - 5478 + - 5483 + - 5514 + - 5523 + - 5558 + - 5631 + - 5648 + - 5659 + - 5664 + - 5674 + - 5678 + - 5683 + - 5688 + - 5698 + - 5703 + - 5716 + - 5728 + - 5768 + - 5773 + - 5779 + - 5793 + - 5799 + - 5914 + - 6018 + - 6080 + - 6089 + - 6098 + - 6123 + - 6153 + - 6198 + - 6213 + - 6298 + - 6358 + - 6434 + - 6483 + - 6634 + - 6678 + - 6713 + - 6787 + - 6837 + - 6907 + - 6937 + - 7017 + - 7057 + - 7064 + - 7167 + - 7178 + - 7214 + - 7262 + - 7282 + - 7328 + - 7342 + - 7412 + - 7455 + - 7477 + - 7488 + - 7578 + - 7608 + - 7623 + - 7627 + - 7638 + - 7655 + - 7677 + - 7683 + - 7709 + - 7717 + - 7723 + - 7749 + - 7765 + - 7772 + - 7812 + - 7828 + - 7833 + - 7855 + - 7869 + - 7892 + - 7903 + - 7907 + - 7948 + - 7968 + - 7997 + - 8004 + - 8008 + - 8047 + - 8073 + - 8080 + - 8105 + - 8124 + - 8143 + - 8149 + - 8160 + - 8167 + - 8185 + - 8189 + - 8192 + - 8253 + - 8278 + - 8287 + - 8324 + - 8369 + - 8372 + - 8382 + - 8398 + - 8417 + - 8427 + - 8433 + - 8437 + - 8462 + - 8467 + - 8485 + - 8487 + - 8494 + - 8512 + - 8514 + - 8518 + - 8520 + - 8523 + - 8525 + - 8527 + - 8529 + - 8533 + - 8535 + - 8537 + - 8540 + - 8541 + - 8545 + - 8552 + - 8554 + - 8555 + - 8556 + - 8560 + - 8561 + - 8567 + - 8569 + - 8570 + - 8586 + - 8590 + - 8597 + - 8598 + - 8610 + - 8616 + - 8618 + - 8620 + - 8622 + - 8625 + - 8626 + - 8643 + - 8644 + - 8657 + - 8660 + - 8663 + - 8674 + - 8683 + - 8686 + - 8695 + - 8697 + - 8700 + - 8718 + - 8722 + - 8724 + - 8742 + - 8746 + - 8748 + - 8756 + - 8770 + - 8774 + - 8778 + - 8788 + - 8790 + - 8792 + - 8797 + - 8804 + - 8806 + - 8812 + - 8814 + - 8823 + - 8826 + - 8834 + - 8836 + - 8846 + - 8868 + - 8872 + - 8880 + - 8890 + - 8893 + - 8901 + - 8906 + - 8912 + - 8914 + - 8916 + - 8922 + - 8924 + - 8926 + - 8933 + - 8934 + - 8942 + - 8953 + - 8960 + - 8964 + - 8967 + - 8968 + - 8974 + - 8980 + - 8982 + - 8986 + - 8992 + - 8998 + - 9002 + - 9004 + - 9012 + - 9018 + - 9020 + - 9024 + - 9026 + - 9028 + - 9030 + - 9039 + - 9040 + - 9042 + - 9044 + - 9046 + - 9048 + - 9054 + - 9056 + - 9058 + - 9060 + - 9063 + - 9066 + - 9075 + - 9076 + - 9086 + - 9109 + - 9115 + - 9116 + - 9124 + - 9126 + - 9128 + - 9130 + - 9140 + - 9146 + - 9148 + - 9151 + - 9156 + - 9158 + - 9160 + - 9177 + - 9179 + - 9182 + - 9184 + - 9186 + - 9192 + - 9205 + - 9214 + - 9216 + - 9220 + - 9224 + - 9226 + - 9228 + - 9238 + - 9242 + - 9246 + - 9248 + - 9253 + - 9254 + - 9284 + - 9289 + - 9290 + - 9302 + - 9306 + - 9309 + - 9312 + - 9314 + - 9327 + - 9334 + - 9337 + - 9347 + - 9350 + - 9361 + - 9363 + - 9367 + - 9396 + - 9399 + - 9404 + - 9426 + - 9451 + - 9467 + - 9512 + - 9541 + - 9609 + - 9657 + - 9661 + - 9665 + - 9702 + - 9712 + - 9730 + - 9771 + - 9784 + - 9790 + - 9809 + - 9830 + - 9847 + - 9862 + - 9867 + - 9872 + - 9876 + - 9902 + - 9917 + - 9936 + - 10012 + - 10039 + - 10076 + - 10085 + - 10096 + - 10099 + - 10124 + - 10135 + - 10142 + - 10155 + - 10164 + - 10170 + - 10225 + - 10237 + - 10251 + - 10256 + - 10270 + - 10280 + - 10292 + - 10307 + - 10316 + - 10331 + - 10340 + - 10356 + - 10362 + - 10371 + - 10381 + - 10385 + - 10506 + - 10517 + - 10612 + - 10627 + - 10651 + - 10666 + - 10692 + - 10697 + - 10721 + - 10776 + - 10810 + - 10816 + - 10822 + - 10832 + - 10852 + - 10939 + - 11071 + - 11106 + - 11255 + - 11287 + - 11362 + - 11417 + - 11442 + - 11447 + - 11492 + - 11577 + - 11656 + - 11687 + - 12042 + - 12082 + - 12086 + - 12151 + - 12157 + - 12171 + - 12292 + - 12294 + - 12325 + - 12336 + - 12352 + - 12355 + - 12365 + - 12382 + - 12396 + - 12421 + - 12446 + - 12461 + - 12467 + - 12486 + - 12504 + - 12511 + - 12537 + - 12541 + - 12581 + - 12607 + - 12617 + - 12622 + - 12626 + - 12629 + - 12671 + - 12712 + - 12722 + - 12735 + - 12751 + - 12792 + - 12796 + - 12800 + - 12805 + - 12814 + - 12834 + - 12844 + - 12852 + - 12857 + - 12891 + - 12896 + - 12930 + - 12951 + - 12956 + - 12961 + - 12967 + - 12971 + - 12981 + - 12995 + - 13013 + - 13035 + - 13042 + - 13046 + - 13049 + - 13067 + - 13107 + - 13114 + - 13132 + - 13141 + - 13147 + - 13174 + - 13185 + - 13197 + - 13200 + - 13214 + - 13220 + - 13221 + - 13234 + - 13275 + - 13281 + - 13285 + - 13297 + - 13305 + - 13313 + - 13323 + - 13326 + - 13329 + - 13330 + - 13335 + - 13341 + - 13342 + - 13348 + - 13353 + - 13359 + - 13362 + - 13369 + - 13375 + - 13379 + - 13381 + - 13383 + - 13391 + - 13393 + - 13397 + - 13399 + - 13403 + - 13411 + - 13413 + - 13419 + - 13421 + - 13427 + - 13431 + - 13435 + - 13439 + - 13449 + - 13451 + - 13457 + - 13459 + - 13467 + - 13477 + - 13481 + - 13489 + - 13497 + - 13511 + - 13513 + - 13517 + - 13519 + - 13521 + - 13523 + - 13525 + - 13527 + - 13528 + - 13533 + - 13535 + - 13547 + - 13553 + - 13573 + - 13579 + - 13581 + - 13587 + - 13591 + - 13593 + - 13598 + - 13603 + - 13605 + - 13606 + - 13612 + - 13615 + - 13619 + - 13621 + - 13622 + - 13626 + - 13629 + - 13633 + - 13641 + - 13643 + - 13647 + - 13655 + - 13657 + - 13661 + - 13665 + - 13673 + - 13677 + - 13689 + - 13695 + - 13696 + - 13703 + - 13707 + - 13708 + - 13719 + - 13723 + - 13724 + - 13731 + - 13739 + - 13743 + - 13746 + - 13749 + - 13751 + - 13753 + - 13757 + - 13765 + - 13769 + - 13770 + - 13773 + - 13781 + - 13783 + - 13793 + - 13795 + - 13798 + - 13802 + - 13804 + - 13807 + - 13809 + - 13818 + - 13825 + - 13835 + - 13839 + - 13841 + - 13849 + - 13851 + - 13853 + - 13854 + - 13857 + - 13859 + - 13863 + - 13871 + - 13883 + - 13885 + - 13886 + - 13893 + - 13895 + - 13897 + - 13915 + - 13917 + - 13918 + - 13921 + - 13933 + - 13937 + - 13947 + - 13951 + - 13959 + - 13963 + - 13965 + - 13967 + - 13971 + - 13983 + - 13993 + - 14003 + - 14009 + - 14015 + - 14016 + - 14019 + - 14030 + - 14032 + - 14035 + - 14041 + - 14053 + - 14057 + - 14063 + - 14068 + - 14074 + - 14126 + - 14151 + - 14157 + - 14191 + - 14201 + - 14242 + - 14274 + - 14277 + - 14306 + - 14347 + - 14436 + - 14457 + - 14467 + - 14476 + - 14491 + - 14536 + - 14566 + - 14611 + - 14621 + - 14661 + - 14671 + - 14692 + - 14696 + - 14701 + - 14716 + - 14852 + - 14916 + - 14978 + - 14996 + - 15011 + - 15077 + - 15127 + - 15152 + - 15161 + - 15177 + - 15183 + - 15201 + - 15231 + - 15236 + - 15281 + - 15289 + - 15331 + - 15358 + - 15383 + - 15403 + - 15411 + - 15423 + - 15503 + - 15519 + - 15559 + - 15581 + - 15603 + - 15617 + - 15628 + - 15643 + - 15689 + - 15709 + - 15721 + - 15727 + - 15731 + - 15742 + - 15762 + - 15777 + - 15783 + - 15788 + - 15846 + - 15866 + - 15877 + - 15883 + - 15921 + - 15938 + - 15942 + - 15947 + - 15951 + - 15982 + - 15987 + - 15997 + - 16013 + - 16026 + - 16032 + - 16046 + - 16056 + - 16216 + - 16223 + - 16276 + - 16396 + - 16411 + - 16416 + - 16486 + - 16581 + - 16617 + - 16656 + - 16696 + - 16727 + - 16771 + - 16776 + - 16791 + - 16812 + - 16827 + - 16836 + - 16946 + - 16967 + - 17034 + - 17052 + - 17059 + - 17171 + - 17191 + - 17236 + - 17277 + - 17281 + - 17397 + - 17436 + - 17443 + - 17467 + - 17497 + - 17541 + - 17554 + - 17587 + - 17606 + - 17647 + - 17656 + - 17666 + - 17676 + - 17721 + - 17728 + - 17806 + - 17841 + - 17881 + - 17961 + - 17986 + - 17990 + - 17994 + - 17995 + - 17997 + - 17998 + - 18000 + - 18002 + - 18003 + - 18005 + - 18006 + - 18010 + - 18019 + - 18020 + - 18021 + - 18024 + - 18032 + - 18034 + - 18057 + - 18060 + - 18065 + - 18067 + - 18070 + - 18072 + - 18074 + - 18077 + - 18080 + - 18088 + - 18092 + - 18094 + - 18100 + - 18106 + - 18121 + - 18123 + - 18126 + - 18128 + - 18142 + - 18145 + - 18156 + - 18162 + - 18173 + - 18180 + - 18184 + - 18188 + - 18192 + - 18198 + - 18200 + - 18202 + - 18210 + - 18212 + - 18221 + - 18223 + - 18243 + - 18248 + - 18250 + - 18252 + - 18256 + - 18261 + - 18262 + - 18266 + - 18294 + - 18298 + - 18300 + - 18303 + - 18306 + - 18308 + - 18316 + - 18320 + - 18322 + - 18324 + - 18327 + - 18328 + - 18332 + - 18340 + - 18344 + - 18350 + - 18352 + - 18365 + - 18366 + - 18381 + - 18387 + - 18390 + - 18394 + - 18401 + - 18404 + - 18406 + - 18418 + - 18423 + - 18424 + - 18431 + - 18435 + - 18436 + - 18438 + - 18442 + - 18447 + - 18448 + - 18455 + - 18456 + - 18460 + - 18466 + - 18469 + - 18470 + - 18473 + - 18477 + - 18482 + - 18484 + - 18488 + - 18491 + - 18492 + - 18496 + - 18500 + - 18510 + - 18514 + - 18518 + - 18522 + - 18530 + - 18533 + - 18534 + - 18540 + - 18543 + - 18545 + - 18546 + - 18556 + - 18566 + - 18578 + - 18582 + - 18584 + - 18592 + - 18598 + - 18601 + - 18602 + - 18617 + - 18620 + - 18622 + - 18624 + - 18645 + - 18654 + - 18656 + - 18662 + - 18664 + - 18667 + - 18676 + - 18678 + - 18680 + - 18682 + - 18684 + - 18690 + - 18694 + - 18710 + - 18712 + - 18714 + - 18718 + - 18726 + - 18732 + - 18736 + - 18740 + - 18744 + - 18747 + - 18748 + - 18750 + - 18755 + - 18756 + - 18760 + - 18767 + - 18770 + - 18780 + - 18782 + - 18784 + - 18788 + - 18796 + - 18798 + - 18802 + - 18804 + - 18806 + - 18808 + - 18814 + - 18819 + - 18829 + - 18831 + - 18832 + - 18873 + - 18893 + - 18946 + - 18953 + - 18983 + - 19013 + - 19065 + - 19107 + - 19117 + - 19126 + - 19132 + - 19148 + - 19173 + - 19181 + - 19192 + - 19200 + - 19240 + - 19248 + - 19268 + - 19272 + - 19290 + - 19326 + - 19355 + - 19361 + - 19372 + - 19377 + - 19388 + - 19397 + - 19433 + - 19436 + - 19443 + - 19463 + - 19468 + - 19556 + - 19577 + - 19582 + - 19637 + - 19681 + - 19697 + - 19702 + - 19712 + - 19718 + - 19732 + - 19738 + - 19752 + - 19755 + - 19776 + - 19787 + - 19792 + - 19806 + - 19853 + - 19860 + - 19923 + - 19933 + - 19993 + - 19996 + - 20028 + - 20043 + - 20046 + - 20067 + - 20081 + - 20107 + - 20148 + - 20166 + - 20200 + - 20212 + - 20223 + - 20283 + - 20302 + - 20358 + - 20442 + - 20493 + - 20507 + - 20525 + - 20543 + - 20552 + - 20723 + - 20731 + - 20742 + - 20762 + - 20802 + - 20848 + - 20883 + - 20943 + - 20988 + - 20991 + - 21022 + - 21091 + - 21098 + - 21138 + - 21143 + - 21167 + - 21238 + - 21240 + - 21273 + - 21331 + - 21351 + - 21363 + - 21407 + - 21413 + - 21423 + - 21468 + - 21493 + - 21543 + - 21548 + - 21562 + - 21567 + - 21577 + - 21606 + - 21657 + - 21698 + - 21743 + - 21753 + - 21758 + - 21862 + - 21868 + - 21878 + - 21903 + - 21905 + - 21938 + - 21952 + - 21963 + - 21973 + - 21980 + - 22006 + - 22018 + - 22033 + - 22046 + - 22061 + - 22103 + - 22118 + - 22140 + - 22193 + - 22197 + - 22200 + - 22206 + - 22216 + - 22240 + - 22262 + - 22267 + - 22286 + - 22296 + - 22325 + - 22365 + - 22371 + - 22380 + - 22392 + - 22397 + - 22407 + - 22427 + - 22476 + - 22493 + - 22497 + - 22507 + - 22522 + - 22553 + - 22556 + - 22576 + - 22588 + - 22590 + - 22602 + - 22635 + - 22639 + - 22652 + - 22654 + - 22660 + - 22662 + - 22677 + - 22681 + - 22683 + - 22704 + - 22722 + - 22734 + - 22739 + - 22742 + - 22745 + - 22752 + - 22755 + - 22759 + - 22765 + - 22787 + - 22791 + - 22793 + - 22794 + - 22809 + - 22811 + - 22815 + - 22816 + - 22818 + - 22821 + - 22826 + - 22828 + - 22831 + - 22835 + - 22840 + - 22845 + - 22847 + - 22853 + - 22857 + - 22865 + - 22867 + - 22871 + - 22873 + - 22876 + - 22879 + - 22881 + - 22883 + - 22891 + - 22892 + - 22917 + - 22919 + - 22921 + - 22925 + - 22927 + - 22931 + - 22938 + - 22951 + - 22961 + - 22963 + - 22977 + - 22979 + - 22991 + - 22996 + - 23003 + - 23009 + - 23010 + - 23015 + - 23019 + - 23021 + - 23022 + - 23032 + - 23035 + - 23041 + - 23043 + - 23045 + - 23047 + - 23051 + - 23052 + - 23055 + - 23061 + - 23063 + - 23065 + - 23067 + - 23071 + - 23077 + - 23089 + - 23105 + - 23111 + - 23113 + - 23115 + - 23117 + - 23120 + - 23123 + - 23129 + - 23131 + - 23133 + - 23137 + - 23138 + - 23141 + - 23163 + - 23165 + - 23167 + - 23170 + - 23173 + - 23175 + - 23177 + - 23181 + - 23191 + - 23196 + - 23199 + - 23205 + - 23206 + - 23209 + - 23213 + - 23215 + - 23217 + - 23226 + - 23231 + - 23232 + - 23235 + - 23242 + - 23247 + - 23251 + - 23263 + - 23268 + - 23271 + - 23275 + - 23277 + - 23278 + - 23283 + - 23293 + - 23295 + - 23312 + - 23317 + - 23333 + - 23335 + - 23342 + - 23347 + - 23349 + - 23351 + - 23355 + - 23359 + - 23360 + - 23363 + - 23373 + - 23381 + - 23383 + - 23384 + - 23387 + - 23389 + - 23392 + - 23395 + - 23397 + - 23403 + - 23417 + - 23427 + - 23430 + - 23435 + - 23439 + - 23443 + - 23445 + - 23447 + - 23455 + - 23465 + - 23467 + - 23468 + - 23471 + - 23481 + - 23487 + - 23492 + - 23498 + - 23500 + - 23530 + - 23690 + - 23720 + - 23802 + - 23826 + - 23882 + - 23952 + - 24005 + - 24026 + - 24080 + - 24090 + - 24141 + - 24176 + - 24230 + - 24310 + - 24341 + - 24345 + - 24366 + - 24390 + - 24460 + - 24502 + - 24526 + - 24530 + - 24615 + - 24626 + - 24633 + - 24645 + - 24661 + - 24665 + - 24675 + - 24683 + - 24695 + - 24725 + - 24737 + - 24750 + - 24766 + - 24802 + - 24826 + - 24865 + - 24900 + - 24920 + - 24995 + - 25015 + - 25025 + - 25070 + - 25100 + - 25131 + - 25171 + - 25176 + - 25260 + - 25275 + - 25296 + - 25337 + - 25346 + - 25367 + - 25410 + - 25439 + - 25450 + - 25492 + - 25502 + - 25510 + - 25545 + - 25570 + - 25585 + - 25625 + - 25670 + - 25695 + - 25745 + - 25750 + - 25781 + - 25791 + - 25826 + - 25838 + - 25840 + - 25864 + - 25887 + - 25900 + - 25956 + - 25970 + - 25980 + - 26035 + - 26095 + - 26125 + - 26145 + - 26150 + - 26166 + - 26170 + - 26176 + - 26187 + - 26201 + - 26247 + - 26250 + - 26273 + - 26286 + - 26295 + - 26306 + - 26315 + - 26320 + - 26330 + - 26336 + - 26348 + - 26361 + - 26372 + - 26391 + - 26446 + - 26460 + - 26465 + - 26475 + - 26487 + - 26551 + - 26556 + - 26562 + - 26621 + - 26625 + - 26631 + - 26652 + - 26659 + - 26661 + - 26667 + - 26669 + - 26675 + - 26677 + - 26686 + - 26688 + - 26694 + - 26704 + - 26706 + - 26714 + - 26722 + - 26724 + - 26743 + - 26747 + - 26755 + - 26759 + - 26765 + - 26767 + - 26777 + - 26788 + - 26794 + - 26798 + - 26800 + - 26804 + - 26807 + - 26808 + - 26810 + - 26815 + - 26816 + - 26818 + - 26835 + - 26836 + - 26838 + - 26840 + - 26844 + - 26862 + - 26876 + - 26886 + - 26892 + - 26897 + - 26898 + - 26900 + - 26912 + - 26922 + - 26924 + - 26932 + - 26942 + - 26946 + - 26950 + - 26960 + - 26970 + - 26974 + - 26978 + - 26982 + - 26984 + - 26986 + - 26990 + - 26992 + - 26995 + - 27002 + - 27006 + - 27008 + - 27011 + - 27012 + - 27018 + - 27024 + - 27030 + - 27032 + - 27036 + - 27038 + - 27044 + - 27050 + - 27052 + - 27056 + - 27065 + - 27068 + - 27070 + - 27075 + - 27076 + - 27080 + - 27093 + - 27100 + - 27102 + - 27110 + - 27114 + - 27120 + - 27125 + - 27127 + - 27128 + - 27132 + - 27136 + - 27139 + - 27140 + - 27143 + - 27144 + - 27150 + - 27160 + - 27167 + - 27169 + - 27177 + - 27184 + - 27186 + - 27188 + - 27191 + - 27192 + - 27194 + - 27196 + - 27198 + - 27200 + - 27202 + - 27210 + - 27220 + - 27224 + - 27228 + - 27231 + - 27234 + - 27237 + - 27243 + - 27244 + - 27253 + - 27260 + - 27265 + - 27267 + - 27271 + - 27288 + - 27292 + - 27298 + - 27300 + - 27304 + - 27306 + - 27312 + - 27320 + - 27324 + - 27326 + - 27328 + - 27330 + - 27346 + - 27348 + - 27350 + - 27358 + - 27360 + - 27362 + - 27364 + - 27374 + - 27376 + - 27378 + - 27382 + - 27390 + - 27402 + - 27415 + - 27423 + - 27430 + - 27446 + - 27448 + - 27450 + - 27453 + - 27454 + - 27456 + - 27460 + - 27462 + - 27466 + - 27474 + - 27482 + - 27490 + - 27494 + - 27499 + - 27500 + - 27502 + - 27504 + - 27510 + - 27515 + - 27517 + - 27520 + - 27528 + - 27533 + - 27534 + +- obs space: + name: Ground GNSS, with thinning scores + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_gauss_thinning_groundgnss_obs.nc4 + simulated variables: [ZTD] + obs filters: + - filter: Gaussian Thinning + ops_compatibility_mode: true + horizontal_mesh: 100 + time_mesh: PT02H00M00S + time_min: 2020-05-11T03:00:00Z + time_max: 2020-05-11T09:00:00Z + priority_variable: + name: thinning_score@MetaData + passedObservationsBenchmark: + - 0 + - 5 + - 25 + - 29 + - 30 + - 34 + - 35 + - 75 + - 80 + - 83 + - 85 + - 95 + - 110 + - 114 + - 125 + - 127 + - 155 + - 165 + - 180 + - 185 + - 189 + - 205 + - 215 + - 219 + - 235 + - 240 + - 254 + - 255 + - 265 + - 275 + - 277 + - 285 + - 295 + - 299 + - 325 + - 365 + - 380 + - 383 + - 420 + - 435 + - 450 + - 490 + - 503 + - 505 + - 515 + - 520 + - 523 + - 560 + - 580 + - 593 + - 600 + - 604 + - 615 + - 616 + - 625 + - 628 + - 665 + - 695 + - 698 + - 715 + - 720 + - 750 + - 753 + - 790 + - 795 + - 796 + - 805 + - 823 + - 850 + - 852 + - 890 + - 900 + - 905 + - 910 + - 920 + - 925 + - 930 + - 932 + - 935 + - 970 + - 975 + - 977 + - 980 + - 984 + - 990 + - 1000 + - 1004 + - 1005 + - 1007 + - 1040 + - 1060 + - 1064 + - 1080 + - 1085 + - 1090 + - 1095 + - 1098 + - 1100 + - 1105 + - 1110 + - 1115 + - 1120 + - 1125 + - 1130 + - 1134 + - 1135 + - 1140 + - 1150 + - 1155 + - 1159 + - 1160 + - 1165 + - 1170 + - 1171 + - 1175 + - 1177 + - 1180 + - 1182 + - 1185 + - 1188 + - 1190 + - 1195 + - 1200 + - 1204 + - 1205 + - 1209 + - 1210 + - 1214 + - 1215 + - 1220 + - 1225 + - 1228 + - 1230 + - 1235 + - 1240 + - 1243 + - 1245 + - 1249 + - 1250 + - 1252 + - 1255 + - 1259 + - 1260 + - 1265 + - 1269 + - 1270 + - 1273 + - 1275 + - 1280 + - 1283 + - 1285 + - 1290 + - 1295 + - 1300 + - 1302 + - 1305 + - 1310 + - 1314 + - 1315 + - 1320 + - 1321 + - 1325 + - 1330 + - 1334 + - 1335 + - 1337 + - 1340 + - 1345 + - 1350 + - 1354 + - 1355 + - 1359 + - 1360 + - 1365 + - 1370 + - 1373 + - 1375 + - 1379 + - 1380 + - 1385 + - 1390 + - 1395 + - 1400 + - 1404 + - 1405 + - 1410 + - 1412 + - 1415 + - 1420 + - 1425 + - 1430 + - 1434 + - 1435 + - 1440 + - 1443 + - 1445 + - 1447 + - 1450 + - 1452 + - 1455 + - 1460 + - 1465 + - 1470 + - 1475 + - 1480 + - 1485 + - 1489 + - 1490 + - 1495 + - 1496 + - 1500 + - 1502 + - 1505 + - 1510 + - 1515 + - 1520 + - 1525 + - 1530 + - 1534 + - 1535 + - 1539 + - 1540 + - 1545 + - 1550 + - 1552 + - 1555 + - 1559 + - 1560 + - 1565 + - 1570 + - 1572 + - 1575 + - 1580 + - 1584 + - 1585 + - 1590 + - 1595 + - 1598 + - 1600 + - 1605 + - 1607 + - 1610 + - 1615 + - 1620 + - 1622 + - 1625 + - 1626 + - 1630 + - 1631 + - 1635 + - 1640 + - 1645 + - 1650 + - 1653 + - 1655 + - 1660 + - 1665 + - 1669 + - 1670 + - 1675 + - 1678 + - 1680 + - 1685 + - 1689 + - 1690 + - 1693 + - 1695 + - 1700 + - 1710 + - 1713 + - 1715 + - 1716 + - 1720 + - 1723 + - 1725 + - 1729 + - 1730 + - 1734 + - 1735 + - 1740 + - 1743 + - 1750 + - 1754 + - 1755 + - 1760 + - 1765 + - 1770 + - 1773 + - 1775 + - 1779 + - 1780 + - 1781 + - 1785 + - 1788 + - 1790 + - 1795 + - 1800 + - 1805 + - 1810 + - 1814 + - 1815 + - 1817 + - 1820 + - 1822 + - 1825 + - 1830 + - 1835 + - 1840 + - 1844 + - 1845 + - 1849 + - 1850 + - 1852 + - 1855 + - 1860 + - 1864 + - 1865 + - 1868 + - 1875 + - 1880 + - 1885 + - 1890 + - 1894 + - 1895 + - 1900 + - 1905 + - 1907 + - 1910 + - 1915 + - 1920 + - 1928 + - 1930 + - 1937 + - 1940 + - 1945 + - 1950 + - 1955 + - 1957 + - 1960 + - 1963 + - 1965 + - 1970 + - 1973 + - 1975 + - 1980 + - 1985 + - 1988 + - 1990 + - 1994 + - 1995 + - 1998 + - 2000 + - 2005 + - 2010 + - 2015 + - 2020 + - 2025 + - 2027 + - 2035 + - 2085 + - 2099 + - 2120 + - 2154 + - 2185 + - 2188 + - 2214 + - 2215 + - 2250 + - 2252 + - 2339 + - 2408 + - 2420 + - 2423 + - 2445 + - 2449 + - 2470 + - 2485 + - 2510 + - 2540 + - 2543 + - 2555 + - 2559 + - 2585 + - 2630 + - 2655 + - 2660 + - 2664 + - 2670 + - 2710 + - 2720 + - 2723 + - 2755 + - 2756 + - 2774 + - 2784 + - 2790 + - 2800 + - 2820 + - 2850 + - 2855 + - 2873 + - 2900 + - 2965 + - 2970 + - 2971 + - 3010 + - 3025 + - 3029 + - 3109 + - 3145 + - 3150 + - 3164 + - 3210 + - 3214 + - 3273 + - 3275 + - 3280 + - 3305 + - 3315 + - 3355 + - 3393 + - 3420 + - 3424 + - 3440 + - 3445 + - 3475 + - 3480 + - 3490 + - 3495 + - 3515 + - 3550 + - 3562 + - 3642 + - 3675 + - 3704 + - 3709 + - 3760 + - 3770 + - 3774 + - 3778 + - 3780 + - 3795 + - 3835 + - 3840 + - 3855 + - 3865 + - 3909 + - 3910 + - 3915 + - 3923 + - 3926 + - 3928 + - 3933 + - 3955 + - 3984 + - 4011 + - 4013 + - 4015 + - 4017 + - 4018 + - 4021 + - 4029 + - 4035 + - 4041 + - 4047 + - 4049 + - 4050 + - 4053 + - 4055 + - 4059 + - 4063 + - 4066 + - 4073 + - 4075 + - 4087 + - 4089 + - 4092 + - 4095 + - 4097 + - 4102 + - 4109 + - 4111 + - 4113 + - 4115 + - 4119 + - 4121 + - 4127 + - 4131 + - 4135 + - 4137 + - 4143 + - 4145 + - 4167 + - 4169 + - 4171 + - 4183 + - 4187 + - 4195 + - 4201 + - 4203 + - 4205 + - 4213 + - 4215 + - 4222 + - 4225 + - 4227 + - 4229 + - 4237 + - 4241 + - 4243 + - 4244 + - 4246 + - 4249 + - 4251 + - 4259 + - 4262 + - 4267 + - 4268 + - 4271 + - 4273 + - 4279 + - 4281 + - 4284 + - 4286 + - 4291 + - 4297 + - 4299 + - 4301 + - 4311 + - 4316 + - 4319 + - 4321 + - 4327 + - 4329 + - 4335 + - 4337 + - 4339 + - 4351 + - 4353 + - 4359 + - 4365 + - 4368 + - 4370 + - 4374 + - 4377 + - 4381 + - 4387 + - 4389 + - 4391 + - 4395 + - 4397 + - 4403 + - 4413 + - 4415 + - 4417 + - 4431 + - 4445 + - 4449 + - 4450 + - 4453 + - 4458 + - 4460 + - 4463 + - 4465 + - 4469 + - 4471 + - 4473 + - 4475 + - 4477 + - 4478 + - 4481 + - 4493 + - 4499 + - 4513 + - 4515 + - 4517 + - 4519 + - 4523 + - 4524 + - 4535 + - 4553 + - 4569 + - 4577 + - 4579 + - 4581 + - 4585 + - 4586 + - 4589 + - 4591 + - 4597 + - 4609 + - 4628 + - 4635 + - 4637 + - 4638 + - 4640 + - 4643 + - 4649 + - 4653 + - 4655 + - 4657 + - 4658 + - 4661 + - 4665 + - 4673 + - 4675 + - 4679 + - 4681 + - 4689 + - 4695 + - 4699 + - 4701 + - 4702 + - 4707 + - 4709 + - 4711 + - 4717 + - 4718 + - 4722 + - 4727 + - 4733 + - 4739 + - 4745 + - 4789 + - 4798 + - 4830 + - 4840 + - 4848 + - 4930 + - 4943 + - 5003 + - 5013 + - 5023 + - 5044 + - 5103 + - 5125 + - 5149 + - 5158 + - 5180 + - 5193 + - 5214 + - 5249 + - 5255 + - 5273 + - 5324 + - 5478 + - 5483 + - 5558 + - 5648 + - 5659 + - 5664 + - 5674 + - 5678 + - 5683 + - 5688 + - 5698 + - 5703 + - 5716 + - 5756 + - 5773 + - 5914 + - 6018 + - 6089 + - 6098 + - 6153 + - 6198 + - 6213 + - 6434 + - 6483 + - 6634 + - 6713 + - 6787 + - 6907 + - 6937 + - 7017 + - 7057 + - 7064 + - 7167 + - 7178 + - 7214 + - 7262 + - 7282 + - 7328 + - 7342 + - 7455 + - 7567 + - 7574 + - 7578 + - 7587 + - 7594 + - 7599 + - 7602 + - 7608 + - 7612 + - 7623 + - 7627 + - 7638 + - 7655 + - 7677 + - 7683 + - 7706 + - 7709 + - 7717 + - 7723 + - 7749 + - 7765 + - 7772 + - 7777 + - 7783 + - 7792 + - 7803 + - 7812 + - 7828 + - 7833 + - 7849 + - 7855 + - 7869 + - 7877 + - 7882 + - 7892 + - 7903 + - 7907 + - 7913 + - 7923 + - 7944 + - 7948 + - 7955 + - 7958 + - 7964 + - 7968 + - 7977 + - 7992 + - 7997 + - 8004 + - 8008 + - 8015 + - 8030 + - 8034 + - 8047 + - 8052 + - 8063 + - 8073 + - 8080 + - 8090 + - 8097 + - 8105 + - 8124 + - 8129 + - 8135 + - 8143 + - 8149 + - 8160 + - 8167 + - 8185 + - 8189 + - 8192 + - 8228 + - 8242 + - 8250 + - 8253 + - 8278 + - 8285 + - 8287 + - 8292 + - 8312 + - 8319 + - 8324 + - 8342 + - 8362 + - 8369 + - 8372 + - 8382 + - 8388 + - 8398 + - 8403 + - 8407 + - 8417 + - 8427 + - 8433 + - 8437 + - 8452 + - 8462 + - 8467 + - 8485 + - 8487 + - 8494 + - 8498 + - 8504 + - 8518 + - 8520 + - 8525 + - 8535 + - 8539 + - 8541 + - 8554 + - 8560 + - 8569 + - 8598 + - 8610 + - 8616 + - 8618 + - 8620 + - 8622 + - 8626 + - 8643 + - 8644 + - 8657 + - 8660 + - 8663 + - 8674 + - 8683 + - 8686 + - 8695 + - 8697 + - 8700 + - 8718 + - 8722 + - 8724 + - 8742 + - 8746 + - 8756 + - 8770 + - 8774 + - 8778 + - 8788 + - 8790 + - 8792 + - 8797 + - 8804 + - 8806 + - 8812 + - 8814 + - 8823 + - 8826 + - 8834 + - 8836 + - 8846 + - 8868 + - 8872 + - 8880 + - 8890 + - 8893 + - 8901 + - 8906 + - 8912 + - 8914 + - 8916 + - 8922 + - 8924 + - 8926 + - 8933 + - 8934 + - 8942 + - 8953 + - 8960 + - 8964 + - 8967 + - 8968 + - 8974 + - 8980 + - 8982 + - 8986 + - 8992 + - 8998 + - 9002 + - 9004 + - 9012 + - 9018 + - 9020 + - 9024 + - 9026 + - 9028 + - 9030 + - 9039 + - 9040 + - 9042 + - 9044 + - 9046 + - 9048 + - 9054 + - 9056 + - 9058 + - 9060 + - 9063 + - 9066 + - 9075 + - 9076 + - 9086 + - 9109 + - 9115 + - 9116 + - 9124 + - 9126 + - 9128 + - 9130 + - 9140 + - 9146 + - 9148 + - 9151 + - 9156 + - 9158 + - 9160 + - 9177 + - 9179 + - 9182 + - 9184 + - 9186 + - 9192 + - 9205 + - 9214 + - 9216 + - 9220 + - 9224 + - 9226 + - 9228 + - 9238 + - 9242 + - 9246 + - 9248 + - 9253 + - 9254 + - 9284 + - 9289 + - 9290 + - 9302 + - 9306 + - 9309 + - 9312 + - 9314 + - 9327 + - 9334 + - 9337 + - 9347 + - 9350 + - 9361 + - 9363 + - 9367 + - 9396 + - 9399 + - 9404 + - 9451 + - 9455 + - 9467 + - 9546 + - 9622 + - 9646 + - 9657 + - 9665 + - 9730 + - 9771 + - 9790 + - 9809 + - 9830 + - 9862 + - 9867 + - 9876 + - 10076 + - 10096 + - 10099 + - 10135 + - 10153 + - 10155 + - 10164 + - 10170 + - 10225 + - 10256 + - 10270 + - 10280 + - 10292 + - 10307 + - 10316 + - 10350 + - 10356 + - 10362 + - 10371 + - 10381 + - 10385 + - 10506 + - 10517 + - 10612 + - 10627 + - 10666 + - 10692 + - 10697 + - 10816 + - 10822 + - 10832 + - 10852 + - 10939 + - 11071 + - 11287 + - 11417 + - 11442 + - 11492 + - 11577 + - 11656 + - 11687 + - 12042 + - 12082 + - 12086 + - 12151 + - 12157 + - 12279 + - 12292 + - 12294 + - 12306 + - 12312 + - 12325 + - 12331 + - 12336 + - 12352 + - 12355 + - 12365 + - 12382 + - 12384 + - 12392 + - 12396 + - 12406 + - 12414 + - 12421 + - 12437 + - 12446 + - 12457 + - 12461 + - 12467 + - 12482 + - 12486 + - 12504 + - 12511 + - 12537 + - 12541 + - 12571 + - 12576 + - 12581 + - 12602 + - 12607 + - 12610 + - 12617 + - 12622 + - 12626 + - 12629 + - 12637 + - 12642 + - 12655 + - 12666 + - 12671 + - 12681 + - 12706 + - 12712 + - 12715 + - 12722 + - 12732 + - 12735 + - 12751 + - 12754 + - 12767 + - 12792 + - 12796 + - 12800 + - 12805 + - 12809 + - 12814 + - 12822 + - 12824 + - 12834 + - 12839 + - 12844 + - 12852 + - 12857 + - 12876 + - 12886 + - 12891 + - 12896 + - 12930 + - 12937 + - 12945 + - 12951 + - 12956 + - 12961 + - 12967 + - 12971 + - 12976 + - 12981 + - 12990 + - 12995 + - 13001 + - 13012 + - 13013 + - 13026 + - 13029 + - 13035 + - 13042 + - 13046 + - 13049 + - 13056 + - 13067 + - 13100 + - 13107 + - 13112 + - 13114 + - 13132 + - 13141 + - 13147 + - 13174 + - 13181 + - 13185 + - 13197 + - 13200 + - 13206 + - 13214 + - 13220 + - 13221 + - 13305 + - 13313 + - 13323 + - 13326 + - 13329 + - 13335 + - 13341 + - 13342 + - 13348 + - 13353 + - 13359 + - 13362 + - 13369 + - 13375 + - 13379 + - 13381 + - 13383 + - 13391 + - 13393 + - 13397 + - 13399 + - 13403 + - 13411 + - 13413 + - 13419 + - 13421 + - 13427 + - 13431 + - 13435 + - 13439 + - 13449 + - 13451 + - 13457 + - 13459 + - 13467 + - 13477 + - 13481 + - 13489 + - 13497 + - 13511 + - 13513 + - 13517 + - 13519 + - 13521 + - 13523 + - 13525 + - 13527 + - 13528 + - 13533 + - 13535 + - 13547 + - 13553 + - 13573 + - 13579 + - 13581 + - 13587 + - 13591 + - 13593 + - 13598 + - 13603 + - 13605 + - 13606 + - 13612 + - 13615 + - 13619 + - 13621 + - 13622 + - 13626 + - 13629 + - 13633 + - 13641 + - 13643 + - 13647 + - 13655 + - 13657 + - 13661 + - 13665 + - 13673 + - 13677 + - 13689 + - 13695 + - 13696 + - 13703 + - 13707 + - 13708 + - 13719 + - 13723 + - 13724 + - 13731 + - 13739 + - 13743 + - 13746 + - 13749 + - 13751 + - 13753 + - 13757 + - 13765 + - 13769 + - 13770 + - 13773 + - 13781 + - 13783 + - 13793 + - 13795 + - 13798 + - 13802 + - 13804 + - 13807 + - 13809 + - 13818 + - 13825 + - 13835 + - 13839 + - 13841 + - 13849 + - 13851 + - 13853 + - 13854 + - 13857 + - 13859 + - 13871 + - 13883 + - 13886 + - 13893 + - 13895 + - 13897 + - 13915 + - 13917 + - 13918 + - 13921 + - 13933 + - 13937 + - 13947 + - 13951 + - 13959 + - 13963 + - 13965 + - 13967 + - 13971 + - 13983 + - 13993 + - 14003 + - 14009 + - 14015 + - 14016 + - 14019 + - 14030 + - 14032 + - 14035 + - 14041 + - 14053 + - 14057 + - 14063 + - 14068 + - 14074 + - 14126 + - 14157 + - 14191 + - 14208 + - 14274 + - 14277 + - 14306 + - 14341 + - 14347 + - 14386 + - 14467 + - 14491 + - 14536 + - 14611 + - 14621 + - 14661 + - 14692 + - 14701 + - 14716 + - 14728 + - 14916 + - 14978 + - 14996 + - 15011 + - 15022 + - 15127 + - 15152 + - 15161 + - 15177 + - 15183 + - 15201 + - 15231 + - 15236 + - 15261 + - 15271 + - 15281 + - 15289 + - 15296 + - 15309 + - 15328 + - 15331 + - 15346 + - 15354 + - 15358 + - 15371 + - 15377 + - 15383 + - 15386 + - 15403 + - 15406 + - 15411 + - 15417 + - 15423 + - 15446 + - 15451 + - 15503 + - 15508 + - 15519 + - 15531 + - 15541 + - 15550 + - 15551 + - 15559 + - 15581 + - 15599 + - 15603 + - 15617 + - 15628 + - 15631 + - 15639 + - 15643 + - 15689 + - 15709 + - 15721 + - 15727 + - 15731 + - 15742 + - 15762 + - 15773 + - 15777 + - 15783 + - 15788 + - 15846 + - 15866 + - 15877 + - 15883 + - 15921 + - 15932 + - 15938 + - 15942 + - 15947 + - 15951 + - 15977 + - 15982 + - 15987 + - 15997 + - 16013 + - 16017 + - 16021 + - 16026 + - 16032 + - 16046 + - 16056 + - 16076 + - 16216 + - 16276 + - 16396 + - 16411 + - 16486 + - 16617 + - 16727 + - 16776 + - 16791 + - 16812 + - 16827 + - 16836 + - 16967 + - 17034 + - 17171 + - 17277 + - 17281 + - 17397 + - 17436 + - 17443 + - 17467 + - 17497 + - 17541 + - 17554 + - 17606 + - 17647 + - 17656 + - 17676 + - 17721 + - 17728 + - 17841 + - 17990 + - 17995 + - 18002 + - 18004 + - 18006 + - 18019 + - 18034 + - 18060 + - 18065 + - 18072 + - 18074 + - 18080 + - 18088 + - 18092 + - 18094 + - 18100 + - 18106 + - 18121 + - 18123 + - 18126 + - 18128 + - 18142 + - 18145 + - 18156 + - 18162 + - 18173 + - 18180 + - 18184 + - 18188 + - 18192 + - 18198 + - 18200 + - 18202 + - 18210 + - 18221 + - 18223 + - 18243 + - 18248 + - 18250 + - 18252 + - 18256 + - 18261 + - 18262 + - 18266 + - 18294 + - 18298 + - 18300 + - 18303 + - 18306 + - 18308 + - 18316 + - 18320 + - 18322 + - 18324 + - 18327 + - 18328 + - 18332 + - 18340 + - 18344 + - 18350 + - 18352 + - 18365 + - 18366 + - 18381 + - 18387 + - 18390 + - 18394 + - 18401 + - 18404 + - 18406 + - 18418 + - 18423 + - 18424 + - 18431 + - 18435 + - 18436 + - 18438 + - 18442 + - 18447 + - 18448 + - 18455 + - 18456 + - 18460 + - 18466 + - 18469 + - 18470 + - 18473 + - 18477 + - 18482 + - 18484 + - 18488 + - 18491 + - 18492 + - 18496 + - 18500 + - 18510 + - 18514 + - 18518 + - 18522 + - 18530 + - 18533 + - 18534 + - 18540 + - 18543 + - 18545 + - 18546 + - 18556 + - 18566 + - 18578 + - 18582 + - 18584 + - 18592 + - 18598 + - 18601 + - 18602 + - 18617 + - 18620 + - 18622 + - 18624 + - 18645 + - 18654 + - 18656 + - 18662 + - 18664 + - 18667 + - 18676 + - 18678 + - 18680 + - 18682 + - 18684 + - 18690 + - 18694 + - 18710 + - 18712 + - 18714 + - 18718 + - 18726 + - 18732 + - 18736 + - 18740 + - 18744 + - 18747 + - 18748 + - 18750 + - 18755 + - 18756 + - 18760 + - 18767 + - 18770 + - 18780 + - 18782 + - 18784 + - 18788 + - 18796 + - 18798 + - 18802 + - 18804 + - 18806 + - 18808 + - 18814 + - 18819 + - 18829 + - 18831 + - 18832 + - 18873 + - 18922 + - 18953 + - 19065 + - 19091 + - 19107 + - 19117 + - 19126 + - 19148 + - 19200 + - 19272 + - 19290 + - 19326 + - 19361 + - 19377 + - 19388 + - 19397 + - 19433 + - 19463 + - 19468 + - 19556 + - 19582 + - 19602 + - 19637 + - 19697 + - 19702 + - 19712 + - 19718 + - 19732 + - 19738 + - 19776 + - 19792 + - 19806 + - 19923 + - 19933 + - 19993 + - 20028 + - 20043 + - 20046 + - 20081 + - 20107 + - 20148 + - 20212 + - 20223 + - 20302 + - 20442 + - 20493 + - 20507 + - 20525 + - 20543 + - 20552 + - 20742 + - 20848 + - 20883 + - 20943 + - 20988 + - 20991 + - 21022 + - 21091 + - 21098 + - 21138 + - 21143 + - 21167 + - 21238 + - 21240 + - 21331 + - 21363 + - 21407 + - 21413 + - 21423 + - 21468 + - 21543 + - 21548 + - 21562 + - 21606 + - 21698 + - 21703 + - 21726 + - 21743 + - 21753 + - 21758 + - 21770 + - 21782 + - 21785 + - 21793 + - 21798 + - 21803 + - 21808 + - 21818 + - 21842 + - 21850 + - 21862 + - 21868 + - 21878 + - 21882 + - 21892 + - 21898 + - 21903 + - 21905 + - 21917 + - 21933 + - 21938 + - 21947 + - 21952 + - 21963 + - 21973 + - 21975 + - 21980 + - 21998 + - 22006 + - 22011 + - 22018 + - 22033 + - 22036 + - 22046 + - 22056 + - 22061 + - 22065 + - 22088 + - 22090 + - 22095 + - 22103 + - 22107 + - 22118 + - 22122 + - 22126 + - 22140 + - 22193 + - 22197 + - 22200 + - 22206 + - 22216 + - 22225 + - 22240 + - 22246 + - 22251 + - 22262 + - 22267 + - 22286 + - 22296 + - 22325 + - 22362 + - 22365 + - 22371 + - 22380 + - 22392 + - 22397 + - 22403 + - 22407 + - 22418 + - 22427 + - 22437 + - 22441 + - 22467 + - 22476 + - 22493 + - 22497 + - 22503 + - 22507 + - 22513 + - 22517 + - 22522 + - 22525 + - 22531 + - 22553 + - 22556 + - 22563 + - 22576 + - 22588 + - 22590 + - 22598 + - 22602 + - 22627 + - 22631 + - 22639 + - 22660 + - 22683 + - 22742 + - 22745 + - 22752 + - 22755 + - 22765 + - 22787 + - 22791 + - 22793 + - 22794 + - 22809 + - 22811 + - 22815 + - 22816 + - 22818 + - 22821 + - 22826 + - 22828 + - 22831 + - 22835 + - 22840 + - 22845 + - 22847 + - 22853 + - 22857 + - 22865 + - 22867 + - 22871 + - 22873 + - 22879 + - 22881 + - 22883 + - 22891 + - 22892 + - 22917 + - 22919 + - 22921 + - 22925 + - 22927 + - 22931 + - 22938 + - 22951 + - 22961 + - 22963 + - 22977 + - 22979 + - 22991 + - 22996 + - 23003 + - 23009 + - 23015 + - 23019 + - 23021 + - 23022 + - 23032 + - 23035 + - 23041 + - 23043 + - 23045 + - 23047 + - 23051 + - 23052 + - 23055 + - 23061 + - 23063 + - 23065 + - 23067 + - 23071 + - 23077 + - 23089 + - 23105 + - 23111 + - 23113 + - 23115 + - 23117 + - 23120 + - 23123 + - 23129 + - 23131 + - 23133 + - 23137 + - 23138 + - 23141 + - 23163 + - 23165 + - 23167 + - 23170 + - 23173 + - 23175 + - 23177 + - 23181 + - 23191 + - 23196 + - 23199 + - 23205 + - 23206 + - 23209 + - 23213 + - 23215 + - 23217 + - 23226 + - 23231 + - 23232 + - 23235 + - 23242 + - 23247 + - 23251 + - 23263 + - 23268 + - 23271 + - 23275 + - 23277 + - 23278 + - 23283 + - 23293 + - 23295 + - 23312 + - 23317 + - 23333 + - 23335 + - 23342 + - 23347 + - 23349 + - 23351 + - 23359 + - 23360 + - 23363 + - 23373 + - 23381 + - 23383 + - 23384 + - 23387 + - 23389 + - 23392 + - 23395 + - 23397 + - 23403 + - 23417 + - 23427 + - 23430 + - 23435 + - 23439 + - 23443 + - 23445 + - 23447 + - 23455 + - 23465 + - 23467 + - 23468 + - 23471 + - 23481 + - 23487 + - 23492 + - 23498 + - 23690 + - 23802 + - 23882 + - 23952 + - 24005 + - 24026 + - 24080 + - 24090 + - 24230 + - 24310 + - 24366 + - 24390 + - 24460 + - 24502 + - 24626 + - 24633 + - 24645 + - 24675 + - 24683 + - 24695 + - 24731 + - 24782 + - 24797 + - 24802 + - 24865 + - 24900 + - 24995 + - 25015 + - 25025 + - 25070 + - 25131 + - 25260 + - 25272 + - 25346 + - 25367 + - 25433 + - 25439 + - 25510 + - 25545 + - 25567 + - 25570 + - 25596 + - 25605 + - 25625 + - 25670 + - 25691 + - 25695 + - 25700 + - 25707 + - 25720 + - 25726 + - 25733 + - 25737 + - 25745 + - 25750 + - 25781 + - 25791 + - 25826 + - 25838 + - 25840 + - 25848 + - 25850 + - 25864 + - 25872 + - 25887 + - 25900 + - 25927 + - 25938 + - 25942 + - 25956 + - 25970 + - 25980 + - 26006 + - 26015 + - 26035 + - 26040 + - 26056 + - 26066 + - 26086 + - 26092 + - 26095 + - 26125 + - 26145 + - 26150 + - 26160 + - 26166 + - 26170 + - 26176 + - 26183 + - 26187 + - 26190 + - 26197 + - 26201 + - 26228 + - 26238 + - 26247 + - 26250 + - 26273 + - 26286 + - 26291 + - 26295 + - 26306 + - 26315 + - 26320 + - 26330 + - 26336 + - 26348 + - 26352 + - 26356 + - 26361 + - 26372 + - 26391 + - 26402 + - 26428 + - 26436 + - 26446 + - 26460 + - 26465 + - 26472 + - 26475 + - 26487 + - 26501 + - 26551 + - 26556 + - 26562 + - 26582 + - 26585 + - 26596 + - 26621 + - 26625 + - 26631 + - 26636 + - 26656 + - 26659 + - 26661 + - 26669 + - 26685 + - 26688 + - 26704 + - 26724 + - 26759 + - 26767 + - 26777 + - 26788 + - 26794 + - 26798 + - 26800 + - 26804 + - 26807 + - 26808 + - 26810 + - 26815 + - 26816 + - 26818 + - 26835 + - 26836 + - 26838 + - 26840 + - 26844 + - 26862 + - 26876 + - 26886 + - 26892 + - 26897 + - 26898 + - 26900 + - 26912 + - 26924 + - 26932 + - 26942 + - 26946 + - 26950 + - 26960 + - 26970 + - 26974 + - 26978 + - 26982 + - 26984 + - 26986 + - 26990 + - 26992 + - 26995 + - 27002 + - 27006 + - 27008 + - 27011 + - 27012 + - 27018 + - 27024 + - 27030 + - 27032 + - 27036 + - 27038 + - 27044 + - 27050 + - 27052 + - 27056 + - 27065 + - 27068 + - 27070 + - 27075 + - 27076 + - 27080 + - 27093 + - 27100 + - 27102 + - 27110 + - 27114 + - 27120 + - 27125 + - 27127 + - 27128 + - 27132 + - 27136 + - 27139 + - 27140 + - 27143 + - 27144 + - 27150 + - 27160 + - 27167 + - 27169 + - 27177 + - 27184 + - 27186 + - 27188 + - 27191 + - 27192 + - 27194 + - 27196 + - 27198 + - 27200 + - 27202 + - 27210 + - 27220 + - 27224 + - 27228 + - 27231 + - 27234 + - 27237 + - 27243 + - 27244 + - 27253 + - 27260 + - 27265 + - 27267 + - 27271 + - 27288 + - 27292 + - 27298 + - 27300 + - 27304 + - 27306 + - 27312 + - 27320 + - 27324 + - 27328 + - 27330 + - 27346 + - 27350 + - 27358 + - 27360 + - 27362 + - 27364 + - 27374 + - 27376 + - 27378 + - 27382 + - 27390 + - 27402 + - 27415 + - 27423 + - 27430 + - 27446 + - 27448 + - 27450 + - 27453 + - 27454 + - 27456 + - 27460 + - 27462 + - 27466 + - 27474 + - 27482 + - 27490 + - 27494 + - 27499 + - 27500 + - 27502 + - 27504 + - 27510 + - 27515 + - 27517 + - 27520 + - 27533 + - 27534 + passedBenchmark: 2192 diff --git a/test/testinput/qc_gauss_thinning_unittests.yaml b/test/testinput/qc_gauss_thinning_unittests.yaml index 7e52f0b55..c0ce6fa5c 100644 --- a/test/testinput/qc_gauss_thinning_unittests.yaml +++ b/test/testinput/qc_gauss_thinning_unittests.yaml @@ -463,3 +463,41 @@ Vertical mesh, single bin, two categories, nonequal priorities, where clause: name: latitude@MetaData maxvalue: 0 expected_thinned_obs_indices: [1, 3, 6, 9, 12, 14] + +ops_compatibility_mode but round_horizontal_bin_count_to_nearest = false: + window begin: 2000-01-01T00:00:00Z + window end: 2030-01-01T00:00:00Z + obs space: + name: Aircraft + simulated variables: [air_temperature] + generate: + list: + lats: [ -2, -1, 0, 1, 2 ] + lons: [ 178, 179, 180, 181, 182 ] + datetimes: [ '2010-01-01T00:04:00Z', '2010-01-01T00:04:12Z', '2010-01-01T00:04:24Z', + '2010-01-01T00:04:36Z', '2010-01-01T00:04:48Z' ] + obs errors: [1.0] + air_pressures: [ 100000, 100000, 100000, 100000, 100000] + GaussianThinning: + ops_compatibility_mode: true + round_horizontal_bin_count_to_nearest: false + on_deserialization_expect_exception_with_message: round_horizontal_bin_count_to_nearest must not be set to false when ops_compatibility_mode is set to true + +ops_compatibility_mode but distance_norm = geodesic: + window begin: 2000-01-01T00:00:00Z + window end: 2030-01-01T00:00:00Z + obs space: + name: Aircraft + simulated variables: [air_temperature] + generate: + list: + lats: [ -2, -1, 0, 1, 2 ] + lons: [ 178, 179, 180, 181, 182 ] + datetimes: [ '2010-01-01T00:04:00Z', '2010-01-01T00:04:12Z', '2010-01-01T00:04:24Z', + '2010-01-01T00:04:36Z', '2010-01-01T00:04:48Z' ] + obs errors: [1.0] + air_pressures: [ 100000, 100000, 100000, 100000, 100000] + GaussianThinning: + ops_compatibility_mode: true + distance_norm: geodesic + on_deserialization_expect_exception_with_message: distance_norm must not be set to 'geodesic' when ops_compatibility_mode is set to true diff --git a/test/ufo/GaussianThinning.h b/test/ufo/GaussianThinning.h index 00cac6c36..2fb0444e4 100644 --- a/test/ufo/GaussianThinning.h +++ b/test/ufo/GaussianThinning.h @@ -65,7 +65,14 @@ void testGaussianThinning(const eckit::LocalConfiguration &conf) { eckit::LocalConfiguration filterConf(conf, "GaussianThinning"); ufo::GaussianThinningParameters filterParameters; - filterParameters.validateAndDeserialize(filterConf); + std::string expectedMessage; + if (conf.get("on_deserialization_expect_exception_with_message", expectedMessage)) { + EXPECT_THROWS_MSG(filterParameters.validateAndDeserialize(filterConf), expectedMessage.c_str()); + return; + } else { + filterParameters.validateAndDeserialize(filterConf); + } + ufo::Gaussian_Thinning filter(obsspace, filterParameters, qcflags, obserr); filter.preProcess(); diff --git a/test/ufo/RecursiveSplitter.h b/test/ufo/RecursiveSplitter.h index 8e9f08f04..9ee692d91 100644 --- a/test/ufo/RecursiveSplitter.h +++ b/test/ufo/RecursiveSplitter.h @@ -227,10 +227,7 @@ CASE("ufo/RecursiveSplitter/Recurse") { // Final sorting of groups categories = {9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; // effectively reverse ordering of final groups - splitter.sortGroupsBy( - [&categories](auto indexA, auto indexB) { - return categories[indexA] < categories[indexB]; - }); + splitter.sortGroupsBy([&categories](auto index) { return categories[index]; }); expected = {{7}, {9, 8}, {6, 5}, {2}, {1, 0}, {4, 3}}; orderedComparison(splitter, expected); } From b152dba7d0de2827ff2fa96520724b248f6e030f Mon Sep 17 00:00:00 2001 From: Carwyn Pelley Date: Tue, 3 Aug 2021 14:53:06 +0100 Subject: [PATCH 65/98] Bilinear interpolation method support for DrawValueFromFile or DrawObsErrorFromFile (#1380) * Minor changes * ENH: First pass UI for the bilinear interpolator * MAINT: Comb. parameters for bilinear UI fail * ENH: Working basic bilinear TODO: - Support range (i.e. extraction). - Test different data type combinations. - Add end-to-end test (yaml). - Extend everything to support >2D arrays? * TEST: Mixed type tests added. - Fixed exception msg. TODO: - Alternate dimension mapping. - range support. - 3D arrays. * ENH: Outside bounds assign missing handling. Next: - Test missing data handling (assign nearest neighbour). * ENH: Range support and tested. * TEST: End-to-end DrawValueFromFile with bilinear * MAINT: Coding norms * BUG: Inline function * MAINT: Review changes * MAINT: Review changes bugfix closest missing * ENH: Split Linear and Bilinear methods * DOC: Neglected to update the doc to reflect previous commit * MAINT: Split payloads into separate files * MAINT: Minor changes for Chris Co-authored-by: Ryan Honeyager --- .../filters/obsfunctions/DrawValueFromFile.cc | 42 ++- .../filters/obsfunctions/DrawValueFromFile.h | 3 +- src/ufo/utils/dataextractor/DataExtractor.cc | 19 +- src/ufo/utils/dataextractor/DataExtractor.h | 263 ++++++++++++++- test/CMakeLists.txt | 9 + test/mains/TestDataExtractor.cc | 15 + .../function_drawvaluefromfile.yaml | 129 ++++++++ test/ufo/DataExtractor.h | 304 ++++++++++++++++++ test/ufo/ObsFunction.h | 7 + 9 files changed, 766 insertions(+), 25 deletions(-) create mode 100644 test/mains/TestDataExtractor.cc create mode 100644 test/ufo/DataExtractor.h diff --git a/src/ufo/filters/obsfunctions/DrawValueFromFile.cc b/src/ufo/filters/obsfunctions/DrawValueFromFile.cc index 2f7f01c06..b5582f3a5 100644 --- a/src/ufo/filters/obsfunctions/DrawValueFromFile.cc +++ b/src/ufo/filters/obsfunctions/DrawValueFromFile.cc @@ -13,10 +13,10 @@ namespace ufo { namespace { // ----------------------------------------------------------------------------- -typedef std::list, - std::vector, - std::vector> - >> ObData; +typedef std::vector, + std::vector, + std::vector> + >> ObData; // ----------------------------------------------------------------------------- @@ -67,16 +67,26 @@ DrawValueFromFile::DrawValueFromFile(const eckit::LocalConfiguration &config) std::vector interpSubConfs; const std::vector &interpolationParameters = options_.interpolation.value(); + int nlin = 0; for (auto intParam = interpolationParameters.begin(); intParam != interpolationParameters.end(); ++intParam) { const ufo::InterpMethod & method = intParam->method.value(); - if ((method == InterpMethod::LINEAR) && (intParam + 1 != interpolationParameters.end())) { + if (method == InterpMethod::BILINEAR) { + nlin++; + } else if (nlin > 0) { + throw eckit::UserError("Bilinear interpolation can only be supplied as the final two " + "arguments.", Here()); + } else if ((method == InterpMethod::LINEAR) && + (intParam + 1 != interpolationParameters.end())) { throw eckit::UserError("Linear interpolation can only be supplied as the very last " "argument.", Here()); } interpSubConfs.push_back(intParam->toConfiguration()); interpMethod_[intParam->name.value()] = method; } + if (nlin > 0 && nlin != 2) { + throw eckit::UserError("Bilinear interpolation requires two variables.", Here()); + } // Get channels from options if (options_.chlist.value() != boost::none) { std::set channels = options_.chlist.value().get(); @@ -98,8 +108,12 @@ class ExtractVisitor : public boost::static_visitor { template void operator()(const std::vector &obDat) { - auto obVal = obDat[iloc]; - interpolator.extract(obVal); + interpolator.extract(obDat[iloc]); + } + + template + void operator()(const std::vector &obDat1, const std::vector &obDat2) { + interpolator.extract(obDat1[iloc], obDat2[iloc]); } DataExtractor &interpolator; @@ -150,10 +164,16 @@ void DrawValueFromFile::compute(const ObsFilterData & in, if (options_.chlist.value() != boost::none) interpolator.extract(channels_[jvar]); - // Perform any extraction methods (exact, nearest and linear interp.) - for (auto &od : obData) { - ExtractVisitor visitor(interpolator, iloc); - boost::apply_visitor(visitor, od.second); + // Perform any extraction methods. + ExtractVisitor visitor(interpolator, iloc); + for (size_t ind=0; ind < obData.size(); ind++) { + const InterpMethod &interpolationMethod = interpMethod_.at(obData[ind].first); + if ((interpolationMethod == InterpMethod::BILINEAR) && (ind == (obData.size()-2))) { + boost::apply_visitor(visitor, obData[ind].second, obData[ind+1].second); + break; + } else { + boost::apply_visitor(visitor, obData[ind].second); + } } out[jvar][iloc] = interpolator.getResult(); } diff --git a/src/ufo/filters/obsfunctions/DrawValueFromFile.h b/src/ufo/filters/obsfunctions/DrawValueFromFile.h index 9010a61ae..80b019b84 100644 --- a/src/ufo/filters/obsfunctions/DrawValueFromFile.h +++ b/src/ufo/filters/obsfunctions/DrawValueFromFile.h @@ -41,7 +41,8 @@ struct InterpMethodParameterTraitsHelper { { InterpMethod::NEAREST, "nearest" }, { InterpMethod::LEAST_UPPER_BOUND, "least upper bound" }, { InterpMethod::GREATEST_LOWER_BOUND, "greatest lower bound" }, - { InterpMethod::LINEAR, "linear" } + { InterpMethod::LINEAR, "linear" }, + { InterpMethod::BILINEAR, "bilinear" } }; }; diff --git a/src/ufo/utils/dataextractor/DataExtractor.cc b/src/ufo/utils/dataextractor/DataExtractor.cc index 7fab07eb8..85b42638b 100644 --- a/src/ufo/utils/dataextractor/DataExtractor.cc +++ b/src/ufo/utils/dataextractor/DataExtractor.cc @@ -14,13 +14,11 @@ #include -#include "eckit/exception/Exceptions.h" #include "eckit/utils/StringTools.h" #include "ioda/Misc/StringFuncs.h" #include "oops/util/Logger.h" -#include "oops/util/missingValues.h" #include "ufo/utils/dataextractor/DataExtractor.h" #include "ufo/utils/dataextractor/DataExtractorCSVBackend.h" @@ -515,9 +513,15 @@ void DataExtractor::sort() { template void DataExtractor::scheduleSort(const std::string &varName, const InterpMethod &method) { - if (method == InterpMethod::LINEAR && !std::is_floating_point::value) - throw eckit::BadParameter("Linear interpolation can be used when extracting floating-point " - "values, but not integers or strings.", Here()); + if (!std::is_floating_point::value) { + std::string msg = "interpolation can be used when extracting floating-point values, but not " + "integers or strings."; + if (method == InterpMethod::LINEAR) { + throw eckit::BadParameter("Linear " + msg, Here()); + } else if (method == InterpMethod::BILINEAR) { + throw eckit::BadParameter("Bilinear " + msg, Here()); + } + } // Map any names of the form var@Group to Group/var const std::string canonicalVarName = ioda::convertV1PathToV2Path(varName); @@ -554,10 +558,9 @@ void DataExtractor::extract(const std::string &obVal) { template template void DataExtractor::extractImpl(const T &obVal) { - if (nextCoordToExtractBy_ == coordsToExtractBy_.cend()) { + if (nextCoordToExtractBy_ == coordsToExtractBy_.cend()) throw eckit::UserError("Too many extract() calls made for the expected number of variables.", Here()); - } // Perform the extraction using the selected method if (nextCoordToExtractBy_->method == InterpMethod::LINEAR) @@ -612,7 +615,7 @@ template <> float DataExtractor::getResult() { // Fetch the result if (resultSet_) { - // This was derived from linear interpolation so return it. + // This was derived from linear/bilinear interpolation so return it. resetExtract(); return result_; } diff --git a/src/ufo/utils/dataextractor/DataExtractor.h b/src/ufo/utils/dataextractor/DataExtractor.h index 4c2e9836c..04405daca 100644 --- a/src/ufo/utils/dataextractor/DataExtractor.h +++ b/src/ufo/utils/dataextractor/DataExtractor.h @@ -8,17 +8,25 @@ #ifndef UFO_UTILS_DATAEXTRACTOR_DATAEXTRACTOR_H_ #define UFO_UTILS_DATAEXTRACTOR_DATAEXTRACTOR_H_ +#include +#include // std::numeric_limits #include // unique_ptr #include #include #include #include +#include #include +#include "eckit/exception/Exceptions.h" + +#include "oops/util/missingValues.h" + #include "ufo/utils/dataextractor/ConstrainedRange.h" #include "ufo/utils/RecursiveSplitter.h" + namespace ioda { class Variable; struct Named_Variable; @@ -27,6 +35,184 @@ struct Named_Variable; namespace ufo { + +/// \brief Perform bilinear interpolation along axis `dimIndex0` and `dimIndex1` at 'location' +/// `obVal0` and `obVal1` corresponding to those axes. +/// +/// \details This function returns the value produced by a bilinear interpolation at point +/// `obVal0, obVal1`. Where any of the neighbouring points used in the calculation are missing, +/// return the nearest of the non-missing neighbouring points. Where all 4 neighbours are missing +/// or the coordinate value is out of bounds, return a missing value. +/// This must be a final call in the sequence calls to extract. This template handles all types +/// except string (see below overloads). +/// +/// \param[in] dimIndex0 +/// One of two axes of `interpolatedArray` along which to interpolate. This axis corresponds to +/// that which `varName0`, `varValues0` describes and to which `obVal0` is to be interpolated. +/// \param[in] dimIndex1 +/// One of two axes of `interpolatedArray` along which to interpolate. This axis corresponds to +/// that which `varName1`, `varValues1` describes and to which `obVal1` is to be interpolated. +/// \param[in] ranges +/// Ranges of slices of `interpolatedArray` matching all constraints considered so far. +/// \param[in] varName0 +/// Name of the coordinate along one axis of two over which to interpolate. +/// \param[in] varName1 +/// Name of the coordinate along one axis of two over which to interpolate. +/// \param[in] varValues0 +/// Vector of values of the `varName0` coordinate. +/// \param[in] varValues1 +/// Vector of values of the `varName1` coordinate. +/// \param[in] obVal0 +/// Interpolation location along the axis corresponding to `varName0`. +/// \param[in] obVal1 +/// Interpolation location along the axis corresponding to `varName1`. +/// \param[in] interpolatedArray +/// Interpolated array. +template +float bilinearInterpolation(const size_t dimIndex0, + const std::string &varName0, + const std::vector &varValues0, + const T &obVal0, + const size_t dimIndex1, + const std::string &varName1, + const std::vector &varValues1, + const R &obVal1, + const std::array &ranges, + const boost::multi_array &interpolatedArray) { + const float missing = util::missingValue(missing); + + // Constrain our index range in the relevant dimension. + const ConstrainedRange &range0 = ranges[dimIndex0]; + const ConstrainedRange &range1 = ranges[dimIndex1]; + + const int nnIndex0 = std::lower_bound(varValues0.begin() + range0.begin(), + varValues0.begin() + range0.end(), obVal0) - + varValues0.begin(); + const int nnIndex1 = std::lower_bound(varValues1.begin() + range1.begin(), + varValues1.begin() + range1.end(), obVal1) - + varValues1.begin(); + + // Convenience function to extract specified index from the multi-array + auto getMultiArrayVal = [&] (int index0, int index1) { + if (dimIndex0 == 0) { + return interpolatedArray[index0][index1]; + } else { + return interpolatedArray[index1][index0]; + } + }; + + if ((varValues0[nnIndex0] == obVal0) && (varValues1[nnIndex1] == obVal1)) { + // No interpolation required. + return getMultiArrayVal(nnIndex0, nnIndex1); + } + + // Setup points + // ------------ + // Indices + const int ix1 = nnIndex0-1 >= 0 ? nnIndex0-1 : nnIndex0; + const int ix2 = ix1 + 1; + const int iy1 = nnIndex1-1 >= 0 ? nnIndex1-1 : nnIndex1; + const int iy2 = iy1 + 1; + + // Coord locations + const T x1 = varValues0[ix1], x2 = varValues0[ix2]; + const R y1 = varValues1[iy1], y2 = varValues1[iy2]; + + // Out of bounds check + if ((obVal0 > varValues0[range0.end()-1]) || (obVal0 < varValues0[range0.begin()])) + return missing; + if ((obVal1 > varValues1[range1.end()-1]) || (obVal1 < varValues1[range1.begin()])) + return missing; + + // Z values at these locations + const float q11 = getMultiArrayVal(ix1, iy1), q12 = getMultiArrayVal(ix1, iy2), + q22 = getMultiArrayVal(ix2, iy2), q21 = getMultiArrayVal(ix2, iy1); + + // Missing data handling + // - Pick nearest non-missing neighbour of the moore neighbourhood (no diagonals) if any of these + // 4 neighbours are missing. + // - Return missing if all 4 of the moore neighbourhood (no diagonals) are missing. + const int nmissing = (q11 == missing) + (q12 == missing) + (q22 == missing) + (q21 == missing); + if (nmissing > 0) { + if (nmissing == 4) + return missing; + const std::array xd = {x1, x1, x2, x2}; + const std::array yd = {y1, y2, y2, y1}; + const std::array qd = {q11, q12, q22, q21}; + + float minq, mindist = std::numeric_limits::max(); + for (size_t ind=0; ind < qd.size(); ind++) { + // Euclidean distance between two points. + const float currdist = std::hypot(obVal0-xd[ind], obVal1-yd[ind]); + if (currdist < mindist && qd[ind] != missing) { + mindist = currdist; + minq = qd[ind]; + } + } + return minq; + } + + // Interpolate at obVal0 and obVal1 + const float denom = static_cast((x2 - x1)*(y2 - y1)); + float res = (((x2 - obVal0)*(y2 - obVal1))) * q11; + res += (((obVal0 - x1)*(y2 - obVal1))) * q21; + res += (((x2 - obVal0)*(obVal1 - y1))) * q12; + res += (((obVal0 - x1)*(obVal1 - y1))) * q22; + return res/denom; +} + + +/// \brief Bilinear interpolation template, templated coord1, string coord2. +template +float bilinearInterpolation(const size_t dimIndex0, + const std::string &varName0, + const std::vector &varValues0, + const T &obVal0, + const size_t dimIndex1, + const std::string &varName1, + const std::vector &varValues1, + const std::string &obVal1, + const std::array &ranges, + const boost::multi_array &interpolatedArray) { + throw eckit::UserError("Bilinear interpolation cannot be performed along coordinate axes indexed " + "by string variables such as " + varName1 + ".", Here()); +} + + +/// \brief Bilinear interpolation template, string coord1, templated coord2. +template +float bilinearInterpolation(const size_t dimIndex0, + const std::string &varName0, + const std::vector &varValues0, + const std::string &obVal0, + const size_t dimIndex1, + const std::string &varName1, + const std::vector &varValues1, + const R &obVal1, + const std::array &ranges, + const boost::multi_array &interpolatedArray) { + throw eckit::UserError("Bilinear interpolation cannot be performed along coordinate axes indexed " + "by string variables such as " + varName0 + ".", Here()); +} + + +// Bilinear interpolation template, string coord1, string coord2. +inline float bilinearInterpolation(const size_t dimIndex0, + const std::string &varName0, + const std::vector &varValues0, + const std::string &obVal0, + const size_t dimIndex1, + const std::string &varName1, + const std::vector &varValues1, + const std::string &obVal1, + const std::array &ranges, + const boost::multi_array &interpolatedArray) { + throw eckit::UserError("Bilinear interpolation cannot be performed along coordinate axes indexed " + "by string variables such as " + varName0 + " or " + + varName1 + ".", Here()); +} + + template class DataExtractorBackend; @@ -63,9 +249,17 @@ enum class InterpMethod { /// \brief Perform a piecewise linear interpolation along the dimension indexed by the ObsSpace /// variable. /// - /// This method can only be used for the last indexing variable. It is supported only when the + /// This method can only be used for the last indexing variable. It is supported only when /// DataExtractor produces values of type `float`, but not `int` or `std::string`. - LINEAR + LINEAR, + + /// \brief Perform a bilinear interpolation along two dimensions indexed by the ObsSpace + /// variables. + /// + /// This method can only be used for the final two indexing variables (see + /// `bilinearInterpolation`). It is supported only when DataExtractor produces values of type + /// `float`, but not `int` or `std::string`. + BILINEAR }; @@ -83,8 +277,8 @@ enum class InterpMethod { /// possible to rapidly extract a value from the payload array corresponding to particular values /// of the coordinates, or to interpolate multiple values from this array. Coordinate matching can /// be exact or approximate (looking for the nearest match). It is also possible to perform a -/// piecewise linear interpolation of the data along one coordinate axis. (Multidimensional linear -/// interpolation is not supported.) +/// piecewise linear interpolation of the data along one coordinate axis or bilinear interpolation +/// along two axes. /// /// Here's how to use this class: /// @@ -144,7 +338,7 @@ class DataExtractor /// \brief Perform extract, given an observation value for the coordinate associated with this /// extract iteration. - /// \details Calls the relevant extract methood (linear, nearest or exact), corresponding to the + /// \details Calls the relevant extract method (linear, nearest or exact), corresponding to the /// coordinate associated with this extract iteration (along with the associated interpolation /// method). The extract then utilises the observation value ('obVal') to perform this /// extraction. @@ -153,6 +347,31 @@ class DataExtractor void extract(int obVal); void extract(const std::string &obVal); + /// \brief Perform extract, given the observation values associated with the current extract + /// iteration and the next. + /// \details Calls the relevant extract methood (linear), corresponding to the coordinate + /// associated with this extract iteration and the next (along with the associated interpolation + /// method). This method actually functions as two iterations, passing the current iteration + /// coordinate and the next iteration coordinate. These are passed to the underlying binary + /// operation (for example bilinear interpolation). + /// \param[in] obValDim0 is the observation value used for the extract operation corresponding + /// to the first coordinate utilised by the underlying method. + /// \param[in] obValDim1 is the observation value used for the extract operation corresponding + /// to the second coordinate utilised by the underlying method. + template + void extract(T obValDim0, R obValDim1) { + if (nextCoordToExtractBy_ == coordsToExtractBy_.cend()) + throw eckit::UserError("Too many extract() calls made for the expected number of variables.", + Here()); + + // Perform the extraction using the selected method + if (nextCoordToExtractBy_->method == InterpMethod::BILINEAR) + maybeExtractByBiLinearInterpolation(obValDim0, obValDim1); + else + throw eckit::UserError("Only bilinear method supports two variables as arguments.", Here()); + ++nextCoordToExtractBy_; + } + /// \brief Fetch the final interpolated value. /// \details This will only be succesful if previous calls to extract() have produced a single /// value to return. @@ -168,6 +387,13 @@ class DataExtractor template void maybeExtractByLinearInterpolation(const T &obVal); + template + void maybeExtractByBiLinearInterpolation(const T &obValDim0, const R &obValDim1) { + // Should never be called -- this error should be detected earlier (scheduleSort). + throw eckit::BadParameter("Bilinear interpolation can be used when extracting floating-point " + "values, but not integers or strings.", Here()); + } + /// \brief Fetch the result produced by previous calls to extract(), none of which may have /// used linear interpolation. /// @@ -231,6 +457,33 @@ class DataExtractor typename std::vector::const_iterator nextCoordToExtractBy_; }; + +// Specialization for ExtractedValue = float. +template <> +template +void DataExtractor::maybeExtractByBiLinearInterpolation( + const T &obValDim0, const R &obValDim1) { + auto &range = constrainedRanges_; + + const size_t dimIndex0 = nextCoordToExtractBy_->payloadDim; + const std::string &varName0 = nextCoordToExtractBy_->name; + const std::vector &varValues0 = boost::get>(nextCoordToExtractBy_->values); + ++nextCoordToExtractBy_; // Consume variable + + if (nextCoordToExtractBy_->method != InterpMethod::BILINEAR) + throw eckit::BadParameter("Second parameter provided to the Bilinear interpolator is not of " + "method 'bilinear'.", Here()); + const size_t dimIndex1 = nextCoordToExtractBy_->payloadDim; + const std::string &varName1 = nextCoordToExtractBy_->name; + + const std::vector &varValues1 = boost::get>(nextCoordToExtractBy_->values); + result_ = bilinearInterpolation(dimIndex0, varName0, varValues0, obValDim0, + dimIndex1, varName1, varValues1, obValDim1, + range, interpolatedArray2D_); + resultSet_ = true; +} + + } // namespace ufo #endif // UFO_UTILS_DATAEXTRACTOR_DATAEXTRACTOR_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4bc8a38c4..d2c6eb565 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -2229,6 +2229,15 @@ ecbuild_add_test( TARGET test_ufo_recursivesplitter LIBS ufo TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_dataextractor + SOURCES mains/TestDataExtractor.cc + # This test doesn't need a configuration file, but oops::Run::Run() requires + # a path to a configuration file to be passed in the first command-line parameter. + ARGS "testinput/empty.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + LIBS ufo + TEST_DEPENDS ufo_get_ufo_test_data ) + ecbuild_add_test( TARGET test_ufo_qc_stuckcheck_unittests SOURCES mains/TestStuckCheck.cc ARGS "testinput/qc_stuckcheck_unittests.yaml" diff --git a/test/mains/TestDataExtractor.cc b/test/mains/TestDataExtractor.cc new file mode 100644 index 000000000..a362bd329 --- /dev/null +++ b/test/mains/TestDataExtractor.cc @@ -0,0 +1,15 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "../ufo/DataExtractor.h" +#include "oops/runs/Run.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + ufo::test::DataExtractor tests; + return run.execute(tests); +} diff --git a/test/testinput/unit_tests/function_drawvaluefromfile.yaml b/test/testinput/unit_tests/function_drawvaluefromfile.yaml index 7c051ebbe..6d644e2a7 100644 --- a/test/testinput/unit_tests/function_drawvaluefromfile.yaml +++ b/test/testinput/unit_tests/function_drawvaluefromfile.yaml @@ -283,6 +283,24 @@ observations: variables: [assigned_string_variable_1] expect compute to throw exception with message: Linear interpolation can be used when extracting floating-point values, but not integers or strings +- obs space: &ObsSpace + name: Attempt to use bilinear interpolation involving an int payload + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@IntObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_advanced2.nc + group: IntResult2D + interpolation: + - name: longitude@MetaData + method: bilinear + - name: latitude@MetaData + method: bilinear + variables: [assigned_float_variable_bilin] + expect compute to throw exception with message: Bilinear interpolation can be used when extracting floating-point values, but not integers or strings + # 4. Tests of DrawValueFromFile@StringObsFunction using a NetCDF input file @@ -366,3 +384,114 @@ observations: method: linear variables: [assigned_string_variable_1] expect compute to throw exception with message: Linear interpolation can be used when extracting floating-point values, but not integers or strings + +- obs space: &ObsSpace + name: Attempt to use bilinear interpolation involving a string payload + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@StringObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_advanced3.nc + group: StringResult2D + interpolation: + - name: longitude@MetaData + method: bilinear + - name: latitude@MetaData + method: bilinear + variables: [assigned_float_variable_bilin] + expect compute to throw exception with message: Bilinear interpolation can be used when extracting floating-point values, but not integers or strings + +# 5. Tests of DrawValueFromFile@ObsFunction (float) using a NetCDF input file + +- obs space: &ObsSpace + name: Attempt to use bilinear interpolation to extract a float variable + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_advanced.nc + group: FloatResult2D + interpolation: + - name: longitude@MetaData + method: bilinear + - name: latitude@MetaData + method: bilinear + variables: [assigned_float_variable_bilin] + tolerance: 1.0e-6 + +- obs space: &ObsSpace + name: Attempt to use bilinear interpolation not as the final calls + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_advanced.nc + group: FloatResult2D + interpolation: + - name: longitude@MetaData + method: bilinear + - name: latitude@MetaData + method: nearest + variables: [assigned_float_variable_bilin] + expect constructor to throw exception with message: Bilinear interpolation can only be supplied as the final two arguments + +- obs space: &ObsSpace + name: Attempt to use bilinear interpolation with only one variable + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_advanced.nc + group: FloatResult2D + interpolation: + - name: longitude@MetaData + method: nearest + - name: latitude@MetaData + method: bilinear + variables: [assigned_float_variable_bilin] + expect constructor to throw exception with message: Bilinear interpolation requires two variables + +- obs space: &ObsSpace + name: Attempt to use linear interpolation not as the final argument + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_advanced.nc + group: FloatResult2D + interpolation: + - name: longitude@MetaData + method: linear + - name: latitude@MetaData + method: nearest + variables: [assigned_float_variable_bilin] + expect constructor to throw exception with message: Linear interpolation can only be supplied as the very last argument + +- obs space: &ObsSpace + name: Attempt to use linear interpolation more than one variable + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_advanced.nc + group: FloatResult2D + interpolation: + - name: longitude@MetaData + method: linear + - name: latitude@MetaData + method: linear + variables: [assigned_float_variable_bilin] + tolerance: 1.0e-6 + expect constructor to throw exception with message: Linear interpolation can only be supplied as the very last argument diff --git a/test/ufo/DataExtractor.h b/test/ufo/DataExtractor.h new file mode 100644 index 000000000..ae13981c8 --- /dev/null +++ b/test/ufo/DataExtractor.h @@ -0,0 +1,304 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef TEST_UFO_DATAEXTRACTOR_H_ +#define TEST_UFO_DATAEXTRACTOR_H_ + +#include "ufo/utils/dataextractor/DataExtractor.h" + +#include +#include +#include +#include +#include + +#include "eckit/testing/Test.h" +#include "oops/runs/Test.h" +#include "oops/util/Expect.h" +#include "oops/util/FloatCompare.h" + +namespace ufo { +namespace test { + +float missing = util::missingValue(missing); + + +template +float run_basic(const T obVal0, const R obVal1, const std::vector &varValues0, + const std::vector &varValues1) { + const int dimIndex0 = 0; + const int dimIndex1 = 1; + const std::string &varName0 = "var0"; + const std::string &varName1 = "var1"; + const std::array ranges { + ConstrainedRange(varValues0.size()), + ConstrainedRange(varValues1.size())}; + assert(varValues0.size() == 5); + assert(varValues1.size() == 3); + + boost::multi_array interpolatedArray(boost::extents[5][3]); + std::vector tmp = {1, 2, 3, 4, 5, + 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15}; + size_t ind = 0; + for (int j=0; j < interpolatedArray.shape()[1]; j++) { + for (int i=0; i < interpolatedArray.shape()[0]; i++) { + interpolatedArray[i][j] = tmp[ind]; + ind++; + } + } + return bilinearInterpolation(dimIndex0, varName0, varValues0, obVal0, + dimIndex1, varName1, varValues1, obVal1, + ranges, interpolatedArray); +} + + +// For int/float calls +template +float run_basic(const T obVal0, const R obVal1) { + const std::vector varValues0 {2, 4, 6, 8, 10}; + const std::vector varValues1 {2, 4, 6}; + return run_basic(obVal0, obVal1, varValues0, varValues1); +} + + +CASE("ufo/DataExtractor/bilinearinterp/float_linear") { + // Effectively becomes linear interpolation along dim1. + const float res = run_basic(4.0, 4.2); + EXPECT(oops::is_close_absolute(res, 7.5f, 1e-5f, 0, + oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); +} + + +CASE("ufo/DataExtractor/bilinearinterp/float_linear_at_lower_boundary_dim0") { + // Check handling where our point is located on the lower boundary. + const float res = run_basic(2, 4); + EXPECT(oops::is_close_absolute(res, 6.0f, 1e-5f, 0, + oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); +} + + +CASE("ufo/DataExtractor/bilinearinterp/float_linear_at_lower_boundary_dim1") { + // Check handling where our point is located on the lower boundary. + const float res = run_basic(4, 2); + EXPECT(oops::is_close_absolute(res, 2.0f, 1e-5f, 0, + oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); +} + + +CASE("ufo/DataExtractor/bilinearinterp/float_blinear") { + // Simple bilinear interpolation. + const float res = run_basic(4.2, 4.2); + EXPECT(oops::is_close_absolute(res, 7.6f, 1e-5f, 0, + oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); +} + + +CASE("ufo/DataExtractor/bilinearinterp/extrapolation_lower_bound_dim0") { + // Lower bound extrapolation dim0 + const float res = run_basic(0, 4.2); + EXPECT_EQUAL(res, missing); +} + + +CASE("ufo/DataExtractor/bilinearinterp/extrapolation_lower_bound_dim1") { + // Lower bound extrapolation dim1 + const float res = run_basic(4.2, 0.0); + EXPECT_EQUAL(res, missing); +} + + +CASE("ufo/DataExtractor/bilinearinterp/extrapolation_upper_bound_dim0") { + // Upper bound extrapolation dim0 + const float res = run_basic(20.0, 4.2); + EXPECT_EQUAL(res, missing); +} + + +CASE("ufo/DataExtractor/bilinearinterp/extrapolation_upper_bound_dim1") { + // Upper bound extrapolation dim1 + const float res = run_basic(4.2, 20); + EXPECT_EQUAL(res, missing); +} + + +CASE("ufo/DataExtractor/bilinearinterp/int_int_dtype") { + const float res = run_basic(3, 3); + EXPECT(oops::is_close_absolute(res, 4.0f, 1e-5f, 0, + oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); +} + + +CASE("ufo/DataExtractor/bilinearinterp/int_float_dtype") { + const float res = run_basic(3, 3.0); + EXPECT(oops::is_close_absolute(res, 4.0f, 1e-5f, 0, + oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); +} + +CASE("ufo/DataExtractor/bilinearinterp/float_int_dtype") { + const float res = run_basic(3.0, 3); + EXPECT(oops::is_close_absolute(res, 4.0f, 1e-5f, 0, + oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); +} + + +CASE("ufo/DataExtractor/bilinearinterp/string_dtype") { + const std::vector strVarValues0 {"2", "4", "6", "8", "10"}; + const std::vector strVarValues1 {"2", "4", "6"}; + const std::vector floatVarValues0 {2, 4, 6, 8, 10}; + const std::vector floatVarValues1 {2, 4, 6}; + const std::string msg = "Bilinear interpolation cannot be performed along coordinate axes " + "indexed by string variables such as "; + EXPECT_THROWS_MSG(run_basic(4.2f, std::string("4.2"), floatVarValues0, strVarValues1), + (msg + "var1.").c_str()); + EXPECT_THROWS_MSG(run_basic(std::string("4.2"), 4.2f, strVarValues0, floatVarValues1), + (msg + "var0.").c_str()); + EXPECT_THROWS_MSG(run_basic(std::string("4.2"), std::string("4.2"), + strVarValues0, strVarValues1), + (msg + "var0 or var1.").c_str()); +} + + +float run_missing(const float obVal0, const float obVal1, const std::vector data) { + const std::vector varValues0 {2, 4, 6, 8, 10}; + const std::vector varValues1 {2, 4, 6}; + const std::array ranges { + ConstrainedRange(varValues0.size()), + ConstrainedRange(varValues1.size())}; + + boost::multi_array interpolatedArray(boost::extents[5][3]); + size_t ind = 0; + for (int j=0; j < interpolatedArray.shape()[1]; j++) { + for (int i=0; i < interpolatedArray.shape()[0]; i++) { + interpolatedArray[i][j] = data[ind]; + ind++; + } + } + return bilinearInterpolation(0, "var0", varValues0, obVal0, + 1, "var1", varValues1, obVal1, + ranges, interpolatedArray); +} + + +CASE("ufo/DataExtractor/bilinearinterp/one_missing") { + // If one missing, pick closes of non missing neighbours. + const std::vector data = {missing, 2, 3, 4, 5, + 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15}; + // Pick closes non-missing neighbour. + float res = run_missing(3.1, 2.5, data); + EXPECT(oops::is_close_absolute(res, 2.0f, 1e-5f, 0, + oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); + + // Different neighbour is closest. + res = run_missing(2.5, 3.1, data); + EXPECT(oops::is_close_absolute(res, 6.0f, 1e-5f, 0, + oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); + + // Avoid missing closest value. + res = run_missing(2.5, 2.1, data); + EXPECT(oops::is_close_absolute(res, 2.0f, 1e-5f, 0, + oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); +} + + +CASE("ufo/DataExtractor/bilinearinterp/all_missing") { + // If one missing, pick closest of non missing neighbours. + const std::vector data = {missing, missing, 3, 4, 5, + missing, missing, 8, 9, 10, + 11, 12, 13, 14, 15}; + const float res = run_missing(3.1, 2.5, data); + EXPECT_EQUAL(res, missing); +} + + +CASE("ufo/DataExtractor/bilinearinterp/range_constrain") { + // Let's constrain the range and change the dimension mapping. + const float obVal0 = 3.0, obVal1 = 3.0; + // Coordinates containing values that would change the answer if not excluded + // via the "range" specified. + const std::vector varValues0 {2, 4, 2, 4, 6}; + const std::vector varValues1 {3, 2, 4}; + + ConstrainedRange con0 = ConstrainedRange(varValues0.size()); + con0.constrain(2, varValues0.size()); // range will ignore 1st two. + ConstrainedRange con1 = ConstrainedRange(varValues1.size()); + con1.constrain(1, varValues1.size()); // range will ignore 1st. + const std::array ranges {con1, con0}; + + const std::vector data = {1, 6, 11, + 2, 7, 12, + 3, 8, 13, + 4, 9, 14, + 5, 10, 15}; + boost::multi_array interpolatedArray(boost::extents[3][5]); + size_t ind = 0; + for (int j=0; j < interpolatedArray.shape()[1]; j++) { + for (int i=0; i < interpolatedArray.shape()[0]; i++) { + interpolatedArray[i][j] = data[ind]; + ind++; + } + } + const float res = bilinearInterpolation(1, "var0", varValues0, obVal0, + 0, "var1", varValues1, obVal1, + ranges, interpolatedArray); + EXPECT(oops::is_close_absolute(res, 11.0f, 1e-5f, 0, + oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); +} + + +CASE("ufo/DataExtractor/bilinearinterp/range_constrain_extrapolation") { + // Constraining the range, such that, what would otherwise be within bounds is now out of bounds + // so returns missing. + const float obVal0 = 5.0, obVal1 = 3.0; + // Coordinates containing values that would change the answer if not excluded + // via the "range" specified. + const std::vector varValues0 {2, 4, 2, 4, 6}; + const std::vector varValues1 {3, 2, 4}; + + ConstrainedRange con0 = ConstrainedRange(varValues0.size()); + con0.constrain(2, varValues0.size()-1); // range will ignore 1st two and the final element. + ConstrainedRange con1 = ConstrainedRange(varValues1.size()); + con1.constrain(1, varValues1.size()); // range will ignore 1st. + const std::array ranges {con1, con0}; + + const std::vector data = {1, 6, 11, + 2, 7, 12, + 3, 8, 13, + 4, 9, 14, + 5, 10, 15}; + boost::multi_array interpolatedArray(boost::extents[3][5]); + size_t ind = 0; + for (int j=0; j < interpolatedArray.shape()[1]; j++) { + for (int i=0; i < interpolatedArray.shape()[0]; i++) { + interpolatedArray[i][j] = data[ind]; + ind++; + } + } + const float res = bilinearInterpolation(1, "var0", varValues0, obVal0, + 0, "var1", varValues1, obVal1, + ranges, interpolatedArray); + EXPECT_EQUAL(res, missing); +} + + +class DataExtractor : public oops::Test { + public: + DataExtractor() {} + + private: + std::string testid() const override {return "ufo::test::DataExtractor";} + + void register_tests() const override {} + + void clear() const override {} +}; + +} // namespace test +} // namespace ufo + +#endif // TEST_UFO_DATAEXTRACTOR_H_ diff --git a/test/ufo/ObsFunction.h b/test/ufo/ObsFunction.h index 6273b9dc5..5f12ca86c 100644 --- a/test/ufo/ObsFunction.h +++ b/test/ufo/ObsFunction.h @@ -48,6 +48,7 @@ namespace test { // ----------------------------------------------------------------------------- const char *expectComputeToThrow = "expect compute to throw exception with message"; +const char *expectConstructorToThrow = "expect constructor to throw exception with message"; // ----------------------------------------------------------------------------- @@ -139,6 +140,12 @@ void doTestFunction(ioda::ObsSpace &ospace, const eckit::Configuration &conf) { Variable funcname(obsfuncconf); /// Setup function + if (conf.has(expectConstructorToThrow)) { + // The constructor is expected to throw an exception containing the specified string. + const std::string expectedMessage = conf.getString(expectConstructorToThrow); + EXPECT_THROWS_MSG(ObsFunction{funcname}, expectedMessage.c_str()); + return; + } ObsFunction obsfunc(funcname); ufo::Variables allfuncvars = obsfunc.requiredVariables(); From bb4728717484db5a0e077aacf2074ebeac518776 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Tue, 3 Aug 2021 09:10:46 -0600 Subject: [PATCH 66/98] changes for turning on Var with bias correction on (#1381) * changes for turning on varbc test: EffectiveError name fix + better print * rollback changes in ObsBias::print * remove cosine_of_lat and sine_of_lat from BC of all sats except SSMIS Co-authored-by: Ryan Honeyager --- ewok/jedi-gdas/amsua_aqua.yaml | 2 -- ewok/jedi-gdas/amsua_metop-a.yaml | 2 -- ewok/jedi-gdas/amsua_metop-b.yaml | 2 -- ewok/jedi-gdas/amsua_metop-c.yaml | 2 -- ewok/jedi-gdas/amsua_n15.yaml | 2 -- ewok/jedi-gdas/amsua_n18.yaml | 2 -- ewok/jedi-gdas/amsua_n19.yaml | 2 -- ewok/jedi-gdas/atms_n20.yaml | 2 -- ewok/jedi-gdas/atms_npp.yaml | 2 -- ewok/jedi-gdas/mhs_metop-b.yaml | 2 -- ewok/jedi-gdas/mhs_metop-c.yaml | 2 -- ewok/jedi-gdas/mhs_n19.yaml | 2 -- src/ufo/ObsBiasCovariance.cc | 11 ++++++----- test/testinput/amsua_crtm_bc.yaml | 2 -- .../instrumentTests/amsua/amsua_aqua_gfs_HofX_bc.yaml | 2 -- .../instrumentTests/amsua/amsua_aqua_gfs_HofX_qc.yaml | 2 -- .../amsua/amsua_metop-a_gfs_HofX_bc.yaml | 2 -- .../amsua/amsua_metop-a_gfs_HofX_qc.yaml | 2 -- .../amsua/amsua_metop-b_gfs_HofX_bc.yaml | 2 -- .../amsua/amsua_metop-b_gfs_HofX_qc.yaml | 2 -- .../amsua/amsua_metop-c_gfs_HofX_bc.yaml | 2 -- .../amsua/amsua_metop-c_gfs_HofX_qc.yaml | 2 -- .../instrumentTests/amsua/amsua_n15_gfs_HofX_bc.yaml | 2 -- .../instrumentTests/amsua/amsua_n15_gfs_HofX_qc.yaml | 2 -- .../instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml | 2 -- .../instrumentTests/amsua/amsua_n18_gfs_HofX_qc.yaml | 2 -- .../instrumentTests/amsua/amsua_n19_gfs_HofX_bc.yaml | 2 -- .../instrumentTests/amsua/amsua_n19_gfs_HofX_qc.yaml | 2 -- .../instrumentTests/atms/atms_n20_gfs_HofX_bc.yaml | 2 -- .../instrumentTests/atms/atms_n20_gfs_HofX_qc.yaml | 2 -- .../instrumentTests/atms/atms_npp_gfs_HofX_bc.yaml | 2 -- .../instrumentTests/atms/atms_npp_gfs_HofX_qc.yaml | 2 -- .../instrumentTests/mhs/mhs_metop-b_gfs_HofX_bc.yaml | 2 -- .../instrumentTests/mhs/mhs_metop-c_gfs_HofX_bc.yaml | 2 -- .../instrumentTests/mhs/mhs_n19_gfs_HofX_bc.yaml | 2 -- test/ufo/ObsBiasCovarianceDetails.h | 2 +- 36 files changed, 7 insertions(+), 74 deletions(-) diff --git a/ewok/jedi-gdas/amsua_aqua.yaml b/ewok/jedi-gdas/amsua_aqua.yaml index a8e2881cd..27bdb8122 100644 --- a/ewok/jedi-gdas/amsua_aqua.yaml +++ b/ewok/jedi-gdas/amsua_aqua.yaml @@ -20,8 +20,6 @@ obs bias: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_aqua_tlapse $(experiment_dir)/{{current_cycle}}/amsua_aqua.{{background_time}}.tlapse.txt diff --git a/ewok/jedi-gdas/amsua_metop-a.yaml b/ewok/jedi-gdas/amsua_metop-a.yaml index 477c7ac96..b55aaca93 100644 --- a/ewok/jedi-gdas/amsua_metop-a.yaml +++ b/ewok/jedi-gdas/amsua_metop-a.yaml @@ -20,8 +20,6 @@ obs bias: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_metop-a_tlapse $(experiment_dir)/{{current_cycle}}/amsua_metop-a.{{background_time}}.tlapse.txt diff --git a/ewok/jedi-gdas/amsua_metop-b.yaml b/ewok/jedi-gdas/amsua_metop-b.yaml index a63505c9c..72e004574 100644 --- a/ewok/jedi-gdas/amsua_metop-b.yaml +++ b/ewok/jedi-gdas/amsua_metop-b.yaml @@ -20,8 +20,6 @@ obs bias: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_metop-b_tlapse $(experiment_dir)/{{current_cycle}}/amsua_metop-b.{{background_time}}.tlapse.txt diff --git a/ewok/jedi-gdas/amsua_metop-c.yaml b/ewok/jedi-gdas/amsua_metop-c.yaml index 0eaa9bff2..960593bcf 100644 --- a/ewok/jedi-gdas/amsua_metop-c.yaml +++ b/ewok/jedi-gdas/amsua_metop-c.yaml @@ -20,8 +20,6 @@ obs bias: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_metop-c_tlapse $(experiment_dir)/{{current_cycle}}/amsua_metop-c.{{background_time}}.tlapse.txt diff --git a/ewok/jedi-gdas/amsua_n15.yaml b/ewok/jedi-gdas/amsua_n15.yaml index e7dc2473b..00705cd92 100644 --- a/ewok/jedi-gdas/amsua_n15.yaml +++ b/ewok/jedi-gdas/amsua_n15.yaml @@ -20,8 +20,6 @@ obs bias: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_n15_tlapse $(experiment_dir)/{{current_cycle}}/amsua_n15.{{background_time}}.tlapse.txt diff --git a/ewok/jedi-gdas/amsua_n18.yaml b/ewok/jedi-gdas/amsua_n18.yaml index 2abcac1dc..bfd714bdf 100644 --- a/ewok/jedi-gdas/amsua_n18.yaml +++ b/ewok/jedi-gdas/amsua_n18.yaml @@ -20,8 +20,6 @@ obs bias: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_n18_tlapse $(experiment_dir)/{{current_cycle}}/amsua_n18.{{background_time}}.tlapse.txt diff --git a/ewok/jedi-gdas/amsua_n19.yaml b/ewok/jedi-gdas/amsua_n19.yaml index e5f8b5dc3..306a8fc8c 100644 --- a/ewok/jedi-gdas/amsua_n19.yaml +++ b/ewok/jedi-gdas/amsua_n19.yaml @@ -20,8 +20,6 @@ obs bias: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_n19_tlapse $(experiment_dir)/{{current_cycle}}/amsua_n19.{{background_time}}.tlapse.txt diff --git a/ewok/jedi-gdas/atms_n20.yaml b/ewok/jedi-gdas/atms_n20.yaml index fea195845..aac174b8a 100644 --- a/ewok/jedi-gdas/atms_n20.yaml +++ b/ewok/jedi-gdas/atms_n20.yaml @@ -20,8 +20,6 @@ obs bias: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &atms_n20_tlapse $(experiment_dir)/{{current_cycle}}/atms_n20.{{background_time}}.tlapse.txt diff --git a/ewok/jedi-gdas/atms_npp.yaml b/ewok/jedi-gdas/atms_npp.yaml index 7fb792ef7..97e84777e 100644 --- a/ewok/jedi-gdas/atms_npp.yaml +++ b/ewok/jedi-gdas/atms_npp.yaml @@ -20,8 +20,6 @@ obs bias: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &atms_npp_tlapse $(experiment_dir)/{{current_cycle}}/atms_npp.{{background_time}}.tlapse.txt diff --git a/ewok/jedi-gdas/mhs_metop-b.yaml b/ewok/jedi-gdas/mhs_metop-b.yaml index f24c798f1..f0015c5de 100644 --- a/ewok/jedi-gdas/mhs_metop-b.yaml +++ b/ewok/jedi-gdas/mhs_metop-b.yaml @@ -18,8 +18,6 @@ obs bias: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &mhs_metop-b_tlapse $(experiment_dir)/{{current_cycle}}/mhs_metop-b.{{background_time}}.tlapse.txt diff --git a/ewok/jedi-gdas/mhs_metop-c.yaml b/ewok/jedi-gdas/mhs_metop-c.yaml index 908da6b53..1fcfc7027 100644 --- a/ewok/jedi-gdas/mhs_metop-c.yaml +++ b/ewok/jedi-gdas/mhs_metop-c.yaml @@ -18,8 +18,6 @@ obs bias: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &mhs_metop-c_tlapse $(experiment_dir)/{{current_cycle}}/mhs_metop-c.{{background_time}}.tlapse.txt diff --git a/ewok/jedi-gdas/mhs_n19.yaml b/ewok/jedi-gdas/mhs_n19.yaml index ac5f24429..9493bc966 100644 --- a/ewok/jedi-gdas/mhs_n19.yaml +++ b/ewok/jedi-gdas/mhs_n19.yaml @@ -18,8 +18,6 @@ obs bias: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &mhs_n19_tlapse $(experiment_dir)/{{current_cycle}}/mhs_n19.{{background_time}}.tlapse.txt diff --git a/src/ufo/ObsBiasCovariance.cc b/src/ufo/ObsBiasCovariance.cc index 6735fcdce..54ab5a112 100644 --- a/src/ufo/ObsBiasCovariance.cc +++ b/src/ufo/ObsBiasCovariance.cc @@ -196,17 +196,17 @@ void ObsBiasCovariance::linearize(const ObsBias & bias, const eckit::Configurati odb_.distribution()->createAccumulator(obs_num_.size()); // Retrieve the QC flags of previous outer loop and recalculate the number of effective obs. - const std::string group_name = "EffectiveQC" + std::to_string(jouter-1); + const std::string qc_group_name = "EffectiveQC" + std::to_string(jouter-1); const std::vector vars = odb_.obsvariables().variables(); std::vector qc_flags(odb_.nlocs(), 999); for (std::size_t jvar = 0; jvar < vars.size(); ++jvar) { - if (odb_.has(group_name, vars[jvar])) { - odb_.get_db(group_name, vars[jvar], qc_flags); + if (odb_.has(qc_group_name, vars[jvar])) { + odb_.get_db(qc_group_name, vars[jvar], qc_flags); for (std::size_t jloc = 0; jloc < qc_flags.size(); ++jloc) if (qc_flags[jloc] == 0) obs_num_accumulator->addTerm(jloc, jvar, 1); } else { - throw eckit::UserError("Unable to find QC flags : " + vars[jvar] + "@" + group_name); + throw eckit::UserError("Unable to find QC flags : " + vars[jvar] + "@" + qc_group_name); } } @@ -217,7 +217,8 @@ void ObsBiasCovariance::linearize(const ObsBias & bias, const eckit::Configurati // compute the hessian contribution from Jo bias terms channel by channel // retrieve the effective error (after QC) for this channel - ioda::ObsVector r_inv(odb_, "EffectiveError"); + const std::string err_group_name = "EffectiveError" + std::to_string(jouter-1); + ioda::ObsVector r_inv(odb_, err_group_name); // compute \mathrm{R}^{-1} std::size_t nvars = r_inv.nvars(); diff --git a/test/testinput/amsua_crtm_bc.yaml b/test/testinput/amsua_crtm_bc.yaml index 4c3afc527..1e1791a09 100644 --- a/test/testinput/amsua_crtm_bc.yaml +++ b/test/testinput/amsua_crtm_bc.yaml @@ -31,8 +31,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua19tlap Data/ufo/testinput_tier_1/amsua_n19_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_bc.yaml index 07f5904cd..e38e6adc8 100644 --- a/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_bc.yaml @@ -24,8 +24,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_aqua_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_aqua_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_qc.yaml index 4a2181461..ad9595068 100644 --- a/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_aqua_gfs_HofX_qc.yaml @@ -26,8 +26,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_aqua_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_aqua_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_bc.yaml index 92c4d0cd1..ea3c4a1d2 100644 --- a/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_bc.yaml @@ -24,8 +24,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-a_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_qc.yaml index 27261643e..59c74f34b 100644 --- a/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_metop-a_gfs_HofX_qc.yaml @@ -26,8 +26,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_metop-a_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-a_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_bc.yaml index cb846aebc..371583757 100644 --- a/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_bc.yaml @@ -24,8 +24,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_metop-b_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-b_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_qc.yaml index 0e05bc985..dc051f549 100644 --- a/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_metop-b_gfs_HofX_qc.yaml @@ -26,8 +26,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_metop-b_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-b_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_bc.yaml index c4e0e8c1f..0381fbd85 100644 --- a/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_bc.yaml @@ -24,8 +24,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_metop-c_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-c_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_qc.yaml index 7abad12b8..924a5f2c3 100644 --- a/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_metop-c_gfs_HofX_qc.yaml @@ -26,8 +26,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_metop-c_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_metop-c_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_bc.yaml index d6bac52d8..95f46992c 100644 --- a/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_bc.yaml @@ -24,8 +24,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_n15_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n15_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_qc.yaml index ec03504c7..21576d2aa 100644 --- a/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_n15_gfs_HofX_qc.yaml @@ -26,8 +26,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_n15_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n15_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml index 4bf2c0b08..bce312af0 100644 --- a/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml @@ -24,8 +24,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_n18_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n18_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_qc.yaml index 0b58e080e..b79d3b7d1 100644 --- a/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_qc.yaml @@ -26,8 +26,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_n18_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n18_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_bc.yaml index 7190a25f4..706d8a3e3 100644 --- a/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_bc.yaml @@ -24,8 +24,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_n19_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n19_tlapmean.txt diff --git a/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_qc.yaml index 0a72965ca..a8c76e955 100644 --- a/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/amsua/amsua_n19_gfs_HofX_qc.yaml @@ -26,8 +26,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &amsua_n19_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n19_tlapmean.txt diff --git a/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_bc.yaml index 088c7d3f3..5a9eb332a 100644 --- a/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_bc.yaml @@ -24,8 +24,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &atms_n20_tlap Data/ufo/testinput_tier_1/instruments/radiance/atms_n20_tlapmean.txt diff --git a/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_qc.yaml index 64ab3e7b6..6c6353a1b 100644 --- a/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/atms/atms_n20_gfs_HofX_qc.yaml @@ -26,8 +26,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &atms_n20_tlap Data/ufo/testinput_tier_1/instruments/radiance/atms_n20_tlapmean.txt diff --git a/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_bc.yaml index f5415e8b5..0bf21060b 100644 --- a/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_bc.yaml @@ -24,8 +24,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &atms_npp_tlap Data/ufo/testinput_tier_1/instruments/radiance/atms_npp_tlapmean.txt diff --git a/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_qc.yaml index 68525b839..44c2f5621 100644 --- a/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/atms/atms_npp_gfs_HofX_qc.yaml @@ -26,8 +26,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &atms_npp_tlap Data/ufo/testinput_tier_1/instruments/radiance/atms_npp_tlapmean.txt diff --git a/test/testinput/instrumentTests/mhs/mhs_metop-b_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/mhs/mhs_metop-b_gfs_HofX_bc.yaml index 06d2204f9..978a34843 100644 --- a/test/testinput/instrumentTests/mhs/mhs_metop-b_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/mhs/mhs_metop-b_gfs_HofX_bc.yaml @@ -22,8 +22,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &mhs_metop-b_tlap Data/ufo/testinput_tier_1/instruments/radiance/mhs_metop-b_tlapmean.txt diff --git a/test/testinput/instrumentTests/mhs/mhs_metop-c_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/mhs/mhs_metop-c_gfs_HofX_bc.yaml index 65146263c..15047b8e0 100644 --- a/test/testinput/instrumentTests/mhs/mhs_metop-c_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/mhs/mhs_metop-c_gfs_HofX_bc.yaml @@ -22,8 +22,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &mhs_metop-c_tlap Data/ufo/testinput_tier_1/instruments/radiance/mhs_metop-c_tlapmean.txt diff --git a/test/testinput/instrumentTests/mhs/mhs_n19_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/mhs/mhs_n19_gfs_HofX_bc.yaml index 52ce8b273..839497fff 100644 --- a/test/testinput/instrumentTests/mhs/mhs_n19_gfs_HofX_bc.yaml +++ b/test/testinput/instrumentTests/mhs/mhs_n19_gfs_HofX_bc.yaml @@ -22,8 +22,6 @@ observations: variational bc: predictors: - name: constant - - name: cosine_of_latitude_times_orbit_node - - name: sine_of_latitude - name: lapse_rate order: 2 tlapse: &mhs_n19_tlap Data/ufo/testinput_tier_1/instruments/radiance/mhs_n19_tlapmean.txt diff --git a/test/ufo/ObsBiasCovarianceDetails.h b/test/ufo/ObsBiasCovarianceDetails.h index 576ab6320..5efe25f70 100644 --- a/test/ufo/ObsBiasCovarianceDetails.h +++ b/test/ufo/ObsBiasCovarianceDetails.h @@ -80,7 +80,7 @@ void testObsBiasCovarianceDetails() { // mimic effective errors const std::vector errs(odb.nlocs(), 1.0); for ( const auto & var : vars) - odb.put_db("EffectiveError", var , errs); + odb.put_db("EffectiveError0", var , errs); // mimic predictors ioda::ObsVector predx(odb); From b0d8de490ff6d52aad811d2b9305f2fa50fa1e88 Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Wed, 4 Aug 2021 10:20:28 -0600 Subject: [PATCH 67/98] change url for crtm coefficient file using env variable (#1405) * first commit * bugfix * clean up * ECBUILD_DOWNLOAD_BASE_URL instead of ECBUILD_DOWNLOAD_BASE_URL_CRTM * bugfix --- test/CMakeLists.txt | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d2c6eb565..d8ac1929b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -379,14 +379,23 @@ function(ADD_DOWNLOAD_TEST repo test_files_dirname output_data_path) elseif( DEFINED GIT_TAG_FUNC) message( STATUS "Tagged version of ${repo} is used" ) # set ARGS for get_${repo}_test_data - set( ECBUILD_DOWNLOAD_BASE_URL https://dashrepo.ucar.edu/api/v1/dataset/147_miesch/version/1.1.0/file ) + + # ECBUILD_DOWNLOAD_BASE_URL env var can be used to force test files and + # crtm coef to be downloaded from other databases such as S3 instead of DASH + # example ECBUILD_DOWNLOAD_BASE_URL=https://jedi-test-files.s3.amazonaws.com + + if( DEFINED ENV{ECBUILD_DOWNLOAD_BASE_URL} ) + set( ECBUILD_DOWNLOAD_BASE_URL "$ENV{ECBUILD_DOWNLOAD_BASE_URL}/${repo}/${GIT_TAG_FUNC}" ) + else() + set( ECBUILD_DOWNLOAD_BASE_URL https://dashrepo.ucar.edu/api/v1/dataset/147_miesch/version/1.1.0/file ) + endif() + set( DIRNAME ${GIT_TAG_FUNC} ) set( checksum "0" ) set( TESTFILE_DIR_REPO "${CMAKE_SOURCE_DIR}/test-data-release/${repo}/${DIRNAME}" ) # Create test-data-release in source directory file( MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/test-data-release ) - # temporary solution for release 1.1.0 if ( ${repo} STREQUAL "crtm" ) list( APPEND REPO_DATA_DOWNLOADER_ARGS @@ -402,9 +411,9 @@ function(ADD_DOWNLOAD_TEST repo test_files_dirname output_data_path) ${test_files_dirname} ${checksum} ) endif() - + message( STATUS "ECBUILD_DOWNLOAD_BASE_URL is set, ${repo} test files will + be downloaded from ${ECBUILD_DOWNLOAD_BASE_URL}/test-data-release") set ( REPO_DATA_DOWNLOADER ufo_data_downloader.py ) - message( STATUS "Test data will be downloaded from: ${ECBUILD_DOWNLOAD_BASE_URL}" ) # Any branch of REPO is being built. # repo-data repository is already cloned by bundle/CMakeLists.txt. From 1ddd85238805b4d127ecb80fcf0d0b7fd9a56e68 Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Thu, 5 Aug 2021 17:58:04 +0100 Subject: [PATCH 68/98] Pressure to height conversion (#1361) * pressure to height conversion * improve comments * add a space * error message * Trigger CI * alternative pressure to height conversion * DerivedValue -> DerivedObsValue * DerivedValue -> DerivedObsValue * NCAR-RAL method: use ICAO atmosphere above 15km * remove unnecessary test; improve comments * modifications following review Co-authored-by: Benjamin Johnson <33431190+BenjaminTJohnson@users.noreply.github.com> --- src/ufo/utils/Constants.h | 21 ++-- src/ufo/variabletransforms/CMakeLists.txt | 2 + .../Cal_HeightFromPressure.cc | 99 +++++++++++++++++++ .../Cal_HeightFromPressure.h | 37 +++++++ src/ufo/variabletransforms/Formulas.cc | 54 ++++++++++ src/ufo/variabletransforms/Formulas.h | 17 ++++ test/CMakeLists.txt | 8 ++ ...bletransforms_SondeHeightFromPressure.yaml | 70 +++++++++++++ 8 files changed, 299 insertions(+), 9 deletions(-) create mode 100644 src/ufo/variabletransforms/Cal_HeightFromPressure.cc create mode 100644 src/ufo/variabletransforms/Cal_HeightFromPressure.h create mode 100644 test/testinput/unit_tests/variabletransforms_SondeHeightFromPressure.yaml diff --git a/src/ufo/utils/Constants.h b/src/ufo/utils/Constants.h index 56ee7bfc2..5fc30d562 100644 --- a/src/ufo/utils/Constants.h +++ b/src/ufo/utils/Constants.h @@ -71,20 +71,23 @@ struct Constants { // https://en.wikipedia.org/wiki/International_Standard_Atmosphere#ICAO_Standard_Atmosphere // with gpm = height in geopotential metres static constexpr double icao_lapse_rate_l = 6.5E-03; // Lapse rate for levels up - // to 11,000 [gpm] - // (isothermal layer) + // to 11,000 gpm + // (lower boundary of the + // isothermal layer) [K/gpm] static constexpr double icao_lapse_rate_u = -1.0E-03; // Lapse rate for levels above - // 20,000 [gpm] - static constexpr double icao_height_l = 11000.0; // Height limit for assumed - // lower lapse rate [m] + // 20,000 gpm + // (upper boundary of the + // isothermal layer) [K/gpm] + static constexpr double icao_height_l = 11000.0; // Height of bottom of isothermal + // layer [gpm] static constexpr double icao_height_u = 20000.0; // Height of top of isothermal - // layer [m] + // layer [gpm] static constexpr double icao_temp_surface = 288.15; // Surface temperature [K] static constexpr double icao_temp_isothermal_layer = 216.65; // Temperature of isothermal // layer [K] - static constexpr double icao_pressure_surface = 1013.25; // Assumed surface pressure - static constexpr double icao_pressure_l = 226.32; // Assumed pressure at 11,000 gpm - static constexpr double icao_pressure_u = 54.7487; // Assumed pressure at 20,000 gpm + static constexpr double icao_pressure_surface = 1013.25; // Assumed surface pressure [hPa] + static constexpr double icao_pressure_l = 226.32; // Assumed pressure at 11,000 gpm [hPa] + static constexpr double icao_pressure_u = 54.7487; // Assumed pressure at 20,000 gpm [hPa] }; //-------------------------------------------------------------------------------------------------- diff --git a/src/ufo/variabletransforms/CMakeLists.txt b/src/ufo/variabletransforms/CMakeLists.txt index b71331cd1..44104bdb3 100644 --- a/src/ufo/variabletransforms/CMakeLists.txt +++ b/src/ufo/variabletransforms/CMakeLists.txt @@ -6,6 +6,8 @@ set ( variabletransforms_files Cal_PressureFromHeight.h Cal_PressureFromHeight.cc + Cal_HeightFromPressure.h + Cal_HeightFromPressure.cc Cal_Humidity.h Cal_Humidity.cc Cal_RemapScanPosition.h diff --git a/src/ufo/variabletransforms/Cal_HeightFromPressure.cc b/src/ufo/variabletransforms/Cal_HeightFromPressure.cc new file mode 100644 index 000000000..e1a8d82a6 --- /dev/null +++ b/src/ufo/variabletransforms/Cal_HeightFromPressure.cc @@ -0,0 +1,99 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/variabletransforms/Cal_HeightFromPressure.h" +#include "ufo/utils/Constants.h" + +namespace ufo { +/************************************************************************************/ +// Cal_HeightFromPressure +/************************************************************************************/ +static TransformMaker + makerCal_HeightFromPressure_("HeightFromPressure"); + +Cal_HeightFromPressure::Cal_HeightFromPressure( + const VariableTransformsParameters &options, ioda::ObsSpace &os, + const std::shared_ptr> &flags, + const std::vector &apply) + : TransformBase(options, os, flags, apply) {} + +/************************************************************************************/ + +void Cal_HeightFromPressure::runTransform() { + oops::Log::trace() << " Retrieve Height From Pressure" << std::endl; + + std::vector geopotentialHeight; + std::vector airPressure; + bool hasBeenUpdated = false; + + const size_t nlocs_ = obsdb_.nlocs(); + + ioda::ObsSpace::RecIdxIter irec; + + // 1. Obtain air pressure from the ObsSpace. + // Two possible pressure variables are searched for: + // - air_pressure, + // - air_pressure_levels. + // If neither is present an exception is thrown. + + // Do the pressure, and derived height, lie on staggered levels? + bool staggeredLevels = false; + + getObservation("MetaData", "air_pressure", airPressure); + if (airPressure.empty()) { + getObservation("MetaData", "air_pressure_levels", airPressure); + if (airPressure.empty()) { + oops::Log::warning() << "Air pressure vector is empty. " + << "Check will not be performed." << std::endl; + throw eckit::BadValue("Air pressure vector is empty ", Here()); + } + staggeredLevels = true; + } + + // 2. Initialise the output array + // ------------------------------------------------------------------------------- + if (staggeredLevels) + getObservation("MetaData", "geopotential_height_levels", geopotentialHeight); + else + getObservation("MetaData", "geopotential_height", geopotentialHeight); + + if (geopotentialHeight.empty()) { + geopotentialHeight = std::vector(nlocs_); + std::fill(geopotentialHeight.begin(), geopotentialHeight.end(), missingValueFloat); + } + + // 3. Loop over each record + // ------------------------------------------------------------------------------------- + for (irec = obsdb_.recidx_begin(); irec != obsdb_.recidx_end(); ++irec) { + const std::vector &rSort = obsdb_.recidx_vector(irec); + size_t ilocs = 0; + + // 3.1 Loop over each record + for (ilocs = 0; ilocs < rSort.size(); ++ilocs) { + // Cycle if the data have been excluded by the where statement + if (!apply_[rSort[ilocs]]) continue; + + // Cycle if geopotential height is valid + if (geopotentialHeight[rSort[ilocs]] != missingValueFloat) continue; + + geopotentialHeight[rSort[ilocs]] = formulas::Pressure_To_Height( + airPressure[rSort[ilocs]], method()); + + hasBeenUpdated = true; + } + } + + if (hasBeenUpdated) { + // If the geopotential height was updated, save it as a DerivedValue. + if (staggeredLevels) + obsdb_.put_db(outputTag, "geopotential_height_levels", geopotentialHeight); + else + obsdb_.put_db(outputTag, "geopotential_height", geopotentialHeight); + } +} +} // namespace ufo + diff --git a/src/ufo/variabletransforms/Cal_HeightFromPressure.h b/src/ufo/variabletransforms/Cal_HeightFromPressure.h new file mode 100644 index 000000000..803c269b9 --- /dev/null +++ b/src/ufo/variabletransforms/Cal_HeightFromPressure.h @@ -0,0 +1,37 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_VARIABLETRANSFORMS_CAL_HEIGHTFROMPRESSURE_H_ +#define UFO_VARIABLETRANSFORMS_CAL_HEIGHTFROMPRESSURE_H_ + +#include +#include +#include +#include + +#include "oops/util/ObjectCounter.h" +#include "ufo/variabletransforms/TransformBase.h" + +namespace ufo { + +/*! +* \brief Converts pressures to heights. +* +* See VariableTransformsParameters for filter setup. +*/ +class Cal_HeightFromPressure : public TransformBase { + public: + Cal_HeightFromPressure(const VariableTransformsParameters &options, + ioda::ObsSpace &os, + const std::shared_ptr> &flags, + const std::vector &apply); + // Run check + void runTransform() override; +}; +} // namespace ufo + +#endif // UFO_VARIABLETRANSFORMS_CAL_HEIGHTFROMPRESSURE_H_ diff --git a/src/ufo/variabletransforms/Formulas.cc b/src/ufo/variabletransforms/Formulas.cc index cad1509b3..733303a9a 100644 --- a/src/ufo/variabletransforms/Formulas.cc +++ b/src/ufo/variabletransforms/Formulas.cc @@ -266,6 +266,60 @@ float Height_To_Pressure_ICAO_atmos(float height, MethodFormulation formulation) return Pressure; } +/* -------------------------------------------------------------------------------------*/ + +float Pressure_To_Height(float pressure, MethodFormulation method) { + const float missingValueFloat = util::missingValue(1.0f); + const float pressure_hPa = pressure * 0.01; + float height = missingValueFloat; + + switch (method) { + case formulas::MethodFormulation::NCAR: + // The NCAR-RAL method: a fast approximation for pressures > 120 hPa. + // Above 120hPa (~15km) use the ICAO atmosphere. + if (pressure == missingValueFloat || pressure <= 0.0f) { + height = missingValueFloat; + } else if (pressure_hPa <= 120.0f && + pressure_hPa > Constants::icao_pressure_u) { + pressure = std::log(Constants::icao_pressure_l) - std::log(pressure_hPa); + height = pressure * Constants::icao_temp_isothermal_layer / Constants::g_over_rd + + Constants::icao_height_l; + } else if (pressure_hPa <= Constants::icao_pressure_u) { + pressure = 1.0 - std::pow(pressure_hPa / Constants::icao_pressure_u, + Constants::icao_lapse_rate_u / Constants::g_over_rd); + height = pressure * Constants::icao_temp_isothermal_layer / Constants::icao_lapse_rate_u + + Constants::icao_height_u; + } else { + height = 44307.692 * (1.0 - std::pow(pressure / 101325.0, 0.190)); + } + break; + + case formulas::MethodFormulation::NOAA: + case formulas::MethodFormulation::UKMO: + default: { + if (pressure == missingValueFloat || pressure <= 0.0f) { + height = missingValueFloat; + } else if (pressure_hPa > Constants::icao_pressure_l) { + pressure = 1.0 - std::pow(pressure_hPa / Constants::icao_pressure_surface, + Constants::icao_lapse_rate_l / Constants::g_over_rd); + height = pressure * Constants::icao_temp_surface / Constants::icao_lapse_rate_l; + } else if (pressure_hPa <= Constants::icao_pressure_l && + pressure_hPa > Constants::icao_pressure_u) { + pressure = std::log(Constants::icao_pressure_l) - std::log(pressure_hPa); + height = pressure * Constants::icao_temp_isothermal_layer / Constants::g_over_rd + + Constants::icao_height_l; + } else { + pressure = 1.0 - std::pow(pressure_hPa / Constants::icao_pressure_u, + Constants::icao_lapse_rate_u / Constants::g_over_rd); + height = pressure * Constants::icao_temp_isothermal_layer / Constants::icao_lapse_rate_u + + Constants::icao_height_u; + } + break; + } + } + return height; +} + float GetWindDirection(float u, float v) { const float missing = util::missingValue(1.0f); float windDirection = missing; // wind direction diff --git a/src/ufo/variabletransforms/Formulas.h b/src/ufo/variabletransforms/Formulas.h index c0a52e007..73fdafaaf 100644 --- a/src/ufo/variabletransforms/Formulas.h +++ b/src/ufo/variabletransforms/Formulas.h @@ -182,6 +182,23 @@ float VirtualTemp_From_Rh_Psat_P_T(float Rh, float Psat, float P, float T, float Height_To_Pressure_ICAO_atmos(float Height, MethodFormulation formulation = formulas::MethodFormulation::DEFAULT); +// ------------------------------------------------------------------------------------- +/*! +* \brief Converts pressure to height. +* +* \b Formulation \b available: +* - NCAR: uses a fast approximation for pressures > 120 hPa and the ICAO atmosphere otherwise. +* - NOAA: as default +* - UKMO: as default +* - DEFAULT: uses the ICAO atmosphere for all pressures. +* +* \param pressure +* observation pressure in Pa +* \return height in geopotential metres +*/ +float Pressure_To_Height(float pressure, + MethodFormulation formulation = formulas::MethodFormulation::DEFAULT); + // ------------------------------------------------------------------------------------- /*! * \brief Converts u and v wind component into wind direction. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d8ac1929b..1c70853b4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -301,6 +301,7 @@ list( APPEND ufo_test_input testinput/unit_tests/variabletransforms_scanposition.yaml testinput/unit_tests/variabletransforms_SondePressureFromHeight.yaml testinput/unit_tests/variabletransforms_SondePressureFromHeightICAO.yaml + testinput/unit_tests/variabletransforms_SondeHeightFromPressure.yaml testinput/unit_tests/variabletransforms_windcomponents.yaml testinput/unit_tests/variabletransforms_windspeedanddirection.yaml ) @@ -2035,6 +2036,13 @@ ecbuild_add_test( TARGET test_ufo_variabletransforms_SondePressureFromHeightICA DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_variabletransforms_SondeHeightFromPressure + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x + ARGS "testinput/unit_tests/variabletransforms_SondeHeightFromPressure.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFilters.x + TEST_DEPENDS ufo_get_ufo_test_data ) + ecbuild_add_test( TARGET test_ufo_conventionalprofileprocessing_average_pressure_obsfilter COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x ARGS "testinput/conventionalprofileprocessing_average_pressure_obsfilter.yaml" diff --git a/test/testinput/unit_tests/variabletransforms_SondeHeightFromPressure.yaml b/test/testinput/unit_tests/variabletransforms_SondeHeightFromPressure.yaml new file mode 100644 index 000000000..c2c726d3b --- /dev/null +++ b/test/testinput/unit_tests/variabletransforms_SondeHeightFromPressure.yaml @@ -0,0 +1,70 @@ +# +#=== Pressure to height conversion for vertical profile ===# +# + +window begin: 2015-01-08T20:30:00Z +window end: 2015-01-09T03:30:00Z + +observations: +# UKMO method (uses ICAO atmosphere). +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_conversion_pressure2height.nc4 + obsgrouping: + group variables: ["station_id"] + sort variable: "datetime" + sort order: "descending" + simulated variables: [air_temperature, relative_humidity, geopotential_height, dew_point_temperature] + obs filters: + - filter: Variable Transforms + Transform: ["HeightFromPressure"] + Method: UKMO + compareVariables: + - test: + name: geopotential_height@DerivedObsValue + reference: + name: geopotential_height_ICAO_reference@ObsValue + relTol: 1.0e-5 + +# NCAR-RAL method (uses approximation for pressures > 120 hPa, ICAO atmosphere otherwise). +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_conversion_pressure2height.nc4 + obsgrouping: + group variables: ["station_id"] + sort variable: "datetime" + sort order: "descending" + simulated variables: [air_temperature, relative_humidity, geopotential_height, dew_point_temperature] + obs filters: + - filter: Variable Transforms + Transform: ["HeightFromPressure"] + Method: NCAR + compareVariables: + - test: + name: geopotential_height@DerivedObsValue + reference: + name: geopotential_height_NCAR_reference@ObsValue + relTol: 1.0e-5 + +# Vertical coordinate is air_pressure_levels. +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_conversion_pressure2height_levels.nc4 + obsgrouping: + group variables: ["station_id"] + sort variable: "datetime" + sort order: "descending" + simulated variables: [air_temperature, relative_humidity, geopotential_height, dew_point_temperature] + obs filters: + - filter: Variable Transforms + Transform: ["HeightFromPressure"] + Method: UKMO + compareVariables: + - test: + name: geopotential_height_levels@DerivedObsValue + reference: + name: geopotential_height_levels_ICAO_reference@ObsValue + relTol: 1.0e-5 From d723318ef2b97fa390a7181a4c6b82e953f88f90 Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Thu, 5 Aug 2021 19:04:04 +0100 Subject: [PATCH 69/98] Compute profile horizontal drift (#1388) * initial code * delete obsfunction * remove obsfunctions * add variable transforms * horizontal drift calculation * unit test * routine that calls drift calculation * make existing methods templated * simplify method/formulation * where clause; check for no valid locations * tests for exceptions * remove functions that were duplicated on merge * reorganise protected section * reorganise protected section * improve comments * correct drift formula * correct typo * modify vector size check * modify comment about method Co-authored-by: Ryan Honeyager --- src/ufo/variabletransforms/CMakeLists.txt | 2 + .../Cal_ProfileHorizontalDrift.cc | 92 +++++++++++++++ .../Cal_ProfileHorizontalDrift.h | 51 ++++++++ src/ufo/variabletransforms/Formulas.cc | 110 ++++++++++++++++++ src/ufo/variabletransforms/Formulas.h | 63 ++++++++++ test/CMakeLists.txt | 8 ++ ...abletransforms_profilehorizontaldrift.yaml | 79 +++++++++++++ 7 files changed, 405 insertions(+) create mode 100644 src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc create mode 100644 src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h create mode 100644 test/testinput/unit_tests/variabletransforms_profilehorizontaldrift.yaml diff --git a/src/ufo/variabletransforms/CMakeLists.txt b/src/ufo/variabletransforms/CMakeLists.txt index 44104bdb3..98d8c0360 100644 --- a/src/ufo/variabletransforms/CMakeLists.txt +++ b/src/ufo/variabletransforms/CMakeLists.txt @@ -14,6 +14,8 @@ set ( variabletransforms_files Cal_RemapScanPosition.cc Cal_Wind.h Cal_Wind.cc + Cal_ProfileHorizontalDrift.h + Cal_ProfileHorizontalDrift.cc LookupTable.h TransformBase.cc TransformBase.h diff --git a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc new file mode 100644 index 000000000..6746389b8 --- /dev/null +++ b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc @@ -0,0 +1,92 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/variabletransforms/Cal_ProfileHorizontalDrift.h" +#include "ufo/utils/Constants.h" + +namespace ufo { + +/************************************************************************************/ +// Cal_ProfileHorizontalDrift +/************************************************************************************/ + +static TransformMaker +makerCal_ProfileHorizontalDrift_("ProfileHorizontalDrift"); + +Cal_ProfileHorizontalDrift::Cal_ProfileHorizontalDrift +(const VariableTransformsParameters &options, + ioda::ObsSpace &os, + const std::shared_ptr> &flags, + const std::vector &apply) + : TransformBase(options, os, flags, apply) {} + +/************************************************************************************/ + +void Cal_ProfileHorizontalDrift::runTransform() { + oops::Log::trace() << " --> Compute horizontal drift lat/lon/time" << std::endl; + oops::Log::trace() << " --> method: " << method() << std::endl; + + // Ensure observations have been grouped into profiles. + if (obsdb_.obs_group_vars().empty()) + throw eckit::UserError("Group variables configuration is empty", Here()); + + // Ensure observations have been sorted by air pressure in descending order. + if (obsdb_.obs_sort_var() != "air_pressure") + throw eckit::UserError("Sort variable must be air_pressure", Here()); + if (obsdb_.obs_sort_order() != "descending") + throw eckit::UserError("Profiles must be sorted in descending order", Here()); + + // Obtain values from ObsSpace. + std::vector latitude_in, longitude_in, wind_speed, wind_from_direction, height; + std::vector datetime_in; + getObservation("MetaData", "latitude", latitude_in, true); + getObservation("MetaData", "longitude", longitude_in, true); + getObservation("MetaData", "datetime", datetime_in, true); + getObservation("ObsValue", "geopotential_height", height, true); + getObservation("ObsValue", "wind_speed", wind_speed, true); + getObservation("ObsValue", "wind_from_direction", wind_from_direction, true); + + if (!oops::allVectorsSameNonZeroSize(latitude_in, longitude_in, datetime_in, + height, wind_speed, wind_from_direction)) { + oops::Log::warning() << "Vector sizes: " + << oops::listOfVectorSizes(latitude_in, longitude_in, datetime_in, + height, wind_speed, wind_from_direction) + << std::endl; + throw eckit::BadValue("At least one vector is the wrong size", Here()); + } + + // Number of locations in the ObsSpace. + const size_t nlocs = obsdb_.nlocs(); + + // Output values are initialised to input values. + std::vector latitude_out = latitude_in; + std::vector longitude_out = longitude_in; + std::vector datetime_out = datetime_in; + + // Get correspondence between record numbers and indices in the total sample. + const std::vector &recnums = obsdb_.recidx_all_recnums(); + + // Number of profiles in the ObsSpace. + const size_t nprofs = recnums.size(); + + // Perform drift calculation for each profile in the sample. + for (size_t jprof = 0; jprof < nprofs; ++jprof) { + const std::vector &locs = obsdb_.recidx_vector(recnums[jprof]); + formulas::horizontalDrift(locs, apply_, + latitude_in, longitude_in, datetime_in, + height, wind_speed, wind_from_direction, + latitude_out, longitude_out, datetime_out); + } + + // Save output values. + obsdb_.put_db("DerivedMetaData", "latitude", latitude_out); + obsdb_.put_db("DerivedMetaData", "longitude", longitude_out); + obsdb_.put_db("DerivedMetaData", "datetime", datetime_out); +} + +} // namespace ufo + diff --git a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h new file mode 100644 index 000000000..f88e01b06 --- /dev/null +++ b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h @@ -0,0 +1,51 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_VARIABLETRANSFORMS_CAL_PROFILEHORIZONTALDRIFT_H_ +#define UFO_VARIABLETRANSFORMS_CAL_PROFILEHORIZONTALDRIFT_H_ + +#include +#include +#include +#include + +#include "oops/util/ObjectCounter.h" +#include "ufo/variabletransforms/TransformBase.h" + +namespace ufo { + +/*! +* \brief Profile horizontal drift calculation. +* +* This computes the horizontal drift positions (and times) given the +* horizontal wind speeds, heights, and an assumed rate of ascent. +* +* This function should only be applied to sondes whose horizontal position was not already +* measured (i.e. not to sondes reporting in BUFR format). +* +* Example: +* +* \code{.yaml} +* obs filters: +* - filter: Variables Transform +* Transform: ["ProfileHorizontalDrift"] +* \endcode +* +* See VariableTransformsParameters for filter setup. +*/ +class Cal_ProfileHorizontalDrift : public TransformBase { + public: + Cal_ProfileHorizontalDrift(const VariableTransformsParameters &options, + ioda::ObsSpace &os, + const std::shared_ptr> &flags, + const std::vector &apply); + // Run variable conversion + void runTransform() override; +}; +} // namespace ufo + +#endif // UFO_VARIABLETRANSFORMS_CAL_PROFILEHORIZONTALDRIFT_H_ diff --git a/src/ufo/variabletransforms/Formulas.cc b/src/ufo/variabletransforms/Formulas.cc index 733303a9a..30ec63f37 100644 --- a/src/ufo/variabletransforms/Formulas.cc +++ b/src/ufo/variabletransforms/Formulas.cc @@ -375,5 +375,115 @@ int RenumberScanPosition(int scanpos) { return newpos; } +/* -------------------------------------------------------------------------------------*/ + +void horizontalDrift +(const std::vector & locs, + const std::vector & apply, + const std::vector & lat_in, + const std::vector & lon_in, + const std::vector & time_in, + const std::vector & height, + const std::vector & windspd, + const std::vector & winddir, + std::vector & lat_out, + std::vector & lon_out, + std::vector & time_out, + MethodFormulation formulation) { + const float missingValueFloat = util::missingValue(1.0f); + + switch (formulation) { + case formulas::MethodFormulation::NCAR: + case formulas::MethodFormulation::NOAA: + case formulas::MethodFormulation::UKMO: + default: { + // Location of the first entry in the profile. + const size_t loc0 = locs.front(); + + // Values of latitude, longitude and datetime at the first entry of the profile. + const double lat0 = lat_in[loc0]; + const double lon0 = lon_in[loc0]; + const util::DateTime time0 = time_in[loc0]; + + // The drift computation is not performed for very high latitude sites. + if (std::abs(lat0) >= 89.0) return; + + // Fill vector of valid locations. + std::vector locs_valid; + for (size_t jloc : locs) { + // If not selected by the where clause. + if (!apply[jloc]) continue; + // The location is classed as valid if the wind speed and height are not missing. + if (windspd[jloc] != missingValueFloat && height[jloc] != missingValueFloat) + locs_valid.push_back(jloc); + } + + // If there are zero or one valid locations, exit the routine. + if (locs_valid.size() < 2) return; + + // Average ascent speed (m/s). + const double ascent_speed = 5.16; + + // Cumulative values of change in time. + // This value is converted to a util::Duration object rather than performing the + // same conversion to each individual change in time. + // This avoids a loss in precision given util::Duration is accurate to the nearest second. + double dt_cumul = 0.0; + + for (size_t k = 0; k < locs_valid.size() - 1; ++k) { + // Locations of the current and next valid observations in the profile. + const size_t loc_current = locs_valid[k]; + const size_t loc_next = locs_valid[k + 1]; + + // Compute changes in latitude, longitude and time between adjacent valid levels. + // Change in height. + const double dh = height[loc_next] - height[loc_current]; + // Change in time. + const double dt = dh / ascent_speed; + // Average eastward and northward wind between the two levels. + // 180 degrees is subtracted from the wind direction in order to account for the different + // conventions used in the observations and in this calculation. + const double avgu = 0.5 * + (windspd[loc_current] * std::sin((winddir[loc_current] - 180.0) * Constants::deg2rad) + + windspd[loc_next] * std::sin((winddir[loc_next] - 180.0) * Constants::deg2rad)); + const double avgv = 0.5 * + (windspd[loc_current] * std::cos((winddir[loc_current] - 180.0) * Constants::deg2rad) + + windspd[loc_next] * std::cos((winddir[loc_next] - 180.0) * Constants::deg2rad)); + // Total height of the observation above the centre of the Earth. + const double totalheight = ufo::Constants::mean_earth_rad * 1000.0 + height[loc_current]; + // Change in latitude. + const double dlat = ufo::Constants::rad2deg * avgv * dt / totalheight; + // Change in longitude. + const double dlon = ufo::Constants::rad2deg * avgu * dt / + (totalheight * std::cos(lat_out[loc_current] * ufo::Constants::Constants::deg2rad)); + + // Fill output values. + lat_out[loc_next] = lat_out[loc_current] + dlat; + lon_out[loc_next] = lon_out[loc_current] + dlon; + // Convert the cumulative change in time to a util::Duration. + dt_cumul += dt; + time_out[loc_next] = time0 + util::Duration(static_cast(dt_cumul)); + } + + // Copy latitude, longitude and time at each valid location to all invalid + // locations that lie between the current valid location and the next one above it. + double lat = lat0; + double lon = lon0; + util::DateTime time = time0; + for (size_t jloc : locs) { + if (std::find(locs_valid.begin(), locs_valid.end(), jloc) != locs_valid.end()) { + lat = lat_out[jloc]; + lon = lon_out[jloc]; + time = time_out[jloc]; + } else { + lat_out[jloc] = lat; + lon_out[jloc] = lon; + time_out[jloc] = time; + } + } + break; + } + } +} } // namespace formulas } // namespace ufo diff --git a/src/ufo/variabletransforms/Formulas.h b/src/ufo/variabletransforms/Formulas.h index 73fdafaaf..9bbf7f5ce 100644 --- a/src/ufo/variabletransforms/Formulas.h +++ b/src/ufo/variabletransforms/Formulas.h @@ -13,6 +13,8 @@ #include #include +#include "oops/util/DateTime.h" +#include "oops/util/Duration.h" #include "oops/util/missingValues.h" #include "ufo/utils/Constants.h" @@ -260,6 +262,67 @@ float GetWind_V(float windSpeed, float windFromDirection); */ int RenumberScanPosition(int scanpos); +// ------------------------------------------------------------------------------------- +/*! +* \brief Compute horizontal drift latitude, longitude and time for an atmospheric profile. +* This formula accepts input and output vectors that correspond to the entire data sample +* as well as a vector of the locations of the current profile in the entire sample. +* The latter vector is used to select the relevant observations from the entire sample +* prior to running the horizontal drift algorithm. +* The outputs of the algorithm are placed in the relevant locations in the entire sample. +* +* \b Formulations \b available: +* - DEFAULT: +* Calculation is using Eq. 4. of Laroche and Sarrazin (2013). +* Reference: "Laroche, S. and Sarrazin, R., +* Impact of Radiosonde Balloon Drift on +* Numerical Weather Prediction and Verification, +* Weather and Forecasting, 28(3), 772-782, 2013." +* - UKMO: as default +* - NCAR: as default +* - NOAA: as default +* +* \param locs +* Vector of locations of the current profile in the entire data sample. +* \param apply +* Vector specifying whether a location should be used or not (governed by the where clause). +* \param lat_in +* Vector of input latitudes in the entire sample [degrees]. +* \param lon_in +* Vector of input longitudes in the entire sample [degrees]. +* \param time_in +* Vector of input datetimes in the entire sample [ISO 8601 format]. +* \param height +* Vector of input heights in the entire sample [m]. +* \param windspd +* Vector of input wind speeds in the entire sample [m/s]. +* \param winddir +* Vector of input wind directions in the entire sample [degrees]. +* Wind direction is defined such that a northerly wind is 0°, an easterly wind is 90°, +* a southerly wind is 180°, and a westerly wind is 270°. +* \param [out] lat_out +* Vector of output latitudes in the entire sample [degrees]. +* \param [out] lon_out +* Vector of output longitudes in the entire sample [degress]. +* \param [out] time_out +* Vector of output datetimes in the entire sample [ISO 8601 format]. +* \param formulation +* Method used to determine the horizontal drift positions. +*/ +void horizontalDrift +(const std::vector & locs, + const std::vector & apply, + const std::vector & lat_in, + const std::vector & lon_in, + const std::vector & time_in, + const std::vector & height, + const std::vector & windspd, + const std::vector & winddir, + std::vector & lat_out, + std::vector & lon_out, + std::vector & time_out, + MethodFormulation formulation = formulas::MethodFormulation::DEFAULT); + } // namespace formulas } // namespace ufo diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1c70853b4..c635e5d72 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -304,6 +304,7 @@ list( APPEND ufo_test_input testinput/unit_tests/variabletransforms_SondeHeightFromPressure.yaml testinput/unit_tests/variabletransforms_windcomponents.yaml testinput/unit_tests/variabletransforms_windspeedanddirection.yaml + testinput/unit_tests/variabletransforms_profilehorizontaldrift.yaml ) @@ -2592,6 +2593,13 @@ ecbuild_add_test( TARGET test_ufo_variabletransforms_humidityspecific DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_variabletransforms_profilehorizontaldrift + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x + ARGS "testinput/unit_tests/variabletransforms_profilehorizontaldrift.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFilters.x + TEST_DEPENDS ufo_get_ufo_test_data ) + # Test Functions ecbuild_add_test( TARGET test_ufo_function_bgddepartureanomaly diff --git a/test/testinput/unit_tests/variabletransforms_profilehorizontaldrift.yaml b/test/testinput/unit_tests/variabletransforms_profilehorizontaldrift.yaml new file mode 100644 index 000000000..193477ec2 --- /dev/null +++ b/test/testinput/unit_tests/variabletransforms_profilehorizontaldrift.yaml @@ -0,0 +1,79 @@ +window begin: 2019-06-14T21:00:00Z +window end: 2019-06-15T03:00:00Z + +observations: +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_profile_horizontal_drift.nc4 + obsgrouping: + group variables: [ "station_id" ] + sort variable: "air_pressure" + sort order: "descending" + simulated variables: [geopotential_height, wind_speed, wind_from_direction] + obs filters: + - filter: Variable Transforms + Transform: ["ProfileHorizontalDrift"] + # Exclude BUFR sondes, whose position has already been measured. + where: + - variable: + name: ObsType@MetaData + is_not_in: 50500 + compareVariables: + - reference: + name: latitude@TestReference + test: + name: latitude@DerivedMetaData + absTol: 1.0e-4 + - reference: + name: longitude@TestReference + test: + name: longitude@DerivedMetaData + absTol: 1.0e-4 + - reference: + name: datetime@TestReference + test: + name: datetime@DerivedMetaData + +# The sample is not grouped into profiles, throwing an exception. +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_profile_horizontal_drift.nc4 + obsgrouping: + sort variable: "air_pressure" + sort order: "descending" + simulated variables: [geopotential_height, wind_speed, wind_from_direction] + obs filters: + - filter: Variable Transforms + Transform: ["ProfileHorizontalDrift"] + expectExceptionWithMessage: Group variables configuration is empty + +# The profiles are not sorted by air pressure, throwing an exception. +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_profile_horizontal_drift.nc4 + obsgrouping: + group variables: [ "station_id" ] + sort order: "descending" + simulated variables: [geopotential_height, wind_speed, wind_from_direction] + obs filters: + - filter: Variable Transforms + Transform: ["ProfileHorizontalDrift"] + expectExceptionWithMessage: Sort variable must be air_pressure + +# The profiles are not sorted in descending order, throwing an exception. +- obs space: + name: Radiosonde + obsdatain: + obsfile: Data/ufo/testinput_tier_1/met_office_profile_horizontal_drift.nc4 + obsgrouping: + group variables: [ "station_id" ] + sort variable: "air_pressure" + sort order: "ascending" + simulated variables: [geopotential_height, wind_speed, wind_from_direction] + obs filters: + - filter: Variable Transforms + Transform: ["ProfileHorizontalDrift"] + expectExceptionWithMessage: Profiles must be sorted in descending order From 185335fba3dc7ebbcda09c335495e0313661bbfa Mon Sep 17 00:00:00 2001 From: Michael Cooke <48374999+mikecooke77@users.noreply.github.com> Date: Thu, 5 Aug 2021 21:48:54 +0100 Subject: [PATCH 70/98] Obs Bound check : Add the ability to just check the good data for flag all filter vars. (#1397) * Add the ability to just check the good data for flag all filter vars in obs bound check * Added an update to ctach if filtervars and testvars are different sizes for the new flag. Also addresses @ctgh comments. * Add back in a test that was accidentally removed Co-authored-by: Ryan Honeyager --- src/ufo/filters/ObsBoundsCheck.cc | 27 +++++-- src/ufo/filters/ObsBoundsCheck.h | 12 ++- test/testinput/qc_boundscheck.yaml | 114 +++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 6 deletions(-) diff --git a/src/ufo/filters/ObsBoundsCheck.cc b/src/ufo/filters/ObsBoundsCheck.cc index e2fff8fc4..e28889c17 100644 --- a/src/ufo/filters/ObsBoundsCheck.cc +++ b/src/ufo/filters/ObsBoundsCheck.cc @@ -34,9 +34,9 @@ namespace { /// closed interval [\p minValue, \p maxValue]. void flagWhereOutOfBounds(const std::vector & apply, const std::vector & testValues, - float minValue, - float maxValue, - bool treatMissingAsOutOfBounds, + const float minValue, + const float maxValue, + const bool treatMissingAsOutOfBounds, std::vector &flagged) { const size_t nlocs = testValues.size(); ASSERT(apply.size() == nlocs); @@ -108,17 +108,34 @@ void ObsBoundsCheck::applyFilter(const std::vector & apply, parameters_.testVariables.value() != boost::none && (parameters_.flagAllFilterVarsIfAnyTestVarOutOfBounds.value() || testvars.nvars() == 1); + const bool onlyTestGoodFilterVarsForFlagAllFilterVars = + parameters_.onlyTestGoodFilterVarsForFlagAllFilterVars; const bool treatMissingAsOutOfBounds = parameters_.treatMissingAsOutOfBounds; // Do the actual work. if (flagAllFilterVarsIfAnyTestVarOutOfBounds) { - std::vector anyTestVarOutOfBounds(obsdb_.nlocs(), false); + // If using test only good filter vars then the number of filter + // vars must equal the number of test vars + if (onlyTestGoodFilterVarsForFlagAllFilterVars && filtervars.nvars() != testvars.nvars()) + throw eckit::UserError("The number of 'primitive' (single-channel) test variables must " + "match that of 'primitive' filter variables when using the " + "'test only filter variables with passed qc when flagging all " + "filter variables' option."); + ASSERT(filtervars.nvars() == flagged.size()); // Loop over all channels of all test variables and record all locations where any of these // channels is out of bounds. + std::vector anyTestVarOutOfBounds(obsdb_.nlocs(), false); + size_t ifiltervar = 0; for (PrimitiveVariable singleChannelTestVar : PrimitiveVariables(testvars, data_)) { + std::vector testAtLocations = apply; + if (onlyTestGoodFilterVarsForFlagAllFilterVars) { + for (size_t iloc=0; iloc < testAtLocations.size(); iloc++) + if ((*flags_)[ifiltervar][iloc] != QCflags::pass) testAtLocations[iloc] = false; + } const std::vector & testValues = singleChannelTestVar.values(); - flagWhereOutOfBounds(apply, testValues, vmin, vmax, treatMissingAsOutOfBounds, + flagWhereOutOfBounds(testAtLocations, testValues, vmin, vmax, treatMissingAsOutOfBounds, anyTestVarOutOfBounds); + ifiltervar++; } // Copy these flags to the flags of all filtered variables. for (std::vector &f : flagged) diff --git a/src/ufo/filters/ObsBoundsCheck.h b/src/ufo/filters/ObsBoundsCheck.h index 0583102d5..b761d10fa 100644 --- a/src/ufo/filters/ObsBoundsCheck.h +++ b/src/ufo/filters/ObsBoundsCheck.h @@ -54,7 +54,10 @@ class ObsBoundsCheckParameters : public FilterParametersBase { /// * If this option is set to a single-element list containing only one single-channel variable /// or the `flag all filter variables if any test variable is out of bounds` option is set to /// `true`, the filter will flag each filter variable at each location where any test variable - /// lies outside the specified bounds. + /// lies outside the specified bounds. If + /// `test only filter variables with passed qc when flagging all filter variables` is set to + /// true and the number of filter vars is not equal to the number of test vars an error + /// will be thrown. /// /// * If this option is set to a list with as many elements as there are filter variables and /// the `flag all filter variables if any test variable is out of bounds` option is set to @@ -69,6 +72,13 @@ class ObsBoundsCheckParameters : public FilterParametersBase { oops::Parameter flagAllFilterVarsIfAnyTestVarOutOfBounds{ "flag all filter variables if any test variable is out of bounds", false, this}; + /// Set this option to `true` to only test the current filter variable if it is flagged + /// as pass qc when using "flag all filter variables if any test variables out of bounds". + /// + /// This option is not used if "flagAllFilterVarsIfAnyTestVarOutOfBounds" is false. + oops::Parameter onlyTestGoodFilterVarsForFlagAllFilterVars{ + "test only filter variables with passed qc when flagging all filter variables", false, this}; + /// By default, the filter flags filter variables at locations where the corresponding test /// variable is set to the missing value indicator. Set this option to `false` to stop it from /// doing so (hence assuming "optimistically" that the test variable was in fact in bounds). diff --git a/test/testinput/qc_boundscheck.yaml b/test/testinput/qc_boundscheck.yaml index 30768e795..a84f8fc00 100644 --- a/test/testinput/qc_boundscheck.yaml +++ b/test/testinput/qc_boundscheck.yaml @@ -266,3 +266,117 @@ observations: minvalue: 2018-04-14T21:00:00Z maxvalue: 2018-04-14T22:00:00Z failedBenchmark: 17 +- obs space: + name: test data + obsdatain: + obsfile: Data/ufo/testinput_tier_1/filters_testdata.nc4 + simulated variables: [variable2, variable3] + obs filters: + - filter: Bounds Check + filter variables: + - name: variable2 + test variables: + - name: ObsValue/variable2 + maxvalue: 24 + - filter: Bounds Check # reject filter variables if the test variable is out of bounds or missing + filter variables: + - name: variable2 + - name: variable3 + test variables: + - name: ObsValue/variable2 + - name: ObsValue/variable3 + treat missing as out of bounds: true + maxvalue: 24 + flag all filter variables if any test variable is out of bounds: true + test only filter variables with passed qc when flagging all filter variables: true +# variable2: 10,12,14,16,18,20,22,24,26,28 +# variable3: 25,24,23,22,21,20,19,18,17,16 +# filter 1 rejects location 9,10 for variable2 +# filter 2 rejects location 1 for both variable2 and variable 3 +# filter 2 does not check variable2 location 9 and 10 because these have previously been rejected. + passedBenchmark: 16 +- obs space: + name: test data + obsdatain: + obsfile: Data/ufo/testinput_tier_1/filters_testdata.nc4 + simulated variables: [variable2, variable3] + obs filters: + - filter: Bounds Check + filter variables: + - name: variable2 + test variables: + - name: ObsValue/variable2 + maxvalue: 24 + - filter: Bounds Check # reject filter variables if the test variable is out of bounds or missing + filter variables: + - name: variable2 + - name: variable3 + test variables: + - name: MetaData/var2 + - name: MetaData/var1 + treat missing as out of bounds: true + minvalue: 6 + flag all filter variables if any test variable is out of bounds: true + test only filter variables with passed qc when flagging all filter variables: true +# variable2: 10,12,14,16,18,20,22,24,26,28 +# variable3: 25,24,23,22,21,20,19,18,17,16 +# MetaData/var1: 1,2,3,4,5,6,7,8,9,10 +# MetaData/var2: 10,9,8,7,6,5,4,3,2,1 +# filter 1 rejects location 9,10 for variable2 +# filter 2 rejects location 1-5 and 6-8 for variable2 and variable 3 +# filter 2 does not check location 9 and 10 because variable 2 has previously been rejected by filter 1. + passedBenchmark: 2 # loc 9,10 for variable3 still valid +- obs space: + name: test data + obsdatain: + obsfile: Data/ufo/testinput_tier_1/filters_testdata.nc4 + simulated variables: [variable2, variable3] + obs filters: + - filter: Bounds Check + filter variables: + - name: variable2 + test variables: + - name: ObsValue/variable2 + maxvalue: 24 + - filter: Bounds Check # reject filter variables if the test variable is out of bounds or missing + filter variables: + - name: variable2 + - name: variable3 + test variables: + - name: MetaData/var2 + - name: MetaData/var1 + treat missing as out of bounds: true + minvalue: 6 + flag all filter variables if any test variable is out of bounds: true + test only filter variables with passed qc when flagging all filter variables: false # default +# variable2: 10,12,14,16,18,20,22,24,26,28 +# variable3: 25,24,23,22,21,20,19,18,17,16 +# MetaData/var1: 1,2,3,4,5,6,7,8,9,10 +# MetaData/var2: 10,9,8,7,6,5,4,3,2,1 +# filter 1 rejects location 9,10 for variable2 +# filter 2 rejects location 1-5 and 6-10 for both variable2 and variable 3 +# filter 2 does check variable2 location 9 and 10 even those these have previously been rejected by filter 1. + passedBenchmark: 0 +- obs space: + name: test data + obsdatain: + obsfile: Data/ufo/testinput_tier_1/filters_testdata.nc4 + simulated variables: [variable2, variable3] + obs filters: + - filter: Bounds Check + filter variables: + - name: variable1 + - name: variable2 + - name: variable3 + test variables: + - name: ObsValue/variable2 + maxvalue: 24 + flag all filter variables if any test variable is out of bounds: true + test only filter variables with passed qc when flagging all filter variables: true +# filter1 fails with an error because filtervars size is not equal to test variables size +# which it needs to be when using +# `test only filter variables with passed qc when flagging all filter variables` + expectExceptionWithMessage: The number of 'primitive' (single-channel) test variables + must match that of 'primitive' filter variables when using + the 'test only filter variables with passed qc when flagging + all filter variables' option. From 84fef69b14ec072f3791201341e82e2d99765522 Mon Sep 17 00:00:00 2001 From: David Davies Date: Fri, 6 Aug 2021 01:58:44 +0100 Subject: [PATCH 71/98] Fix icpc compile failure. (#1419) Co-authored-by: Ryan Honeyager --- src/ufo/filters/obsfunctions/DrawValueFromFile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ufo/filters/obsfunctions/DrawValueFromFile.cc b/src/ufo/filters/obsfunctions/DrawValueFromFile.cc index b5582f3a5..068584645 100644 --- a/src/ufo/filters/obsfunctions/DrawValueFromFile.cc +++ b/src/ufo/filters/obsfunctions/DrawValueFromFile.cc @@ -167,7 +167,7 @@ void DrawValueFromFile::compute(const ObsFilterData & in, // Perform any extraction methods. ExtractVisitor visitor(interpolator, iloc); for (size_t ind=0; ind < obData.size(); ind++) { - const InterpMethod &interpolationMethod = interpMethod_.at(obData[ind].first); + ufo::InterpMethod interpolationMethod = interpMethod_.at(obData[ind].first); if ((interpolationMethod == InterpMethod::BILINEAR) && (ind == (obData.size()-2))) { boost::apply_visitor(visitor, obData[ind].second, obData[ind+1].second); break; From 76f06bed18639f5901f5c97938b1eaec3651d9b9 Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan <2984440+yaswant@users.noreply.github.com> Date: Fri, 6 Aug 2021 15:07:09 +0100 Subject: [PATCH 72/98] fix CMAKE_CURRENT_SOURCE_DIR expansion (#1408) * fix CMAKE_CURRENT_SOURCE_DIR expansion * Update sources of the autogenerated filter to accommodate recent changes to the oops::ObsFilter interface. (#1409) Co-authored-by: Wojciech Smigaj Co-authored-by: Ryan Honeyager --- tools/CMakeLists.txt | 2 +- tools/new_qc/example/Example.cc | 5 ++--- tools/new_qc/example/Example.h | 17 +++++++++-------- tools/new_qc/test/TestAutogeneratedFilter.cc | 7 +++---- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 1ac618cab..4da8d2372 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -9,7 +9,7 @@ foreach(FILENAME ${test_files}) ${CMAKE_BINARY_DIR}/bin/${PROJECT_NAME}_${FILENAME} ) endforeach(FILENAME) -execute_process( COMMAND test -w @CMAKE_CURRENT_SOURCE_DIR RESULT_VARIABLE NO_WRITE ) +execute_process( COMMAND test -w ${CMAKE_CURRENT_SOURCE_DIR} RESULT_VARIABLE NO_WRITE ) if ( NO_WRITE ) message( WARNING "Source dir ${CMAKE_CURRENT_SOURCE_DIR} is not writeable - autogenerated templates for obsoperator and filters - new_obsop and new_qc - wont be produced in ufo build." ) else() diff --git a/tools/new_qc/example/Example.cc b/tools/new_qc/example/Example.cc index ffabf29a5..f0198d3c3 100644 --- a/tools/new_qc/example/Example.cc +++ b/tools/new_qc/example/Example.cc @@ -13,7 +13,6 @@ #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" #include "oops/base/Variables.h" -#include "oops/interface/ObsFilter.h" #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" #include "ufo/ObsDiagnostics.h" @@ -41,14 +40,14 @@ Example::~Example() { // ----------------------------------------------------------------------------- -void Example::priorFilter(const GeoVaLs & gv) const { +void Example::priorFilter(const GeoVaLs & gv) { oops::Log::trace() << "Example priorFilter" << std::endl; ufo_example_prior_f90(key_, obsdb_, gv.toFortran()); } // ----------------------------------------------------------------------------- -void Example::postFilter(const ioda::ObsVector & hofxb, const ObsDiagnostics & diags) const { +void Example::postFilter(const ioda::ObsVector & hofxb, const ObsDiagnostics & diags) { oops::Log::trace() << "Example postFilter" << std::endl; ufo_example_post_f90(key_, obsdb_, hofxb.nvars(), hofxb.nlocs(), hofxb.toFortran(), diags.toFortran()); diff --git a/tools/new_qc/example/Example.h b/tools/new_qc/example/Example.h index 48ea9f0ac..00acc944d 100644 --- a/tools/new_qc/example/Example.h +++ b/tools/new_qc/example/Example.h @@ -15,8 +15,9 @@ #include "ioda/ObsDataVector.h" #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" -#include "oops/util/Printable.h" +#include "oops/interface/ObsFilterBase.h" #include "tools/new_qc/example/Example.interface.h" +#include "ufo/ObsTraits.h" namespace eckit { class Configuration; @@ -33,7 +34,7 @@ namespace ufo { /// Example filter -class Example : public util::Printable, +class Example : public oops::interface::ObsFilterBase, private util::ObjectCounter { public: static const std::string classname() {return "ufo::Example";} @@ -43,15 +44,15 @@ class Example : public util::Printable, std::shared_ptr >); ~Example(); - void preProcess() const {} - void priorFilter(const GeoVaLs &) const; - void postFilter(const ioda::ObsVector &, const ObsDiagnostics &) const; + void preProcess() override {} + void priorFilter(const GeoVaLs &) override; + void postFilter(const ioda::ObsVector &, const ObsDiagnostics &) override; - const oops::Variables & requiredVars() const {return geovars_;} - const oops::Variables & requiredHdiagnostics() const {return diagnostics_;} + oops::Variables requiredVars() const override {return geovars_;} + oops::Variables requiredHdiagnostics() const override {return diagnostics_;} private: - void print(std::ostream &) const; + void print(std::ostream &) const override; F90check key_; ioda::ObsSpace & obsdb_; diff --git a/tools/new_qc/test/TestAutogeneratedFilter.cc b/tools/new_qc/test/TestAutogeneratedFilter.cc index 95801326c..951199df6 100644 --- a/tools/new_qc/test/TestAutogeneratedFilter.cc +++ b/tools/new_qc/test/TestAutogeneratedFilter.cc @@ -5,16 +5,15 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#include "oops/interface/ObsFilter.h" +#include "oops/interface/ObsFilterBase.h" #include "oops/runs/Run.h" #include "ufo/filters/AutogeneratedFilter.h" #include "ufo/instantiateObsFilterFactory.h" #include "ufo/ObsTraits.h" #include "test/ufo/ObsFilters.h" -int main(int argc, char ** argv) { - oops::FilterMaker > - maker("Autogenerated"); +int main(int argc, char ** argv) { + oops::interface::FilterMaker maker("Autogenerated"); oops::Run run(argc, argv); ufo::instantiateObsFilterFactory(); From 0bcc3d9a5de6e1869a6343ee8f150621af26a8c6 Mon Sep 17 00:00:00 2001 From: Neill Bowler <45592137+neill-b@users.noreply.github.com> Date: Fri, 6 Aug 2021 16:35:34 +0100 Subject: [PATCH 73/98] Use the parameters class in Met Office's GNSS-RO bending angle operator and in 1D-Var filter (#1394) * Use Parameters class in the Met Office's GNSS-RO bending angle operator. * Remove redundant parameters, as in separate pull request. * First working use of parameters in 1D-Var code. * Various ways to tidy up the code. Co-authored-by: Ryan Honeyager --- .../filters/gnssroonedvarcheck/CMakeLists.txt | 1 + .../gnssroonedvarcheck/GNSSROOneDVarCheck.cc | 36 ++++++-- .../gnssroonedvarcheck/GNSSROOneDVarCheck.h | 16 +++- .../GNSSROOneDVarCheck.interface.F90 | 62 +++++++++++-- .../GNSSROOneDVarCheck.interface.h | 23 ++++- .../GNSSROOneDVarCheckParameters.h | 77 ++++++++++++++++ .../ufo_gnssroonedvarcheck_do1dvar_mod.f90 | 4 - .../ufo_gnssroonedvarcheck_mod.f90 | 87 ++++++++++++------- .../BendMetOffice/ObsGnssroBendMetOffice.cc | 14 +-- .../BendMetOffice/ObsGnssroBendMetOffice.h | 4 +- .../ObsGnssroBendMetOffice.interface.F90 | 15 ++-- .../ObsGnssroBendMetOffice.interface.h | 2 +- .../ObsGnssroBendMetOfficeParameters.h | 49 +++++++++++ .../ObsGnssroBendMetOfficeTLAD.cc | 16 ++-- .../ObsGnssroBendMetOfficeTLAD.h | 2 + .../ObsGnssroBendMetOfficeTLAD.interface.F90 | 15 ++-- .../ObsGnssroBendMetOfficeTLAD.interface.h | 2 +- .../ufo_gnssro_bendmetoffice_mod.F90 | 14 +-- .../ufo_gnssro_bendmetoffice_tlad_mod.F90 | 14 +-- .../RefMetOffice/ObsGnssroRefMetOffice.cc | 4 +- .../RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc | 5 +- test/testinput/gnssrobendmetoffice_qc.yaml | 4 - 22 files changed, 360 insertions(+), 106 deletions(-) create mode 100644 src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheckParameters.h create mode 100644 src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeParameters.h diff --git a/src/ufo/filters/gnssroonedvarcheck/CMakeLists.txt b/src/ufo/filters/gnssroonedvarcheck/CMakeLists.txt index 740da9e46..9fd3eebb9 100644 --- a/src/ufo/filters/gnssroonedvarcheck/CMakeLists.txt +++ b/src/ufo/filters/gnssroonedvarcheck/CMakeLists.txt @@ -8,6 +8,7 @@ set ( gnssroonedvarcheck_files GNSSROOneDVarCheck.cc GNSSROOneDVarCheck.interface.F90 GNSSROOneDVarCheck.interface.h + GNSSROOneDVarCheckParameters.h ufo_gnssroonedvarcheck_do1dvar_mod.f90 ufo_gnssroonedvarcheck_eval_derivs_mod.f90 ufo_gnssroonedvarcheck_get_bmatrix_mod.f90 diff --git a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.cc b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.cc index bf2098b2b..82593324e 100644 --- a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.cc +++ b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.cc @@ -20,24 +20,43 @@ #include "ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.interface.h" #include "ufo/GeoVaLs.h" -#include "eckit/config/Configuration.h" - #include "oops/util/IntSetParser.h" namespace ufo { // ----------------------------------------------------------------------------- -GNSSROOneDVarCheck::GNSSROOneDVarCheck(ioda::ObsSpace & obsdb, const eckit::Configuration & config, - std::shared_ptr > flags, - std::shared_ptr > obserr) - : FilterBase(obsdb, config, flags, obserr), config_(config) +GNSSROOneDVarCheck::GNSSROOneDVarCheck(ioda::ObsSpace & obsdb, + const Parameters_ & parameters, + std::shared_ptr > flags, + std::shared_ptr > obserr) + : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::debug() << "GNSSROOneDVarCheck contructor starting" << std::endl; + // convert from std::string to char pointer to pass to Fortran routine + // pass each variable as a char pointer + corresponding variable length + char * filename = new char[parameters_.bmatrix_filename.value().size()+1]; + strcpy(filename, parameters_.bmatrix_filename.value().c_str()); + // Setup fortran object - const eckit::Configuration * conf = &config_; - ufo_gnssroonedvarcheck_create_f90(key_, obsdb, conf, GNSSROOneDVarCheck::qcFlag()); + ufo_gnssroonedvarcheck_create_f90(key_, + obsdb, + parameters_.bmatrix_filename.value().size(), + filename, + parameters_.capsupersat.value(), + parameters_.cost_funct_test.value(), + parameters_.Delta_ct2.value(), + parameters_.Delta_factor.value(), + parameters_.min_temp_grad.value(), + parameters_.n_iteration_test.value(), + parameters_.OB_test.value(), + parameters_.pseudo_ops.value(), + parameters_.vert_interp_ops.value(), + parameters_.y_test.value(), + GNSSROOneDVarCheck::qcFlag()); + + delete[] filename; oops::Log::debug() << "GNSSROOneDVarCheck contructor complete. " << std::endl; } @@ -69,7 +88,6 @@ void GNSSROOneDVarCheck::applyFilter(const std::vector & apply, flags_->save("FortranQC"); // temporary measure as per ROobserror qc // Pass it all to fortran - const eckit::Configuration * conf = &config_; ufo_gnssroonedvarcheck_apply_f90(key_, gvals->toFortran(), apply_char.size(), apply_char[0]); diff --git a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.h b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.h index a2c697392..91be961e5 100644 --- a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.h +++ b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.h @@ -20,6 +20,7 @@ #include "oops/util/Printable.h" #include "ufo/filters/FilterBase.h" #include "ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.interface.h" +#include "ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheckParameters.h" #include "ufo/filters/QCflags.h" namespace eckit { @@ -48,21 +49,28 @@ namespace ufo { class GNSSROOneDVarCheck : public FilterBase, private util::ObjectCounter { public: + /// The type of parameters accepted by the constructor of this filter. + /// This typedef is used by the FilterFactory. + typedef GNSSROOneDVarCheckParameters Parameters_; + static const std::string classname() {return "ufo::GNSSROOneDVarCheck";} - GNSSROOneDVarCheck(ioda::ObsSpace &, const eckit::Configuration &, - std::shared_ptr >, - std::shared_ptr >); + GNSSROOneDVarCheck(ioda::ObsSpace &, + const Parameters_ &, + std::shared_ptr >, + std::shared_ptr >); ~GNSSROOneDVarCheck(); private: void print(std::ostream &) const override; - void applyFilter(const std::vector &, const Variables &, + void applyFilter(const std::vector &, + const Variables &, std::vector> &) const override; int qcFlag() const override {return QCflags::onedvar;} F90onedvarcheck key_; const eckit::LocalConfiguration config_; + GNSSROOneDVarCheckParameters parameters_; }; } // namespace ufo diff --git a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.interface.F90 b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.interface.F90 index dcf0fbac7..5c7a2a54a 100644 --- a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.interface.F90 +++ b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.interface.F90 @@ -31,7 +31,21 @@ module ufo_gnssroonedvarcheck_mod_c #include "oops/util/linkedList_c.f" ! ------------------------------------------------------------------------------------------------ -subroutine ufo_gnssroonedvarcheck_create_c(c_self, c_obspace, c_conf, c_onedvarflag) & +subroutine ufo_gnssroonedvarcheck_create_c(c_self, & + c_obspace, & + filename_length, & + input_filename, & + capsupersat, & + cost_funct_test, & + Delta_ct2, & + Delta_factor, & + min_temp_grad, & + n_iteration_test, & + OB_test, & + pseudo_ops, & + vert_interp_ops, & + y_test, & + c_onedvarflag) & bind(c,name='ufo_gnssroonedvarcheck_create_f90') !> \brief Interface to the Fortran create method @@ -41,18 +55,50 @@ subroutine ufo_gnssroonedvarcheck_create_c(c_self, c_obspace, c_conf, c_onedvarf !! \date 09/06/2020: Created !! implicit none -integer(c_int), intent(inout) :: c_self !< self - inout -type(c_ptr), value, intent(in) :: c_obspace !< obsspace - input -type(c_ptr), value, intent(in) :: c_conf !< yaml configuration - input -integer(c_int), intent(in) :: c_onedvarflag !< flag for qc manager logging - input + +integer(c_int), intent(inout) :: c_self !< self - inout +type(c_ptr), value, intent(in) :: c_obspace !< obsspace - input +integer(c_int), intent(in) :: filename_length !< Length of the filename string +character(c_char), intent(in) :: input_filename(filename_length) !< B-matrix filename +logical(c_bool), intent(in) :: capsupersat !< Whether to remove super-saturation (wrt ice?) +real(c_float), intent(in) :: cost_funct_test !< Threshold value for the cost function convergence test +real(c_float), intent(in) :: Delta_ct2 !< Threshold used in calculating convergence +real(c_float), intent(in) :: Delta_factor !< Threshold used in calculating convergence +real(c_float), intent(in) :: min_temp_grad !< The minimum vertical temperature gradient allowed +integer(c_int), intent(in) :: n_iteration_test !< Maximum number of iterations in the 1DVar +real(c_float), intent(in) :: OB_test !< Threshold for the O-B throughout the profile +logical(c_bool), intent(in) :: pseudo_ops !< Whether to use pseudo levels in forward operator +logical(c_bool), intent(in) :: vert_interp_ops !< Whether to use ln(p) or exner in vertical interpolation +real(c_float), intent(in) :: y_test !< Threshold on distance between observed and solution bending angles +integer(c_int), intent(in) :: c_onedvarflag !< flag for qc manager logging - input + +character(len=filename_length) :: bmatrix_filename ! Location of the B-matrix file +integer :: ifname ! Loop variable for filename type(ufo_gnssroonedvarcheck), pointer :: self -type(fckit_configuration) :: f_conf call ufo_gnssroonedvarcheck_registry%setup(c_self, self) -f_conf = fckit_configuration(c_conf) -call ufo_gnssroonedvarcheck_create(self, c_obspace, f_conf, c_onedvarflag) +! copy over the char* into a Fortran character +do ifname = 1, filename_length + if (input_filename(ifname) == c_null_char) exit + bmatrix_filename(ifname:ifname) = input_filename(ifname) +end do + +call ufo_gnssroonedvarcheck_create(self, & + c_obspace, & + bmatrix_filename, & + capsupersat, & + cost_funct_test, & + Delta_ct2, & + Delta_factor, & + min_temp_grad, & + n_iteration_test, & + OB_test, & + pseudo_ops, & + vert_interp_ops, & + y_test, & + c_onedvarflag) end subroutine ufo_gnssroonedvarcheck_create_c diff --git a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.interface.h b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.interface.h index bdb5094ef..f7f5f4c7c 100644 --- a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.interface.h +++ b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.interface.h @@ -26,11 +26,26 @@ typedef int F90onedvarcheck; /// Interface to Fortran routines extern "C" { - void ufo_gnssroonedvarcheck_create_f90(F90onedvarcheck &, const ioda::ObsSpace &, - const eckit::Configuration *, const int &); + void ufo_gnssroonedvarcheck_create_f90(F90onedvarcheck &, + const ioda::ObsSpace &, + const int &, + const char *, + const bool &, + const float &, + const float &, + const float &, + const float &, + const int &, + const float &, + const bool &, + const bool &, + const float &, + const int &); void ufo_gnssroonedvarcheck_delete_f90(F90onedvarcheck &); - void ufo_gnssroonedvarcheck_apply_f90(const F90onedvarcheck &, const F90goms &, const int &, - const char &); + void ufo_gnssroonedvarcheck_apply_f90(const F90onedvarcheck &, + const F90goms &, + const int &, + const char &); } // extern C } // namespace ufo diff --git a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheckParameters.h b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheckParameters.h new file mode 100644 index 000000000..8980d9456 --- /dev/null +++ b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheckParameters.h @@ -0,0 +1,77 @@ +/* + * (C) Copyright 2021 UK Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_FILTERS_GNSSROONEDVARCHECK_GNSSROONEDVARCHECKPARAMETERS_H_ +#define UFO_FILTERS_GNSSROONEDVARCHECK_GNSSROONEDVARCHECKPARAMETERS_H_ + +#include +#include + +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" + +namespace ufo { + +/// Configuration options recognized by the gnss-ro 1DVar check. +class GNSSROOneDVarCheckParameters : public FilterParametersBase { + OOPS_CONCRETE_PARAMETERS(GNSSROOneDVarCheckParameters, FilterParametersBase) + + public: + /// Filename of the static B-matrix to be used in the 1D-Var minimisation + oops::RequiredParameter bmatrix_filename{"bmatrix_filename", this}; + + /// If true calculate saturation vapour pressure with respect to water and ice + /// (below zero degrees), else calculate it with respect to water everywhere. + oops::Parameter capsupersat{"capsupersat", false, this}; + + /// The profile is rejected if the final cost-function is larger than + /// cost_funct_test times the number of observations. + oops::Parameter cost_funct_test{"cost_funct_test", 2.0, this}; + + /// The minimisation is considered to have converged if the absolute value of + /// the change in the solution for an iteration, divided by the gradient in + /// the cost-function is less than Delta_ct2 times the number of observations + /// in the profile, divided by 200. + oops::Parameter Delta_ct2{"Delta_ct2", 1, this}; + + /// The minimisation is considered to have converged if the absolute change + /// in the cost-function at this iteration is less than Delta_factor times + /// either the previous value of the cost-function or the number of + /// observations (whichever is the smaller). That is, the minimisation has + /// converged if: ABS(J_new - J_old) < Delta_factor * min(J_old, nObs) + oops::Parameter Delta_factor{"Delta_factor", 0.01, this}; + + /// Threshold for the minimum temperature gradient before a profile is + /// considered isothermal (units: K per model level). Only applies if + /// pseudo-levels are being used. + oops::Parameter min_temp_grad{"min_temp_grad", 1.0e-6, this}; + + /// The maximum number of iterations - the profile is rejected if it does not + /// converge in time. + oops::Parameter n_iteration_test{"n_iteration_test", 20, this}; + + /// If the RMS difference between the observations and the background bending + /// angle is greater than OB_test then the whole profile is rejected. + oops::Parameter OB_test{"OB_test", 2.5, this}; + + /// Whether to use pseudo-levels to reduce interpolation errors in the + /// forward model. + oops::Parameter pseudo_ops{"pseudo_ops", true, this}; + + /// If true use linear interpolation in ln(pressure), otherwise use linear + /// interpolation in exner. + oops::Parameter vert_interp_ops{"vert_interp_ops", true, this}; + + /// If an observation is more than y_test times the observation error away + /// from the solution bending angle, then the observation (not the whole + /// profile) is rejected. + oops::Parameter y_test{"y_test", 5, this}; +}; + +} // namespace ufo +#endif // UFO_FILTERS_GNSSROONEDVARCHECK_GNSSROONEDVARCHECKPARAMETERS_H_ diff --git a/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_do1dvar_mod.f90 b/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_do1dvar_mod.f90 index 0f0756a6a..069aff31c 100644 --- a/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_do1dvar_mod.f90 +++ b/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_do1dvar_mod.f90 @@ -30,8 +30,6 @@ SUBROUTINE Ops_GPSRO_Do1DVar_BA (nlevp, & GPSRO_pseudo_ops, & GPSRO_vert_interp_ops, & GPSRO_min_temp_grad, & - GPSRO_Zmin, & - GPSRO_Zmax, & GPSRO_cost_funct_test, & ! Threshold value for the cost function convergence test GPSRO_y_test, & ! Threshold value for the yobs-ysol tes GPSRO_n_iteration_test, & ! Maximum number of iterations @@ -67,8 +65,6 @@ SUBROUTINE Ops_GPSRO_Do1DVar_BA (nlevp, & LOGICAL, INTENT(IN) :: GPSRO_pseudo_ops LOGICAL, INTENT(IN) :: GPSRO_vert_interp_ops REAL(kind_real), INTENT(IN) :: GPSRO_min_temp_grad -REAL(kind_real), INTENT(IN) :: GPSRO_Zmin -REAL(kind_real), INTENT(IN) :: GPSRO_Zmax REAL(kind_real), INTENT(IN) :: GPSRO_cost_funct_test REAL(kind_real), INTENT(IN) :: GPSRO_y_test INTEGER, INTENT(IN) :: GPSRO_n_iteration_test diff --git a/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_mod.f90 b/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_mod.f90 index 5bd679884..696d01bba 100644 --- a/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_mod.f90 +++ b/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_mod.f90 @@ -43,8 +43,6 @@ module ufo_gnssroonedvarcheck_mod integer :: n_iteration_test !< Maximum number of iterations in the 1DVar real(kind_real) :: OB_test !< Threshold for the O-B throughout the profile real(kind_real) :: y_test !< Threshold on distance between observed and solution bending angles - real(kind_real) :: Zmin !< Minimum height at for assimilation of data - real(kind_real) :: Zmax !< Maximum height at for assimilation of data character(len=800) :: bmatrix_filename !< Location of the B-matrix file logical :: pseudo_ops !< Whether to use pseudo levels in forward operator logical :: vert_interp_ops !< Whether to use ln(p) or exner in vertical interpolation @@ -62,34 +60,69 @@ module ufo_gnssroonedvarcheck_mod !! !! \date 09/06/2020: Created !! -subroutine ufo_gnssroonedvarcheck_create(self, obsspace, f_conf, onedvarflag) +subroutine ufo_gnssroonedvarcheck_create(self, obsspace, bmatrix_filename, & + capsupersat, cost_funct_test, & + Delta_ct2, Delta_factor, min_temp_grad, & + n_iteration_test, OB_test, pseudo_ops, & + vert_interp_ops, y_test, onedvarflag) implicit none - type(ufo_gnssroonedvarcheck), intent(inout) :: self !< gnssroonedvarcheck main object - type(c_ptr), value, intent(in) :: obsspace !< observation database pointer - type(fckit_configuration), intent(in) :: f_conf !< yaml file contents - integer(c_int), intent(in) :: onedvarflag !< flag for qc manager - - character(len=:), allocatable :: temp_str1 + type(ufo_gnssroonedvarcheck), intent(inout) :: self !< gnssroonedvarcheck main object + type(c_ptr), value, intent(in) :: obsspace !< observation database pointer + character(len=*), intent(in) :: bmatrix_filename !< Location of the B-matrix file + logical(c_bool), intent(in) :: capsupersat !< Whether to remove super-saturation (wrt ice?) + real(c_float), intent(in) :: cost_funct_test !< Threshold value for the cost function convergence test + real(c_float), intent(in) :: Delta_ct2 !< Threshold used in calculating convergence + real(c_float), intent(in) :: Delta_factor !< Threshold used in calculating convergence + real(c_float), intent(in) :: min_temp_grad !< The minimum vertical temperature gradient allowed + integer(c_int), intent(in) :: n_iteration_test !< Maximum number of iterations in the 1DVar + real(c_float), intent(in) :: OB_test !< Threshold for the O-B throughout the profile + logical(c_bool), intent(in) :: pseudo_ops !< Whether to use pseudo levels in forward operator + logical(c_bool), intent(in) :: vert_interp_ops !< Whether to use ln(p) or exner in vertical interpolation + real(c_float), intent(in) :: y_test !< Threshold on distance between observed and solution bending angles + integer(c_int), intent(in) :: onedvarflag !< flag for qc manager + + character(len=800) :: message self % obsdb = obsspace - self % conf = f_conf self % onedvarflag = onedvarflag - call f_conf%get_or_die("capsupersat", self % capsupersat) - call f_conf%get_or_die("Zmin", self % Zmin) - call f_conf%get_or_die("Zmax", self % Zmax) - call f_conf%get_or_die("cost_funct_test", self % cost_funct_test) - call f_conf%get_or_die("y_test", self % y_test) - call f_conf%get_or_die("n_iteration_test", self % n_iteration_test) - call f_conf%get_or_die("Delta_ct2", self % Delta_ct2) - call f_conf%get_or_die("Delta_factor", self % Delta_factor) - call f_conf%get_or_die("OB_test", self % OB_test) - call f_conf%get_or_die("bmatrix_filename", temp_str1) - self % bmatrix_filename = temp_str1 - call f_conf%get_or_die("pseudo_ops", self % pseudo_ops) - call f_conf%get_or_die("vert_interp_ops", self % vert_interp_ops) - call f_conf%get_or_die("min_temp_grad", self % min_temp_grad) + self % bmatrix_filename = bmatrix_filename + self % capsupersat = capsupersat + self % cost_funct_test = cost_funct_test + self % Delta_ct2 = Delta_ct2 + self % Delta_factor = Delta_factor + self % min_temp_grad = min_temp_grad + self % n_iteration_test = n_iteration_test + self % OB_test = OB_test + self % pseudo_ops = pseudo_ops + self % vert_interp_ops = vert_interp_ops + self % y_test = y_test + + write(message, '(A)') 'GNSS-RO 1D-Var check: input parameters are:' + call fckit_log % debug(message) + write(message, '(2A)') 'bmatrix_filename = ', bmatrix_filename + call fckit_log % debug(message) + write(message, '(A,L1)') 'capsupersat = ', capsupersat + call fckit_log % debug(message) + write(message, '(A,F16.8)') 'cost_funct_test = ', cost_funct_test + call fckit_log % debug(message) + write(message, '(A,F16.8)') 'Delta_ct2 = ', Delta_ct2 + call fckit_log % debug(message) + write(message, '(A,F16.8)') 'Delta_factor = ', Delta_factor + call fckit_log % debug(message) + write(message, '(A,F16.8)') 'min_temp_grad = ', min_temp_grad + call fckit_log % debug(message) + write(message, '(A,I7)') 'n_iteration_test = ', n_iteration_test + call fckit_log % debug(message) + write(message, '(A,F16.8)') 'OB_test = ', OB_test + call fckit_log % debug(message) + write(message, '(A,L1)') 'pseudo_ops = ', pseudo_ops + call fckit_log % debug(message) + write(message, '(A,L1)') 'vert_interp_ops = ', vert_interp_ops + call fckit_log % debug(message) + write(message, '(A,F16.8)') 'y_test = ', y_test + call fckit_log % debug(message) end subroutine ufo_gnssroonedvarcheck_create @@ -221,10 +254,6 @@ subroutine ufo_gnssroonedvarcheck_apply(self, geovals, apply) call fckit_log % info(Message) WRITE (Message, '(A,I0)') 'Sat ID ', obsSatid(index_vals(start_point)) call fckit_log % info(Message) - WRITE (Message, '(A,F12.2)') 'GPSRO_Zmin ', self % Zmin - call fckit_log % info(Message) - WRITE (Message, '(A,F12.2)') 'GPSRO_Zmax ', self % Zmax - call fckit_log % info(Message) ! Work out which observations belong to the current profile do current_point = start_point, nobs @@ -273,8 +302,6 @@ subroutine ufo_gnssroonedvarcheck_apply(self, geovals, apply) self % pseudo_ops, & ! Whether to use pseudo-levels in calculation self % vert_interp_ops, & ! Whether to interpolate using ln(p) or exner self % min_temp_grad, & ! Minimum vertical temperature gradient allowed - self % Zmin, & ! Minimum vertical level to accept observations - self % Zmax, & ! Maximum vertical level to accept observations self % cost_funct_test, & ! Threshold value for the cost function convergence test self % y_test, & ! Threshold value for the yobs-ysol tes self % n_iteration_test, & ! Maximum number of iterations diff --git a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.cc b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.cc index b5100d9a5..4b0ae58f5 100644 --- a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.cc +++ b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.cc @@ -5,8 +5,6 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#include "ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.h" - #include #include #include @@ -17,6 +15,8 @@ #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" +#include "ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.h" +#include "ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeParameters.h" #include "ufo/ObsDiagnostics.h" namespace ufo { @@ -29,13 +29,17 @@ ObsGnssroBendMetOffice::ObsGnssroBendMetOffice(const ioda::ObsSpace & odb, const eckit::Configuration & config) : ObsOperatorBase(odb, config), keyOperGnssroBendMetOffice_(0), odb_(odb), varin_() { + parameters_.validateAndDeserialize(config); + ObsGnssroBendMetOfficeOptions obsOptions = parameters_.obsOptions.value(); + const std::vector vv{"air_pressure_levels", "specific_humidity", "geopotential_height", "geopotential_height_levels"}; varin_.reset(new oops::Variables(vv)); - const eckit::LocalConfiguration obsOptions(config, "obs options"); - const eckit::Configuration *configc = &obsOptions; - ufo_gnssro_bendmetoffice_setup_f90(keyOperGnssroBendMetOffice_, &configc); + ufo_gnssro_bendmetoffice_setup_f90(keyOperGnssroBendMetOffice_, + obsOptions.vertInterpOPS, + obsOptions.pseudoLevels, + obsOptions.minTempGrad); oops::Log::trace() << "ObsGnssroBendMetOffice created." << std::endl; } diff --git a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.h b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.h index 3e9d63d92..0c531bf43 100644 --- a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.h +++ b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.h @@ -16,6 +16,7 @@ #include "oops/util/ObjectCounter.h" #include "ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.interface.h" #include "ufo/ObsOperatorBase.h" +#include "ObsGnssroBendMetOfficeParameters.h" namespace eckit { class Configuration; @@ -31,8 +32,8 @@ namespace ufo { class ObsDiagnostics; // ----------------------------------------------------------------------------- - /// GnssroBendMetOffice observation operator +// ----------------------------------------------------------------------------- class ObsGnssroBendMetOffice : public ObsOperatorBase, private util::ObjectCounter { public: @@ -55,6 +56,7 @@ class ObsGnssroBendMetOffice : public ObsOperatorBase, F90hop keyOperGnssroBendMetOffice_; const ioda::ObsSpace& odb_; std::unique_ptr varin_; + ObsGnssroBendMetOfficeParameters parameters_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.interface.F90 b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.interface.F90 index 8befcb37d..1019ddc59 100644 --- a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.interface.F90 +++ b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.interface.F90 @@ -32,19 +32,22 @@ module ufo_gnssro_bendmetoffice_mod_c #include "oops/util/linkedList_c.f" ! ------------------------------------------------------------------------------ - -subroutine ufo_gnssro_bendmetoffice_setup_c(c_key_self, c_conf) bind(c,name='ufo_gnssro_bendmetoffice_setup_f90') + +subroutine ufo_gnssro_bendmetoffice_setup_c(c_key_self, & + vert_interp_ops, & + pseudo_ops, & + min_temp_grad) bind(c,name='ufo_gnssro_bendmetoffice_setup_f90') implicit none integer(c_int), intent(inout) :: c_key_self -type(c_ptr), intent(in) :: c_conf +logical(c_bool), intent(in) :: vert_interp_ops +logical(c_bool), intent(in) :: pseudo_ops +real(c_float), intent(in) :: min_temp_grad type(ufo_gnssro_BendMetOffice), pointer :: self -type(fckit_configuration) :: f_conf call ufo_gnssro_bendmetoffice_registry%setup(c_key_self, self) -f_conf = fckit_configuration(c_conf) -call self%setup(f_conf) +call self%setup(vert_interp_ops, pseudo_ops, min_temp_grad) end subroutine ufo_gnssro_bendmetoffice_setup_c diff --git a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.interface.h b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.interface.h index ab39d48a9..486e3a973 100644 --- a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.interface.h +++ b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOffice.interface.h @@ -23,7 +23,7 @@ extern "C" { // ----------------------------------------------------------------------------- // Gnssro bending angle observation operators - (Met Office 1D) // ----------------------------------------------------------------------------- - void ufo_gnssro_bendmetoffice_setup_f90(F90hop &, const eckit::Configuration * const *); + void ufo_gnssro_bendmetoffice_setup_f90(F90hop &, const bool &, const bool &, const float &); void ufo_gnssro_bendmetoffice_delete_f90(F90hop &); void ufo_gnssro_bendmetoffice_simobs_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, const int &, double &, const F90goms &); diff --git a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeParameters.h b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeParameters.h new file mode 100644 index 000000000..e473feac6 --- /dev/null +++ b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeParameters.h @@ -0,0 +1,49 @@ +/* + * (C) Copyright 2021 UK Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_GNSSRO_BENDMETOFFICE_OBSGNSSROBENDMETOFFICEPARAMETERS_H_ +#define UFO_GNSSRO_BENDMETOFFICE_OBSGNSSROBENDMETOFFICEPARAMETERS_H_ + +#include +#include + +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" + +namespace ufo { + +/// Configuration options recognized by the bending angle operator. +class ObsGnssroBendMetOfficeOptions : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsGnssroBendMetOfficeOptions, Parameters) + + public: + /// If true assume that pressure varies exponentially with height when + /// interpolating. Otherwise assume that exner varies linearly with height, + /// and derive pressure from this. + oops::Parameter vertInterpOPS{"vert_interp_ops", true, this}; + /// Whether to use pseudo-levels in the calculation. + oops::Parameter pseudoLevels{"pseudo_ops", true, this}; + /// The minimum temperature gradient permitted before a profile is considered + /// isothermal. Only used if pseudo-levels are also used. + oops::Parameter minTempGrad{"min_temp_grad", 1.0e-6, this}; +}; + +/// Configuration options recognized by the bending angle operator. +class ObsGnssroBendMetOfficeParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsGnssroBendMetOfficeParameters, Parameters) + + public: + /// Operator name. In future will be moved to a base class for parameters of all ObsOperators. + oops::OptionalParameter name{"name", this}; + /// Obs Options - settings for the observation operator + oops::Parameter obsOptions{"obs options", + ObsGnssroBendMetOfficeOptions(), this}; +}; + +} // namespace ufo +#endif // UFO_GNSSRO_BENDMETOFFICE_OBSGNSSROBENDMETOFFICEPARAMETERS_H_ diff --git a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.cc b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.cc index aa099f09b..8e48c651c 100644 --- a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.cc +++ b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.cc @@ -5,8 +5,6 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#include "ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.h" - #include #include #include @@ -19,6 +17,8 @@ #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" +#include "ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeParameters.h" +#include "ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.h" namespace ufo { @@ -31,15 +31,19 @@ ObsGnssroBendMetOfficeTLAD::ObsGnssroBendMetOfficeTLAD(const ioda::ObsSpace & od const eckit::Configuration & config) : LinearObsOperatorBase(odb), keyOperGnssroBendMetOffice_(0), varin_() { - const eckit::LocalConfiguration obsOptions(config, "obs options"); - const eckit::Configuration * configc = &obsOptions; + parameters_.validateAndDeserialize(config); + ObsGnssroBendMetOfficeOptions obsOptions = parameters_.obsOptions.value(); - ufo_gnssro_bendmetoffice_tlad_setup_f90(keyOperGnssroBendMetOffice_, &configc); const std::vector vv{"air_pressure_levels", "specific_humidity", "geopotential_height", "geopotential_height_levels"}; - varin_.reset(new oops::Variables(vv)); oops::Log::info() << "ObsGnssroBendMetOfficeTLAD vars: " << *varin_ << std::endl; + + ufo_gnssro_bendmetoffice_tlad_setup_f90(keyOperGnssroBendMetOffice_, + obsOptions.vertInterpOPS, + obsOptions.pseudoLevels, + obsOptions.minTempGrad); + oops::Log::trace() << "ObsGnssroBendMetOfficeTLAD created" << std::endl; } diff --git a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.h b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.h index c6d7d35b6..2562480ad 100644 --- a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.h +++ b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.h @@ -16,6 +16,7 @@ #include "oops/util/ObjectCounter.h" #include "ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.interface.h" #include "ufo/LinearObsOperatorBase.h" +#include "ObsGnssroBendMetOfficeParameters.h" // Forward declarations namespace eckit { @@ -56,6 +57,7 @@ class ObsGnssroBendMetOfficeTLAD : public LinearObsOperatorBase, void print(std::ostream &) const override; F90hop keyOperGnssroBendMetOffice_; std::unique_ptr varin_; + ObsGnssroBendMetOfficeParameters parameters_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.interface.F90 b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.interface.F90 index 900c17ecc..01b6339cb 100644 --- a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.interface.F90 +++ b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.interface.F90 @@ -27,19 +27,22 @@ module ufo_gnssro_bendmetoffice_tlad_mod_c #include "oops/util/linkedList_c.f" ! ------------------------------------------------------------------------------ - -subroutine ufo_gnssro_bendmetoffice_tlad_setup_c(c_key_self, c_conf) bind(c,name='ufo_gnssro_bendmetoffice_tlad_setup_f90') + +subroutine ufo_gnssro_bendmetoffice_tlad_setup_c(c_key_self, & + vert_interp_ops, & + pseudo_ops, & + min_temp_grad) bind(c,name='ufo_gnssro_bendmetoffice_tlad_setup_f90') implicit none integer(c_int), intent(inout) :: c_key_self -type(c_ptr), intent(in) :: c_conf +logical(c_bool), intent(in) :: vert_interp_ops +logical(c_bool), intent(in) :: pseudo_ops +real(c_float), intent(in) :: min_temp_grad type(ufo_gnssro_bendmetoffice_tlad), pointer :: self -type(fckit_configuration) :: f_conf call ufo_gnssro_bendmetoffice_tlad_registry%setup(c_key_self, self) -f_conf = fckit_configuration(c_conf) -call self%setup(f_conf) +call self%setup(vert_interp_ops, pseudo_ops, min_temp_grad) end subroutine ufo_gnssro_bendmetoffice_tlad_setup_c diff --git a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.interface.h b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.interface.h index dc523e5a9..5d6348e6f 100644 --- a/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.interface.h +++ b/src/ufo/gnssro/BendMetOffice/ObsGnssroBendMetOfficeTLAD.interface.h @@ -23,7 +23,7 @@ extern "C" { // ----------------------------------------------------------------------------- // Gnssro bending angle tl/ad observation operators - (ROPP1D) // ----------------------------------------------------------------------------- - void ufo_gnssro_bendmetoffice_tlad_setup_f90(F90hop &, const eckit::Configuration * const *); + void ufo_gnssro_bendmetoffice_tlad_setup_f90(F90hop &, const bool &, const bool &, const float &); void ufo_gnssro_bendmetoffice_tlad_delete_f90(F90hop &); void ufo_gnssro_bendmetoffice_tlad_settraj_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &); diff --git a/src/ufo/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_mod.F90 b/src/ufo/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_mod.F90 index f92da7f00..1e01397ea 100644 --- a/src/ufo/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_mod.F90 +++ b/src/ufo/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_mod.F90 @@ -42,16 +42,18 @@ module ufo_gnssro_bendmetoffice_mod ! Get the optional settings for the forward model, and save them in the object ! so that they can be used in the code. ! ------------------------------------------------------------------------------ -subroutine ufo_gnssro_bendmetoffice_setup(self, f_conf) +subroutine ufo_gnssro_bendmetoffice_setup(self, vert_interp_ops, pseudo_ops, min_temp_grad) -use fckit_configuration_module, only: fckit_configuration implicit none + class(ufo_gnssro_BendMetOffice), intent(inout) :: self -type(fckit_configuration), intent(in) :: f_conf +logical(c_bool), intent(in) :: vert_interp_ops +logical(c_bool), intent(in) :: pseudo_ops +real(c_float), intent(in) :: min_temp_grad -call f_conf%get_or_die("vert_interp_ops", self % vert_interp_ops) -call f_conf%get_or_die("pseudo_ops", self % pseudo_ops) -call f_conf%get_or_die("min_temp_grad", self % min_temp_grad) +self % vert_interp_ops = vert_interp_ops +self % pseudo_ops = pseudo_ops +self % min_temp_grad = min_temp_grad end subroutine ufo_gnssro_bendmetoffice_setup diff --git a/src/ufo/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 b/src/ufo/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 index 08d0f04b6..74ec07a94 100644 --- a/src/ufo/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 +++ b/src/ufo/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 @@ -50,16 +50,18 @@ module ufo_gnssro_bendmetoffice_tlad_mod ! Get the optional settings for the forward model, and save them in the object ! so that they can be used in the code. ! ------------------------------------------------------------------------------ -subroutine ufo_gnssro_bendmetoffice_setup(self, f_conf) +subroutine ufo_gnssro_bendmetoffice_setup(self, vert_interp_ops, pseudo_ops, min_temp_grad) -use fckit_configuration_module, only: fckit_configuration implicit none + class(ufo_gnssro_bendmetoffice_tlad), intent(inout) :: self -type(fckit_configuration), intent(in) :: f_conf +logical(c_bool), intent(in) :: vert_interp_ops +logical(c_bool), intent(in) :: pseudo_ops +real(c_float), intent(in) :: min_temp_grad -call f_conf%get_or_die("vert_interp_ops", self % vert_interp_ops) -call f_conf%get_or_die("pseudo_ops", self % pseudo_ops) -call f_conf%get_or_die("min_temp_grad", self % min_temp_grad) +self % vert_interp_ops = vert_interp_ops +self % pseudo_ops = pseudo_ops +self % min_temp_grad = min_temp_grad end subroutine ufo_gnssro_bendmetoffice_setup diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc index 08f209254..c75fc3daa 100644 --- a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc @@ -5,8 +5,6 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#include "ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h" - #include #include #include @@ -17,6 +15,8 @@ #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" +#include "ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h" +#include "ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeParameters.h" #include "ufo/ObsDiagnostics.h" namespace ufo { diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc index 18efa6752..1630d0155 100644 --- a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc @@ -5,13 +5,10 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#include "ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.h" - #include #include #include - #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" @@ -19,6 +16,8 @@ #include "oops/util/Logger.h" #include "ufo/GeoVaLs.h" +#include "ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeParameters.h" +#include "ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.h" namespace ufo { diff --git a/test/testinput/gnssrobendmetoffice_qc.yaml b/test/testinput/gnssrobendmetoffice_qc.yaml index c07d02245..a5183c283 100644 --- a/test/testinput/gnssrobendmetoffice_qc.yaml +++ b/test/testinput/gnssrobendmetoffice_qc.yaml @@ -75,8 +75,6 @@ observations: n_iteration_test: 20 OB_test: 2.5 y_test: 5 - Zmin: 0 - Zmax: 60000 bmatrix_filename: ../resources/bmatrix/gnssro/gnssro_bmatrix.txt filter variables: - name: bending_angle @@ -154,8 +152,6 @@ observations: n_iteration_test: 20 OB_test: 2.5 y_test: 5 - Zmin: 0 - Zmax: 60000 bmatrix_filename: ../resources/bmatrix/gnssro/gnssro_bmatrix.txt filter variables: - name: bending_angle From 3c5daaa437f3361d097963c9236fd0c83cdc746d Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Fri, 6 Aug 2021 17:26:36 +0100 Subject: [PATCH 74/98] Slant path location finding utility (#1349) * modifications to algorithm * Modifications to algorithm; fill H(x) * refactor to use data handling class * improve documentation * coding norms * more unit tests * more error checking; improved error messages * coding norms * coding norms * add extra tests to trigger exceptions * Trigger CI * clear stringstreams correctly * improve comments; change function name * no longer clear stringstream * allow for different vertical coordinate; test for T * add test with RH * add variables option * coding norms * Trigger CI * correct include path * update doxygen comments * change 'location' to 'level' in a couple of places * correct treatment of operator variable indices * profile average TL/AD code * add expected simulateObsTL exceptions * move GeoVaL caching to data object * slant path locations in a standalone function; adapt code to use this * Trigger CI * coding norms * coding norms * update comment * Improve comments * rename vertical coordinate parameters and functions * change iterator name * comment * modifications following review Co-authored-by: Benjamin Johnson <33431190+BenjaminTJohnson@users.noreply.github.com> Co-authored-by: Ryan Honeyager --- src/ufo/profile/CMakeLists.txt | 2 + src/ufo/profile/DataHandlerParameters.h | 18 +++++ src/ufo/profile/ObsProfileAverageData.cc | 82 ++++++----------------- src/ufo/profile/ProfileDataHandler.cc | 49 +++++++++++--- src/ufo/profile/ProfileDataHandler.h | 4 ++ src/ufo/profile/SlantPathLocations.cc | 85 ++++++++++++++++++++++++ src/ufo/profile/SlantPathLocations.h | 46 +++++++++++++ 7 files changed, 215 insertions(+), 71 deletions(-) create mode 100644 src/ufo/profile/SlantPathLocations.cc create mode 100644 src/ufo/profile/SlantPathLocations.h diff --git a/src/ufo/profile/CMakeLists.txt b/src/ufo/profile/CMakeLists.txt index 0570c7e04..f9dd0ca35 100644 --- a/src/ufo/profile/CMakeLists.txt +++ b/src/ufo/profile/CMakeLists.txt @@ -81,6 +81,8 @@ set ( profile_files ProfileVerticalInterpolation.h ProfileWindProfilerFlags.cc ProfileWindProfilerFlags.h + SlantPathLocations.cc + SlantPathLocations.h VariableNames.cc VariableNames.h ) diff --git a/src/ufo/profile/DataHandlerParameters.h b/src/ufo/profile/DataHandlerParameters.h index ac7dd4cbd..84c571d06 100644 --- a/src/ufo/profile/DataHandlerParameters.h +++ b/src/ufo/profile/DataHandlerParameters.h @@ -8,6 +8,7 @@ #ifndef UFO_PROFILE_DATAHANDLERPARAMETERS_H_ #define UFO_PROFILE_DATAHANDLERPARAMETERS_H_ +#include #include #include @@ -94,6 +95,23 @@ namespace ufo { oops::Parameter ModelLevelsDerivedValuesFilename {"ModelLevelsDerivedValuesFilename", "ModelLevelsDerivedValues.nc4", this}; + /// Default vertical coordinate to use in the slant path location algorithm. + /// This can be overridden for each variable by using the \p alternativeVerticalCoordinate + /// option. + oops::Parameter defaultVerticalCoordinate + {"defaultVerticalCoordinate", "air_pressure", this}; + + /// Alternative vertical coordinate(s) to use in the slant path location algorithm. + /// The first string in each pair is the name of the variable whose slanted profile is to + /// be determined, and the second string is the vertical coordinate that should be used to + /// find the slant path locations for the variable. + /// This will typically be useful for models whose variables appear on staggered vertical levels + /// (e.g. with vertical coordinates 'air_pressure' and 'air_pressure_levels'). + oops::Parameter> alternativeVerticalCoordinate + {"alternativeVerticalCoordinate", + {{"eastward_wind", "air_pressure_levels"}, {"northward_wind", "air_pressure_levels"}, + {"ExnerPA", "air_pressure_levels"}, {"LogPA", "air_pressure_levels"}}, this}; + /// Parameters related to the model. ModelParameters ModParameters{this}; }; diff --git a/src/ufo/profile/ObsProfileAverageData.cc b/src/ufo/profile/ObsProfileAverageData.cc index 7a8a4b711..437b4fb30 100644 --- a/src/ufo/profile/ObsProfileAverageData.cc +++ b/src/ufo/profile/ObsProfileAverageData.cc @@ -9,6 +9,7 @@ #include "oops/util/missingValues.h" #include "ufo/profile/ObsProfileAverageData.h" +#include "ufo/profile/SlantPathLocations.h" #include "ufo/utils/OperatorUtils.h" // for getOperatorVariables namespace ufo { @@ -76,74 +77,29 @@ namespace ufo { (const std::vector & locsOriginal, const std::vector & locsExtended) const { - const float missing = util::missingValue(missing); - - // Get observed pressure. - std::vector pressure_obs(odb_.nlocs()); - odb_.get_db("MetaData", "air_pressure", pressure_obs); - - // Set up GeoVaLs and H(x) vectors. - // Number of levels for model pressure. - const std::size_t nlevs_p = cachedGeoVaLs_->nlevs(modelVerticalCoord_); - // Vector storing location for each level along the slant path. - // Initially the first location in the profile is used everywhere. - std::vector slant_path_location(nlevs_p, locsOriginal.front()); - // Vector of slanted pressures, used for comparisons with OPS. - std::vector slant_pressure; - // Vector used to store different pressure GeoVaLs. - std::vector pressure_gv(nlevs_p); - - // Loop over model levels and find intersection of profile with model layer boundary. - // This can be performed multiple times in order to account for slanted model levels. - std::size_t idxstart = 0; // Starting index for loop over levels. - // This counter records locations in the slanted profile. - // It is initialised at the first location in the original profile. - std::size_t jlocslant = locsOriginal.front(); - // Maximum iteration value. - const int itermax = options_.numIntersectionIterations.value() - 1; - // Loop over each model level in turn. - for (std::size_t mlev = 0; mlev < nlevs_p; ++mlev) { - for (int iter = 0; iter <= itermax; ++iter) { - // Get the GeoVaL that corresponds to the current slanted profile location. - cachedGeoVaLs_->getAtLocation(pressure_gv, modelVerticalCoord_, jlocslant); - // Define an iteration-specific location that is initialised to the - // current slanted profile location. - std::size_t jlociter = jlocslant; - // Determine the intersection of the observed profile with the current model level. - // The intersection is taken to be the location with the largest observed pressure - // that is less than or equal to the model pressure at this level. - for (std::size_t idx = idxstart; idx < locsOriginal.size(); ++idx) { - // Intersection location. - const std::size_t jlocintersect = locsOriginal[idx]; - // If pressure has not been recorded, move to the next level. - if (pressure_obs[jlocintersect] == missing) continue; - // Break from the loop if the observed pressure is lower than - // the pressure of this model level. - if (pressure_obs[jlocintersect] <= pressure_gv[mlev]) break; - // Update the iteration-specific location with the new intersection location. - jlociter = jlocintersect; - // Update the loop starting index when the last iteration is reached. - if (iter == itermax) idxstart = idx; - } - // Modify the slanted location in the original profile. - jlocslant = jlociter; - if (iter == itermax) { - // Record the value of the slant path location at this model level and all above. - // This ensures that missing values are dealt with correctly. - for (std::size_t mlevcolumn = mlev; mlevcolumn < nlevs_p; ++mlevcolumn) - slant_path_location[mlevcolumn] = jlocslant; - // Fill the slant pressure if the comparison with OPS will be performed. - if (options_.compareWithOPS.value()) - slant_pressure.push_back(pressure_gv[mlev]); - } - } - } + const std::vector slant_path_location = + ufo::getSlantPathLocations(odb_, + *cachedGeoVaLs_, + locsOriginal, + modelVerticalCoord_, + options_.numIntersectionIterations.value() - 1); // If required, compare slant path locations and slant pressure with OPS output. - if (options_.compareWithOPS.value()) + if (options_.compareWithOPS.value()) { + // Vector of slanted pressures, used for comparisons with OPS. + std::vector slant_pressure; + // Number of levels for model pressure. + const std::size_t nlevs_p = cachedGeoVaLs_->nlevs(modelVerticalCoord_); + // Vector used to store different pressure GeoVaLs. + std::vector pressure_gv(nlevs_p); + for (std::size_t mlev = 0; mlev < nlevs_p; ++mlev) { + cachedGeoVaLs_->getAtLocation(pressure_gv, modelVerticalCoord_, slant_path_location[mlev]); + slant_pressure.push_back(pressure_gv[mlev]); + } this->compareAuxiliaryReferenceVariables(locsExtended, slant_path_location, slant_pressure); + } return slant_path_location; } diff --git a/src/ufo/profile/ProfileDataHandler.cc b/src/ufo/profile/ProfileDataHandler.cc index f599a6792..5e5b1e96c 100644 --- a/src/ufo/profile/ProfileDataHandler.cc +++ b/src/ufo/profile/ProfileDataHandler.cc @@ -13,6 +13,7 @@ #include "ufo/profile/ProfileCheckBase.h" #include "ufo/profile/ProfileDataHandler.h" #include "ufo/profile/ProfileDataHolder.h" +#include "ufo/profile/SlantPathLocations.h" #include "ufo/profile/VariableNames.h" namespace ufo { @@ -192,6 +193,21 @@ namespace ufo { } } + std::string ProfileDataHandler::getAssociatedVerticalCoordinate + (const std::string & variableName) const + { + // Obtain the map between non-default vertical coordinates and variable names. + const auto & alternativeVerticalCoordinate = options_.alternativeVerticalCoordinate.value(); + auto it_altCoord = alternativeVerticalCoordinate.find(variableName); + if (it_altCoord != alternativeVerticalCoordinate.end()) { + // This variable has an associated alternative vertical coordinate. + return it_altCoord->second; + } else { + // This variable uses the default vertical coordinate. + return options_.defaultVerticalCoordinate.value(); + } + } + std::vector & ProfileDataHandler::getGeoVaLVector(const std::string &variableName) { auto it_GeoVaLData = GeoVaLData_.find(variableName); @@ -205,16 +221,33 @@ namespace ufo { if (geovals_ && obsdb_.nlocs() > 0 && geovals_->has(variableName)) { - // Location at which to retrieve the GeoVaL. - // This assumes each model column for each observation in a profile is identical - // so takes the first entry in each case. - // todo(ctgh): this is an approximation that should be revisited - // when considering horizontal drift. - const size_t jloc = profileIndices_->getProfileIndices()[0]; + // Locations at which to retrieve the GeoVaL. + const std::vector slant_path_location = + ufo::getSlantPathLocations(obsdb_, + *geovals_, + profileIndices_->getProfileIndices(), + this->getAssociatedVerticalCoordinate(variableName)); // Vector storing GeoVaL data for current profile. vec_GeoVaL_column.assign(geovals_->nlevs(variableName), 0.0); - // Get GeoVaLs at the specified location. - geovals_->getAtLocation(vec_GeoVaL_column, variableName, jloc); + // Check the number of entries in the slant path location vector is equal + // to the number of entries in the GeoVaL for this variable. + // If not, the GeoVaL at the first location in the profile is used; + // in other words, drift is not accounted for. + // todo(ctgh): revisit this choice in a future PR. + if (slant_path_location.size() == vec_GeoVaL_column.size()) { + std::vector vec_GeoVaL_loc(geovals_->nlevs(variableName)); + // Take the GeoVaL at each slant path location and copy the relevant + // value from each GeoVaL into the output vector. + for (std::size_t mlev = 0; mlev < slant_path_location.size(); ++mlev) { + const std::size_t jloc = slant_path_location[mlev]; + geovals_->getAtLocation(vec_GeoVaL_loc, variableName, jloc); + vec_GeoVaL_column[mlev] = vec_GeoVaL_loc[mlev]; + } + } else { + // Take the GeoVaL at the first location. + const std::size_t jloc = profileIndices_->getProfileIndices()[0]; + geovals_->getAtLocation(vec_GeoVaL_column, variableName, jloc); + } } // Add GeoVaL vector to map (even if it is empty). GeoVaLData_.emplace(variableName, std::move(vec_GeoVaL_column)); diff --git a/src/ufo/profile/ProfileDataHandler.h b/src/ufo/profile/ProfileDataHandler.h index 25386210a..c961c2f6a 100644 --- a/src/ufo/profile/ProfileDataHandler.h +++ b/src/ufo/profile/ProfileDataHandler.h @@ -203,6 +203,10 @@ namespace ufo { /// Get indices in entire sample corresponding to current profile. void getProfileIndicesInEntireSample(const std::string& groupname); + /// Get the name of the vertical coordinate that is used to determine the slant path + /// locations for the variable \p variableName. + std::string getAssociatedVerticalCoordinate(const std::string & variableName) const; + private: // members /// Container of each variable in the current profile. std::unordered_map + +#include "ioda/ObsSpace.h" + +#include "oops/util/missingValues.h" + +#include "ufo/GeoVaLs.h" +#include "ufo/profile/SlantPathLocations.h" + +namespace ufo { + + std::vector getSlantPathLocations(const ioda::ObsSpace & odb, + const GeoVaLs & gv, + const std::vector & locs, + const std::string & modelVerticalCoord, + const int itermax) { + const float missing = util::missingValue(missing); + + // Get observed pressure. + std::vector pressure_obs(odb.nlocs()); + odb.get_db("MetaData", "air_pressure", pressure_obs); + + // If the pressure GeoVaL is not present, return an empty vector. + // todo(ctgh): eventually throw an exception here. + // This will be dealt with in a future PR. + if (!gv.has(modelVerticalCoord)) return {}; + // Number of levels for model pressure. + const std::size_t nlevs_p = gv.nlevs(modelVerticalCoord); + // Vector storing location for each level along the slant path. + // Initially the first location in the profile is used everywhere. + std::vector slant_path_location(nlevs_p, locs.front()); + // Vector used to store different pressure GeoVaLs. + std::vector pressure_gv(nlevs_p); + + // Loop over model levels and find intersection of profile with model layer boundary. + // This can be performed multiple times in order to account for slanted model levels. + std::size_t idxstart = 0; // Starting index for loop over levels. + // This counter records locations in the slanted profile. + // It is initialised at the first location in the original profile. + std::size_t jlocslant = locs.front(); + // Loop over each model level in turn. + for (std::size_t mlev = 0; mlev < nlevs_p; ++mlev) { + for (int iter = 0; iter <= itermax; ++iter) { + // Get the GeoVaL that corresponds to the current slanted profile location. + gv.getAtLocation(pressure_gv, modelVerticalCoord, jlocslant); + // Define an iteration-specific location that is initialised to the + // current slanted profile location. + std::size_t jlociter = jlocslant; + // Determine the intersection of the observed profile with the current model level. + // The intersection is taken to be the location with the largest observed pressure + // that is less than or equal to the model pressure at this level. + for (std::size_t idx = idxstart; idx < locs.size(); ++idx) { + // Intersection location. + const std::size_t jlocintersect = locs[idx]; + // If pressure has not been recorded, move to the next level. + if (pressure_obs[jlocintersect] == missing) continue; + // Break from the loop if the observed pressure is lower than + // the pressure of this model level. + if (pressure_obs[jlocintersect] <= pressure_gv[mlev]) break; + // Update the iteration-specific location with the new intersection location. + jlociter = jlocintersect; + // Update the loop starting index when the last iteration is reached. + if (iter == itermax) idxstart = idx; + } + // Modify the slanted location in the original profile. + jlocslant = jlociter; + if (iter == itermax) { + // Record the value of the slant path location at this model level and all above. + // This ensures that missing values are dealt with correctly. + for (std::size_t mlevcolumn = mlev; mlevcolumn < nlevs_p; ++mlevcolumn) + slant_path_location[mlevcolumn] = jlocslant; + } + } + } + + return slant_path_location; + } +} // namespace ufo diff --git a/src/ufo/profile/SlantPathLocations.h b/src/ufo/profile/SlantPathLocations.h new file mode 100644 index 000000000..982a61a19 --- /dev/null +++ b/src/ufo/profile/SlantPathLocations.h @@ -0,0 +1,46 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_PROFILE_SLANTPATHLOCATIONS_H_ +#define UFO_PROFILE_SLANTPATHLOCATIONS_H_ + +#include +#include + +namespace ioda { + class ObsSpace; +} + +namespace ufo { + class GeoVaLs; +} + +namespace ufo { + /// Get slant path locations. This determines, for each model level, the location that + /// corresponds to the intersection of the observed profile with that level. + /// + /// \param odb + /// ObsSpace. + /// \param gv + /// GeoVaLs. + /// \param locs + /// All locations in the profile. + /// \param modelVerticalCoord + /// Name of the vertical coordinate used in the model. + /// \param itermax + /// Maximum number of interations that will be used to find the intersections + /// between observed pressures and model levels. + /// + /// \returns A vector of the slant path locations. + std::vector getSlantPathLocations(const ioda::ObsSpace & odb, + const GeoVaLs & gv, + const std::vector & locs, + const std::string & modelVerticalCoord, + const int itermax = 3); +} // namespace ufo + +#endif // UFO_PROFILE_SLANTPATHLOCATIONS_H_ From 93c0fd22a2106dd54bfe6b4dd32731a7a478a159 Mon Sep 17 00:00:00 2001 From: James Cotton <64855970+james-cotton@users.noreply.github.com> Date: Fri, 6 Aug 2021 18:51:55 +0100 Subject: [PATCH 75/98] Feature/satwind pressure errors (#1414) * Update SatwindIndivErrors@ObsFunction to read in pressure errors variable. Use yaml to set errors which vary with satellite and channel * Complete the channel information for GOES-16 * Read in pressure errors from nc file * Update SatName filter to have a single missing value indicator and add a diagnostic variable * Update file locations * Fix indent and set min pressure error to 5 hPa * Update to use oops missing string value * Typos and obsdb_ -> obsdb * Add comment for minimum pressure error * Restart CI Co-authored-by: Ryan Honeyager --- src/ufo/filters/SatName.cc | 243 ++++++++++++------ src/ufo/filters/SatName.h | 2 + .../obsfunctions/SatwindIndivErrors.cc | 57 ++-- .../filters/obsfunctions/SatwindIndivErrors.h | 16 +- .../function_satwind_indiv_errors.yaml | 55 +++- 5 files changed, 265 insertions(+), 108 deletions(-) diff --git a/src/ufo/filters/SatName.cc b/src/ufo/filters/SatName.cc index d106849c0..5b02061ad 100644 --- a/src/ufo/filters/SatName.cc +++ b/src/ufo/filters/SatName.cc @@ -4,83 +4,60 @@ * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -//------------------------------------------------------------------------------- -// -// This filter is the first step in the AMV processing. -// It creates a string for classifying Atmospheric Motion Vectors (AMV)'s into groups. -// This is then used for later AMV processing modules -// -// Inputs: -// data from an AMV WMO bufr file in netcdf4 format -// -// Required : -// -// "MetaData", "sensor_central_frequency", cfreq -// "MetaData", "originating_subcentre", orsub -// "MetaData", "satellite_identifier", satid -// "MetaData", "wind_computation_method", compm -// -// Outputs: -// "MetaData", "satwind_id", wind_id -//------------------------------------------------------------------------------- -// -// Method: -// -// There are a number of satellite series (both geostationary and polar) that currently provide AMVs -// Each satellite or satellite pairs, provides AMVs using difference channels in the visible -// infrared and water vapour bands. -// -// -// satellite frequency and computational method are used to generate the AMV character string, -// an example is: -// "NOAA16" and "ir108" are combined "NOAA16ir108" and added to the output NETCDF4 file -// as a variable satwind_id -// -// -// satids channel values -// -// hrvis vis06 vis08 -// ir16 ir38 ir87 ir97 ir108 ir120 -// wv62 wv67 wv73 -// cswv62 cswv67 cswv73 -// mixwv62 mixwv67 mixwv73 -// misstyp unknown satellite identified -// -// + #include "ufo/filters/SatName.h" + #include +#include #include #include +#include +#include #include +#include "ioda/distribution/Accumulator.h" #include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" #include "oops/util/Logger.h" -#include "oops/util/missingValues.h" #include "ufo/filters/QCflags.h" #include "ufo/utils/StringUtils.h" + namespace ufo { -std::string Sat_Characteristics(int SatID, float centralFrequency, int satobchannel, - const std::vector &SatIDRanges) { - std::string Missing_Sat_windChan = "misstyp"; - for (const SatIDRangeParameters &SatIDRange : SatIDRanges) - if (SatIDRange.minSatID <= SatID && SatID <= SatIDRange.maxSatID) - for (const FrequencyBandParameters &frequencyBand : SatIDRange.Satellite_comp.value()) + +// With floats std::to_string may yield unexpected results, so use stringstream instead +template < typename Type > std::string to_str(const Type & t) +{ + std::ostringstream os; + os << t; + return os.str (); +} + +std::string get_channel_name(int SatID, float centralFrequency, int satobchannel, + const std::vector &SatIDRanges) { + for (const SatIDRangeParameters &SatIDRange : SatIDRanges) { + if (SatIDRange.minSatID <= SatID && SatID <= SatIDRange.maxSatID) { + for (const FrequencyBandParameters &frequencyBand : SatIDRange.Satellite_comp.value()) { if (frequencyBand.minFrequency <= centralFrequency && - centralFrequency <= frequencyBand.maxFrequency && - (frequencyBand.satobchannel.value() == boost::none || + centralFrequency <= frequencyBand.maxFrequency && + (frequencyBand.satobchannel.value() == boost::none || satobchannel == *frequencyBand.satobchannel.value())) { - return frequencyBand.windChannel; + return frequencyBand.windChannel; + } + } + } } - return Missing_Sat_windChan; + return missing_value_string; } -std::string sat_id(int SatID, - const std::vector &SatIDRanges) { - std::string satmn = "misatid"; - for (const SatIDRangeParameters &SatIDRange : SatIDRanges) - for (const SatnameParameters &SatNames : SatIDRange.Satellite_id.value()) - if (SatID == SatNames.Satnumber.value()) {return SatNames.Satname;} - return satmn; +std::string get_sat_name(int SatID, const std::vector &SatIDRanges) { + for (const SatIDRangeParameters &SatIDRange : SatIDRanges) { + for (const SatnameParameters &SatNames : SatIDRange.Satellite_id.value()) { + if (SatID == SatNames.Satnumber.value()) { + return SatNames.Satname; + } + } + } + return missing_value_string; } + // ----------------------------------------------------------------------------- SatName::SatName(ioda::ObsSpace & obsdb, const Parameters_ & parameters, std::shared_ptr > flags, @@ -89,36 +66,146 @@ SatName::SatName(ioda::ObsSpace & obsdb, const Parameters_ & parameters, { oops::Log::debug() << "SatName: config (constructor) = " << parameters_ << std::endl; } + // ----------------------------------------------------------------------------- SatName::~SatName() {} // ----------------------------------------------------------------------------- +/*! \brief A filter that creates a string variable that makes it simpler to + * identify Atmospheric Motion Vector (AMV) / Satwind observations by + * combining satellite and channel information. This is useful for later + * AMV processing where we want to apply filters to subsets of observations. + * + * \details To identify the type of motion that has been tracked, AMV BUFR observations + * are supplied with a channel central frequency (Hz) and a wind computation method: + * + * 002023 SATELLITE DERIVED WIND COMPUTATION METHOD + * * 0 Reserved + * * 1 INFRARED MOTION OBSERVED IN THE INFRARED CHANNEL + * * 2 VISIBLE MOTION OBSERVED IN THE VISIBLE CHANNEL + * * 3 VAPOUR CLOUD MOTION OBSERVED IN THE WATER VAPOUR CHANNEL + * * 4 COMBINATION MOTION OBSERVED IN A COMBINATION OF SPECTRAL CHANNELS + * * 5 VAPOUR CLEAR MOTION OBSERVED IN THE WATER VAPOUR CHANNEL IN CLEAR AIR + * * 6 OZONE MOTION OBSERVED IN THE OZONE CHANNEL + * * 7 VAPOUR MOTION OBSERVED IN WATER VAPOUR CHANNEL (CLOUD OR CLEAR) + * * 13 Root-mean-square + * + * The most common use of the wind computation method is to distinguish between clear-sky and + * cloudy water vapour targets. + * + * This filter combines this channel information, together with the satellite name, to + * create a string that defines the satellite/channel combination of each observation. + * We also output a diagnostic variable which provides information on unidentified + * satellites or channels. + * + * Required : + * * "MetaData", "sensor_central_frequency" + * * "MetaData", "satellite_identifier" + * * "MetaData", "wind_computation_method" + * + * Outputs: + * * "MetaData", "satwind_id" + * * "Diag", "satwind_id" + * + * Example: + * The following yaml will attempt to identify two infrared channels with computation method + * value of 1 and central frequencies falling betwen the min and max frequency bounds. + * If there are observations that match these conditions they are labelled with the respective + * "wind channel" string. + * If observations are identified from GOES-16 (platform number 270) they are also labelled with + * the respective "Sat name" string. + * This will fill MetaData/satwind_id with values "GOES16ir112","GOES16ir38" if these are present + * in the observations. + * If either the satellite or channel are not identified, then MetaData/satwind_id is set to + * "MISSING". To help track down why observations are set to missing we also output a diagnostic + * string variable, Diag/satwind_id. When the observation is not identified, this has the form: + * id_comp_freq. + * E.g. if the satellite is identified but the channel is not: "GOES16_comp3_freq0.484317e14", + * if the satellite is not identified but the channel is: "id270ir112". + * + * \code{.unparsed} + * obs filters: + * - filter: satname + * SatName assignments: + * - min WMO Satellite id: 1 + * max WMO Satellite id: 999 + * Satellite_comp: + * - satobchannel: 1 + * min frequency: 2.6e+13 + * max frequency: 2.7e+13 + * wind channel: ir112 + * - satobchannel: 1 + * min frequency: 7.5e+13 + * max frequency: 8.2e+13 + * wind channel: ir38 + * Satellite_id: + * - Sat ID: 270 + * Sat name: GOES16 + * \endcode + * + */ + void SatName::applyFilter(const std::vector & apply, - const Variables & filtervars, - std::vector> & flagged) const { + const Variables & filtervars, + std::vector> & flagged) const { std::vector cfreq(obsdb_.nlocs()); - std::vector orsub(obsdb_.nlocs()); - std::vector orgin(obsdb_.nlocs()); std::vector satid(obsdb_.nlocs()); std::vector compm(obsdb_.nlocs()); - std::vector wind_id(obsdb_.nlocs()); + // initialise output vectors to missing data string + std::vector wind_id(obsdb_.nlocs(), missing_value_string); + std::vector diag_id(obsdb_.nlocs(), missing_value_string); + // get variables from ObsSpace obsdb_.get_db("MetaData", "sensor_central_frequency", cfreq); - obsdb_.get_db("MetaData", "originating_subcentre", orsub); - obsdb_.get_db("MetaData", "originating_centre", orgin); obsdb_.get_db("MetaData", "satellite_identifier", satid); obsdb_.get_db("MetaData", "wind_computation_method", compm); -// + // define counter variables to be summed over all processors at the end of the routine + std::unique_ptr> countSatAccumulator = + obsdb_.distribution()->createAccumulator(); + std::unique_ptr> countChanAccumulator = + obsdb_.distribution()->createAccumulator(); + for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - wind_id[jobs] = sat_id(satid[jobs], parameters_.SatNameAssignments.value()) - + Sat_Characteristics(satid[jobs], cfreq[jobs], compm[jobs], - parameters_.SatNameAssignments.value()); + std::string satellite_name; + std::string channel_name; + satellite_name = get_sat_name(satid[jobs], parameters_.SatNameAssignments.value()); + channel_name = get_channel_name(satid[jobs], cfreq[jobs], compm[jobs], + parameters_.SatNameAssignments.value()); + // if both satellite and channel have been identified, then combine and fill wind_id + if (satellite_name != missing_value_string && + channel_name != missing_value_string) { + wind_id[jobs] = satellite_name + channel_name; + } + // if the satellite has not been identified then output the satid number to the diagnostic, + // otherwise output the found satellite name + std::string satellite_diag; + if (satellite_name == missing_value_string) { + satellite_diag = "id" + to_str(satid[jobs]); + countSatAccumulator->addTerm(jobs, 1); + } else { + satellite_diag = satellite_name; + } + // if the channel has not been identified then output the computation method and + // central frequency to the diagnostic, otherwise output the found channel name + std::string channel_diag; + if (channel_name == missing_value_string) { + channel_diag = "_comp" + to_str(compm[jobs]) + + "_freq" + to_str(cfreq[jobs]/1.0e14) + "e14"; + countChanAccumulator->addTerm(jobs, 1); + } else { + channel_diag = channel_name; + } + // combine diagnostic strings and fill diag_id + diag_id[jobs] = satellite_diag + channel_diag; } obsdb_.put_db("MetaData", "satwind_id", wind_id); -// only print the first 10 observations while testing - for (size_t ii = 0; ii < 10 ; ++ii) { - oops::Log::trace() << " freq " << cfreq[ii] << "orsub " << orsub[ii] << "orgin " - << orgin[ii] << " satid " << satid[ii] << " compm " << compm[ii] - << " wind id "<< wind_id[ii] << std::endl;} -} + obsdb_.put_db("Diag", "satwind_id", diag_id); + // sum number of unidentified satellites and channels + const std::size_t count_missing_sat = countSatAccumulator->computeResult(); + const std::size_t count_missing_chan = countChanAccumulator->computeResult(); + oops::Log::info() << "SatName: " << count_missing_sat + << " observations with unidentified satellite id" << std::endl; + oops::Log::info() << "SatName: " << count_missing_chan + << " observations with unidentified channel" << std::endl; + } // ----------------------------------------------------------------------------- void SatName::print(std::ostream & os) const { os << "SatName: config = " << parameters_ << std::endl; diff --git a/src/ufo/filters/SatName.h b/src/ufo/filters/SatName.h index af9fb2652..d812521a0 100644 --- a/src/ufo/filters/SatName.h +++ b/src/ufo/filters/SatName.h @@ -13,6 +13,7 @@ #include #include +#include "oops/util/missingValues.h" #include "oops/util/ObjectCounter.h" #include "oops/util/parameters/OptionalParameter.h" #include "oops/util/parameters/RequiredParameter.h" @@ -28,6 +29,7 @@ namespace ioda { class ObsSpace; } namespace ufo { + static const std::string missing_value_string = util::missingValue(missing_value_string); // // table in yaml file to relate satellite name and wmo number label in yaml "Satellite_id" // diff --git a/src/ufo/filters/obsfunctions/SatwindIndivErrors.cc b/src/ufo/filters/obsfunctions/SatwindIndivErrors.cc index 1c8c3a663..e712c4de8 100644 --- a/src/ufo/filters/obsfunctions/SatwindIndivErrors.cc +++ b/src/ufo/filters/obsfunctions/SatwindIndivErrors.cc @@ -10,9 +10,12 @@ #include #include +#include #include +#include "ioda/distribution/Accumulator.h" #include "ioda/ObsDataVector.h" +#include "ioda/ObsSpace.h" #include "oops/util/missingValues.h" #include "ufo/filters/Variable.h" @@ -36,8 +39,9 @@ SatwindIndivErrors::SatwindIndivErrors(const eckit::LocalConfiguration & conf) // Include list of required data from ObsSpace invars_ += Variable(vcoord+"@MetaData"); - invars_ += Variable("percent_confidence_2@MetaData"); invars_ += Variable(profile+"@hofx"); + invars_ += options_.pressure_error; + invars_ += options_.quality_index; // Include list of required data from GeoVaLs invars_ += Variable(vcoord+"@GeoVaLs"); @@ -58,7 +62,7 @@ SatwindIndivErrors::~SatwindIndivErrors() {} * estimates of the vector error and height error from the data producers. * * Currently the vector error estimate \f$E_{vector}\f$ - * is based on the model-independent quality index (QI) and is calculated as: + * is based on the quality index (QI) (ideally model-independent) and is calculated as: * \f[ * E_{vector} = \text{EuMult}\left(QI \times 0.01\right) + \text{EuAdd} * \f] @@ -102,11 +106,13 @@ SatwindIndivErrors::~SatwindIndivErrors() {} void SatwindIndivErrors::compute(const ObsFilterData & in, ioda::ObsDataVector & obserr) const { + // Get obs space + auto & obsdb = in.obsspace(); + // Get parameters from options float const eu_add = options_.eu_add.value(); float const eu_mult = options_.eu_mult.value(); - float const default_err_p = options_.default_err_p.value(); // Pa - float const min_press = options_.min_press.value().value_or(10000); // Pa + float const min_press = options_.min_press.value(); // Pa std::string const profile = options_.profile.value(); std::string const vcoord = options_.vcoord.value(); oops::Log::debug() << "Wind profile for calculating observation errors is " @@ -136,14 +142,13 @@ void SatwindIndivErrors::compute(const ObsFilterData & in, // Get variables from ObsSpace if present. If not, throw an exception std::vector ob_p; - std::vector ob_qi; std::vector bg_windcomponent; + std::vector pressure_error; + std::vector ob_qi; in.get(Variable(vcoord+"@MetaData"), ob_p); - in.get(Variable("percent_confidence_2@MetaData"), ob_qi); in.get(Variable(profile+"@hofx"), bg_windcomponent); - - // Set pressure error (use default 100 hPa for now) - std::vector err_p(nlocs, default_err_p); + in.get(options_.pressure_error, pressure_error); + in.get(options_.quality_index, ob_qi); // Get variables from GeoVals // Get pressure [Pa] (nlevs, nlocs) @@ -158,6 +163,10 @@ void SatwindIndivErrors::compute(const ObsFilterData & in, in.get(Variable(profile+"@GeoVaLs"), ilev, cx_windcomponent[ilev]); } + // diagnostic variables to be summed over all processors at the end of the routine + std::unique_ptr> countQiAccumulator = + obsdb.distribution()->createAccumulator(); + // Loop through locations for (size_t iloc=0; iloc < nlocs; ++iloc) { // Initialize at each location @@ -168,18 +177,26 @@ void SatwindIndivErrors::compute(const ObsFilterData & in, double sum_weight = 0.0; obserr[0][iloc] = missing; - // check for valid pressure error estimate - float const min_err_p = 100; // 1 Pa - if ( err_p[iloc] == missing || err_p[iloc] < min_err_p ) { - errString << "Pressure error estimate invalid: " << err_p[iloc] << " Pa" << std::endl; + // Check for valid pressure error estimate. + // The choice of minimum pressure error is somewhat arbitrary. + // Having a minimum helps catch cases where we may have forgotten to specify in Pa + // and instead used hPa, e.g. 60 Pa instead of 6000 Pa. + float const min_pressure_error = 500; // 5 hPa + float const max_pressure_error = 50000; // 500 hPa + if ( pressure_error[iloc] == missing || + pressure_error[iloc] < min_pressure_error || + pressure_error[iloc] > max_pressure_error ) { + errString << "Pressure error invalid: " << pressure_error[iloc] << " Pa" << std::endl; throw eckit::BadValue(errString.str()); } // Calculate vector error using QI - if ( (ob_qi[iloc] != missing) && (ob_qi[iloc] > 0.0) ) { + if ( (ob_qi[iloc] != missing) && + (ob_qi[iloc] > 0.0) && + (ob_qi[iloc] <= 100.0)) { error_vector = eu_mult * (ob_qi[iloc] * 0.01) + eu_add; } else { - oops::Log::warning() << "No valid QI. Using default vector err" << error_vector << std::endl; + countQiAccumulator->addTerm(iloc, 1); } // Calculate the error in vector due to error in pressure @@ -190,9 +207,9 @@ void SatwindIndivErrors::compute(const ObsFilterData & in, continue; } // Calculate weight for each background level, avoiding zero divide. - if (err_p[iloc] > 0) { + if (pressure_error[iloc] > 0) { weight = exp(-0.5 * pow(cx_p[ilev][iloc] - ob_p[iloc], 2) / - pow(err_p[iloc], 2) ) + pow(pressure_error[iloc], 2) ) * std::abs(cx_p[ilev][iloc] - cx_p[ilev - 1][iloc]); } else { weight = 0.0; @@ -215,6 +232,12 @@ void SatwindIndivErrors::compute(const ObsFilterData & in, obserr[0][iloc] = sqrt(pow(error_vector, 2) + pow(error_press, 2) ); } + // sum number of bad QI values + const std::size_t countQi = countQiAccumulator->computeResult(); + if (countQi > 0) { + oops::Log::warning() << "Satwind Indiv Errors: " << countQi + << " observations with bad/missing QI. Using default vector err in these cases" << std::endl; + } } // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/obsfunctions/SatwindIndivErrors.h b/src/ufo/filters/obsfunctions/SatwindIndivErrors.h index 63efe73fb..c3b39c3d2 100644 --- a/src/ufo/filters/obsfunctions/SatwindIndivErrors.h +++ b/src/ufo/filters/obsfunctions/SatwindIndivErrors.h @@ -35,21 +35,23 @@ class SatwindIndivErrorsParameters : public oops::Parameters { oops::RequiredParameter eu_add{"verror add", this}; /// Vector error estimate multiply oops::RequiredParameter eu_mult{"verror mult", this}; - /// Profile we are calculating error for + /// String containing the name of the wind component we are calculating the error for oops::RequiredParameter profile{"wind component", this}; - /// vertical coordinate to use + /// String containing the vertical coordinate to use for the wind component oops::RequiredParameter vcoord{"vertical coordinate", this}; - /// default pressure error (Pa) - oops::RequiredParameter default_err_p{"default pressure error", this}; - /// ignore contribution above height of minimum pressure (Pa) - oops::OptionalParameter min_press{"minimum pressure", this}; + /// Ignore contribution above height of minimum pressure (Pa) + oops::Parameter min_press{"minimum pressure", 10000.0, this}; + /// Name of the variable containing the input height error estimates (Pa) + oops::RequiredParameter pressure_error{"pressure error", this}; + /// Name of the variable containing quality index values for use in the vector error calculation + oops::RequiredParameter quality_index{"quality index", this}; }; // ----------------------------------------------------------------------------- /// /// \brief Function calculates individual observation errors for Satwind u and v winds -/// dependent on an input height (pressure) error estimate and the wind shear. +/// dependent on an input pressure error estimate and the model wind shear. /// class SatwindIndivErrors : public ObsFunctionBase { diff --git a/test/testinput/function_satwind_indiv_errors.yaml b/test/testinput/function_satwind_indiv_errors.yaml index bc9c219c9..671cbf9c2 100644 --- a/test/testinput/function_satwind_indiv_errors.yaml +++ b/test/testinput/function_satwind_indiv_errors.yaml @@ -9,13 +9,50 @@ observations: name: Satwind obsdatain: obsfile: Data/ufo/testinput_tier_1/satwind_obs_1d_2020100106_noinv.nc4 - obsdataout: - obsfile: Data/satwind_obs_1d_2020100106_noinv_errors.nc4 simulated variables: [eastward_wind, northward_wind] geovals: filename: Data/ufo/testinput_tier_1/satwind_geoval_20201001T0600Z_noinv.nc4 obs filters: - - filter: Bounds Check + # Create MetaData/satwind_id string variable containing channel information + #-------------------------------------------------------------------------- + - filter: satname + SatName assignments: + - min WMO Satellite id: 1 + max WMO Satellite id: 999 + Satellite_comp: + - satobchannel: 1 + min frequency: 2.6e+13 + max frequency: 2.7e+13 + wind channel: ir112 + - satobchannel: 1 + min frequency: 7.5e+13 + max frequency: 8.2e+13 + wind channel: ir38 + Satellite_id: + - Sat ID: 270 + Sat name: GOES16 + # Assign estimated pressure errors for each satellite and channel combination + # Errors are defined as a function of: + # air_pressure_levels@MetaData + # satwind_id@MetaData + #-------------------------------------------------------------------------- + - filter: Variable Assignment + assignments: + - name: air_pressure_levels@MetaDataError + type: float + function: + name: DrawValueFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/satwind_pressure_errors.nc4 + group: MetaDataError + interpolation: + - name: satwind_id@MetaData + method: exact + - name: air_pressure_levels@MetaData + method: linear + # Post filter - Assign individual observation errors + #--------------------------------------------------- + - filter: Perform Action defer to post: true filter variables: - name: eastward_wind @@ -24,12 +61,15 @@ observations: error function: name: SatwindIndivErrors@ObsFunction options: - default pressure error: 10000 verror add: 7.5 verror mult: -5.0 wind component: eastward_wind vertical coordinate: air_pressure_levels - - filter: Bounds Check + pressure error: + name: air_pressure_levels@MetaDataError + quality index: + name: percent_confidence_2@MetaData + - filter: Perform Action defer to post: true filter variables: - name: northward_wind @@ -38,11 +78,14 @@ observations: error function: name: SatwindIndivErrors@ObsFunction options: - default pressure error: 10000 verror add: 7.5 verror mult: -5.0 wind component: northward_wind vertical coordinate: air_pressure_levels + pressure error: + name: air_pressure_levels@MetaDataError + quality index: + name: percent_confidence_2@MetaData compareVariables: # test output matches precalculated values - reference: name: eastward_wind@TestReference From 697b3f61d5866628023df763bcd9794f65dd7fc8 Mon Sep 17 00:00:00 2001 From: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> Date: Fri, 6 Aug 2021 15:02:09 -0400 Subject: [PATCH 76/98] JEDI-GEOS ewok files (#1426) * add yamls for geos observations (copy of gdas) * update geos yamls Co-authored-by: danholdaway Co-authored-by: Ryan Honeyager --- ewok/jedi-geos/aircraft.yaml | 395 +++++++++++++++++++++ ewok/jedi-geos/airs_aqua.yaml | 338 ++++++++++++++++++ ewok/jedi-geos/amsua_aqua.yaml | 323 +++++++++++++++++ ewok/jedi-geos/amsua_metop-a.yaml | 323 +++++++++++++++++ ewok/jedi-geos/amsua_metop-b.yaml | 323 +++++++++++++++++ ewok/jedi-geos/amsua_metop-c.yaml | 323 +++++++++++++++++ ewok/jedi-geos/amsua_n15.yaml | 323 +++++++++++++++++ ewok/jedi-geos/amsua_n18.yaml | 323 +++++++++++++++++ ewok/jedi-geos/amsua_n19.yaml | 323 +++++++++++++++++ ewok/jedi-geos/atms_n20.yaml | 355 +++++++++++++++++++ ewok/jedi-geos/atms_npp.yaml | 355 +++++++++++++++++++ ewok/jedi-geos/avhrr3_metop-a.yaml | 175 ++++++++++ ewok/jedi-geos/avhrr3_n18.yaml | 176 ++++++++++ ewok/jedi-geos/cris-fsr_n20.yaml | 436 +++++++++++++++++++++++ ewok/jedi-geos/cris-fsr_npp.yaml | 436 +++++++++++++++++++++++ ewok/jedi-geos/gnssrobndnbam.yaml | 45 +++ ewok/jedi-geos/iasi_metop-a.yaml | 533 +++++++++++++++++++++++++++++ ewok/jedi-geos/iasi_metop-b.yaml | 533 +++++++++++++++++++++++++++++ ewok/jedi-geos/mhs_metop-b.yaml | 33 ++ ewok/jedi-geos/mhs_metop-c.yaml | 33 ++ ewok/jedi-geos/mhs_n19.yaml | 33 ++ ewok/jedi-geos/omi_aura.yaml | 12 + ewok/jedi-geos/rass_tv.yaml | 9 + ewok/jedi-geos/satwind.yaml | 315 +++++++++++++++++ ewok/jedi-geos/scatwind.yaml | 113 ++++++ ewok/jedi-geos/seviri_m11.yaml | 138 ++++++++ ewok/jedi-geos/sfc.yaml | 120 +++++++ ewok/jedi-geos/sfcship.yaml | 129 +++++++ ewok/jedi-geos/sondes.yaml | 295 ++++++++++++++++ ewok/jedi-geos/ssmis_f17.yaml | 164 +++++++++ ewok/jedi-geos/vadwind.yaml | 154 +++++++++ 31 files changed, 7586 insertions(+) create mode 100644 ewok/jedi-geos/aircraft.yaml create mode 100644 ewok/jedi-geos/airs_aqua.yaml create mode 100644 ewok/jedi-geos/amsua_aqua.yaml create mode 100644 ewok/jedi-geos/amsua_metop-a.yaml create mode 100644 ewok/jedi-geos/amsua_metop-b.yaml create mode 100644 ewok/jedi-geos/amsua_metop-c.yaml create mode 100644 ewok/jedi-geos/amsua_n15.yaml create mode 100644 ewok/jedi-geos/amsua_n18.yaml create mode 100644 ewok/jedi-geos/amsua_n19.yaml create mode 100644 ewok/jedi-geos/atms_n20.yaml create mode 100644 ewok/jedi-geos/atms_npp.yaml create mode 100644 ewok/jedi-geos/avhrr3_metop-a.yaml create mode 100644 ewok/jedi-geos/avhrr3_n18.yaml create mode 100644 ewok/jedi-geos/cris-fsr_n20.yaml create mode 100644 ewok/jedi-geos/cris-fsr_npp.yaml create mode 100644 ewok/jedi-geos/gnssrobndnbam.yaml create mode 100644 ewok/jedi-geos/iasi_metop-a.yaml create mode 100644 ewok/jedi-geos/iasi_metop-b.yaml create mode 100644 ewok/jedi-geos/mhs_metop-b.yaml create mode 100644 ewok/jedi-geos/mhs_metop-c.yaml create mode 100644 ewok/jedi-geos/mhs_n19.yaml create mode 100644 ewok/jedi-geos/omi_aura.yaml create mode 100644 ewok/jedi-geos/rass_tv.yaml create mode 100644 ewok/jedi-geos/satwind.yaml create mode 100644 ewok/jedi-geos/scatwind.yaml create mode 100644 ewok/jedi-geos/seviri_m11.yaml create mode 100644 ewok/jedi-geos/sfc.yaml create mode 100644 ewok/jedi-geos/sfcship.yaml create mode 100644 ewok/jedi-geos/sondes.yaml create mode 100644 ewok/jedi-geos/ssmis_f17.yaml create mode 100644 ewok/jedi-geos/vadwind.yaml diff --git a/ewok/jedi-geos/aircraft.yaml b/ewok/jedi-geos/aircraft.yaml new file mode 100644 index 000000000..9e5f13e31 --- /dev/null +++ b/ewok/jedi-geos/aircraft.yaml @@ -0,0 +1,395 @@ +obs operator: + name: VertInterp +obs space: + name: Aircraft + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/aircraft.{{window_begin}}.nc4 + obsgrouping: + group variables: ["station_id"] + sort variable: "air_pressure" + sort order: "descending" + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).aircraft.{{window_begin}}.nc4 + simulated variables: [eastward_wind, northward_wind, air_temperature] +obs filters: +#-------------------------------------------------------------------------------------------------------------------- +# WINDS +#-------------------------------------------------------------------------------------------------------------------- +# +# Begin by assigning all ObsError to a constant value. These will get overwritten (as needed) for specific types. +- filter: BlackList + filter variables: + - name: eastward_wind + - name: northward_wind + action: + name: assign error + error parameter: 2.0 # 2.0 m/s +# +# Assign intial ObsError specific to AIREP/ACARS +- filter: BlackList + filter variables: + - name: eastward_wind + - name: northward_wind + action: + name: assign error + error parameter: 3.6 # 3.6 m/s + where: + - variable: + name: eastward_wind@ObsType + is_in: 230 +# +# Assign intial ObsError specific to AMDAR +- filter: BlackList + filter variables: + - name: eastward_wind + - name: northward_wind + action: + name: assign error + error parameter: 3.0 # 3.0 m/s + where: + - variable: + name: eastward_wind@ObsType + is_in: 231 +# +# Assign intial ObsError specific to MDCRS +- filter: BlackList + filter variables: + - name: eastward_wind + - name: northward_wind + action: + name: assign error + error parameter: 2.5 # 2.5 m/s + where: + - variable: + name: eastward_wind@ObsType + is_in: 233 +# +# Assign the initial ObsError, based on height/pressure for RECON aircraft +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: air_pressure@MetaData + xvals: [70000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000, 7500, 5000] + errors: [2.4, 2.5, 2.6, 2.7, 2.8, 2.95, 3.1, 3.25, 3.4, 3.175, 2.95, 2.725, 2.5, 2.6, 2.7] + where: + - variable: + name: eastward_wind@ObsType + is_in: 232 +# +# Reject all obs with PreQC mark already set above 3 +- filter: PreQC + maxvalue: 3 + action: + name: reject +# +# Observation Range Sanity Check: either wind component or velocity exceeds 135 m/s +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + minvalue: -135 + maxvalue: 135 + action: + name: reject +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: Velocity@ObsFunction + maxvalue: 135.0 + action: + name: reject +# +# Reject when pressure is less than 126 mb. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: air_pressure@MetaData + minvalue: 12600 + action: + name: reject +# +# Reject when difference of wind direction is more than 50 degrees. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: WindDirAngleDiff@ObsFunction + maxvalue: 50.0 + action: + name: reject +# +# When multiple obs exist within a single vertical model level, inflate ObsError +- filter: BlackList + filter variables: + - name: eastward_wind + action: + name: inflate error + inflation variable: + name: ObsErrorFactorConventional@ObsFunction + options: + test QCflag: PreQC + inflate variables: [eastward_wind] + defer to post: true +# +- filter: BlackList + filter variables: + - name: northward_wind + action: + name: inflate error + inflation variable: + name: ObsErrorFactorConventional@ObsFunction + options: + test QCflag: PreQC + inflate variables: [northward_wind] + defer to post: true +# +# If background check is largely different than obs, inflate ObsError +- filter: Background Check + filter variables: + - name: eastward_wind + - name: northward_wind + absolute threshold: 7.5 + action: + name: inflate error + inflation factor: 3.0 + defer to post: true +# +- filter: Bounds Check + filter variables: + - name: eastward_wind + action: + name: reject + maxvalue: 7.0 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: eastward_wind@ObsErrorData # After inflation step + denominator: + name: eastward_wind@ObsError + defer to post: true +# +# If ObsError inflation factor is larger than threshold, reject obs +- filter: Bounds Check + filter variables: + - name: northward_wind + action: + name: reject + maxvalue: 7.0 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: northward_wind@ObsErrorData # After inflation step + denominator: + name: northward_wind@ObsError + defer to post: true +#-------------------------------------------------------------------------------------------------------------------- +# TEMPERATURE +#-------------------------------------------------------------------------------------------------------------------- +# +# Begin by assigning all ObsError to a constant value. These will get overwritten for specific types. +- filter: BlackList + filter variables: + - name: air_temperature + action: + name: assign error + error parameter: 2.0 # 2.0 K +# +# Assign the initial observation error, based on pressure (for AIREP/ACARS; itype=130) +- filter: Bounds Check + filter variables: + - name: air_temperature + minvalue: 195 + maxvalue: 327 + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: air_pressure@MetaData + xvals: [100000, 95000, 90000, 85000, 80000] + errors: [2.5, 2.3, 2.1, 1.9, 1.7] + where: + - variable: + name: air_temperature@ObsType + is_in: 130 +# +# Assign the initial observation error, based on pressure (for AMDAR and MDCRS; itype=131,133) +- filter: Bounds Check + filter variables: + - name: air_temperature + minvalue: 195 + maxvalue: 327 + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: air_pressure@MetaData + xvals: [100000, 95000, 90000, 85000, 80000] + errors: [1.4706, 1.3529, 1.2353, 1.1176, 1.0] + where: + - variable: + name: air_temperature@ObsType + is_in: 131,133 +# +# Assign the initial observation error, based on pressure (for RECON aircraft; itype=132) +- filter: Bounds Check + filter variables: + - name: air_temperature + minvalue: 195 + maxvalue: 327 + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: air_pressure@MetaData + xvals: [100000, 95000, 90000, 85000, 35000, 30000, 25000, 20000, 15000, 10000, 7500, 5000, 4000, 3200, 2000, 1000] + errors: [1.2, 1.1, 0.9, 0.8, 0.8, 0.9, 1.2, 1.2, 1.0, 0.8, 0.8, 0.9, 0.95, 1.0, 1.25, 1.5] + where: + - variable: + name: air_temperature@ObsType + is_in: 132 +# +# Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: air_temperature + minvalue: 195 + maxvalue: 327 + action: + name: reject +# +# Reject all obs with PreQC mark already set above 3 +- filter: PreQC + maxvalue: 3 + action: + name: reject +# +# When multiple obs exist within a single vertical model level, inflate ObsError +- filter: BlackList + filter variables: + - name: air_temperature + action: + name: inflate error + inflation variable: + name: ObsErrorFactorConventional@ObsFunction + options: +# test QCflag: PreQC + inflate variables: [air_temperature] + defer to post: true +# +# If background check is largely different than obs, inflate ObsError +- filter: Background Check + filter variables: + - name: air_temperature + absolute threshold: 4.0 + action: + name: inflate error + inflation factor: 3.0 + defer to post: true +# +# If ObsError inflation factor is larger than threshold, reject obs +- filter: Bounds Check + filter variables: + - name: air_temperature + action: + name: reject + maxvalue: 7.0 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: air_temperature@ObsErrorData # After inflation step + denominator: + name: air_temperature@ObsError + defer to post: true +# #-------------------------------------------------------------------------------------------------------------------- +# # MOISTURE +# #-------------------------------------------------------------------------------------------------------------------- +# # +# # Assign the initial observation error, based on height/pressure ONLY MDCRS +# - filter: Bounds Check +# filter variables: +# - name: specific_humidity +# minvalue: 1.0E-7 +# maxvalue: 0.34999999 +# action: +# name: assign error +# error function: +# name: ObsErrorModelStepwiseLinear@ObsFunction +# options: +# xvar: +# name: air_pressure@MetaData +# xvals: [110000, 105000, 100000, 95000, 90000, 85000, 80000, 75000, 70000, 65000, 60000, 55000, +# 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000, 7500, 5000, 4000, 3000] +# errors: [.19455, .19062, .18488, .17877, .17342, .16976, .16777, .16696, .16605, .16522, .16637, .17086, +# .17791, .18492, .18996, .19294, .19447, .19597, .19748, .19866, .19941, .19979, .19994, .19999, .2] +# scale_factor_var: specific_humidity@ObsValue +# where: +# - variable: +# name: specific_humidity@ObsType +# is_in: 133 +# # +# # Observation Range Sanity Check +# - filter: Bounds Check +# filter variables: +# - name: specific_humidity +# minvalue: 1.0E-7 +# maxvalue: 0.34999999 +# action: +# name: reject +# # +# # Reject all obs with PreQC mark already set above 3 +# - filter: PreQC +# maxvalue: 3 +# action: +# name: reject +# # +# # When multiple obs exist within a single vertical model level, inflate ObsError +# - filter: BlackList +# filter variables: +# - name: specific_humidity +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorConventional@ObsFunction +# options: +# test QCflag: PreQC +# inflate variables: [specific_humidity] +# defer to post: true +# # +# # If ObsError inflation factor is larger than threshold, reject obs +# - filter: Bounds Check +# filter variables: +# - name: specific_humidity +# action: +# name: reject +# maxvalue: 8.0 +# test variables: +# - name: ObsErrorFactorQuotient@ObsFunction +# options: +# numerator: +# name: specific_humidity@ObsErrorData # After inflation step +# denominator: +# name: specific_humidity@ObsError +# defer to post: true diff --git a/ewok/jedi-geos/airs_aqua.yaml b/ewok/jedi-geos/airs_aqua.yaml new file mode 100644 index 000000000..714963842 --- /dev/null +++ b/ewok/jedi-geos/airs_aqua.yaml @@ -0,0 +1,338 @@ +obs space: + name: airs_aqua + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/airs_aqua.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).airs_aqua.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &airs_aqua_channels 1, 6, 7, 10, 11, 15, 16, 17, 20, 21, 22, 24, + 27, 28, 30, 36, 39, 40, 42, 51, 52, 54, 55, 56, 59, 62, 63, 68, 69, 71, + 72, 73, 74, 75, 76, 77, 78, 79, 80, 82, 83, 84, 86, 92, 93, 98, 99, 101, + 104, 105, 108, 110, 111, 113, 116, 117, 123, 124, 128, 129, 138, 139, + 144, 145, 150, 151, 156, 157, 159, 162, 165, 168, 169, 170, 172, 173, + 174, 175, 177, 179, 180, 182, 185, 186, 190, 192, 198, 201, 204, 207, + 210, 215, 216, 221, 226, 227, 232, 252, 253, 256, 257, 261, 262, 267, + 272, 295, 299, 300, 305, 310, 321, 325, 333, 338, 355, 362, 375, 453, + 475, 484, 497, 528, 587, 672, 787, 791, 843, 870, 914, 950, 1003, 1012, + 1019, 1024, 1030, 1038, 1048, 1069, 1079, 1082, 1083, 1088, 1090, 1092, + 1095, 1104, 1111, 1115, 1116, 1119, 1120, 1123, 1130, 1138, 1142, 1178, + 1199, 1206, 1221, 1237, 1252, 1260, 1263, 1266, 1285, 1301, 1304, 1329, + 1371, 1382, 1415, 1424, 1449, 1455, 1466, 1477, 1500, 1519, 1538, 1545, + 1565, 1574, 1583, 1593, 1614, 1627, 1636, 1644, 1652, 1669, 1674, 1681, + 1694, 1708, 1717, 1723, 1740, 1748, 1751, 1756, 1763, 1766, 1771, 1777, + 1780, 1783, 1794, 1800, 1803, 1806, 1812, 1826, 1843, 1852, 1865, 1866, + 1868, 1869, 1872, 1873, 1876, 1881, 1882, 1883, 1911, 1917, 1918, 1924, + 1928, 1937, 1941, 2099, 2100, 2101, 2103, 2104, 2106, 2107, 2108, 2109, + 2110, 2111, 2112, 2113, 2114, 2115, 2116, 2117, 2118, 2119, 2120, 2121, + 2122, 2123, 2128, 2134, 2141, 2145, 2149, 2153, 2164, 2189, 2197, 2209, + 2226, 2234, 2280, 2318, 2321, 2325, 2328, 2333, 2339, 2348, 2353, 2355, + 2357, 2363, 2370, 2371, 2377 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: airs_aqua + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/airs_aqua.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &airs_aqua_tlapse $(experiment_dir)/{{current_cycle}}/airs_aqua.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *airs_aqua_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +# Wavenumber Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: 2122, 2123, 2128, 2134, 2141, 2145, 2149, 2153, 2164, 2189, 2197, 2209, + 2226, 2234, 2280, 2318, 2321, 2325, 2328, 2333, 2339, 2348, 2353, 2355, + 2357, 2363, 2370, 2371, 2377 + where: + - variable: + name: solar_zenith_angle@MetaData + maxvalue: 88.9999 + - variable: + name: water_area_fraction@GeoVaLs + minvalue: 1.0e-12 + action: + name: reject +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *airs_aqua_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorWavenumIR@ObsFunction + channels: *airs_aqua_channels + options: + channels: *airs_aqua_channels +# Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *airs_aqua_channels + minvalue: 50.00001 + maxvalue: 449.99999 + action: + name: reject +# Topography Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *airs_aqua_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *airs_aqua_channels + options: + channels: *airs_aqua_channels + sensor: airs_aqua +# Transmittance Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *airs_aqua_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *airs_aqua_channels + options: + channels: *airs_aqua_channels +# Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *airs_aqua_channels + test variables: + - name: CloudDetectMinResidualIR@ObsFunction + channels: *airs_aqua_channels + options: + channels: *airs_aqua_channels + use_flag: [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, + 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, + 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, + 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1 ] + use_flag_clddet: [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, + 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, + 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, + 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1 ] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + maxvalue: 1.0e-12 + action: + name: reject +# NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *airs_aqua_channels + test variables: + - name: NearSSTRetCheckIR@ObsFunction + channels: *airs_aqua_channels + options: + channels: *airs_aqua_channels + use_flag: [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, + 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, + 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, + 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1 ] + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + maxvalue: 1.0e-12 + action: + name: reject +# Surface Jacobians Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *airs_aqua_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *airs_aqua_channels + options: + channels: *airs_aqua_channels + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *airs_aqua_channels + function absolute threshold: + - name: ObsErrorBoundIR@ObsFunction + channels: *airs_aqua_channels + options: + channels: *airs_aqua_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *airs_aqua_channels + options: + channels: *airs_aqua_channels + obserr_bound_max: [ 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, + 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, + 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, + 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, + 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, + 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, + 3.5, 3.5, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, + 3.0, 3.0, 3.0, 3.0, 3.5, 3.5, 3.5, 3.0, 3.0, 3.5, + 3.5, 3.0, 3.0, 3.0, 3.5, 3.0, 3.0, 3.0, 3.0, 3.0, + 3.5, 3.0, 3.5, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, + 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, + 3.5, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, + 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, + 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, + 3.5, 3.5, 3.5, 3.5, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, + 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, + 3.0, 3.5, 3.0, 3.5, 3.0, 3.0, 3.0, 3.0, 3.5, 3.0, + 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, + 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, + 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, + 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, + 4.5, 4.5, 4.5, 4.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, + 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 3.5, + 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, + 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, + 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, + 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, + 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, + 3.0 ] + action: + name: reject +# Useflag Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *airs_aqua_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *airs_aqua_channels + options: + channels: *airs_aqua_channels + use_flag: [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, + 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, + 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, + 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1 ] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/amsua_aqua.yaml b/ewok/jedi-geos/amsua_aqua.yaml new file mode 100644 index 000000000..27bdb8122 --- /dev/null +++ b/ewok/jedi-geos/amsua_aqua.yaml @@ -0,0 +1,323 @@ +obs space: + name: amsua_aqua + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/amsua_aqua.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).amsua_aqua.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &amsua_aqua_channels 1-15 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: amsua_aqua + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/amsua_aqua.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_aqua_tlapse $(experiment_dir)/{{current_cycle}}/amsua_aqua.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *amsua_aqua_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_aqua_channels + action: + name: assign error + error function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_aqua_channels + options: + channels: *amsua_aqua_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.000, 2.000, 0.500, 0.400, + 0.400, 0.500, 0.300, 0.350, 0.350, + 0.450, 1.000, 1.500, 2.500, 2.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.550, + 0.400, 0.500, 0.300, 0.350, 0.350, + 0.450, 1.000, 1.500, 2.500, 18.000] +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + maxvalue: 999.0 + action: + name: reject +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + maxvalue: 999.0 + action: + name: reject +# Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_aqua_channels + test variables: + - name: HydrometeorCheckAMSUA@ObsFunction + channels: *amsua_aqua_channels + options: + channels: *amsua_aqua_channels + obserr_clearsky: [ 2.500, 2.000, 2.000, 0.500, 0.400, + 0.400, 0.500, 0.300, 0.350, 0.350, + 0.450, 1.000, 1.500, 2.500, 2.500] + clwret_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_aqua_channels + options: + channels: *amsua_aqua_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.000, 2.000, 0.500, 0.400, + 0.400, 0.500, 0.300, 0.350, 0.350, + 0.450, 1.000, 1.500, 2.500, 2.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.550, + 0.400, 0.500, 0.300, 0.350, 0.350, + 0.450, 1.000, 1.500, 2.500, 18.000] + maxvalue: 0.0 + action: + name: reject +# Topography check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_aqua_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_aqua_channels + options: + sensor: amsua_aqua + channels: *amsua_aqua_channels +# Transmittnace Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_aqua_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_aqua_channels + options: + channels: *amsua_aqua_channels +# Surface Jacobian check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_aqua_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *amsua_aqua_channels + options: + channels: *amsua_aqua_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] +# Situation dependent Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_aqua_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *amsua_aqua_channels + options: + sensor: amsua_aqua + channels: *amsua_aqua_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + scatobs_function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + bias_application: HofX + clwmatchidx_function: + name: CLWMatchIndexMW@ObsFunction + channels: *amsua_aqua_channels + options: + channels: *amsua_aqua_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + obserr_clearsky: [2.500, 2.000, 2.000, 0.500, 0.400, + 0.400, 0.500, 0.300, 0.350, 0.350, + 0.450, 1.000, 1.500, 2.500, 2.500] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *amsua_aqua_channels + function absolute threshold: + - name: ObsErrorBoundMW@ObsFunction + channels: *amsua_aqua_channels + options: + sensor: amsua_aqua + channels: *amsua_aqua_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_aqua_channels + options: + channels: *amsua_aqua_channels + obserr_bound_topo: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_aqua_channels + options: + channels: *amsua_aqua_channels + sensor: amsua_aqua + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_aqua_channels + options: + channels: *amsua_aqua_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + bias_application: HofX + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.000, 2.000, 0.500, 0.400, + 0.400, 0.500, 0.300, 0.350, 0.350, + 0.450, 1.000, 1.500, 2.500, 2.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.550, + 0.400, 0.500, 0.300, 0.350, 0.350, + 0.450, 1.000, 1.500, 2.500, 18.000] + obserr_bound_max: [4.5, 4.5, 4.5, 3.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 3.0, 3.5, 4.5, 4.5, 4.5] + action: + name: reject +# Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_aqua_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *amsua_aqua_channels + options: + channels: *amsua_aqua_channels + sensor: amsua_aqua + use_flag: [ -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, + 1, 1, 1, -1, -1 ] + maxvalue: 1.0e-12 + action: + name: reject +# Useflag check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_aqua_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *amsua_aqua_channels + options: + channels: *amsua_aqua_channels + use_flag: [ -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, + 1, 1, 1, -1, -1 ] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/amsua_metop-a.yaml b/ewok/jedi-geos/amsua_metop-a.yaml new file mode 100644 index 000000000..b55aaca93 --- /dev/null +++ b/ewok/jedi-geos/amsua_metop-a.yaml @@ -0,0 +1,323 @@ +obs space: + name: amsua_metop-a + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/amsua_metop-a.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).amsua_metop-a.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &amsua_metop-a_channels 1-15 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: amsua_metop-a + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/amsua_metop-a.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-a_tlapse $(experiment_dir)/{{current_cycle}}/amsua_metop-a.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *amsua_metop-a_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-a_channels + action: + name: assign error + error function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + maxvalue: 999.0 + action: + name: reject +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + maxvalue: 999.0 + action: + name: reject +# Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_metop-a_channels + test variables: + - name: HydrometeorCheckAMSUA@ObsFunction + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + clwret_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] + maxvalue: 0.0 + action: + name: reject +# Topography check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_metop-a_channels + options: + sensor: amsua_metop-a + channels: *amsua_metop-a_channels +# Transmittnace Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels +# Surface Jacobian check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] +# Situation dependent Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *amsua_metop-a_channels + options: + sensor: amsua_metop-a + channels: *amsua_metop-a_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + scatobs_function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + bias_application: HofX + clwmatchidx_function: + name: CLWMatchIndexMW@ObsFunction + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *amsua_metop-a_channels + function absolute threshold: + - name: ObsErrorBoundMW@ObsFunction + channels: *amsua_metop-a_channels + options: + sensor: amsua_metop-a + channels: *amsua_metop-a_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + obserr_bound_topo: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + sensor: amsua_metop-a + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + bias_application: HofX + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + action: + name: reject +# Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_metop-a_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + sensor: amsua_metop-a + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, -1, 1 ] + maxvalue: 1.0e-12 + action: + name: reject +# Useflag check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_metop-a_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, -1, 1 ] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/amsua_metop-b.yaml b/ewok/jedi-geos/amsua_metop-b.yaml new file mode 100644 index 000000000..72e004574 --- /dev/null +++ b/ewok/jedi-geos/amsua_metop-b.yaml @@ -0,0 +1,323 @@ +obs space: + name: amsua_metop-b + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/amsua_metop-b.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).amsua_metop-b.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &amsua_metop-b_channels 1-15 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: amsua_metop-b + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/amsua_metop-b.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-b_tlapse $(experiment_dir)/{{current_cycle}}/amsua_metop-b.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *amsua_metop-b_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-b_channels + action: + name: assign error + error function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + maxvalue: 999.0 + action: + name: reject +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + maxvalue: 999.0 + action: + name: reject +# Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_metop-b_channels + test variables: + - name: HydrometeorCheckAMSUA@ObsFunction + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + clwret_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] + maxvalue: 0.0 + action: + name: reject +# Topography check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_metop-b_channels + options: + sensor: amsua_metop-b + channels: *amsua_metop-b_channels +# Transmittnace Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels +# Surface Jacobian check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] +# Situation dependent Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *amsua_metop-b_channels + options: + sensor: amsua_metop-b + channels: *amsua_metop-b_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + scatobs_function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + bias_application: HofX + clwmatchidx_function: + name: CLWMatchIndexMW@ObsFunction + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *amsua_metop-b_channels + function absolute threshold: + - name: ObsErrorBoundMW@ObsFunction + channels: *amsua_metop-b_channels + options: + sensor: amsua_metop-b + channels: *amsua_metop-b_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + obserr_bound_topo: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + sensor: amsua_metop-b + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + bias_application: HofX + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + action: + name: reject +# Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_metop-b_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + sensor: amsua_metop-b + use_flag: [-1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, + 1, 1, 1, -1, -1] + maxvalue: 1.0e-12 + action: + name: reject +# Useflag check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_metop-b_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + use_flag: [-1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, + 1, 1, 1, -1, -1] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/amsua_metop-c.yaml b/ewok/jedi-geos/amsua_metop-c.yaml new file mode 100644 index 000000000..960593bcf --- /dev/null +++ b/ewok/jedi-geos/amsua_metop-c.yaml @@ -0,0 +1,323 @@ +obs space: + name: amsua_metop-c + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/amsua_metop-c.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).amsua_metop-c.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &amsua_metop-c_channels 1-15 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: amsua_metop-c + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/amsua_metop-c.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-c_tlapse $(experiment_dir)/{{current_cycle}}/amsua_metop-c.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *amsua_metop-c_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-c_channels + action: + name: assign error + error function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + maxvalue: 999.0 + action: + name: reject +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + maxvalue: 999.0 + action: + name: reject +# Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_metop-c_channels + test variables: + - name: HydrometeorCheckAMSUA@ObsFunction + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + clwret_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] + maxvalue: 0.0 + action: + name: reject +# Topography check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-c_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_metop-c_channels + options: + sensor: amsua_metop-c + channels: *amsua_metop-c_channels +# Transmittnace Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-c_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels +# Surface Jacobian check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-c_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] +# Situation dependent Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_metop-c_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *amsua_metop-c_channels + options: + sensor: amsua_metop-c + channels: *amsua_metop-c_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + scatobs_function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + bias_application: HofX + clwmatchidx_function: + name: CLWMatchIndexMW@ObsFunction + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *amsua_metop-c_channels + function absolute threshold: + - name: ObsErrorBoundMW@ObsFunction + channels: *amsua_metop-c_channels + options: + sensor: amsua_metop-c + channels: *amsua_metop-c_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + obserr_bound_topo: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + sensor: amsua_metop-c + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + bias_application: HofX + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + action: + name: reject +# Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_metop-c_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + sensor: amsua_metop-c + use_flag: [-1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1] + maxvalue: 1.0e-12 + action: + name: reject +# Useflag check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_metop-c_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + use_flag: [-1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/amsua_n15.yaml b/ewok/jedi-geos/amsua_n15.yaml new file mode 100644 index 000000000..00705cd92 --- /dev/null +++ b/ewok/jedi-geos/amsua_n15.yaml @@ -0,0 +1,323 @@ +obs space: + name: amsua_n15 + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/amsua_n15.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).amsua_n15.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &amsua_n15_channels 1-15 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: amsua_n15 + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/amsua_n15.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n15_tlapse $(experiment_dir)/{{current_cycle}}/amsua_n15.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *amsua_n15_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n15_channels + action: + name: assign error + error function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 2.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 2.000, 18.000] +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + maxvalue: 999.0 + action: + name: reject +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + maxvalue: 999.0 + action: + name: reject +# Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_n15_channels + test variables: + - name: HydrometeorCheckAMSUA@ObsFunction + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + obserr_clearsky: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 2.000, 3.500] + clwret_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 2.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 2.000, 18.000] + maxvalue: 0.0 + action: + name: reject +# Topography check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n15_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_n15_channels + options: + sensor: amsua_n15 + channels: *amsua_n15_channels +# Transmittnace Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n15_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels +# Surface Jacobian check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n15_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] +# Situation dependent Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n15_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *amsua_n15_channels + options: + sensor: amsua_n15 + channels: *amsua_n15_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + scatobs_function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + bias_application: HofX + clwmatchidx_function: + name: CLWMatchIndexMW@ObsFunction + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + obserr_clearsky: [3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 2.000, 3.500] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *amsua_n15_channels + function absolute threshold: + - name: ObsErrorBoundMW@ObsFunction + channels: *amsua_n15_channels + options: + sensor: amsua_n15 + channels: *amsua_n15_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + obserr_bound_topo: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + sensor: amsua_n15 + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + bias_application: HofX + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 2.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 2.000, 18.000] + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + action: + name: reject +# Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_n15_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + sensor: amsua_n15 + use_flag: [ 1, 1, 1, 1, 1, + -1, 1, 1, 1, 1, + -1, 1, 1, -1, 1 ] + maxvalue: 1.0e-12 + action: + name: reject +# Useflag check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_n15_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + use_flag: [ 1, 1, 1, 1, 1, + -1, 1, 1, 1, 1, + -1, 1, 1, -1, 1 ] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/amsua_n18.yaml b/ewok/jedi-geos/amsua_n18.yaml new file mode 100644 index 000000000..bfd714bdf --- /dev/null +++ b/ewok/jedi-geos/amsua_n18.yaml @@ -0,0 +1,323 @@ +obs space: + name: amsua_n18 + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/amsua_n18.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).amsua_n18.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &amsua_n18_channels 1-15 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: amsua_n18 + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/amsua_n18.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n18_tlapse $(experiment_dir)/{{current_cycle}}/amsua_n18.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *amsua_n18_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n18_channels + action: + name: assign error + error function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + maxvalue: 999.0 + action: + name: reject +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + maxvalue: 999.0 + action: + name: reject +# Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_n18_channels + test variables: + - name: HydrometeorCheckAMSUA@ObsFunction + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + clwret_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] + maxvalue: 0.0 + action: + name: reject +# Topography check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n18_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_n18_channels + options: + sensor: amsua_n18 + channels: *amsua_n18_channels +# Transmittnace Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n18_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels +# Surface Jacobian check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n18_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] +# Situation dependent Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n18_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *amsua_n18_channels + options: + sensor: amsua_n18 + channels: *amsua_n18_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + scatobs_function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + bias_application: HofX + clwmatchidx_function: + name: CLWMatchIndexMW@ObsFunction + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *amsua_n18_channels + function absolute threshold: + - name: ObsErrorBoundMW@ObsFunction + channels: *amsua_n18_channels + options: + sensor: amsua_n18 + channels: *amsua_n18_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + obserr_bound_topo: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + sensor: amsua_n18 + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + bias_application: HofX + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + action: + name: reject +# Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_n18_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + sensor: amsua_n18 + use_flag: [ 1, 1, 1, 1, -1, + 1, 1, -1, -1, 1, + 1, 1, 1, -1, 1 ] + maxvalue: 1.0e-12 + action: + name: reject +# Useflag check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_n18_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + use_flag: [ 1, 1, 1, 1, -1, + 1, 1, -1, -1, 1, + 1, 1, 1, -1, 1 ] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/amsua_n19.yaml b/ewok/jedi-geos/amsua_n19.yaml new file mode 100644 index 000000000..306a8fc8c --- /dev/null +++ b/ewok/jedi-geos/amsua_n19.yaml @@ -0,0 +1,323 @@ +obs space: + name: amsua_n19 + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/amsua_n19.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).amsua_n19.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &amsua_n19_channels 1-15 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: amsua_n19 + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/amsua_n19.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n19_tlapse $(experiment_dir)/{{current_cycle}}/amsua_n19.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *amsua_n19_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n19_channels + action: + name: assign error + error function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + maxvalue: 999.0 + action: + name: reject +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-6, 15 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + maxvalue: 999.0 + action: + name: reject +# Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_n19_channels + test variables: + - name: HydrometeorCheckAMSUA@ObsFunction + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + clwret_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] + maxvalue: 0.0 + action: + name: reject +# Topography check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n19_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_n19_channels + options: + sensor: amsua_n19 + channels: *amsua_n19_channels +# Transmittnace Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n19_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels +# Surface Jacobian check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n19_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] +# Situation dependent Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *amsua_n19_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *amsua_n19_channels + options: + sensor: amsua_n19 + channels: *amsua_n19_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + scatobs_function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + bias_application: HofX + clwmatchidx_function: + name: CLWMatchIndexMW@ObsFunction + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + bias_application: HofX + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *amsua_n19_channels + function absolute threshold: + - name: ObsErrorBoundMW@ObsFunction + channels: *amsua_n19_channels + options: + sensor: amsua_n19 + channels: *amsua_n19_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + obserr_bound_topo: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + sensor: amsua_n19 + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + bias_application: HofX + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 3.000, 18.000] + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + action: + name: reject +# Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_n19_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + sensor: amsua_n19 + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, -1, 1 ] + maxvalue: 1.0e-12 + action: + name: reject +# Useflag check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *amsua_n19_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, -1, 1 ] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/atms_n20.yaml b/ewok/jedi-geos/atms_n20.yaml new file mode 100644 index 000000000..aac174b8a --- /dev/null +++ b/ewok/jedi-geos/atms_n20.yaml @@ -0,0 +1,355 @@ +obs space: + name: atms_n20 + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/atms_n20.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).atms_n20.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &atms_n20_channels 1-22 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: atms_n20 + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/atms_n20.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &atms_n20_tlapse $(experiment_dir)/{{current_cycle}}/atms_n20.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *atms_n20_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *atms_n20_channels + action: + name: assign error + error function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + maxvalue: 999.0 + action: + name: reject +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + maxvalue: 999.0 + action: + name: reject +# Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *atms_n20_channels + test variables: + - name: HydrometeorCheckATMS@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + maxvalue: 0.0 + action: + name: reject +# Topography check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *atms_n20_channels + options: + sensor: atms_n20 + channels: *atms_n20_channels +# Transmittnace Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels +# Surface Jacobian check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] +# Situation dependent Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *atms_n20_channels + options: + sensor: atms_n20 + channels: *atms_n20_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + scatobs_function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 16 + scatret_types: [ObsValue] + clwmatchidx_function: + name: CLWMatchIndexMW@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *atms_n20_channels + function absolute threshold: + - name: ObsErrorBoundMW@ObsFunction + channels: *atms_n20_channels + options: + sensor: atms_n20 + channels: *atms_n20_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + obserr_bound_topo: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + sensor: atms_n20 + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 2.0, 4.5, + 4.5, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0] + action: + name: reject +# Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *atms_n20_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + sensor: atms_n20 + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, -1, + 1, 1, 1, 1, 1, + 1, 1] + maxvalue: 1.0e-12 + action: + name: reject +# Useflag check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *atms_n20_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, -1, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/atms_npp.yaml b/ewok/jedi-geos/atms_npp.yaml new file mode 100644 index 000000000..97e84777e --- /dev/null +++ b/ewok/jedi-geos/atms_npp.yaml @@ -0,0 +1,355 @@ +obs space: + name: atms_npp + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/atms_npp.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).atms_npp.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &atms_npp_channels 1-22 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: atms_npp + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/atms_npp.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &atms_npp_tlapse $(experiment_dir)/{{current_cycle}}/atms_npp.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *atms_npp_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *atms_npp_channels + action: + name: assign error + error function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + maxvalue: 999.0 + action: + name: reject +# CLW Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + maxvalue: 999.0 + action: + name: reject +# Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *atms_npp_channels + test variables: + - name: HydrometeorCheckATMS@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + maxvalue: 0.0 + action: + name: reject +# Topography check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *atms_npp_channels + options: + sensor: atms_npp + channels: *atms_npp_channels +# Transmittnace Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels +# Surface Jacobian check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] +# Situation dependent Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *atms_npp_channels + options: + sensor: atms_npp + channels: *atms_npp_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + scatobs_function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 16 + scatret_types: [ObsValue] + clwmatchidx_function: + name: CLWMatchIndexMW@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + clwobs_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + clwbkg_function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *atms_npp_channels + function absolute threshold: + - name: ObsErrorBoundMW@ObsFunction + channels: *atms_npp_channels + options: + sensor: atms_npp + channels: *atms_npp_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + obserr_bound_topo: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + sensor: atms_npp + obserr_function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + xvar: + name: CLWRetSymmetricMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue, HofX] + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 2.0, 4.5, + 4.5, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0] + action: + name: reject +# Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *atms_npp_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + sensor: atms_npp + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, -1, + 1, 1, 1, 1, 1, + 1, 1] + maxvalue: 1.0e-12 + action: + name: reject +# Useflag check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *atms_npp_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, -1, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/avhrr3_metop-a.yaml b/ewok/jedi-geos/avhrr3_metop-a.yaml new file mode 100644 index 000000000..a037fce04 --- /dev/null +++ b/ewok/jedi-geos/avhrr3_metop-a.yaml @@ -0,0 +1,175 @@ +obs space: + name: avhrr3_metop-a + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/avhrr3_metop-a.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).avhrr3_metop-a.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &avhrr3_metop-a_channels 3,4,5 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: avhrr3_metop-a + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/avhrr3_metop-a.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &avhrr3_metop-a_tlapse $(experiment_dir)/{{current_cycle}}/avhrr3_metop-a.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *avhrr3_metop-a_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +# Wavenumber Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: 3 + where: + - variable: + name: solar_zenith_angle@MetaData + maxvalue: 88.9999 + - variable: + name: water_area_fraction@GeoVaLs + minvalue: 1.0e-12 + action: + name: reject +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *avhrr3_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorWavenumIR@ObsFunction + channels: *avhrr3_metop-a_channels + options: + channels: *avhrr3_metop-a_channels +# Topography Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *avhrr3_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *avhrr3_metop-a_channels + options: + channels: *avhrr3_metop-a_channels + sensor: avhrr3_metop-a +# Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *avhrr3_metop-a_channels + minvalue: 0.00001 + maxvalue: 1000.0 + action: + name: reject +# Transmittance Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *avhrr3_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *avhrr3_metop-a_channels + options: + channels: *avhrr3_metop-a_channels +# Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *avhrr3_metop-a_channels + test variables: + - name: CloudDetectMinResidualAVHRR@ObsFunction + channels: *avhrr3_metop-a_channels + options: + channels: *avhrr3_metop-a_channels + use_flag: [ 1, 1, 1 ] + use_flag_clddet: [ 1, 1, 1 ] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *avhrr3_metop-a_channels + test variables: + - name: NearSSTRetCheckIR@ObsFunction + channels: *avhrr3_metop-a_channels + options: + channels: *avhrr3_metop-a_channels + use_flag: [ 1, 1, 1 ] + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# Surface Jacobians Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *avhrr3_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *avhrr3_metop-a_channels + options: + channels: *avhrr3_metop-a_channels + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *avhrr3_metop-a_channels + function absolute threshold: + - name: ObsErrorBoundIR@ObsFunction + channels: *avhrr3_metop-a_channels + options: + channels: *avhrr3_metop-a_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *avhrr3_metop-a_channels + options: + channels: *avhrr3_metop-a_channels + obserr_bound_max: [ 6.0, 6.0, 6.0 ] + action: + name: reject +# Useflag Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *avhrr3_metop-a_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *avhrr3_metop-a_channels + options: + channels: *avhrr3_metop-a_channels + use_flag: [ 1, 1, 1 ] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/avhrr3_n18.yaml b/ewok/jedi-geos/avhrr3_n18.yaml new file mode 100644 index 000000000..05d86b281 --- /dev/null +++ b/ewok/jedi-geos/avhrr3_n18.yaml @@ -0,0 +1,176 @@ +obs space: + name: avhrr3_n18 + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/avhrr3_n18.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).avhrr3_n18.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &avhrr3_n18_channels 3,4,5 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: avhrr3_n18 + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/avhrr3_n18.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &avhrr3_n18_tlapse $(experiment_dir)/{{current_cycle}}/avhrr3_n18.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *avhrr3_n18_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +# Wavenumber Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: 3 + where: + - variable: + name: solar_zenith_angle@MetaData + maxvalue: 88.9999 + - variable: + name: water_area_fraction@GeoVaLs + minvalue: 1.0e-12 + action: + name: reject +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *avhrr3_n18_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorWavenumIR@ObsFunction + channels: *avhrr3_n18_channels + options: + channels: *avhrr3_n18_channels +# Topography Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *avhrr3_n18_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *avhrr3_n18_channels + options: + channels: *avhrr3_n18_channels + sensor: avhrr3_n18 +# Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *avhrr3_n18_channels + minvalue: 0.00001 + maxvalue: 1000.0 + action: + name: reject +# Transmittance Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *avhrr3_n18_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *avhrr3_n18_channels + options: + channels: *avhrr3_n18_channels +# Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *avhrr3_n18_channels + test variables: + - name: CloudDetectMinResidualAVHRR@ObsFunction + channels: *avhrr3_n18_channels + options: + channels: *avhrr3_n18_channels + use_flag: [ 1, 1, 1 ] + use_flag_clddet: [ 1, 1, 1 ] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *avhrr3_n18_channels + test variables: + - name: NearSSTRetCheckIR@ObsFunction + channels: *avhrr3_n18_channels + options: + channels: *avhrr3_n18_channels + use_flag: [ 1, 1, 1 ] + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# Surface Jacobians Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *avhrr3_n18_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *avhrr3_n18_channels + options: + channels: *avhrr3_n18_channels + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *avhrr3_n18_channels + function absolute threshold: + - name: ObsErrorBoundIR@ObsFunction + channels: *avhrr3_n18_channels + options: + channels: *avhrr3_n18_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *avhrr3_n18_channels + options: + channels: *avhrr3_n18_channels + obserr_bound_max: [ 6.0, 6.0, 6.0 ] + action: + name: reject +# Useflag Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *avhrr3_n18_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *avhrr3_n18_channels + options: + channels: *avhrr3_n18_channels + use_flag: [ 1, 1, 1 ] + minvalue: 1.0e-12 + action: + name: reject + diff --git a/ewok/jedi-geos/cris-fsr_n20.yaml b/ewok/jedi-geos/cris-fsr_n20.yaml new file mode 100644 index 000000000..0110522d8 --- /dev/null +++ b/ewok/jedi-geos/cris-fsr_n20.yaml @@ -0,0 +1,436 @@ +obs space: + name: cris-fsr_n20 + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/cris-fsr_n20.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).cris-fsr_n20.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &cris-fsr_n20_channels 19, 24, 26, 27, 28, 31, 32, 33, 37, 39, 42, 44, 47, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, + 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, + 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, + 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, + 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 208, + 211, 216, 224, 234, 236, 238, 239, 242, 246, 248, 255, 264, 266, 268, + 275, 279, 283, 285, 291, 295, 301, 305, 311, 332, 342, 389, 400, 402, + 404, 406, 410, 427, 439, 440, 441, 445, 449, 455, 458, 461, 464, 467, + 470, 473, 475, 482, 486, 487, 490, 493, 496, 499, 501, 503, 505, 511, + 513, 514, 518, 519, 520, 522, 529, 534, 563, 568, 575, 592, 594, 596, + 598, 600, 602, 604, 611, 614, 616, 618, 620, 622, 626, 631, 638, 646, + 648, 652, 659, 673, 675, 678, 684, 688, 694, 700, 707, 710, 713, 714, + 718, 720, 722, 725, 728, 735, 742, 748, 753, 762, 780, 784, 798, 849, + 860, 862, 866, 874, 882, 890, 898, 906, 907, 908, 914, 937, 972, 973, + 978, 980, 981, 988, 995, 998, 1000, 1003, 1008, 1009, 1010, 1014, 1017, + 1018, 1020, 1022, 1024, 1026, 1029, 1030, 1032, 1034, 1037, 1038, 1041, + 1042, 1044, 1046, 1049, 1050, 1053, 1054, 1058, 1060, 1062, 1064, 1066, + 1069, 1076, 1077, 1080, 1086, 1091, 1095, 1101, 1109, 1112, 1121, 1128, + 1133, 1163, 1172, 1187, 1189, 1205, 1211, 1219, 1231, 1245, 1271, 1289, + 1300, 1313, 1316, 1325, 1329, 1346, 1347, 1473, 1474, 1491, 1499, 1553, + 1570, 1596, 1602, 1619, 1624, 1635, 1939, 1940, 1941, 1942, 1943, 1944, + 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952, 1953, 1954, 1955, 1956, + 1957, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, + 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, + 1981, 1982, 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, 2153, + 2158, 2161, 2168, 2171, 2175, 2182 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: cris-fsr_n20 + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/cris-fsr_n20.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &cris-fsr_n20_tlapse $(experiment_dir)/{{current_cycle}}/cris-fsr_n20.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *cris-fsr_n20_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +# Wavenumber Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, + 1982, 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, + 2153, 2158, 2161, 2168, 2171, 2175, 2182 + where: + - variable: + name: solar_zenith_angle@MetaData + maxvalue: 88.9999 + - variable: + name: water_area_fraction@GeoVaLs + minvalue: 1.0e-12 + action: + name: reject +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *cris-fsr_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorWavenumIR@ObsFunction + channels: *cris-fsr_n20_channels + options: + channels: *cris-fsr_n20_channels +# Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *cris-fsr_n20_channels + minvalue: 50.00001 + maxvalue: 449.99999 + action: + name: reject +# Topography Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *cris-fsr_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *cris-fsr_n20_channels + options: + channels: *cris-fsr_n20_channels + sensor: cris-fsr_n20 +# Transmittance Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *cris-fsr_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *cris-fsr_n20_channels + options: + channels: *cris-fsr_n20_channels +# Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *cris-fsr_n20_channels + test variables: + - name: CloudDetectMinResidualIR@ObsFunction + channels: *cris-fsr_n20_channels + options: + channels: *cris-fsr_n20_channels + use_flag: [ -1, 1, 1, -1, 1, -1, 1, -1, 1, 1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, + 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1] + use_flag_clddet: [ -1, 1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, 1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# Surface Temperature Jacobian Check over Land +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *cris-fsr_n20_channels + where: + - variable: + name: water_area_fraction@GeoVaLs + maxvalue: 0.99 + test variables: + - name: brightness_temperature_jacobian_surface_temperature@ObsDiag + channels: *cris-fsr_n20_channels + maxvalue: 0.2 +# NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *cris-fsr_n20_channels + test variables: + - name: NearSSTRetCheckIR@ObsFunction + channels: *cris-fsr_n20_channels + options: + channels: *cris-fsr_n20_channels + use_flag: [ -1, 1, 1, -1, 1, -1, 1, -1, 1, 1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, + 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1 ] + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# Surface Jacobians Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *cris-fsr_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *cris-fsr_n20_channels + options: + channels: *cris-fsr_n20_channels + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *cris-fsr_n20_channels + function absolute threshold: + - name: ObsErrorBoundIR@ObsFunction + channels: *cris-fsr_n20_channels + options: + channels: *cris-fsr_n20_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *cris-fsr_n20_channels + options: + channels: *cris-fsr_n20_channels + obserr_bound_max: [ 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0 ] + action: + name: reject +# Useflag Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *cris-fsr_n20_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *cris-fsr_n20_channels + options: + channels: *cris-fsr_n20_channels + use_flag: [ -1, 1, 1, -1, 1, -1, 1, -1, 1, 1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, + 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1 ] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/cris-fsr_npp.yaml b/ewok/jedi-geos/cris-fsr_npp.yaml new file mode 100644 index 000000000..c9a2311b4 --- /dev/null +++ b/ewok/jedi-geos/cris-fsr_npp.yaml @@ -0,0 +1,436 @@ +obs space: + name: cris-fsr_npp + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/cris-fsr_npp.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).cris-fsr_npp.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &cris-fsr_npp_channels 19, 24, 26, 27, 28, 31, 32, 33, 37, 39, 42, 44, 47, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, + 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, + 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, + 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, + 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 208, + 211, 216, 224, 234, 236, 238, 239, 242, 246, 248, 255, 264, 266, 268, + 275, 279, 283, 285, 291, 295, 301, 305, 311, 332, 342, 389, 400, 402, + 404, 406, 410, 427, 439, 440, 441, 445, 449, 455, 458, 461, 464, 467, + 470, 473, 475, 482, 486, 487, 490, 493, 496, 499, 501, 503, 505, 511, + 513, 514, 518, 519, 520, 522, 529, 534, 563, 568, 575, 592, 594, 596, + 598, 600, 602, 604, 611, 614, 616, 618, 620, 622, 626, 631, 638, 646, + 648, 652, 659, 673, 675, 678, 684, 688, 694, 700, 707, 710, 713, 714, + 718, 720, 722, 725, 728, 735, 742, 748, 753, 762, 780, 784, 798, 849, + 860, 862, 866, 874, 882, 890, 898, 906, 907, 908, 914, 937, 972, 973, + 978, 980, 981, 988, 995, 998, 1000, 1003, 1008, 1009, 1010, 1014, 1017, + 1018, 1020, 1022, 1024, 1026, 1029, 1030, 1032, 1034, 1037, 1038, 1041, + 1042, 1044, 1046, 1049, 1050, 1053, 1054, 1058, 1060, 1062, 1064, 1066, + 1069, 1076, 1077, 1080, 1086, 1091, 1095, 1101, 1109, 1112, 1121, 1128, + 1133, 1163, 1172, 1187, 1189, 1205, 1211, 1219, 1231, 1245, 1271, 1289, + 1300, 1313, 1316, 1325, 1329, 1346, 1347, 1473, 1474, 1491, 1499, 1553, + 1570, 1596, 1602, 1619, 1624, 1635, 1939, 1940, 1941, 1942, 1943, 1944, + 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952, 1953, 1954, 1955, 1956, + 1957, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, + 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, + 1981, 1982, 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, 2153, + 2158, 2161, 2168, 2171, 2175, 2182 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: cris-fsr_npp + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/cris-fsr_npp.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &cris-fsr_npp_tlapse $(experiment_dir)/{{current_cycle}}/cris-fsr_npp.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *cris-fsr_npp_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +# Wavenumber Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, + 1982, 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, + 2153, 2158, 2161, 2168, 2171, 2175, 2182 + where: + - variable: + name: solar_zenith_angle@MetaData + maxvalue: 88.9999 + - variable: + name: water_area_fraction@GeoVaLs + minvalue: 1.0e-12 + action: + name: reject +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *cris-fsr_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorWavenumIR@ObsFunction + channels: *cris-fsr_npp_channels + options: + channels: *cris-fsr_npp_channels +# Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *cris-fsr_npp_channels + minvalue: 50.00001 + maxvalue: 449.99999 + action: + name: reject +# Topography Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *cris-fsr_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *cris-fsr_npp_channels + options: + channels: *cris-fsr_npp_channels + sensor: cris-fsr_npp +# Transmittance Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *cris-fsr_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *cris-fsr_npp_channels + options: + channels: *cris-fsr_npp_channels +# Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *cris-fsr_npp_channels + test variables: + - name: CloudDetectMinResidualIR@ObsFunction + channels: *cris-fsr_npp_channels + options: + channels: *cris-fsr_npp_channels + use_flag: [ -1, 1, 1, -1, 1, -1, 1, -1, 1, 1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, + 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1] + use_flag_clddet: [ -1, 1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, 1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# Surface Temperature Jacobian Check over Land +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *cris-fsr_npp_channels + where: + - variable: + name: water_area_fraction@GeoVaLs + maxvalue: 0.99 + test variables: + - name: brightness_temperature_jacobian_surface_temperature@ObsDiag + channels: *cris-fsr_npp_channels + maxvalue: 0.2 +# NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *cris-fsr_npp_channels + test variables: + - name: NearSSTRetCheckIR@ObsFunction + channels: *cris-fsr_npp_channels + options: + channels: *cris-fsr_npp_channels + use_flag: [ -1, 1, 1, -1, 1, -1, 1, -1, 1, 1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, + 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1 ] + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# Surface Jacobians Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *cris-fsr_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *cris-fsr_npp_channels + options: + channels: *cris-fsr_npp_channels + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *cris-fsr_npp_channels + function absolute threshold: + - name: ObsErrorBoundIR@ObsFunction + channels: *cris-fsr_npp_channels + options: + channels: *cris-fsr_npp_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *cris-fsr_npp_channels + options: + channels: *cris-fsr_npp_channels + obserr_bound_max: [ 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0 ] + action: + name: reject +# Useflag Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *cris-fsr_npp_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *cris-fsr_npp_channels + options: + channels: *cris-fsr_npp_channels + use_flag: [ -1, 1, 1, -1, 1, -1, 1, -1, 1, 1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, + 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1 ] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/gnssrobndnbam.yaml b/ewok/jedi-geos/gnssrobndnbam.yaml new file mode 100644 index 000000000..914b46922 --- /dev/null +++ b/ewok/jedi-geos/gnssrobndnbam.yaml @@ -0,0 +1,45 @@ +obs space: + name: gnssrobndnbam + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/gnssrobndnbam.{{window_begin}}.nc4 + obsgrouping: + group variables: [ 'record_number' ] + sort variable: 'impact_height' + sort order: 'ascending' + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).gnssrobndnbam.{{window_begin}}.nc4 + simulated variables: [bending_angle] +obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM +obs filters: +- filter: Domain Check + filter variables: + - name: [bending_angle] + where: + - variable: + name: impact_height@MetaData + minvalue: 0 + maxvalue: 50000 +- filter: Bounds Check + filter variables: + - name: [bending_angle] + where: + - variable: + name: occulting_sat_id@MetaData + is_in: 3-5 + test variables: + - name: impact_height@MetaData + minvalue: 8000 +- filter: ROobserror + filter variables: + - name: bending_angle + errmodel: NBAM +- filter: Background Check RONBAM + filter variables: + - name: [bending_angle] + threshold: 3 diff --git a/ewok/jedi-geos/iasi_metop-a.yaml b/ewok/jedi-geos/iasi_metop-a.yaml new file mode 100644 index 000000000..5fd9dc91b --- /dev/null +++ b/ewok/jedi-geos/iasi_metop-a.yaml @@ -0,0 +1,533 @@ +obs space: + name: iasi_metop-a + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/iasi_metop-a.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).iasi_metop-a.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &iasi_metop-a_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-a + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/iasi_metop-a.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-a_tlapse $(experiment_dir)/{{current_cycle}}/iasi_metop-a.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *iasi_metop-a_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +# Wavenumber Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, + 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, + 7436, 7444, 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, + 7865, 7885, 7888, 7912, 7950, 7972, 7980, 7995, 8007, 8015, + 8055, 8078 + where: + - variable: + name: solar_zenith_angle@MetaData + maxvalue: 88.9999 + - variable: + name: water_area_fraction@GeoVaLs + minvalue: 1.0e-12 + action: + name: reject +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorWavenumIR@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels +# Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *iasi_metop-a_channels + minvalue: 50.00001 + maxvalue: 449.99999 + action: + name: reject +# Topography Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + sensor: iasi_metop-a +# Transmittance Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels +# Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *iasi_metop-a_channels + test variables: + - name: CloudDetectMinResidualIR@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + use_flag_clddet: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *iasi_metop-a_channels + test variables: + - name: NearSSTRetCheckIR@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# Surface Jacobians Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *iasi_metop-a_channels + function absolute threshold: + - name: ObsErrorBoundIR@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_bound_max: [ 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 4.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 4.0, 4.0, + 3.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 3.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.5, 2.0, 2.5, 2.5, 3.0, 2.5, + 2.5, 2.5, 2.5, 3.5, 2.5, 2.5, 3.0, 3.5, 3.0, 4.0, + 4.0, 4.0, 4.0, 4.0, 4.0, 4.5, 4.5, 4.5, 4.5, 4.5, + 4.0, 4.5, 4.0, 4.0, 4.5, 2.5, 3.0, 2.5, 3.0, 2.5, + 3.0, 2.0, 2.5, 2.5, 3.0, 3.0, 2.5, 3.0, 3.0, 3.0, + 2.5, 2.5, 4.0, 4.5, 4.5, 5.0, 4.0, 4.0, 5.0, 5.0, + 5.0, 5.0, 5.5, 5.5, 4.0, 5.0, 4.0, 4.5, 5.5, 5.5, + 6.0, 4.5, 4.5, 4.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.5, 4.5, 6.0, + 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, + 6.0, 6.0, 4.0, 6.0, 6.0, 6.0, 6.0, 4.5, 6.0, 6.0, + 4.5, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, 6.0, 6.0, + 5.0, 6.0, 6.0, 5.0, 6.0, 5.0, 6.0, 6.0, 6.0, 5.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0] + action: + name: reject +# Useflag Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *iasi_metop-a_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/iasi_metop-b.yaml b/ewok/jedi-geos/iasi_metop-b.yaml new file mode 100644 index 000000000..a96bb47ed --- /dev/null +++ b/ewok/jedi-geos/iasi_metop-b.yaml @@ -0,0 +1,533 @@ +obs space: + name: iasi_metop-b + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/iasi_metop-b.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).iasi_metop-b.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &iasi_metop-b_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-b + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/iasi_metop-b.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-b_tlapse $(experiment_dir)/{{current_cycle}}/iasi_metop-b.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *iasi_metop-b_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +# Wavenumber Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, + 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, + 7436, 7444, 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, + 7865, 7885, 7888, 7912, 7950, 7972, 7980, 7995, 8007, 8015, + 8055, 8078 + where: + - variable: + name: solar_zenith_angle@MetaData + maxvalue: 88.9999 + - variable: + name: water_area_fraction@GeoVaLs + minvalue: 1.0e-12 + action: + name: reject +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorWavenumIR@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels +# Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *iasi_metop-b_channels + minvalue: 50.00001 + maxvalue: 449.99999 + action: + name: reject +# Topography Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + sensor: iasi_metop-b +# Transmittance Top Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels +# Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *iasi_metop-b_channels + test variables: + - name: CloudDetectMinResidualIR@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + use_flag_clddet: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *iasi_metop-b_channels + test variables: + - name: NearSSTRetCheckIR@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# Surface Jacobians Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *iasi_metop-b_channels + function absolute threshold: + - name: ObsErrorBoundIR@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + obserr_bound_latitude: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + obserr_bound_max: [ 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 4.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 4.0, 4.0, + 3.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 3.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.5, 2.0, 2.5, 2.5, 3.0, 2.5, + 2.5, 2.5, 2.5, 3.5, 2.5, 2.5, 3.0, 3.5, 3.0, 4.0, + 4.0, 4.0, 4.0, 4.0, 4.0, 4.5, 4.5, 4.5, 4.5, 4.5, + 4.0, 4.5, 4.0, 4.0, 4.5, 2.5, 3.0, 2.5, 3.0, 2.5, + 3.0, 2.0, 2.5, 2.5, 3.0, 3.0, 2.5, 3.0, 3.0, 3.0, + 2.5, 2.5, 4.0, 4.5, 4.5, 5.0, 4.0, 4.0, 5.0, 5.0, + 5.0, 5.0, 5.5, 5.5, 4.0, 5.0, 4.0, 4.5, 5.5, 5.5, + 6.0, 4.5, 4.5, 4.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.5, 4.5, 6.0, + 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, + 6.0, 6.0, 4.0, 6.0, 6.0, 6.0, 6.0, 4.5, 6.0, 6.0, + 4.5, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, 6.0, 6.0, + 5.0, 6.0, 6.0, 5.0, 6.0, 5.0, 6.0, 6.0, 6.0, 5.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0] + action: + name: reject +# Useflag Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *iasi_metop-b_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/mhs_metop-b.yaml b/ewok/jedi-geos/mhs_metop-b.yaml new file mode 100644 index 000000000..f0015c5de --- /dev/null +++ b/ewok/jedi-geos/mhs_metop-b.yaml @@ -0,0 +1,33 @@ +obs space: + name: mhs_metop-b + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/mhs_metop-b.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).mhs_metop-b.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: 1-5 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: mhs_metop-b + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/mhs_metop-b.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &mhs_metop-b_tlapse $(experiment_dir)/{{current_cycle}}/mhs_metop-b.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *mhs_metop-b_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle diff --git a/ewok/jedi-geos/mhs_metop-c.yaml b/ewok/jedi-geos/mhs_metop-c.yaml new file mode 100644 index 000000000..1fcfc7027 --- /dev/null +++ b/ewok/jedi-geos/mhs_metop-c.yaml @@ -0,0 +1,33 @@ +obs space: + name: mhs_metop-c + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/mhs_metop-c.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).mhs_metop-c.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: 1-5 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: mhs_metop-c + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/mhs_metop-c.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &mhs_metop-c_tlapse $(experiment_dir)/{{current_cycle}}/mhs_metop-c.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *mhs_metop-c_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle diff --git a/ewok/jedi-geos/mhs_n19.yaml b/ewok/jedi-geos/mhs_n19.yaml new file mode 100644 index 000000000..9493bc966 --- /dev/null +++ b/ewok/jedi-geos/mhs_n19.yaml @@ -0,0 +1,33 @@ +obs space: + name: mhs_n19 + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/mhs_n19.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).mhs_n19.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: 1-5 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: mhs_n19 + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/mhs_n19.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &mhs_n19_tlapse $(experiment_dir)/{{current_cycle}}/mhs_n19.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *mhs_n19_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle diff --git a/ewok/jedi-geos/omi_aura.yaml b/ewok/jedi-geos/omi_aura.yaml new file mode 100644 index 000000000..8824711ca --- /dev/null +++ b/ewok/jedi-geos/omi_aura.yaml @@ -0,0 +1,12 @@ +obs space: + name: omi_aura + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/omi_aura.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).omi_aura.{{window_begin}}.nc4 + simulated variables: [integrated_layer_ozone_in_air] +obs operator: + name: AtmVertInterpLay + geovals: [mole_fraction_of_ozone_in_air] + coefficients: [0.007886131] # convert from ppmv to DU + nlevels: [1] diff --git a/ewok/jedi-geos/rass_tv.yaml b/ewok/jedi-geos/rass_tv.yaml new file mode 100644 index 000000000..d316542fb --- /dev/null +++ b/ewok/jedi-geos/rass_tv.yaml @@ -0,0 +1,9 @@ +obs operator: + name: VertInterp +obs space: + name: rass_tv + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/rass_tv.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).rass_tv.{{window_begin}}.nc4 + simulated variables: [virtual_temperature] diff --git a/ewok/jedi-geos/satwind.yaml b/ewok/jedi-geos/satwind.yaml new file mode 100644 index 000000000..d5969395c --- /dev/null +++ b/ewok/jedi-geos/satwind.yaml @@ -0,0 +1,315 @@ +obs operator: + name: VertInterp +obs space: + name: satwind + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/satwind.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).satwind.{{window_begin}}.nc4 + simulated variables: [eastward_wind, northward_wind] +obs filters: +# +# Reject all obs with PreQC mark already set above 3 +- filter: PreQC + maxvalue: 3 + action: + name: reject +# +# Assign the initial observation error, based on height/pressure +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: air_pressure@MetaData + xvals: [100000, 95000, 80000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000] #Pressure (Pa) + errors: [1.4, 1.5, 1.6, 1.8, 1.9, 2.0, 2.1, 2.3, 2.6, 2.8, 3.0, 3.2, 2.7, 2.4, 2.1] +# +# Observation Range Sanity Check: either wind component or velocity exceeds 135 m/s +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + minvalue: -135 + maxvalue: 135 + action: + name: reject +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: Velocity@ObsFunction + maxvalue: 135.0 + action: + name: reject +# +# All satellite platforms, reject when pressure greater than 950 mb. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: air_pressure@MetaData + maxvalue: 95000 + action: + name: reject +# +# Difference check surface_pressure and air_pressure@ObsValue, if less than 100 hPa, reject. +# Starting with 730029 values, 338418 missing (half?), 50883 rejected by difference check, leaving 340728 +- filter: Difference Check + filter variables: + - name: eastward_wind + - name: northward_wind + reference: surface_pressure@GeoVaLs + value: air_pressure@MetaData + maxvalue: -10000 +# +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +- filter: Difference Check + filter variables: + - name: eastward_wind + - name: northward_wind + reference: TropopauseEstimate@ObsFunction + value: air_pressure@MetaData + minvalue: -5000 # 50 hPa above tropopause level, negative p-diff + action: + name: reject +# +# GOES WV (non-cloudy; itype=247) reject when difference of wind direction is more than 50 degrees. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: WindDirAngleDiff@ObsFunction + maxvalue: 50.0 + action: + name: reject +# +# GOES IR (245), EUMET IR (253), JMA IR (252) reject when pressure between 400 and 800 mb. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: air_pressure@MetaData + minvalue: 40000 + maxvalue: 80000 + where: + - variable: + name: eastward_wind@ObsType + is_in: 245, 252, 253 + action: + name: reject +# +# GOES WV (246, 250, 254), reject when pressure greater than 400 mb. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: air_pressure@MetaData + maxvalue: 40000 + where: + - variable: + name: eastward_wind@ObsType + is_in: 246, 250, 254 + action: + name: reject +# +# EUMET (242) and JMA (243) vis, reject when pressure less than 700 mb. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: air_pressure@MetaData + minvalue: 70000 + where: + - variable: + name: eastward_wind@ObsType + is_in: 242, 243 + action: + name: reject +# +# MODIS-Aqua/Terra (257) and (259), reject when pressure less than 250 mb. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: air_pressure@MetaData + minvalue: 25000 + where: + - variable: + name: eastward_wind@ObsType + is_in: 257-259 + action: + name: reject +# +# MODIS-Aqua/Terra (258) and (259), reject when pressure greater than 600 mb. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: air_pressure@MetaData + maxvalue: 60000 + where: + - variable: + name: eastward_wind@ObsType + is_in: 258, 259 + action: + name: reject +# +# AVHRR (244), MODIS (257,258,259), VIIRS (260), GOES (247) use a LNVD check. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: SatWindsLNVDCheck@ObsFunction + maxvalue: 3 + where: + - variable: + name: eastward_wind@ObsType + is_in: 244, 247, 257-260 + action: + name: reject +# +# AVHRR and MODIS (ObsType=244,257,258,259) use a SPDB check. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: SatWindsSPDBCheck@ObsFunction + options: + error_min: 1.4 + error_max: 20.0 + maxvalue: 1.75 + where: + - variable: + name: eastward_wind@ObsType + is_in: 244, 257, 258, 259 + action: + name: reject +# +# GOES (ObsType=245,246,253,254) use a SPDB check only between 300-400 mb. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: SatWindsSPDBCheck@ObsFunction + options: + error_min: 1.4 + error_max: 20.0 + maxvalue: 1.75 + where: + - variable: + name: eastward_wind@ObsType + is_in: 244, 257, 258, 259 + - variable: + name: air_pressure@MetaData + minvalue: 30000 + maxvalue: 40000 + action: + name: reject +# +- filter: Background Check + filter variables: + - name: eastward_wind + - name: northward_wind + absolute threshold: 7.5 + action: + name: inflate error + inflation factor: 3.0 + defer to post: true +# +# If the total inflation factor is too big, reject. +- filter: Bounds Check + filter variables: + - name: eastward_wind + action: + name: reject + maxvalue: 2.5 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: eastward_wind@ObsErrorData # After inflation step + denominator: + name: eastward_wind@ObsError + where: + - variable: + name: eastward_wind@ObsType + is_in: 240, 241, 242, 244, 247, 248, 249, 250, 252, 255-260 + defer to post: true +# +- filter: Bounds Check + filter variables: + - name: northward_wind + action: + name: reject + maxvalue: 2.5 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: northward_wind@ObsErrorData # After inflation step + denominator: + name: northward_wind@ObsError + where: + - variable: + name: northward_wind@ObsType + is_in: 240, 241, 242, 244, 247, 248, 249, 250, 252, 255-260 + defer to post: true +# +# Some satellite platforms have a lower threshold of inflation factor of 1.5 +- filter: Bounds Check + filter variables: + - name: eastward_wind + action: + name: reject + maxvalue: 1.5 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: eastward_wind@ObsErrorData # After inflation step + denominator: + name: eastward_wind@ObsError + where: + - variable: + name: eastward_wind@ObsType + is_in: 243, 245, 246, 251, 253, 254 + defer to post: true +# +- filter: Bounds Check + filter variables: + - name: northward_wind + action: + name: reject + maxvalue: 1.5 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: northward_wind@ObsErrorData # After inflation step + denominator: + name: northward_wind@ObsError + where: + - variable: + name: eastward_wind@ObsType + is_in: 243, 245, 246, 251, 253, 254 + defer to post: true + diff --git a/ewok/jedi-geos/scatwind.yaml b/ewok/jedi-geos/scatwind.yaml new file mode 100644 index 000000000..7df4b345a --- /dev/null +++ b/ewok/jedi-geos/scatwind.yaml @@ -0,0 +1,113 @@ +obs operator: + name: GSISfcModel +obs space: + name: scatwind + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/scatwind.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).scatwind.{{window_begin}}.nc4 + simulated variables: [eastward_wind, northward_wind] +obs filters: +# Reject all obs with PreQC mark already set above 3 +- filter: PreQC + maxvalue: 3 + action: + name: reject +# +# WindSat (289), ASCAT (290), or OSCAT (291) either wind component or velocity greater than 20 m/s, then reject. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + minvalue: -20 + maxvalue: 20 +# +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: Velocity@ObsFunction + maxvalue: 20.0 + where: + - variable: + name: eastward_wind@ObsType + is_in: 289, 290, 291 + action: + name: reject +# +# Similar to satellite winds AMV, reject when obs wind direction is more than 50 degrees different than model. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: WindDirAngleDiff@ObsFunction + maxvalue: 50.0 + action: + name: reject +# +# ASCAT (290), RAPIDSCAT (296) use a LNVD check. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: SatWindsLNVDCheck@ObsFunction + maxvalue: 3 + where: + - variable: + name: eastward_wind@ObsType + is_in: 290, 296 + action: + name: reject +# +# Assign the initial observation error (constant value, 3.5 m/s right now). +- filter: BlackList + filter variables: + - name: eastward_wind + - name: northward_wind + action: + name: assign error + error parameter: 3.5 +# +# For OSCAT, ASCAT, RAPIDSCAT, or WindSat, reject when either wind component differs by at least 6 m/s from model. +- filter: Background Check + filter variables: + - name: eastward_wind + - name: northward_wind + absolute threshold: 6.0 + where: + - variable: + name: eastward_wind@ObsType + is_in: 289, 290, 291, 296 + action: + name: reject +# +- filter: Bounds Check + filter variables: + - name: eastward_wind + action: + name: reject + maxvalue: 5.0 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: eastward_wind@ObsErrorData # After inflation step + denominator: + name: eastward_wind@ObsError +# +- filter: Bounds Check + filter variables: + - name: northward_wind + action: + name: reject + maxvalue: 5.0 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: northward_wind@ObsErrorData # After inflation step + denominator: + name: northward_wind@ObsError diff --git a/ewok/jedi-geos/seviri_m11.yaml b/ewok/jedi-geos/seviri_m11.yaml new file mode 100644 index 000000000..321a221f7 --- /dev/null +++ b/ewok/jedi-geos/seviri_m11.yaml @@ -0,0 +1,138 @@ +obs space: + name: seviri_m11 + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/seviri_m11.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).seviri_m11.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &seviri_m11_channels 4-11 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: seviri_m11 + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/seviri_m11.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &seviri_m11_tlapse $(experiment_dir)/{{current_cycle}}/seviri_m11.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *seviri_m11_tlapse + - name: emissivity + - name: scan_angle + var_name: scan_position + order: 4 + - name: scan_angle + var_name: scan_position + order: 3 + - name: scan_angle + var_name: scan_position + order: 2 + - name: scan_angle + var_name: scan_position +obs filters: +# Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *seviri_m11_channels + minvalue: 50.00001 + maxvalue: 449.99999 + action: + name: reject +# Surface Check:use chn 2 and 3 over both sea and land while other IR chns only over sea +# ch2 and ch3 in GSI should be the original seviri ch5 and ch6 +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: 4,7-11 + where: + - variable: + name: water_area_fraction@GeoVaLs + maxvalue: 0.99 +# Do not use ch5,6 over snow +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *seviri_m11_channels + where: + - variable: + name: surface_snow_area_fraction@GeoVaLs + minvalue: 0.01 +# Do not use ch5,6 over ice +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *seviri_m11_channels + where: + - variable: + name: ice_area_fraction@GeoVaLs + minvalue: 0.01 +# Do not use over mixed surface +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *seviri_m11_channels + where: + - variable: + name: land_area_fraction@GeoVaLs + maxvalue: 0.99 + - variable: + name: water_area_fraction@GeoVaLs + maxvalue: 0.99 + - variable: + name: ice_area_fraction@GeoVaLs + maxvalue: 0.99 + - variable: + name: surface_snow_area_fraction@GeoVaLs + maxvalue: 0.99 +# QC_terrain: If seviri and terrain height > 1km. do not use +- filter: Domain Check + filter variables: + - name: brightness_temperature + channels: *seviri_m11_channels + where: + - variable: + name: surface_geopotential_height@GeoVaLs + maxvalue: 1000.0 +# Gross check +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *seviri_m11_channels + absolute threshold: 2.0 + action: + name: reject +# Surface Jacobians Check +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *seviri_m11_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *seviri_m11_channels + options: + channels: *seviri_m11_channels + obserr_demisf: [0.01, 0.02, 0.02, 0.02, 0.02] + obserr_dtempf: [0.50, 2.00, 3.00, 3.00, 5.00] +# Useflag Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *seviri_m11_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *seviri_m11_channels + options: + channels: *seviri_m11_channels + use_flag: [ -1, 1, 1, -1, -1, -1, -1, -1 ] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/sfc.yaml b/ewok/jedi-geos/sfc.yaml new file mode 100644 index 000000000..25eafa60f --- /dev/null +++ b/ewok/jedi-geos/sfc.yaml @@ -0,0 +1,120 @@ +obs operator: + name: SfcPCorrected + da_psfc_scheme: UKMO +# geovar_geomz: geopotential_height +# geovar_sfc_geomz: surface_geopotential_height +obs space: + name: sfc + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/sfc.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).sfc.{{window_begin}}.nc4 + simulated variables: [surface_pressure] #, air_temperature] +obs filters: +# Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: surface_pressure + minvalue: 37499 + maxvalue: 106999 + action: + name: reject +# Reject all obs with PreQC mark already set above 3 +- filter: PreQC + maxvalue: 3 + action: + name: reject +# Assign obsError. +- filter: BlackList + filter variables: + - name: surface_pressure + action: + name: assign error + error parameter: 120 # 120 Pa (1.2mb) +# +- filter: BlackList + filter variables: + - name: surface_pressure + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: surface_pressure@ObsValue + xvals: [80000, 75000] + errors: [110, 120] + where: + - variable: + name: surface_pressure@ObsType + is_in: 181 # Type is SYNOP +# +- filter: BlackList + filter variables: + - name: surface_pressure + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: surface_pressure@ObsValue + xvals: [85000, 80000] + errors: [120, 140] + where: + - variable: + name: surface_pressure@ObsType + is_in: 187 # Type is METAR +# +# Inflate ObsError as it is done with GSI +- filter: BlackList + filter variables: + - name: surface_pressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSfcPressure@ObsFunction + options: +# original_obserr: ObsError + error_min: 100 # 1 mb + error_max: 300 # 3 mb + geovar_sfc_geomz: surface_geopotential_height +# +# If the ObsError is inflated too much, reject the obs +- filter: Bounds Check + filter variables: + - name: surface_pressure + action: + name: reject + maxvalue: 3.6 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: surface_pressure@ObsErrorData # After inflation step + denominator: + name: surface_pressure@ObsError + defer to post: true + where: + - variable: + name: surface_pressure@ObsType + is_in: 181 # Type is SYNOP +# +- filter: Bounds Check + filter variables: + - name: surface_pressure + action: + name: reject + maxvalue: 4.0 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: surface_pressure@ObsErrorData # After inflation step + denominator: + name: surface_pressure@ObsError + defer to post: true + where: + - variable: + name: surface_pressure@ObsType + is_in: 187 # Type is METAR diff --git a/ewok/jedi-geos/sfcship.yaml b/ewok/jedi-geos/sfcship.yaml new file mode 100644 index 000000000..4a222f204 --- /dev/null +++ b/ewok/jedi-geos/sfcship.yaml @@ -0,0 +1,129 @@ +obs operator: + name: SfcPCorrected + da_psfc_scheme: UKMO +obs space: + name: sfcship + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/sfcship.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).sfcship.{{window_begin}}.nc4 + simulated variables: [surface_pressure] #, air_temperature, specific_humidity] +obs filters: +# Observation Range Sanity Check +#- filter: Bounds Check +# filter variables: +# - name: air_temperature +# minvalue: 195 +# maxvalue: 327 +# action: +# name: reject +# +- filter: Bounds Check + filter variables: + - name: surface_pressure + minvalue: 37499 + maxvalue: 106999 + action: + name: reject +# +#- filter: Bounds Check +# filter variables: +# - name: specific_humidity +# minvalue: 1.0E-7 +# maxvalue: 0.034999999 +# action: +# name: reject +# Reject all obs with PreQC mark already set above 3 +- filter: PreQC + maxvalue: 3 + action: + name: reject +# Assign obsError, first temperature +#- filter: BlackList +# filter variables: +# - name: air_temperature +# action: +# name: assign error +# error parameter: 2.5 +# Assign obsError, next moisture +#- filter: BlackList +# filter variables: +# - name: specific_humidity +# action: +# name: assign error +# error function: +# name: ObsErrorModelStepwiseLinear@ObsFunction +# options: +# xvar: +# name: surface_pressure@ObsValue +# xvals: [110000, 10] +# errors: [.2, .2] +# scale_factor_var: specific_humidity@ObsValue +# Assign obsError, last surface pressure +- filter: BlackList + filter variables: + - name: surface_pressure + action: + name: assign error + error parameter: 130 # 130 Pa (1.3mb) +# +# Inflate ObsError as it is done with GSI +- filter: BlackList + filter variables: + - name: surface_pressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSfcPressure@ObsFunction + options: + error_min: 100 # 1 mb + error_max: 300 # 3 mb + geovar_sfc_geomz: surface_geopotential_height +# +- filter: Background Check + filter variables: + - name: surface_pressure + absolute threshold: 500.0 # 5 mb + action: + name: inflate error + inflation factor: 3.0 + defer to post: true +# +#- filter: Background Check +# filter variables: +# - name: air_temperature +# absolute threshold: 4.0 +# action: +# name: inflate error +# inflation factor: 3.0 +# defer to post: true +# +#- filter: Bounds Check +# filter variables: +# - name: air_temperature +# action: +# name: reject +# maxvalue: 4.5 +# test variables: +# - name: ObsErrorFactorQuotient@ObsFunction +# options: +# numerator: +# name: air_temperature@ObsErrorData # After inflation step +# denominator: +# name: air_temperature@ObsError +# defer to post: true +# +- filter: Bounds Check + filter variables: + - name: surface_pressure + action: + name: reject + maxvalue: 4.0 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: surface_pressure@ObsErrorData # After inflation step + denominator: + name: surface_pressure@ObsError + defer to post: true diff --git a/ewok/jedi-geos/sondes.yaml b/ewok/jedi-geos/sondes.yaml new file mode 100644 index 000000000..175f4a02a --- /dev/null +++ b/ewok/jedi-geos/sondes.yaml @@ -0,0 +1,295 @@ +obs space: + name: sondes + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/sondes.{{window_begin}}.nc4 + obsgrouping: + group variables: ["station_id", "LaunchTime"] + sort variable: "air_pressure" + sort order: "descending" + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).sondes.{{window_begin}}.nc4 + simulated variables: [air_temperature, specific_humidity, eastward_wind, northward_wind, surface_pressure] +obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: air_temperature + - name: specific_humidity + - name: eastward_wind + - name: northward_wind + - name: SfcPCorrected + variables: + - name: surface_pressure + da_psfc_scheme: UKMO +# geovar_geomz: geopotential_height +# geovar_sfc_geomz: surface_geopotential_height +obs filters: +# Reject all obs with PreQC mark already set above 3 +- filter: PreQC + maxvalue: 3 + action: + name: reject +# +# Observation Range Sanity Check: temperature, surface_pressure, moisture, winds +- filter: Bounds Check + filter variables: + - name: air_temperature + minvalue: 185 + maxvalue: 327 + action: + name: reject +# +- filter: Bounds Check + filter variables: + - name: surface_pressure + minvalue: 37499 + maxvalue: 106999 + action: + name: reject +# +- filter: Bounds Check + filter variables: + - name: specific_humidity + minvalue: 1.0E-8 + maxvalue: 0.034999999 + action: + name: reject +# +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + minvalue: -135 + maxvalue: 135 + action: + name: reject +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: Velocity@ObsFunction + maxvalue: 135.0 + action: + name: reject +# +# Reject when difference of wind direction is more than 50 degrees. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: WindDirAngleDiff@ObsFunction + maxvalue: 50.0 + action: + name: reject +# +# Assign the initial observation error, based on height/pressure +- filter: Perform Action + filter variables: + - name: air_temperature + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: air_pressure@MetaData + xvals: [100000, 95000, 90000, 85000, 35000, 30000, 25000, 20000, 15000, 10000, 7500, 5000, 4000, 3000, 2000, 1000] + errors: [1.2, 1.1, 0.9, 0.8, 0.8, 0.9, 1.2, 1.2, 1.0, 0.8, 0.8, 0.9, 0.95, 1.0, 1.25, 1.5] +# +- filter: Perform Action + filter variables: + - name: surface_pressure + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: surface_pressure@ObsValue + xvals: [80000, 75000] + errors: [110, 120] # 1.1 mb below 800 mb and 1.2 mb agove 750 mb +# +- filter: Perform Action + filter variables: + - name: surface_pressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSfcPressure@ObsFunction + options: + error_min: 100 # 1 mb + error_max: 300 # 3 mb + geovar_sfc_geomz: surface_geopotential_height +# +- filter: Perform Action + filter variables: + - name: specific_humidity + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: air_pressure@MetaData + xvals: [25000, 20000, 10] + errors: [0.2, 0.4, 0.8] # 20% RH up to 250 mb, then increased rapidly above + scale_factor_var: specific_humidity@ObsValue +# +- filter: Perform Action + filter variables: + - name: eastward_wind + - name: northward_wind + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: air_pressure@MetaData + xvals: [100000, 95000, 80000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000] #Pressure (Pa) + errors: [1.4, 1.5, 1.6, 1.8, 1.9, 2.0, 2.1, 2.3, 2.6, 2.8, 3.0, 3.2, 2.7, 2.4, 2.1] +# +# Inflate obserror when multiple obs exist inside vertical model layers. +- filter: Perform Action + filter variables: + - name: specific_humidity + action: + name: inflate error + inflation variable: + name: ObsErrorFactorConventional@ObsFunction + options: + test QCflag: PreQC + inflate variables: [specific_humidity] + defer to post: true +- filter: Perform Action + filter variables: + - name: air_temperature + action: + name: inflate error + inflation variable: + name: ObsErrorFactorConventional@ObsFunction + options: + test QCflag: PreQC + inflate variables: [air_temperature] + defer to post: true +- filter: Perform Action + filter variables: + - name: eastward_wind + action: + name: inflate error + inflation variable: + name: ObsErrorFactorConventional@ObsFunction + options: + test QCflag: PreQC + inflate variables: [eastward_wind] + defer to post: true +# +- filter: Perform Action + filter variables: + - name: northward_wind + action: + name: inflate error + inflation variable: + name: ObsErrorFactorConventional@ObsFunction + options: + test QCflag: PreQC + inflate variables: [northward_wind] + defer to post: true +# +- filter: Background Check + filter variables: + - name: air_temperature + absolute threshold: 4.0 + action: + name: inflate error + inflation factor: 3.0 + defer to post: true +# +- filter: Background Check + filter variables: + - name: eastward_wind + - name: northward_wind + absolute threshold: 7.5 + action: + name: inflate error + inflation factor: 3.0 + defer to post: true +# +# If the total inflation factor is too big, reject. +- filter: Bounds Check + filter variables: + - name: air_temperature + action: + name: reject + maxvalue: 8.0 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: air_temperature@ObsErrorData # After inflation step + denominator: + name: air_temperature@ObsError + defer to post: true +# +- filter: Bounds Check + filter variables: + - name: specific_humidity + action: + name: reject + maxvalue: 8.0 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: specific_humidity@ObsErrorData # After inflation step + denominator: + name: specific_humidity@ObsError + defer to post: true +# +- filter: Bounds Check + filter variables: + - name: eastward_wind + action: + name: reject + maxvalue: 8.0 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: eastward_wind@ObsErrorData # After inflation step + denominator: + name: eastward_wind@ObsError + defer to post: true +# +- filter: Bounds Check + filter variables: + - name: northward_wind + action: + name: reject + maxvalue: 8.0 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: northward_wind@ObsErrorData # After inflation step + denominator: + name: northward_wind@ObsError + defer to post: true +# +- filter: Bounds Check + filter variables: + - name: surface_pressure + action: + name: reject + maxvalue: 4.0 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: surface_pressure@ObsErrorData # After inflation step + denominator: + name: surface_pressure@ObsError + defer to post: true diff --git a/ewok/jedi-geos/ssmis_f17.yaml b/ewok/jedi-geos/ssmis_f17.yaml new file mode 100644 index 000000000..6cf87fa31 --- /dev/null +++ b/ewok/jedi-geos/ssmis_f17.yaml @@ -0,0 +1,164 @@ +obs space: + name: ssmis_f17 + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/ssmis_f17.{{window_begin}}.nc4 + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).ssmis_f17.{{window_begin}}.nc4 + simulated variables: [brightness_temperature] + channels: &ssmis_f17_channels 1-24 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: ssmis_f17 + EndianType: little_endian + CoefficientPath: $(jedi_build)/ufo/test/Data/ +obs bias: + input file: $(experiment_dir)/{{current_cycle}}/ssmis_f17.{{background_time}}.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloud_liquid_water + satellite: SSMIS + ch19h: 12 + ch19v: 13 + ch22v: 14 + ch37h: 15 + ch37v: 16 + ch91v: 17 + ch91h: 18 + - name: cosine_of_latitude_times_orbit_node + - name: sine_of_latitude + - name: lapse_rate + order: 2 + tlapse: &ssmis_f17_tlapse $(experiment_dir)/{{current_cycle}}/ssmis_f17.{{background_time}}.tlapse.txt + - name: lapse_rate + tlapse: *ssmis_f17_tlapse + - name: emissivity + - name: scan_angle + var_name: scan_position + order: 4 + - name: scan_angle + var_name: scan_position + order: 3 + - name: scan_angle + var_name: scan_position + order: 2 + - name: scan_angle + var_name: scan_position +obs filters: +#step1: Gross check (setuprad) +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *ssmis_f17_channels + threshold: 1.5 + action: + name: reject +#step1: Gross check(qcmod) +- filter: Background Check + filter variables: + - name: brightness_temperature + channels: *ssmis_f17_channels + absolute threshold: 3.5 + bias correction parameter: 1.0 + action: + name: reject +# #step2: clw check +# Keep the CLW check in yaml for further improvement. +# The test case using 2020110112 global SSMIS data shows that CLW check is not activated in GSI. +#- filter: Bounds Check +# filter variables: +# - name: brightness_temperature +# channels: 1 +# test variables: +# - name: CLWRetMW_SSMIS@ObsFunction +# options: +# satellite: SSMIS +# ch19h: 12 +# ch19v: 13 +# ch22v: 14 +# ch37h: 15 +# ch37v: 16 +# ch91v: 17 +# ch91h: 18 +# varGroup: ObsValue +# minvalue: 0.0 +# maxvalue: 0.1 +# where: +# - variable: +# name: water_area_fraction@GeoVaLs +# minvalue: 0.99 +# action: +# name: reject +#step3: +- filter: Difference Check + filter variables: + - name: brightness_temperature + channels: 1-2,12-16 + reference: brightness_temperature_2@ObsValue + value: brightness_temperature_2@HofX + minvalue: -1.5 + maxvalue: 1.5 + where: + - variable: + name: water_area_fraction@GeoVaLs + maxvalue: 0.99 +#QC_terrain: If ssmis and terrain height > 2km. do not use +- filter: Domain Check + filter variables: + - name: brightness_temperature + channels: *ssmis_f17_channels + where: + - variable: + name: height_above_mean_sea_level@MetaData + maxvalue: 2000.0 +#Do not use over mixed surface +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: 1-3,8-18 + where: + - variable: + name: land_area_fraction@GeoVaLs + maxvalue: 0.99 + - variable: + name: water_area_fraction@GeoVaLs + maxvalue: 0.99 + - variable: + name: ice_area_fraction@GeoVaLs + maxvalue: 0.99 + - variable: + name: surface_snow_area_fraction@GeoVaLs + maxvalue: 0.99 +#step4: Generate q.c. bounds and modified variances +- filter: BlackList + filter variables: + - name: brightness_temperature + channels: *ssmis_f17_channels + action: + name: inflate error + inflation variable: +#Surface Jacobian check + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *ssmis_f17_channels + options: + channels: *ssmis_f17_channels + obserr_demisf: [0.010, 0.010, 0.010, 0.010, 0.010] + obserr_dtempf: [0.500, 0.500, 0.500, 0.500, 0.500] +#Useflag Check +- filter: Bounds Check + filter variables: + - name: brightness_temperature + channels: *ssmis_f17_channels + test variables: + - name: ChannelUseflagCheckRad@ObsFunction + channels: *ssmis_f17_channels + options: + channels: *ssmis_f17_channels + use_flag: [ 1, -1, -1, -1, 1 , 1, 1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1] + minvalue: 1.0e-12 + action: + name: reject diff --git a/ewok/jedi-geos/vadwind.yaml b/ewok/jedi-geos/vadwind.yaml new file mode 100644 index 000000000..f4bf74fa4 --- /dev/null +++ b/ewok/jedi-geos/vadwind.yaml @@ -0,0 +1,154 @@ +obs operator: + name: VertInterp +obs space: + name: vadwind + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/vadwind.{{window_begin}}.nc4 + obsgrouping: + group variables: ["station_id", "datetime"] + sort variable: "air_pressure" + sort order: "descending" + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/$(experiment).vadwind.{{window_begin}}.nc4 + simulated variables: [eastward_wind, northward_wind] +#-------------------------------------------------------------------------------------------------------------------- +obs filters: +# Begin by assigning all ObsError to a constant value. These might get overwritten later. +- filter: BlackList + filter variables: + - name: eastward_wind + - name: northward_wind + action: + name: assign error + error parameter: 2.0 # 2.0 m/s +# +# Assign the initial ObsError, based on height/pressure +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: air_pressure@MetaData + xvals: [100000, 95000, 85000, 80000, 70000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000] + errors: [1.4, 1.5, 1.5, 1.6, 1.6, 1.8, 1.9, 2.0, 2.1, 2.3, 2.6, 2.8, 3.0, 3.2, 2.7, 2.4, 2.1] +# +# Reject all obs with PreQC mark already set above 3 +- filter: PreQC + maxvalue: 3 + action: + name: reject +# +# Reject when pressure is less than 226 mb. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: air_pressure@MetaData + minvalue: 22600 + action: + name: reject +# +# Observation Range Sanity Check: either wind component or velocity exceeds 135 m/s +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + minvalue: -135 + maxvalue: 135 + action: + name: reject +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: Velocity@ObsFunction + maxvalue: 135.0 + action: + name: reject +# +# Reject when difference of wind direction is more than 50 degrees. +- filter: Bounds Check + filter variables: + - name: eastward_wind + - name: northward_wind + test variables: + - name: WindDirAngleDiff@ObsFunction + maxvalue: 50.0 + action: + name: reject + defer to post: true +# +# Inflate obserror when multiple obs exist inside vertical model layers. +- filter: BlackList + filter variables: + - name: eastward_wind + action: + name: inflate error + inflation variable: + name: ObsErrorFactorConventional@ObsFunction + options: + test QCflag: PreQC + inflate variables: [eastward_wind] + defer to post: true +# +- filter: BlackList + filter variables: + - name: northward_wind + action: + name: inflate error + inflation variable: + name: ObsErrorFactorConventional@ObsFunction + options: + test QCflag: PreQC + inflate variables: [northward_wind] + defer to post: true +# +- filter: Background Check + filter variables: + - name: eastward_wind + - name: northward_wind + absolute threshold: 7.5 + action: + name: inflate error + inflation factor: 2.5 + defer to post: true +# +# If the total inflation factor is too big, reject. +- filter: Bounds Check + filter variables: + - name: eastward_wind + action: + name: reject + maxvalue: 6.5 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: eastward_wind@ObsErrorData # After inflation step + denominator: + name: eastward_wind@ObsError + defer to post: true +# +- filter: Bounds Check + filter variables: + - name: northward_wind + action: + name: reject + maxvalue: 6.5 + test variables: + - name: ObsErrorFactorQuotient@ObsFunction + options: + numerator: + name: northward_wind@ObsErrorData # After inflation step + denominator: + name: northward_wind@ObsError + defer to post: true From 1a0ecc21dcc89550c938ffeb58bf167592847ab4 Mon Sep 17 00:00:00 2001 From: Stuart Newman <50145691+smnewman@users.noreply.github.com> Date: Mon, 9 Aug 2021 21:55:16 +0100 Subject: [PATCH 77/98] Thinning of multiple filter variables where QC flags may differ (#1378) * Commit changes for review * Minor update for code comments * More tidying of comments * Changes in response to @wsmigaj comments * Correct doc comments --- src/ufo/filters/GaussianThinningParameters.h | 7 ++ src/ufo/filters/Gaussian_Thinning.cc | 3 +- src/ufo/filters/ObsAccessor.cc | 38 +++++- src/ufo/filters/ObsAccessor.h | 47 ++++++-- src/ufo/filters/PoissonDiskThinning.cc | 5 +- src/ufo/filters/PoissonDiskThinning.h | 1 + src/ufo/filters/TemporalThinning.cc | 6 +- src/ufo/filters/TemporalThinning.h | 1 + src/ufo/filters/TrackCheck.cc | 3 +- src/ufo/filters/TrackCheckShip.cc | 3 +- test/CMakeLists.txt | 8 ++ test/testinput/qc_multichannel_thinning.yaml | 118 +++++++++++++++++++ 12 files changed, 220 insertions(+), 20 deletions(-) create mode 100644 test/testinput/qc_multichannel_thinning.yaml diff --git a/src/ufo/filters/GaussianThinningParameters.h b/src/ufo/filters/GaussianThinningParameters.h index 6b8e2c685..a08f047ef 100644 --- a/src/ufo/filters/GaussianThinningParameters.h +++ b/src/ufo/filters/GaussianThinningParameters.h @@ -169,6 +169,13 @@ class GaussianThinningParameters : public FilterParametersBase { /// enough. oops::Parameter opsCompatibilityMode{"ops_compatibility_mode", false, this}; + /// Option to choose how to treat observations where there are multiple filter variables. If true, + /// treats an observation location as valid if any filter variables have not been rejected. + /// If false, observations are treated as valid only if all filter variables have passed QC. + /// This is an optional parameter, if omitted the default value is true. + oops::Parameter + thinIfAnyFilterVariablesAreValid{"thin_if_any_filter_variables_are_valid", true, this}; + private: static float defaultHorizontalMesh() { return static_cast(2 * M_PI * Constants::mean_earth_rad / 360.0); diff --git a/src/ufo/filters/Gaussian_Thinning.cc b/src/ufo/filters/Gaussian_Thinning.cc index 7073e858f..df5253478 100644 --- a/src/ufo/filters/Gaussian_Thinning.cc +++ b/src/ufo/filters/Gaussian_Thinning.cc @@ -53,7 +53,8 @@ void Gaussian_Thinning::applyFilter(const std::vector & apply, std::vector> & flagged) const { ObsAccessor obsAccessor = createObsAccessor(); - std::vector validObsIds = obsAccessor.getValidObservationIds(apply, *flags_); + std::vector validObsIds = obsAccessor.getValidObservationIds(apply, *flags_, + filtervars, options_.thinIfAnyFilterVariablesAreValid.value()); if (options_.opsCompatibilityMode) { // Sort observations by latitude diff --git a/src/ufo/filters/ObsAccessor.cc b/src/ufo/filters/ObsAccessor.cc index 0577c96e3..8f6988b8e 100644 --- a/src/ufo/filters/ObsAccessor.cc +++ b/src/ufo/filters/ObsAccessor.cc @@ -12,7 +12,6 @@ #include #include "ioda/distribution/InefficientDistribution.h" -#include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" #include "ufo/filters/QCflags.h" #include "ufo/utils/RecursiveSplitter.h" @@ -98,11 +97,21 @@ ObsAccessor ObsAccessor::toObservationsSplitIntoIndependentGroupsByVariable( } std::vector ObsAccessor::getValidObservationIds( - const std::vector &apply, const ioda::ObsDataVector &flags) const { + const std::vector &apply, const ioda::ObsDataVector &flags, + const ufo::Variables &filtervars, bool validIfAnyFilterVariablePassedQC) const { // TODO(wsmigaj): use std::vector to save space std::vector globalApply(apply.size()); + std::vector> filterVariableFlags; + // Select flags for respective filtervars + for (size_t ivar = 0; ivar < filtervars.nvars(); ++ivar) { + std::string filterVariableName = filtervars.variable(ivar).variable(); + auto it = std::find(flags.varnames().variables().begin(), flags.varnames().variables().end(), + filterVariableName); + filterVariableFlags.push_back(flags[*it]); + } for (size_t obsId = 0; obsId < apply.size(); ++obsId) - globalApply[obsId] = apply[obsId] && flags[0][obsId] == QCflags::pass; + globalApply[obsId] = apply[obsId] + && isValid(filterVariableFlags, obsId, validIfAnyFilterVariablePassedQC); obsDistribution_->allGatherv(globalApply); std::vector validObsIds; @@ -162,6 +171,29 @@ size_t ObsAccessor::totalNumObservations() const { return obsdb_->globalNumLocs(); } +bool ObsAccessor::isValid(const std::vector> &flags, size_t obsId, + bool validIfAnyFilterVariablePassedQC) const { + bool obIsNotFlagged; + if (validIfAnyFilterVariablePassedQC) { + obIsNotFlagged = false; + for (size_t irow = 0; irow < flags.size(); ++irow) { + if (flags[irow][obsId] == QCflags::pass) { + obIsNotFlagged = true; + break; + } + } + } else { + obIsNotFlagged = true; + for (size_t irow = 0; irow < flags.size(); ++irow) { + if (flags[irow][obsId] != QCflags::pass) { + obIsNotFlagged = false; + break; + } + } + } + return obIsNotFlagged; +} + RecursiveSplitter ObsAccessor::splitObservationsIntoIndependentGroups( const std::vector &validObsIds, bool opsCompatibilityMode) const { RecursiveSplitter splitter(validObsIds.size(), opsCompatibilityMode); diff --git a/src/ufo/filters/ObsAccessor.h b/src/ufo/filters/ObsAccessor.h index 1c3de6d8a..cbc495458 100644 --- a/src/ufo/filters/ObsAccessor.h +++ b/src/ufo/filters/ObsAccessor.h @@ -14,8 +14,10 @@ #include +#include "ioda/ObsDataVector.h" #include "oops/util/DateTime.h" #include "ufo/filters/Variable.h" +#include "ufo/filters/Variables.h" namespace ioda { class Distribution; @@ -85,9 +87,17 @@ class ObsAccessor { /// An ObsDataVector holding the QC flags (set by any filters run previously) /// of observations held on the current MPI rank. /// + /// \param filtervars + /// List of filter variables. + /// + /// \param validIfAnyFilterVariablePassedQC + /// Boolean switch to treat an observation as valid if any filter variable has not been + /// rejected. By default this is true; if false, the observation is only treated as valid + /// if all filter variables have passed QC. + /// /// An observation location is treated as valid if (a) it has been selected by the \c where - /// clause and (b) its QC flag for the first filtered variable is set to \c pass (see note - /// below). + /// clause and (b) its QC flag(s) for (some/all) filtered variable(s) are set to \c pass + /// (see below). /// /// If each independent group of observations is stored entirely on a single MPI rank, the /// returned vector contains local IDs of valid observation locations held on the current rank @@ -96,15 +106,15 @@ class ObsAccessor { /// nlocs(0) + nlocs(1) - 1 corresponding to locations held on rank 1 and so on, where nlocs(i) /// denotes the number of locations held on ith rank. /// - /// \note It is an open question what should be done if there's more than one filtered variable - /// and their QC flags differ. Should only observation locations where none of the filtered - /// variables have so far been rejected be treated as candidates for retaining? Or all locations - /// where at least one of these variables has not yet been rejected? For the moment we duck the - /// question and check only the QC flags of the first filtered variable, implicitly assuming that - /// if multiple variables are thinned together, they all have the same QC flags. It might be best - /// to make this user-configurable. + /// If there is more than one filtered variable, and their QC flags differ, there is a choice + /// as to whether to treat observation locations as valid (i) where none of the filtered variables + /// have so far been rejected, or (ii) where at least one of these variables has not yet been + /// rejected. The latter choice (ii) is the default, configurable via the switch + /// \c validIfAnyFilterVariablePassedQC. std::vector getValidObservationIds(const std::vector &apply, - const ioda::ObsDataVector &flags) const; + const ioda::ObsDataVector &flags, + const Variables &filtervars, + bool validIfAnyFilterVariablePassedQC = true) const; /// \brief Return the IDs of both flagged and unflagged observation locations selected by the /// where clause. @@ -194,6 +204,23 @@ class ObsAccessor { void groupObservationsByCategoryVariable(const std::vector &validObsIds, RecursiveSplitter &splitter) const; + /// \brief Return true if filtered variable(s) have passed QC, otherwise false. + /// + /// \param flags + /// A vector of type ObsDataRow holding the QC flags for the subset of simulated variables + /// present in the list of filtered variables. + /// + /// \param ObsId + /// Index of observation location. + /// + /// \param validIfAnyFilterVariablePassedQC + /// Boolean variable to decide how to treat observation locations where QC flags of filtered + /// variables differ. + /// If true, consider that observation has passed QC if any filtered variable has passed QC. + /// If false, consider that observation has passed QC only if all filtered variables passed QC. + bool isValid(const std::vector> &flags, size_t ObsId, + bool validIfAnyFilterVariablePassedQC) const; + private: const ioda::ObsSpace *obsdb_; std::shared_ptr obsDistribution_; diff --git a/src/ufo/filters/PoissonDiskThinning.cc b/src/ufo/filters/PoissonDiskThinning.cc index 04783ee1f..0883c3574 100644 --- a/src/ufo/filters/PoissonDiskThinning.cc +++ b/src/ufo/filters/PoissonDiskThinning.cc @@ -161,7 +161,7 @@ void PoissonDiskThinning::applyFilter(const std::vector & apply, std::vector> & flagged) const { ObsAccessor obsAccessor = createObsAccessor(); - const std::vector validObsIds = getValidObservationIds(apply, obsAccessor); + const std::vector validObsIds = getValidObservationIds(apply, filtervars, obsAccessor); int numSpatialDims, numNonspatialDims; ObsData obsData = getObsData(obsAccessor, numSpatialDims, numNonspatialDims); @@ -288,8 +288,9 @@ void PoissonDiskThinning::validateSpacings( std::vector PoissonDiskThinning::getValidObservationIds( const std::vector & apply, + const Variables & filtervars, const ObsAccessor &obsAccessor) const { - std::vector validObsIds = obsAccessor.getValidObservationIds(apply, *flags_); + std::vector validObsIds = obsAccessor.getValidObservationIds(apply, *flags_, filtervars); if (!options_.shuffle) { // The user wants to process observations in fixed (non-random) order. Ensure the filter diff --git a/src/ufo/filters/PoissonDiskThinning.h b/src/ufo/filters/PoissonDiskThinning.h index 140a02a8f..f3d07bb6d 100644 --- a/src/ufo/filters/PoissonDiskThinning.h +++ b/src/ufo/filters/PoissonDiskThinning.h @@ -87,6 +87,7 @@ class PoissonDiskThinning : public FilterBase, const std::string ¶meterName) const; std::vector getValidObservationIds(const std::vector &apply, + const Variables &filtervars, const ObsAccessor &obsAccessor) const; /// Initialise random number generators used for shuffling with the same seed on diff --git a/src/ufo/filters/TemporalThinning.cc b/src/ufo/filters/TemporalThinning.cc index 84a1701ca..d58f529a1 100644 --- a/src/ufo/filters/TemporalThinning.cc +++ b/src/ufo/filters/TemporalThinning.cc @@ -320,7 +320,7 @@ void TemporalThinning::applyFilter(const std::vector & apply, std::vector> & flagged) const { ObsAccessor obsAccessor = createObsAccessor(); - const std::vector isThinned = identifyThinnedObservations(apply, obsAccessor); + const std::vector isThinned = identifyThinnedObservations(apply, filtervars, obsAccessor); obsAccessor.flagRejectedObservations(isThinned, flagged); @@ -344,8 +344,10 @@ ObsAccessor TemporalThinning::createObsAccessor() const { std::vector TemporalThinning::identifyThinnedObservations( const std::vector & apply, + const Variables & filtervars, const ObsAccessor &obsAccessor) const { - const std::vector validObsIds = obsAccessor.getValidObservationIds(apply, *flags_); + const std::vector validObsIds + = obsAccessor.getValidObservationIds(apply, *flags_, filtervars); RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups(validObsIds); diff --git a/src/ufo/filters/TemporalThinning.h b/src/ufo/filters/TemporalThinning.h index 4d8560758..ee1cd0615 100644 --- a/src/ufo/filters/TemporalThinning.h +++ b/src/ufo/filters/TemporalThinning.h @@ -64,6 +64,7 @@ class TemporalThinning : public FilterBase, ObsAccessor createObsAccessor() const; std::vector identifyThinnedObservations(const std::vector &apply, + const Variables &filtervars, const ObsAccessor &obsAccessor) const; boost::optional> getObservationPriorities( diff --git a/src/ufo/filters/TrackCheck.cc b/src/ufo/filters/TrackCheck.cc index 4611d3533..5964219ae 100644 --- a/src/ufo/filters/TrackCheck.cc +++ b/src/ufo/filters/TrackCheck.cc @@ -111,7 +111,8 @@ void TrackCheck::applyFilter(const std::vector & apply, std::vector> & flagged) const { ObsAccessor obsAccessor = TrackCheckUtils::createObsAccessor(options_.stationIdVariable, obsdb_); - const std::vector validObsIds = obsAccessor.getValidObservationIds(apply, *flags_); + const std::vector validObsIds + = obsAccessor.getValidObservationIds(apply, *flags_, filtervars); RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups(validObsIds); TrackCheckUtils::sortTracksChronologically(validObsIds, obsAccessor, splitter); diff --git a/src/ufo/filters/TrackCheckShip.cc b/src/ufo/filters/TrackCheckShip.cc index 7eaf76b23..5a00fbd41 100644 --- a/src/ufo/filters/TrackCheckShip.cc +++ b/src/ufo/filters/TrackCheckShip.cc @@ -157,7 +157,8 @@ void TrackCheckShip::applyFilter(const std::vector & apply, std::vector> & flagged) const { ObsAccessor obsAccessor = TrackCheckUtils::createObsAccessor(options_.stationIdVariable, obsdb_); - const std::vector validObsIds = obsAccessor.getValidObservationIds(apply, *flags_); + const std::vector validObsIds + = obsAccessor.getValidObservationIds(apply, *flags_, filtervars); RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups(validObsIds); TrackCheckUtils::sortTracksChronologically(validObsIds, obsAccessor, splitter); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c635e5d72..0de0154c4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -190,6 +190,7 @@ list( APPEND ufo_test_input testinput/qc_met_office_buddy_pair_finder.yaml testinput/qc_modelbestfitpressure.yaml testinput/qc_modelobthreshold.yaml + testinput/qc_multichannel_thinning.yaml testinput/qc_oceancolor_preqc.yaml testinput/qc_performaction.yaml testinput/qc_poisson_disk_thinning.yaml @@ -1828,6 +1829,13 @@ ecbuild_add_test( TARGET test_ufo_qc_met_office_buddy_check DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_qc_multichannel_thinning + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x + ARGS "testinput/qc_multichannel_thinning.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFilters.x + TEST_DEPENDS ufo_get_ufo_test_data ) + ecbuild_add_test( TARGET test_ufo_qc_oceancolor_preqc COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x ARGS "testinput/qc_oceancolor_preqc.yaml" diff --git a/test/testinput/qc_multichannel_thinning.yaml b/test/testinput/qc_multichannel_thinning.yaml new file mode 100644 index 000000000..44df5d0c9 --- /dev/null +++ b/test/testinput/qc_multichannel_thinning.yaml @@ -0,0 +1,118 @@ +window begin: 2019-12-29T21:00:00Z +window end: 2019-12-30T03:00:00Z + +observations: +# +# Example with all 22 channels valid (no QC rejections before thinning). +- obs space: + name: ATMS + obsdatain: + obsfile: Data/ufo/testinput_tier_1/atms_n20_obs_20191230T0000_rttov.nc4 + simulated variables: [brightness_temperature] + channels: 1-22 + obs filters: + - filter: Gaussian Thinning + horizontal_mesh: 1111.949266 #km = 10 deg at equator + passedBenchmark: 88 # 4 ob locations pass QC, 4*22 channels = 88 +# +# Example with 3 channels blacklisted, remaining 19 channels undergo thinning. +- obs space: + name: ATMS + obsdatain: + obsfile: Data/ufo/testinput_tier_1/atms_n20_obs_20191230T0000_rttov.nc4 + simulated variables: [brightness_temperature] + channels: 1-22 + obs filters: + - filter: BlackList + filter variables: + - name: brightness_temperature + channels: 1-3 + - filter: Gaussian Thinning + horizontal_mesh: 1111.949266 #km = 10 deg at equator + passedBenchmark: 76 # 4 ob locations pass QC, 4*19 valid channels = 76 +# +# Option thin_if_any_filter_variables_are_valid is true (default) meaning +# that obs undergo thinning so long as at least one filter variable is valid. +- obs space: + name: ATMS + obsdatain: + obsfile: Data/ufo/testinput_tier_1/atms_n20_obs_20191230T0000_rttov.nc4 + simulated variables: [brightness_temperature] + channels: 1-22 + obs filters: + - filter: BlackList + filter variables: + - name: brightness_temperature + channels: 1-3 + - filter: Gaussian Thinning + horizontal_mesh: 1111.949266 #km = 10 deg at equator + thin_if_any_filter_variables_are_valid: true # default is true + passedBenchmark: 76 # 4 ob locations pass QC, 4*19 valid channels = 76 +# +# Option thin_if_any_filter_variables_are_valid is false meaning +# that obs undergo thinning only if all filter variables have passed QC. +- obs space: + name: ATMS + obsdatain: + obsfile: Data/ufo/testinput_tier_1/atms_n20_obs_20191230T0000_rttov.nc4 + simulated variables: [brightness_temperature] + channels: 1-22 + obs filters: + - filter: BlackList + filter variables: + - name: brightness_temperature + channels: 1-3 + - filter: Gaussian Thinning + horizontal_mesh: 1111.949266 #km = 10 deg at equator + thin_if_any_filter_variables_are_valid: false # only thin if all valid + passedBenchmark: 1900 # every ob location has filter variables failing QC so + # none of 100 obs thinned, 100*19 valid channels = 1900 +# +# Example where selected channels for some ob locations fail Domain Check QC +# and thin_if_any_filter_variables_are_valid is true. +- obs space: + name: ATMS + obsdatain: + obsfile: Data/ufo/testinput_tier_1/atms_n20_obs_20191230T0000_rttov.nc4 + simulated variables: [brightness_temperature] + channels: 1-22 + obs filters: + - filter: Domain Check + filter variables: + - name: brightness_temperature + channels: 1-3 + where: + - variable: + name: latitude@MetaData + minvalue: 61.0 + maxvalue: 64.0 # 75 out of 100 obs fall outside domain for channels 1-3 + - filter: Gaussian Thinning + horizontal_mesh: 1111.949266 #km = 10 deg at equator + thin_if_any_filter_variables_are_valid: true # default is true + passedBenchmark: 79 # 75 out of domain, 24 thinned out for channels 1-3 + # 0 out of domain, 96 thinned out for channels 4-22 + # 1*3 + 4*19 = 79 +# +# Example where list of filter variables (channels) differs from list of +# simulated variables. +- obs space: + name: ATMS + obsdatain: + obsfile: Data/ufo/testinput_tier_1/atms_n20_obs_20191230T0000_rttov.nc4 + simulated variables: [brightness_temperature] + channels: 1-22 + obs filters: + - filter: BlackList + filter variables: + - name: brightness_temperature + channels: 1-3 + - filter: Gaussian Thinning + filter variables: + - name: brightness_temperature + channels: 4-6 + horizontal_mesh: 1111.949266 #km = 10 deg at equator + thin_if_any_filter_variables_are_valid: false # default is true + passedBenchmark: 1612 # filter variable channels 4-6 all pass QC + # 4 out of 100 ob locations pass thinning + # 16 channels (7-22) neither blacklisted nor thinned + # 4*3 + 100*16 = 1612 From a015cf5df4d9c83aa19f27e5fe092cb60789b497 Mon Sep 17 00:00:00 2001 From: Susan Date: Tue, 10 Aug 2021 21:08:22 +0100 Subject: [PATCH 78/98] Modify Background Check to allow option of threshold relative to BG error (#1385) * Modifying BackgroundCheck... * Make PreQC a subclass of FilterBase. Do work in applyFilter() rather than the constructor. * Added a FinalCheck filter. * Apply the FinalCheck filter at the end of the ObsFilters test. * QCmanager: don't reject observations with missing obs errors. Initially reject all derived obs. * Allow copying variables using the Variable Assignment filter. * FinalCheck: Inherit from ObsProcessorBase. * VariableAssignment: Extra documentation. * VariableAssignment: do int comparisons before float comparisons. * QCmanager: Add documentation. remove unused variable. * QCmanager: Look both at ObsValue and DerivedObsValue. * Rewritten an error message in FinalCheck. * Removed duplicate anchors. * Replace DerivedValue by DerivedObsValue in an example. * Added tests verifying correct handling of derived ObsValues and derived simulated variables. * QCmanager: If the ObsValue array of a derived simulated variable already exists, treat it like a non-derived variable. * Removed inaccurate comments. * Test the "source variable" option of Variable Assignment. * Save point before testing derived_variables changes. * yaml file doing as expected for the moment * Undoing accidental change to qc_variableassignment.yaml * Simplifying and correcting the ctest that tests modified BG check * Changes for review round 1. * Restarting CI Co-authored-by: Wojciech Smigaj Co-authored-by: Ryan Honeyager --- src/ufo/filters/BackgroundCheck.cc | 30 ++++++++++++++++++++++++-- src/ufo/filters/BackgroundCheck.h | 16 +++++++++++++- test/testinput/qc_backgroundcheck.yaml | 17 +++++++++++++++ 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/ufo/filters/BackgroundCheck.cc b/src/ufo/filters/BackgroundCheck.cc index a835b5885..cba4216fc 100644 --- a/src/ufo/filters/BackgroundCheck.cc +++ b/src/ufo/filters/BackgroundCheck.cc @@ -46,6 +46,12 @@ BackgroundCheck::BackgroundCheck(ioda::ObsSpace & obsdb, const Parameters_ & par allvars_ += var; } allvars_ += Variables(filtervars_, test_hofx); + // Add BG error variable if threshold is required wrt BG error: + for (size_t jv = 0; jv < filtervars_.size(); ++jv) { + if (parameters_.thresholdWrtBGerror.value()) { + allvars_ += backgrErrVariable(filtervars_[jv]); + } + } ASSERT(parameters_.threshold.value() || parameters_.absoluteThreshold.value() || parameters_.functionAbsoluteThreshold.value()); @@ -54,6 +60,9 @@ BackgroundCheck::BackgroundCheck(ioda::ObsSpace & obsdb, const Parameters_ & par !parameters_.absoluteThreshold.value()); ASSERT(!parameters_.functionAbsoluteThreshold.value()->empty()); } + if (parameters_.thresholdWrtBGerror.value()) { + ASSERT(parameters_.threshold.value()); + } } // ----------------------------------------------------------------------------- @@ -62,6 +71,14 @@ BackgroundCheck::~BackgroundCheck() { oops::Log::trace() << "BackgroundCheck destructed" << std::endl; } +// ----------------------------------------------------------------------------- +/// Return the name of the variable containing the background error estimate of the +/// specified filter variable. + +Variable BackgroundCheck::backgrErrVariable(const Variable &filterVariable) const { + return Variable(filterVariable.variable() + "_background_error@ObsDiag"); +} + // ----------------------------------------------------------------------------- void BackgroundCheck::applyFilter(const std::vector & apply, @@ -70,7 +87,7 @@ void BackgroundCheck::applyFilter(const std::vector & apply, oops::Log::trace() << "BackgroundCheck postFilter" << std::endl; const oops::Variables observed = obsdb_.obsvariables(); const float missing = util::missingValue(missing); - oops::Log::debug() << "BackgroundCheck obserr: " << *obserr_; + oops::Log::debug() << "BackgroundCheck obserr: " << *obserr_ << std::endl; ioda::ObsDataVector obs(obsdb_, filtervars.toOopsVariables(), "ObsValue"); ioda::ObsDataVector obsbias(obsdb_, filtervars.toOopsVariables(), "ObsBias", false); @@ -112,6 +129,12 @@ void BackgroundCheck::applyFilter(const std::vector & apply, // H(x) std::vector hofx; data_.get(varhofx.variable(jv), hofx); +// H(x) error + std::vector hofxerr; + bool thresholdWrtBGerror = parameters_.thresholdWrtBGerror.value(); + if (thresholdWrtBGerror) { + data_.get(backgrErrVariable(filtervars[jv]), hofxerr); + } // Threshold for current variable std::vector abs_thr(obsdb_.nlocs(), std::numeric_limits::max()); @@ -137,9 +160,12 @@ void BackgroundCheck::applyFilter(const std::vector & apply, } ASSERT(hofx[jobs] != util::missingValue(hofx[jobs])); + const std::vector &errorMultiplier = thresholdWrtBGerror ? + hofxerr : (*obserr_)[iv]; // Threshold for current observation float zz = (thr[jobs] == std::numeric_limits::max()) ? abs_thr[jobs] : - std::min(abs_thr[jobs], thr[jobs] * (*obserr_)[iv][jobs]); + std::min(abs_thr[jobs], thr[jobs] * errorMultiplier[jobs]); + ASSERT(zz < std::numeric_limits::max() && zz > 0.0); // Check distance from background diff --git a/src/ufo/filters/BackgroundCheck.h b/src/ufo/filters/BackgroundCheck.h index 7a6feb72c..d224d0c3a 100644 --- a/src/ufo/filters/BackgroundCheck.h +++ b/src/ufo/filters/BackgroundCheck.h @@ -38,11 +38,22 @@ class BackgroundCheckParameters : public FilterParametersBase { public: /// The filter will flag observations whose bias-corrected value differs from its model equivalent - /// by more than `threshold` times the current estimate of the observation error. + /// by more than `threshold` times the current estimate of the observation error. Or if the + /// option "threshold wrt background error" is true, the `threshold` is multiplied by the + /// background error rather than observation error. E.g. + /// + /// filter variables: + /// - name: sea_surface_height + /// threshold wrt background error: true + /// threshold: 3.0 /// /// `threshold` can be a real number or the name of a variable. oops::OptionalParameter threshold{"threshold", this}; + /// A switch indicating whether threshold must be multiplied by background error rather than + /// observation error. If true, `threshold` must have a value. + oops::Parameter thresholdWrtBGerror{"threshold wrt background error", false, this}; + /// The filter will flag observations whose bias-corrected value differs from its model equivalent /// by more than `absolute threshold` /// @@ -90,6 +101,9 @@ class BackgroundCheck : public FilterBase, void applyFilter(const std::vector &, const Variables &, std::vector> &) const override; int qcFlag() const override {return QCflags::fguess;} + /// \brief Return the name of the variable containing the background error estimate of the + /// specified filter variable. + Variable backgrErrVariable(const Variable & filterVariable) const; Parameters_ parameters_; }; diff --git a/test/testinput/qc_backgroundcheck.yaml b/test/testinput/qc_backgroundcheck.yaml index 43b2a8953..bdec2c401 100644 --- a/test/testinput/qc_backgroundcheck.yaml +++ b/test/testinput/qc_backgroundcheck.yaml @@ -141,3 +141,20 @@ observations: # variable3@HofX = 25, 23, 21, 19, 17, 15, 13, 11, 9, 7 # variable3@ObsError = 2, 4, 2, 4, 2, 4, 2, 4, 2, 4 passedBenchmark: 30 +- obs space: + name: Altimeter + obsdatain: + obsfile: Data/ufo/testinput_tier_1/sataltimeter_sla_testdata.nc4 + obsdataout: + obsfile: Data/altimeter_testout.nc4 + simulated variables: [sea_surface_height] + HofX: HofX + obs diagnostics: + filename: Data/ufo/testinput_tier_1/background_errors_for_altimeter_test.nc4 + obs filters: + - filter: Background Check # BG check with threshold wrt BG error + filter variables: + - name: sea_surface_height + threshold wrt background error: true + threshold: 3.0 + passedBenchmark: 12 # [0:7] accepted; [7:11] missing; [16:22] BG-rejected From 40be4626e3a412a11682538e3fbc15c4b1c8aeb5 Mon Sep 17 00:00:00 2001 From: James Cotton <64855970+james-cotton@users.noreply.github.com> Date: Wed, 11 Aug 2021 18:15:42 +0100 Subject: [PATCH 79/98] Add choice of pressure coordinate to Gaussian thinning (#1418) * Add vertical coordinate option * New line * Remove references to pressure in favour of vertical coordinate Co-authored-by: Ryan Honeyager --- src/ufo/filters/GaussianThinningParameters.h | 10 +++++---- src/ufo/filters/Gaussian_Thinning.cc | 23 ++++++++++---------- src/ufo/filters/Gaussian_Thinning.h | 14 ++++++------ 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/ufo/filters/GaussianThinningParameters.h b/src/ufo/filters/GaussianThinningParameters.h index a08f047ef..7d087d714 100644 --- a/src/ufo/filters/GaussianThinningParameters.h +++ b/src/ufo/filters/GaussianThinningParameters.h @@ -85,15 +85,17 @@ class GaussianThinningParameters : public FilterParametersBase { // Vertical grid - /// Cell size (in Pa) in the vertical direction. Thinning in the vertical direction is disabled + /// Cell size in the vertical direction. Thinning in the vertical direction is disabled /// if this parameter is not specified or negative. oops::Parameter verticalMesh{"vertical_mesh", -1.0f, this}; - /// Lower bound of the pressure interval split into cells of size \c vertical_mesh. + /// Lower bound of the vertical coordinate interval split into cells of size \c vertical_mesh. oops::Parameter verticalMin{"vertical_min", 100.0f, this}; - /// Upper bound of the pressure interval split into cells of size \c vertical_mesh. + /// Upper bound of the vertical coordinate interval split into cells of size \c vertical_mesh. /// This parameter is rounded upwards to the nearest multiple of \c vertical_mesh starting from /// \c vertical_min. oops::Parameter verticalMax{"vertical_max", 110000.0f, this}; + /// Observation vertical coordinate. + oops::Parameter verticalCoord{"vertical_coordinate", "air_pressure", this}; // Temporal grid @@ -134,7 +136,7 @@ class GaussianThinningParameters : public FilterParametersBase { /// Allowed values: /// /// - \c geodesic: retain the observation closest to the cell centre in the horizontal direction - /// (air pressure and time are ignored) + /// (the vertical coordinate and time are ignored) /// - \c maximum: retain the observation lying furthest from the cell's bounding box in the /// system of coordinates in which the cell is a unit cube (all dimensions along which thinning /// is enabled are taken into account). diff --git a/src/ufo/filters/Gaussian_Thinning.cc b/src/ufo/filters/Gaussian_Thinning.cc index df5253478..53703a93e 100644 --- a/src/ufo/filters/Gaussian_Thinning.cc +++ b/src/ufo/filters/Gaussian_Thinning.cc @@ -67,8 +67,8 @@ void Gaussian_Thinning::applyFilter(const std::vector & apply, RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups( validObsIds, options_.opsCompatibilityMode); - groupObservationsByPressure(validObsIds, *distanceCalculator, obsAccessor, - splitter, distancesToBinCenter); + groupObservationsByVerticalCoordinate(validObsIds, *distanceCalculator, obsAccessor, + splitter, distancesToBinCenter); groupObservationsByTime(validObsIds, *distanceCalculator, obsAccessor, splitter, distancesToBinCenter); groupObservationsBySpatialLocation(validObsIds, *distanceCalculator, obsAccessor, @@ -204,13 +204,13 @@ boost::optional Gaussian_Thinning::makeSpatialBinSelector( // ----------------------------------------------------------------------------- -void Gaussian_Thinning::groupObservationsByPressure( +void Gaussian_Thinning::groupObservationsByVerticalCoordinate( const std::vector &validObsIds, const DistanceCalculator &distanceCalculator, const ObsAccessor &obsAccessor, RecursiveSplitter &splitter, std::vector &distancesToBinCenter) const { - std::unique_ptr binSelector = makePressureBinSelector(options_); + std::unique_ptr binSelector = makeVerticalBinSelector(options_); if (!binSelector) return; @@ -218,23 +218,24 @@ void Gaussian_Thinning::groupObservationsByPressure( oops::Log::debug() << "Gaussian_Thinning: number of vertical bins = " << *binSelector->numBins() << std::endl; - std::vector pres = obsAccessor.getFloatVariableFromObsSpace("MetaData", "air_pressure"); + std::vector vcoord = obsAccessor.getFloatVariableFromObsSpace( + "MetaData", options_.verticalCoord); std::vector bins; bins.reserve(validObsIds.size()); for (size_t obsId : validObsIds) { - bins.push_back(binSelector->bin(pres[obsId])); + bins.push_back(binSelector->bin(vcoord[obsId])); } splitter.groupBy(bins); - oops::Log::debug() << "Gaussian_Thinning: pressures = " << pres << std::endl; - oops::Log::debug() << "Gaussian_Thinning: pressure bins = " << bins << std::endl; + oops::Log::debug() << "Gaussian_Thinning: vertical coords = " << vcoord << std::endl; + oops::Log::debug() << "Gaussian_Thinning: vertical coord bins = " << bins << std::endl; for (size_t validObsIndex = 0; validObsIndex < validObsIds.size(); ++validObsIndex) { const size_t obsId = validObsIds[validObsIndex]; const float component = distanceCalculator.nonspatialDistanceComponent( - pres[obsId], binSelector->binCenter(bins[validObsIndex]), + vcoord[obsId], binSelector->binCenter(bins[validObsIndex]), binSelector->inverseBinWidth()); distancesToBinCenter[validObsIndex] = distanceCalculator.combineDistanceComponents( distancesToBinCenter[validObsIndex], component); @@ -243,7 +244,7 @@ void Gaussian_Thinning::groupObservationsByPressure( // ----------------------------------------------------------------------------- -std::unique_ptr Gaussian_Thinning::makePressureBinSelector( +std::unique_ptr Gaussian_Thinning::makeVerticalBinSelector( const GaussianThinningParameters &options) { if (options.verticalMesh <= 0) return nullptr; @@ -256,7 +257,7 @@ std::unique_ptr Gaussian_Thinning::makePressureBinSel 1, static_cast(std::ceil((options.verticalMax - options.verticalMin) / options.verticalMesh))); - // Adjust verticalMax upwards to make the range of pressures + // Adjust verticalMax upwards to make the range of vertical coordinates // evenly divisible into bins of width verticalMesh. const float adjustedVerticalMax = options.verticalMin + numVerticalBins * options.verticalMesh; diff --git a/src/ufo/filters/Gaussian_Thinning.h b/src/ufo/filters/Gaussian_Thinning.h index 9f2b3f0ea..ed914406d 100644 --- a/src/ufo/filters/Gaussian_Thinning.h +++ b/src/ufo/filters/Gaussian_Thinning.h @@ -47,7 +47,7 @@ class SpatialBinSelector; /// /// Cell assignment can be based on an arbitrary combination of: /// - horizontal position -/// - vertical position (in terms of air pressure) +/// - vertical position (in terms of height or pressure) /// - time /// - category (arbitrary integer associated with each observation). /// @@ -83,11 +83,11 @@ class Gaussian_Thinning : public FilterBase, RecursiveSplitter &splitter, std::vector &distancesToBinCenter) const; - void groupObservationsByPressure(const std::vector &validObsIds, - const DistanceCalculator &distanceCalculator, - const ObsAccessor &obsAccessor, - RecursiveSplitter &splitter, - std::vector &distancesToBinCenter) const; + void groupObservationsByVerticalCoordinate(const std::vector &validObsIds, + const DistanceCalculator &distanceCalculator, + const ObsAccessor &obsAccessor, + RecursiveSplitter &splitter, + std::vector &distancesToBinCenter) const; void groupObservationsByTime(const std::vector &validObsIds, const DistanceCalculator &distanceCalculator, @@ -109,7 +109,7 @@ class Gaussian_Thinning : public FilterBase, static boost::optional makeSpatialBinSelector( const GaussianThinningParameters &options); - static std::unique_ptr makePressureBinSelector( + static std::unique_ptr makeVerticalBinSelector( const GaussianThinningParameters &options); static std::unique_ptr makeTimeBinSelector( From 36c7474a24ae3f22adc29598373c34a420d3213f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Wed, 11 Aug 2021 19:15:38 +0100 Subject: [PATCH 80/98] Variable Assignment filter: extend handling of the "source variable" option. (#1422) * Variable Assignment filter: extend handling of the "source variable" option. This option copies one variable into another. Integer variables can now be copied to floating-point ones and vice versa. String and DateTime variables can be copied to variables of the same type. * Added extra tests to cover paths throwing exceptions. * Modify expected exception message. * Modify expected exception message. * When converting float variables to int variables, round values to the nearest integer and check for out-of-range errors. Co-authored-by: Ryan Honeyager --- src/ufo/filters/VariableAssignment.cc | 126 +++++++++---- test/testinput/qc_variableassignment.yaml | 214 +++++++++++++++++++++- 2 files changed, 291 insertions(+), 49 deletions(-) diff --git a/src/ufo/filters/VariableAssignment.cc b/src/ufo/filters/VariableAssignment.cc index 7afc88682..d90c1c046 100644 --- a/src/ufo/filters/VariableAssignment.cc +++ b/src/ufo/filters/VariableAssignment.cc @@ -15,6 +15,7 @@ #include #include +#include #include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" @@ -32,6 +33,23 @@ namespace ufo { namespace { +/// Convert a float \p x to an int by rounding. If \p is equal to \p missingIn, return \p +/// missingOut. If the value to be returned is too large to be represented by an int, throw an +/// exception. +int safeCast(float x, float missingIn, int missingOut) { + if (x == missingIn) + return missingOut; + // Throws boost::math::rounding_error if x is outside the range representable by ints + return boost::math::iround(x); +} + +/// Cast an int \p x to a float. If \p is equal to \p missingIn, return \p missingOut. +float safeCast(int x, int missingIn, float missingOut) { + if (x == missingIn) + return missingOut; + return x; +} + /// Convert \p valueAsString to `VariableType` and for each vector in \p values assign that value /// at locations selected by the `where` statement. template @@ -51,24 +69,65 @@ void assignValue(const std::string &valueAsString, } } -/// Retrieve the variable \p variable and assign its components (channels) to successive -/// vectors in \p values (only at locations selected by the `where` statement). +/// For each location selected by the `where` statement, copy the corresponding element of +/// \p source to \p destination. +/// +/// This two-parameter template is selected by the compiler when SourceVariableType is +/// different from DestinationVariableType. It takes care of converting missing value indicators +/// of type SourceVariableType to ones of type DestinationVariableType. +template +void assignObsDataVector(const std::vector &apply, + const ioda::ObsDataVector &source, + ioda::ObsDataVector &destination) { + ASSERT(source.nvars() == destination.nvars()); + + const SourceVariableType missingSource = util::missingValue(SourceVariableType()); + const DestinationVariableType missingDestination = util::missingValue(DestinationVariableType()); + + for (size_t ival = 0; ival < source.nvars(); ++ival) { + const ioda::ObsDataRow ¤tSource = source[ival]; + ioda::ObsDataRow ¤tDestination = destination[ival]; + for (size_t iloc = 0; iloc < apply.size(); ++iloc) { + if (apply[iloc]) { + currentDestination[iloc] = safeCast(currentSource[iloc], missingSource, missingDestination); + } + } + } +} + +/// For each location selected by the `where` statement, copy the corresponding element of +/// \p source to \p destination. +/// +/// This single-parameter template is selected by the compiler when \p source and \p destination +/// are of the same type. template +void assignObsDataVector(const std::vector &apply, + const ioda::ObsDataVector &source, + ioda::ObsDataVector &destination) { + ASSERT(source.nvars() == destination.nvars()); + + for (size_t ival = 0; ival < source.nvars(); ++ival) { + const ioda::ObsDataRow ¤tSource = source[ival]; + ioda::ObsDataRow ¤tDestination = destination[ival]; + for (size_t iloc = 0; iloc < apply.size(); ++iloc) { + if (apply[iloc]) { + currentDestination[iloc] = currentSource[iloc]; + } + } + } +} + +/// Retrieve the variable \p variable of type \c SourceVariableType and assign its components +/// (channels) to successive vectors in \p values (only at locations selected by the `where` +/// statement). +template void assignVariable(const ufo::Variable &variable, const std::vector &apply, const ObsFilterData &data, - ioda::ObsDataVector &values) { - ioda::ObsDataVector newValues(data.obsspace(), variable.toOopsVariables()); + ioda::ObsDataVector &values) { + ioda::ObsDataVector newValues(data.obsspace(), variable.toOopsVariables()); data.get(variable, newValues); - - for (size_t ival = 0; ival < values.nvars(); ++ival) { - std::vector ¤tValues = values[ival]; - const ioda::ObsDataRow ¤tNewValues = newValues[ival]; - for (size_t iloc = 0; iloc < apply.size(); ++iloc) { - if (apply[iloc]) - currentValues[iloc] = currentNewValues[iloc]; - } - } + assignObsDataVector(apply, newValues, values); } /// Evaluate the ObsFunction \p function and assign the vectors it produced to successive vectors @@ -81,30 +140,7 @@ void assignFunction(const ufo::Variable &function, ioda::ObsDataVector &values) { ioda::ObsDataVector newValues(data.obsspace(), variable.toOopsVariables()); data.get(function, newValues); - - const VariableType missingVariableValue = util::missingValue(VariableType()); - const FunctionValueType missingFunctionValue = util::missingValue(FunctionValueType()); - const FunctionValueType maxFunctionValue = std::numeric_limits::max(); - for (size_t ival = 0; ival < values.nvars(); ++ival) { - std::vector ¤tValues = values[ival]; - const ioda::ObsDataRow ¤tNewValues = newValues[ival]; - for (size_t iloc = 0; iloc < apply.size(); ++iloc) { - if (apply[iloc]) { - if (currentNewValues[iloc] != missingFunctionValue) - // The call to min() below is a hack preventing clang from generating optimized code - // that speculatively converts currentNewValues[iloc] to VariableType regardless of - // whether it's equal to missingFunctionValue or not. This causes a floating exception - // and thus breaks the test suite. After migration to a more recent version of clang - // we may be able to remove the call to min() and use the -ffp-exception-behavior=maytrap - // flag instead to instruct the compiler to avoid transformations that may raise - // floating-point exceptions that would not have been raised by the original code. - currentValues[iloc] = - static_cast(std::min(currentNewValues[iloc], maxFunctionValue)); - else - currentValues[iloc] = missingVariableValue; - } - } - } + assignObsDataVector(apply, newValues, values); } /// Assign values to a numeric variable (of type float or int). @@ -117,7 +153,17 @@ void assignNumericValues(const AssignmentParameters ¶ms, if (params.value_.value() != boost::none) { assignValue(*params.value_.value(), apply, values); } else if (params.sourceVariable.value() != boost::none) { - assignVariable(*params.sourceVariable.value(), apply, data, values); + switch (data.dtype(*params.sourceVariable.value())) { + case ioda::ObsDtype::Float: + assignVariable(*params.sourceVariable.value(), apply, data, values); + break; + case ioda::ObsDtype::Integer: + assignVariable(*params.sourceVariable.value(), apply, data, values); + break; + default: + throw eckit::BadParameter(params.sourceVariable.value()->fullName() + + " is not a numeric variable", Here()); + } } else { ASSERT(params.function.value() != boost::none); if (params.function.value()->group() == ObsFunctionTraits::groupName) @@ -139,8 +185,10 @@ void assignNonnumericValues(const AssignmentParameters ¶ms, ioda::ObsDataVector &values) { if (params.value_.value() != boost::none) { assignValue(*params.value_.value(), apply, values); - // TODO(wsmigaj): Handle sourceVariable once PR #1280 is merged in. + } else if (params.sourceVariable.value() != boost::none) { + assignVariable(*params.sourceVariable.value(), apply, data, values); } else { + ASSERT(params.function.value() != boost::none); assignFunction(*params.function.value(), variable, apply, data, values); } } diff --git a/test/testinput/qc_variableassignment.yaml b/test/testinput/qc_variableassignment.yaml index fe2dec5f8..45ed413a2 100644 --- a/test/testinput/qc_variableassignment.yaml +++ b/test/testinput/qc_variableassignment.yaml @@ -264,7 +264,7 @@ observations: - obs space: <<: *ObsSpace - name: Copy a variable to another (pre-existing) float variable + name: Copy a float variable to another (pre-existing) float variable obs filters: - filter: Variable Assignment where: *Where @@ -280,7 +280,7 @@ observations: - obs space: <<: *ObsSpace - name: Copy a variable to a new float variable + name: Copy a float variable to a new float variable obs filters: - filter: Variable Assignment where: *Where @@ -297,14 +297,14 @@ observations: - obs space: <<: *ObsSpace - name: Copy a variable to another (pre-existing) int variable + name: Copy an int variable to another (pre-existing) int variable obs filters: - filter: Variable Assignment - where: *Where + where: *Where # selects observations 1, 2 and 3 assignments: - - name: int_variable_1@MetaData + - name: int_variable_1@MetaData # [1, 75, 75, 75, 5] source variable: - name: assigned_int_variable_3@TestReference + name: assigned_int_variable_3@TestReference # [1, 2, 3, 4, 5] compareVariables: - test: name: int_variable_1@MetaData @@ -313,20 +313,101 @@ observations: - obs space: <<: *ObsSpace - name: Copy a variable to new int variable + name: Copy an int variable to a new int variable obs filters: - filter: Variable Assignment - where: *Where + where: *Where # selects observations 1, 2 and 3 assignments: - name: new_variable@MetaData type: int source variable: - name: assigned_int_variable_3@TestReference + name: assigned_int_variable_3@TestReference # [1, 2, 3, 4, 5] compareVariables: - test: name: new_variable@MetaData reference: - name: assigned_int_variable_4@TestReference + name: assigned_int_variable_4@TestReference # [_, 2, 3, 4, _] + +- obs space: + <<: *ObsSpace + name: Copy a string variable to another (pre-existing) string variable + obs filters: + - filter: Variable Assignment + where: *Where # selects observations 1, 2 and 3 + assignments: + - name: string_variable_1@MetaData # ["ABC", "DEF", "GHI", "JKL", "MNO"] + source variable: + name: assigned_string_variable_2@TestReference # ["*** MISSING ***", "XYZ", "XYZ", "XYZ", "*** MISSING ***"] + compareVariables: + - test: + name: string_variable_1@MetaData + reference: + name: assigned_string_variable_1@TestReference # ["ABC", "XYZ", "XYZ", "XYZ", "MNO"] + +- obs space: + <<: *ObsSpace + name: Copy a string variable to a new string variable + obs filters: + - filter: Variable Assignment + where: *Where # selects observations 1, 2 and 3 + assignments: + - name: new_variable@MetaData + type: string + source variable: + name: assigned_string_variable_1@TestReference # ["ABC", "XYZ", "XYZ", "XYZ", "MNO"] + compareVariables: + - test: + name: new_variable@MetaData + reference: + name: assigned_string_variable_2@TestReference # ["*** MISSING ***", "XYZ", "XYZ", "XYZ", "*** MISSING ***"] + +- obs space: + <<: *ObsSpace + name: Copy a datetime variable to another (pre-existing) datetime variable + obs filters: + - filter: Variable Assignment + where: *Where # selects observations 1, 2 and 3 + assignments: + - name: datetime@MetaData # ["2018-04-15T06:00:00Z", "2018-04-16T15:00:00Z", "2018-04-17T06:00:00Z", "2018-04-18T15:00:00Z", "2018-04-19T06:00:00Z"] + source variable: + name: assigned_datetime_variable_2@TestReference # ["9996-02-29T23:58:57Z", "2000-01-02T03:04:05Z", "2000-01-02T03:04:05Z", "2000-01-02T03:04:05Z", "9996-02-29T23:58:57Z"] + compareVariables: + - test: + name: datetime@MetaData + reference: + name: assigned_datetime_variable_1@TestReference # ["2018-04-15T06:00:00Z", "2000-01-02T03:04:05Z", "2000-01-02T03:04:05Z", 2000-01-02T03:04:05Z", "2018-04-19T06:00:00Z"] + +- obs space: + <<: *ObsSpace + name: Copy a datetime variable to a new datetime variable + obs filters: + - filter: Variable Assignment + where: *Where # selects observations 1, 2 and 3 + assignments: + - name: new_variable@MetaData + type: datetime + source variable: + name: assigned_datetime_variable_1@TestReference # ["2018-04-15T06:00:00Z", "2000-01-02T03:04:05Z", "2000-01-02T03:04:05Z", 2000-01-02T03:04:05Z", "2018-04-19T06:00:00Z"] + compareVariables: + - test: + name: new_variable@MetaData + reference: + name: assigned_datetime_variable_2@TestReference # ["9996-02-29T23:58:57Z", "2000-01-02T03:04:05Z", "2000-01-02T03:04:05Z", "2000-01-02T03:04:05Z", "9996-02-29T23:58:57Z"] + +- obs space: + <<: *ObsSpace + name: Copy a float variable to a (pre-existing) int variable + obs filters: + - filter: Variable Assignment + assignments: + - name: int_variable_1@MetaData # [1, 2, 3, 4, 5] + source variable: + name: assigned_float_variable_7@TestReference # [_, 74.5, 74.9, 75.4, _] + compareVariables: + - test: + name: int_variable_1@MetaData + reference: + name: assigned_int_variable_2@TestReference # [_, 75, 75, 75, _] ## Test Conditional ObsFunction "add on" - Example 1 ## Float variable assignment including multiple where in a case @@ -395,6 +476,56 @@ observations: reference: name: assigned_int_variable_2@TestReference +- obs space: &ObsSpace + name: Assign values produced by a string-valued ObsFunction to a string variable + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs filters: + - filter: Variable Assignment + assignments: + - name: string_variable_1@MetaData + function: + name: Conditional@StringObsFunction + options: + cases: + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 2-4 + value: XYZ + compareVariables: + - test: + name: string_variable_1@MetaData + reference: + name: assigned_string_variable_2@TestReference # ["*** MISSING ***", "XYZ", "XYZ", "XYZ", "*** MISSING ***"] + +- obs space: &ObsSpace + name: Assign values produced by a datetime-valued ObsFunction to a datetime variable + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs filters: + - filter: Variable Assignment + assignments: + - name: datetime@MetaData + function: + name: Conditional@DateTimeObsFunction + options: + cases: + - where: + - variable: + # [1, 2, 3, 4, 5] + name: int_variable_1@MetaData + is_in: 2-4 + value: 2000-01-02T03:04:05Z + compareVariables: + - test: + name: datetime@MetaData + reference: + name: assigned_datetime_variable_2@TestReference # ["9996-02-29T23:58:57Z", "2000-01-02T03:04:05Z", "2000-01-02T03:04:05Z", "2000-01-02T03:04:05Z", "9996-02-29T23:58:57Z"] + ## Tests on the obsspace with multiple channels - obs space: &MultichannelObsSpace @@ -646,6 +777,58 @@ observations: value: XYZ expectExceptionWithMessage: Conditional@StringObsFunction is not a function producing numeric values +- obs space: &ObsSpace + <<: *ObsSpace + name: Try to copy a string variable into a float variable + obs filters: + - filter: Variable Assignment + assignments: + - name: float_variable_1@MetaData + source variable: + name: string_variable_1@MetaData + expectExceptionWithMessage: string_variable_1@MetaData is not a numeric variable + +- obs space: &ObsSpace + <<: *ObsSpace + name: Try to copy a string variable into an int variable + obs filters: + - filter: Variable Assignment + assignments: + - name: int_variable_1@MetaData + source variable: + name: string_variable_1@MetaData + expectExceptionWithMessage: string_variable_1@MetaData is not a numeric variable + +- obs space: &ObsSpace + <<: *ObsSpace + name: Try to copy a float variable into a string variable + obs filters: + - filter: Variable Assignment + assignments: + - name: string_variable_1@MetaData + source variable: + name: float_variable_1@MetaData + # Unfortunately in this case ioda throws a nested exception; the message attached to the + # innermost exception is fairly clear (Requested data type not equal to storage datatype), + # but the EXPECT_THROWS_MSG() macro sees only the outermost exception message, which is + # rather generic and only includes the code location where the exception was thrown. + expectExceptionWithMessage: ::read + +- obs space: &ObsSpace + <<: *ObsSpace + name: Try to copy a float variable into a datetime variable + obs filters: + - filter: Variable Assignment + assignments: + - name: datetime@MetaData + source variable: + name: float_variable_1@MetaData + # Unfortunately in this case ioda throws a nested exception; the message attached to the + # innermost exception is fairly clear (Requested data type not equal to storage datatype), + # but the EXPECT_THROWS_MSG() macro sees only the outermost exception message, which is + # rather generic and only includes the code location where the exception was thrown. + expectExceptionWithMessage: ::read + - obs space: <<: *ObsSpace name: Try to assign values to an existing variable from the ObsValue group @@ -676,3 +859,14 @@ observations: - name: new_variable@MetaData value: ABCDEF expectExceptionWithMessage: You need to specify the type of the variable to be created + +- obs space: + <<: *ObsSpace + name: Try to assign out-of-range floating-point values to an int variable + obs filters: + - filter: Variable Assignment + assignments: + - name: new_variable@MetaData + type: int + source variable: float_variable_3@MetaData # contains 1e20 + expectExceptionWithMessage: Value 1.00000002e+20 can not be represented in the target integer type From 663cf0e486a38bf2b86b028104e14425810f4bc0 Mon Sep 17 00:00:00 2001 From: JoanneAWaller <52206323+JoanneAWaller@users.noreply.github.com> Date: Wed, 11 Aug 2021 21:18:03 +0100 Subject: [PATCH 81/98] Allow variable transforms to access geovals (#1428) * Modified varaible transforms to take ufo::ObsFilterData& data rahter than ioda::ObsSpace& os * Added requiredVars vitrual method to TransformBase * Removed apply from transform base, and added to run transform function. * Coding norms and tidy * Modify HeightToPressure and HorizontalDrift transforms * Deleted unneed comments from test yaml * Fix poor indent * Respinding to Wojciech's commetns * Tidied requiredVariables method * Coding norms * Addressing Chris' comments Co-authored-by: Ryan Honeyager --- src/ufo/filters/VariableTransforms.cc | 15 +++++-- .../Cal_HeightFromPressure.cc | 12 +++--- .../Cal_HeightFromPressure.h | 7 ++-- src/ufo/variabletransforms/Cal_Humidity.cc | 39 +++++++++---------- src/ufo/variabletransforms/Cal_Humidity.h | 20 +++++----- .../Cal_PressureFromHeight.cc | 31 +++++++-------- .../Cal_PressureFromHeight.h | 18 ++++----- .../Cal_ProfileHorizontalDrift.cc | 12 +++--- .../Cal_ProfileHorizontalDrift.h | 7 ++-- .../Cal_RemapScanPosition.cc | 13 +++---- .../Cal_RemapScanPosition.h | 7 ++-- src/ufo/variabletransforms/Cal_Wind.cc | 27 +++++++------ src/ufo/variabletransforms/Cal_Wind.h | 14 +++---- src/ufo/variabletransforms/TransformBase.cc | 15 ++++--- src/ufo/variabletransforms/TransformBase.h | 37 +++++++++++------- 15 files changed, 136 insertions(+), 138 deletions(-) diff --git a/src/ufo/filters/VariableTransforms.cc b/src/ufo/filters/VariableTransforms.cc index f0f542976..6d2c19578 100644 --- a/src/ufo/filters/VariableTransforms.cc +++ b/src/ufo/filters/VariableTransforms.cc @@ -38,6 +38,14 @@ VariableTransforms::VariableTransforms( options_->deserialize(config); allvars_ += Variables(filtervars_); + // Add any required geovals to allvars_ + for (const auto& cal_ : options_->Transform.value()) { + std::unique_ptr Transform = + TransformFactory::create(cal_, *options_, data_, flags_); + Variables gvars = Transform->requiredVariables(); + allvars_ += Variables(gvars); + } + oops::Log::debug() << "VariableTransforms: config = " << config << std::endl; } @@ -56,11 +64,10 @@ void VariableTransforms::applyFilter( // Run all calculations requested for (const auto& cal_ : options_->Transform.value()) { - std::cout << " estimate: " << cal_ << std::endl; + oops::Log::debug() << " estimate: " << cal_ << std::endl; std::unique_ptr Transform = - TransformFactory::create(cal_, *options_, obsdb_, flags_, apply); - - Transform->runTransform(); + TransformFactory::create(cal_, *options_, data_, flags_); + Transform->runTransform(apply); } } diff --git a/src/ufo/variabletransforms/Cal_HeightFromPressure.cc b/src/ufo/variabletransforms/Cal_HeightFromPressure.cc index e1a8d82a6..acb576450 100644 --- a/src/ufo/variabletransforms/Cal_HeightFromPressure.cc +++ b/src/ufo/variabletransforms/Cal_HeightFromPressure.cc @@ -16,14 +16,14 @@ static TransformMaker makerCal_HeightFromPressure_("HeightFromPressure"); Cal_HeightFromPressure::Cal_HeightFromPressure( - const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply) - : TransformBase(options, os, flags, apply) {} + const VariableTransformsParameters &options, + const ObsFilterData &data, + const std::shared_ptr> &flags) + : TransformBase(options, data, flags) {} /************************************************************************************/ -void Cal_HeightFromPressure::runTransform() { +void Cal_HeightFromPressure::runTransform(const std::vector &apply) { oops::Log::trace() << " Retrieve Height From Pressure" << std::endl; std::vector geopotentialHeight; @@ -75,7 +75,7 @@ void Cal_HeightFromPressure::runTransform() { // 3.1 Loop over each record for (ilocs = 0; ilocs < rSort.size(); ++ilocs) { // Cycle if the data have been excluded by the where statement - if (!apply_[rSort[ilocs]]) continue; + if (!apply[rSort[ilocs]]) continue; // Cycle if geopotential height is valid if (geopotentialHeight[rSort[ilocs]] != missingValueFloat) continue; diff --git a/src/ufo/variabletransforms/Cal_HeightFromPressure.h b/src/ufo/variabletransforms/Cal_HeightFromPressure.h index 803c269b9..c92059272 100644 --- a/src/ufo/variabletransforms/Cal_HeightFromPressure.h +++ b/src/ufo/variabletransforms/Cal_HeightFromPressure.h @@ -26,11 +26,10 @@ namespace ufo { class Cal_HeightFromPressure : public TransformBase { public: Cal_HeightFromPressure(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply); + const ObsFilterData &data, + const std::shared_ptr> &flags); // Run check - void runTransform() override; + void runTransform(const std::vector &apply) override; }; } // namespace ufo diff --git a/src/ufo/variabletransforms/Cal_Humidity.cc b/src/ufo/variabletransforms/Cal_Humidity.cc index e792b9119..86f2c433a 100644 --- a/src/ufo/variabletransforms/Cal_Humidity.cc +++ b/src/ufo/variabletransforms/Cal_Humidity.cc @@ -17,14 +17,14 @@ static TransformMaker makerCal_RelativeHumidity_("RelativeHumidity"); Cal_RelativeHumidity::Cal_RelativeHumidity( - const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply) - : TransformBase(options, os, flags, apply) {} + const VariableTransformsParameters &options, + const ObsFilterData &data, + const std::shared_ptr> &flags) + : TransformBase(options, data, flags) {} /**************************************************************************************************/ -void Cal_RelativeHumidity::runTransform() { +void Cal_RelativeHumidity::runTransform(const std::vector &apply) { oops::Log::trace() << " --> Retrieve Relative humidity" << std::endl; oops::Log::trace() << " --> method: " << method() << std::endl; @@ -34,13 +34,13 @@ void Cal_RelativeHumidity::runTransform() { // Get the right method switch (method()) { case formulas::MethodFormulation::UKMO: { - methodUKMO(); + methodUKMO(apply); break; } case formulas::MethodFormulation::NCAR: case formulas::MethodFormulation::NOAA: default: { - methodDEFAULT(); + methodDEFAULT(apply); break; } } @@ -65,7 +65,7 @@ Method: - If the pressure, dew point or temperature take extreme values or are missing, the relative humidity is set to missing data. */ -void Cal_RelativeHumidity::methodUKMO() { +void Cal_RelativeHumidity::methodUKMO(const std::vector &apply) { const size_t nlocs_ = obsdb_.nlocs(); // return if no data if (obsdb_.nlocs() == 0) { @@ -166,7 +166,7 @@ void Cal_RelativeHumidity::methodUKMO() { // 3.1 Loop over each record for (size_t iloc : rSort) { // if the data have been excluded by the where statement - if (!apply_[iloc]) continue; + if (!apply[iloc]) continue; // store some variables pressure = airPressure[iloc]; @@ -226,7 +226,7 @@ void Cal_RelativeHumidity::methodUKMO() { /**************************************************************************************************/ -void Cal_RelativeHumidity::methodDEFAULT() { +void Cal_RelativeHumidity::methodDEFAULT(const std::vector &apply) { const size_t nlocs = obsdb_.nlocs(); float esat, qvs, qv, satVaporPres; @@ -262,7 +262,7 @@ void Cal_RelativeHumidity::methodDEFAULT() { // Loop over all obs for (size_t jobs = 0; jobs < nlocs; ++jobs) { // if the data have been excluded by the where statement - if (!apply_[jobs]) continue; + if (!apply[jobs]) continue; if (specificHumidity[jobs] != missingValueFloat && airTemperature[jobs] != missingValueFloat && pressure[jobs] != missingValueFloat) { @@ -292,14 +292,14 @@ static TransformMaker makerCal_SpecificHumidity_("SpecificHumidity"); Cal_SpecificHumidity::Cal_SpecificHumidity( - const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply) - : TransformBase(options, os, flags, apply) {} + const VariableTransformsParameters &options, + const ObsFilterData &data, + const std::shared_ptr> &flags) + : TransformBase(options, data, flags) {} /************************************************************************************/ -void Cal_SpecificHumidity::runTransform() { +void Cal_SpecificHumidity::runTransform(const std::vector &apply) { oops::Log::trace() << " Retrieve Specific Humidity" << std::endl; oops::Log::trace() << " --> method: " << method() << std::endl; oops::Log::trace() << " --> formulation: " << formulation() << std::endl; @@ -311,14 +311,14 @@ void Cal_SpecificHumidity::runTransform() { case formulas::MethodFormulation::NOAA: case formulas::MethodFormulation::UKMO: default: { - methodDEFAULT(); + methodDEFAULT(apply); break; } } } /************************************************************************************/ -void Cal_SpecificHumidity::methodDEFAULT() { +void Cal_SpecificHumidity::methodDEFAULT(const std::vector &apply) { const size_t nlocs = obsdb_.nlocs(); float esat, qvs, qv, satVaporPres; std::vector relativeHumidity; @@ -370,8 +370,5 @@ void Cal_SpecificHumidity::methodDEFAULT() { } putObservation("specific_humidity", specificHumidity); } - - - } // namespace ufo diff --git a/src/ufo/variabletransforms/Cal_Humidity.h b/src/ufo/variabletransforms/Cal_Humidity.h index d2d05d867..1d48ba0e5 100644 --- a/src/ufo/variabletransforms/Cal_Humidity.h +++ b/src/ufo/variabletransforms/Cal_Humidity.h @@ -40,16 +40,15 @@ namespace ufo { class Cal_RelativeHumidity : public TransformBase { public: Cal_RelativeHumidity(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply); + const ObsFilterData &data, + const std::shared_ptr> &flags); // Run variable conversion - void runTransform() override; + void runTransform(const std::vector &apply) override; private: // list of specific implementation(s) - This is controlled by "method" - void methodDEFAULT(); - void methodUKMO(); + void methodDEFAULT(const std::vector &apply); + void methodUKMO(const std::vector &apply); }; @@ -75,15 +74,14 @@ class Cal_RelativeHumidity : public TransformBase { class Cal_SpecificHumidity : public TransformBase { public: Cal_SpecificHumidity(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply); + const ObsFilterData &data, + const std::shared_ptr> &flags); // Run check - void runTransform() override; + void runTransform(const std::vector &apply) override; private: // list of specific implementation(s) - This is controlled by "method" - void methodDEFAULT(); + void methodDEFAULT(const std::vector &apply); }; } // namespace ufo diff --git a/src/ufo/variabletransforms/Cal_PressureFromHeight.cc b/src/ufo/variabletransforms/Cal_PressureFromHeight.cc index 3f7e5cdea..65a1f269c 100644 --- a/src/ufo/variabletransforms/Cal_PressureFromHeight.cc +++ b/src/ufo/variabletransforms/Cal_PressureFromHeight.cc @@ -17,14 +17,13 @@ static TransformMaker makerCal_PressureFromHeightForProfile_("PressureFromHeightForProfile"); Cal_PressureFromHeightForProfile::Cal_PressureFromHeightForProfile( - const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply) - : TransformBase(options, os, flags, apply) {} + const VariableTransformsParameters &options, const ObsFilterData &data, + const std::shared_ptr> &flags) + : TransformBase(options, data, flags) {} /************************************************************************************/ -void Cal_PressureFromHeightForProfile::runTransform() { +void Cal_PressureFromHeightForProfile::runTransform(const std::vector &apply) { oops::Log::trace() << " --> Retrieve Pressure From Height (Profile)" << std::endl; oops::Log::trace() << " --> method: " << method() << std::endl; @@ -37,7 +36,7 @@ void Cal_PressureFromHeightForProfile::runTransform() { case formulas::MethodFormulation::NOAA: case formulas::MethodFormulation::UKMO: default: { - methodUKMO(); + methodUKMO(apply); break; } } @@ -45,7 +44,7 @@ void Cal_PressureFromHeightForProfile::runTransform() { /************************************************************************************/ -void Cal_PressureFromHeightForProfile::methodUKMO() { +void Cal_PressureFromHeightForProfile::methodUKMO(const std::vector &apply) { std::vector airTemperature; std::vector airTemperatureSurface; std::vector geopotentialHeight; @@ -185,7 +184,7 @@ void Cal_PressureFromHeightForProfile::methodUKMO() { // 4.2 Loop over the length of the profile for (ilocs = 0; ilocs < rSort.size(); ++ilocs) { // if the data have been excluded by the where statement - if (!apply_[rSort[ilocs]]) continue; + if (!apply[rSort[ilocs]]) continue; // Take current level values Zcurrent = geopotentialHeight[rSort[ilocs]]; @@ -245,14 +244,13 @@ static TransformMaker makerCal_PressureFromHeightForICAO_("PressureFromHeightForICAO"); Cal_PressureFromHeightForICAO::Cal_PressureFromHeightForICAO( - const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply) - : TransformBase(options, os, flags, apply) {} + const VariableTransformsParameters &options, const ObsFilterData &data, + const std::shared_ptr> &flags) + : TransformBase(options, data, flags) {} /************************************************************************************/ -void Cal_PressureFromHeightForICAO::runTransform() { +void Cal_PressureFromHeightForICAO::runTransform(const std::vector &apply) { oops::Log::trace() << " Retrieve Pressure From Height (ICAO)" << std::endl; oops::Log::trace() << " --> method: " << method() << std::endl; oops::Log::trace() << " --> formulation: " << formulation() << std::endl; @@ -264,14 +262,14 @@ void Cal_PressureFromHeightForICAO::runTransform() { case formulas::MethodFormulation::NOAA: case formulas::MethodFormulation::UKMO: default: { - methodUKMO(); + methodUKMO(apply); break; } } } /************************************************************************************/ -void Cal_PressureFromHeightForICAO::methodUKMO() { +void Cal_PressureFromHeightForICAO::methodUKMO(const std::vector &apply) { std::vector geopotentialHeight; std::vector airPressure; std::vector airPressure_ref; @@ -312,7 +310,7 @@ void Cal_PressureFromHeightForICAO::methodUKMO() { // 3.1 Loop over each record for (ilocs = 0; ilocs < rSort.size(); ++ilocs) { // if the data have been excluded by the where statement - if (!apply_[rSort[ilocs]]) continue; + if (!apply[rSort[ilocs]]) continue; // Cycle if airPressure is valid if (airPressure[rSort[ilocs]] != missingValueFloat) continue; @@ -330,6 +328,5 @@ void Cal_PressureFromHeightForICAO::methodUKMO() { putObservation("air_pressure", airPressure); } } - } // namespace ufo diff --git a/src/ufo/variabletransforms/Cal_PressureFromHeight.h b/src/ufo/variabletransforms/Cal_PressureFromHeight.h index 38b93f0c2..0d476a5df 100644 --- a/src/ufo/variabletransforms/Cal_PressureFromHeight.h +++ b/src/ufo/variabletransforms/Cal_PressureFromHeight.h @@ -30,15 +30,14 @@ namespace ufo { class Cal_PressureFromHeightForProfile : public TransformBase { public: Cal_PressureFromHeightForProfile(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply); + const ObsFilterData &data, + const std::shared_ptr> &flags); // Run variable conversion - void runTransform() override; + void runTransform(const std::vector &apply) override; private: // list of specific implementation(s) - This is controlled by "method" - void methodUKMO(); + void methodUKMO(const std::vector &apply); }; /*! @@ -51,15 +50,14 @@ class Cal_PressureFromHeightForProfile : public TransformBase { class Cal_PressureFromHeightForICAO : public TransformBase { public: Cal_PressureFromHeightForICAO(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply); + const ObsFilterData &data, + const std::shared_ptr> &flags); // Run check - void runTransform() override; + void runTransform(const std::vector &apply) override; private: // list of specific implementation(s) - This is controled by "method" - void methodUKMO(); + void methodUKMO(const std::vector &apply); }; } // namespace ufo diff --git a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc index 6746389b8..ed06bb5c6 100644 --- a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc +++ b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc @@ -19,14 +19,13 @@ makerCal_ProfileHorizontalDrift_("ProfileHorizontalDrift"); Cal_ProfileHorizontalDrift::Cal_ProfileHorizontalDrift (const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply) - : TransformBase(options, os, flags, apply) {} + const ObsFilterData &data, + const std::shared_ptr> &flags) + : TransformBase(options, data, flags) {} /************************************************************************************/ -void Cal_ProfileHorizontalDrift::runTransform() { +void Cal_ProfileHorizontalDrift::runTransform(const std::vector &apply) { oops::Log::trace() << " --> Compute horizontal drift lat/lon/time" << std::endl; oops::Log::trace() << " --> method: " << method() << std::endl; @@ -76,7 +75,7 @@ void Cal_ProfileHorizontalDrift::runTransform() { // Perform drift calculation for each profile in the sample. for (size_t jprof = 0; jprof < nprofs; ++jprof) { const std::vector &locs = obsdb_.recidx_vector(recnums[jprof]); - formulas::horizontalDrift(locs, apply_, + formulas::horizontalDrift(locs, apply, latitude_in, longitude_in, datetime_in, height, wind_speed, wind_from_direction, latitude_out, longitude_out, datetime_out); @@ -87,6 +86,5 @@ void Cal_ProfileHorizontalDrift::runTransform() { obsdb_.put_db("DerivedMetaData", "longitude", longitude_out); obsdb_.put_db("DerivedMetaData", "datetime", datetime_out); } - } // namespace ufo diff --git a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h index f88e01b06..94f5da73d 100644 --- a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h +++ b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h @@ -40,11 +40,10 @@ namespace ufo { class Cal_ProfileHorizontalDrift : public TransformBase { public: Cal_ProfileHorizontalDrift(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply); + const ObsFilterData &data, + const std::shared_ptr> &flags); // Run variable conversion - void runTransform() override; + void runTransform(const std::vector &apply) override; }; } // namespace ufo diff --git a/src/ufo/variabletransforms/Cal_RemapScanPosition.cc b/src/ufo/variabletransforms/Cal_RemapScanPosition.cc index 2b9eb208c..a645ad29b 100644 --- a/src/ufo/variabletransforms/Cal_RemapScanPosition.cc +++ b/src/ufo/variabletransforms/Cal_RemapScanPosition.cc @@ -17,14 +17,14 @@ static TransformMaker makerCal_RemapScanPosition_("RemapScanPosition"); Cal_RemapScanPosition::Cal_RemapScanPosition( - const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply) - : TransformBase(options, os, flags, apply) {} + const VariableTransformsParameters &options, + const ObsFilterData &data, + const std::shared_ptr> &flags) + : TransformBase(options, data, flags) {} /************************************************************************************/ -void Cal_RemapScanPosition::runTransform() { +void Cal_RemapScanPosition::runTransform(const std::vector &apply) { oops::Log::trace() << " --> Renumber satellite scan position" << std::endl; oops::Log::trace() << " --> method: " << method() << std::endl; @@ -41,7 +41,7 @@ void Cal_RemapScanPosition::runTransform() { // Loop over all obs for (size_t jobs = 0; jobs < nlocs; ++jobs) { // if the data have been excluded by the where statement - if (!apply_[jobs]) continue; + if (!apply[jobs]) continue; // Calculate wind vector if (original_scan_position[jobs] != missingValueInt) { @@ -51,6 +51,5 @@ void Cal_RemapScanPosition::runTransform() { // Overwrite variable at existing locations obsdb_.put_db("MetaData", "scan_position", remapped_scan_position); } - } // namespace ufo diff --git a/src/ufo/variabletransforms/Cal_RemapScanPosition.h b/src/ufo/variabletransforms/Cal_RemapScanPosition.h index c8f16c9d8..aa7b70db5 100644 --- a/src/ufo/variabletransforms/Cal_RemapScanPosition.h +++ b/src/ufo/variabletransforms/Cal_RemapScanPosition.h @@ -34,11 +34,10 @@ namespace ufo { class Cal_RemapScanPosition : public TransformBase { public: Cal_RemapScanPosition(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply); + const ObsFilterData &data, + const std::shared_ptr> &flags); // Run variable conversion - void runTransform() override; + void runTransform(const std::vector &apply) override; }; } // namespace ufo diff --git a/src/ufo/variabletransforms/Cal_Wind.cc b/src/ufo/variabletransforms/Cal_Wind.cc index e6b84795c..b5d1d5476 100644 --- a/src/ufo/variabletransforms/Cal_Wind.cc +++ b/src/ufo/variabletransforms/Cal_Wind.cc @@ -8,6 +8,9 @@ #include "ufo/variabletransforms/Cal_Wind.h" #include "ufo/utils/Constants.h" +#include "ufo/filters/VariableTransformsParameters.h" + + namespace ufo { /************************************************************************************/ @@ -18,14 +21,14 @@ static TransformMaker makerCal_WindSpeedAndDirection_("WindSpeedAndDirection"); Cal_WindSpeedAndDirection::Cal_WindSpeedAndDirection( - const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply) - : TransformBase(options, os, flags, apply) {} + const VariableTransformsParameters &options, const ObsFilterData &data, + const std::shared_ptr> &flags) + : TransformBase(options, data, flags) { +} /************************************************************************************/ -void Cal_WindSpeedAndDirection::runTransform() { +void Cal_WindSpeedAndDirection::runTransform(const std::vector &apply) { oops::Log::trace() << " --> Retrieve wind speed and direction" << std::endl; oops::Log::trace() << " --> method: " << method() << std::endl; @@ -54,7 +57,7 @@ void Cal_WindSpeedAndDirection::runTransform() { // Loop over all obs for (size_t jobs = 0; jobs < nlocs; ++jobs) { // if the data have been excluded by the where statement - if (!apply_[jobs]) continue; + if (!apply[jobs]) continue; // Calculate wind vector if (u[jobs] != missingValueFloat && v[jobs] != missingValueFloat) { @@ -74,14 +77,14 @@ static TransformMaker makerCal_WindComponents_("WindComponents"); Cal_WindComponents::Cal_WindComponents( - const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply) - : TransformBase(options, os, flags, apply) {} + const VariableTransformsParameters &options, + const ObsFilterData &data, + const std::shared_ptr> &flags) + : TransformBase(options, data, flags) {} /************************************************************************************/ -void Cal_WindComponents::runTransform() { +void Cal_WindComponents::runTransform(const std::vector &apply) { oops::Log::trace() << " --> Retrieve wind component" << std::endl; oops::Log::trace() << " --> method: " << method() << std::endl; @@ -110,7 +113,7 @@ void Cal_WindComponents::runTransform() { // Loop over all obs for (size_t jobs = 0; jobs < nlocs; ++jobs) { // if the data have been excluded by the where statement - if (!apply_[jobs]) continue; + if (!apply[jobs]) continue; // Calculate wind vector if (windFromDirection[jobs] != missingValueFloat && diff --git a/src/ufo/variabletransforms/Cal_Wind.h b/src/ufo/variabletransforms/Cal_Wind.h index 68c2e02f0..20b5367fb 100644 --- a/src/ufo/variabletransforms/Cal_Wind.h +++ b/src/ufo/variabletransforms/Cal_Wind.h @@ -31,11 +31,10 @@ namespace ufo { class Cal_WindSpeedAndDirection : public TransformBase { public: Cal_WindSpeedAndDirection(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply); + const ObsFilterData &data, + const std::shared_ptr> &flags); // Run variable conversion - void runTransform() override; + void runTransform(const std::vector &apply) override; }; /*! @@ -50,11 +49,10 @@ class Cal_WindSpeedAndDirection : public TransformBase { class Cal_WindComponents : public TransformBase { public: Cal_WindComponents(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply); + const ObsFilterData &data, + const std::shared_ptr> &flags); // Run check - void runTransform() override; + void runTransform(const std::vector &apply) override; }; } // namespace ufo diff --git a/src/ufo/variabletransforms/TransformBase.cc b/src/ufo/variabletransforms/TransformBase.cc index d52309eac..8a0b9b181 100644 --- a/src/ufo/variabletransforms/TransformBase.cc +++ b/src/ufo/variabletransforms/TransformBase.cc @@ -13,16 +13,15 @@ namespace ufo { TransformBase::TransformBase(const VariableTransformsParameters& options, - ioda::ObsSpace& os, - const std::shared_ptr>& flags, - const std::vector &apply) - : options_(options), obsdb_(os) , flags_(*flags), apply_(apply) { + const ObsFilterData& data, + const std::shared_ptr>& flags) + : options_(options), data_(data) , flags_(*flags) { method_ = formulas::resolveMethods(options.Method.value()); formulation_ = formulas::resolveFormulations(options.Formulation.value(), options.Method.value()); UseValidDataOnly_ = options.UseValidDataOnly.value(); AllowSuperSaturation_ = options.AllowSuperSaturation.value(); - obsName_ = os.obsname(); + obsName_ = data_.obsspace().obsname(); } TransformFactory::TransformFactory(const std::string& name) { @@ -35,8 +34,8 @@ TransformFactory::TransformFactory(const std::string& name) { std::unique_ptr TransformFactory::create( const std::string& name, const VariableTransformsParameters& options, - ioda::ObsSpace& os, const std::shared_ptr>& flags, - const std::vector& apply) { + const ObsFilterData& data, + const std::shared_ptr>& flags) { oops::Log::trace() << " --> TransformFactory::create" << std::endl; oops::Log::trace() << " --> name: " << name << std::endl; @@ -60,7 +59,7 @@ std::unique_ptr TransformFactory::create( Here()); } - std::unique_ptr ptr = jloc->second->make(options, os, flags, apply); + std::unique_ptr ptr = jloc->second->make(options, data, flags); oops::Log::trace() << "TransformBase::create done" << std::endl; return ptr; diff --git a/src/ufo/variabletransforms/TransformBase.h b/src/ufo/variabletransforms/TransformBase.h index 2bcb5b85f..15cc6d1af 100644 --- a/src/ufo/variabletransforms/TransformBase.h +++ b/src/ufo/variabletransforms/TransformBase.h @@ -28,10 +28,13 @@ #include "oops/util/ObjectCounter.h" #include "oops/util/PropertiesOfNVectors.h" +#include "ufo/filters/ObsFilterData.h" #include "ufo/filters/QCflags.h" +#include "ufo/filters/Variables.h" #include "ufo/filters/VariableTransformsParameters.h" #include "ufo/variabletransforms/Formulas.h" + namespace ioda { template class ObsDataVector; @@ -41,6 +44,7 @@ class ObsVector; namespace ufo { class VariableTransformsParameters; +class Variables; } namespace ufo { @@ -49,13 +53,14 @@ namespace ufo { class TransformBase { public: TransformBase(const VariableTransformsParameters &options, - ioda::ObsSpace &os, - const std::shared_ptr>& flags, - const std::vector& apply); + const ObsFilterData& data, + const std::shared_ptr>& flags); /// Destructor virtual ~TransformBase() {} /// Run variable conversion - virtual void runTransform() = 0; + virtual void runTransform(const std::vector &apply) = 0; + /// Return list of required geovals + virtual Variables requiredVariables() const { return Variables(); } private: /// templated function for float, int data types @@ -138,10 +143,12 @@ class TransformBase { std::string obsName() const { return obsName_; } /// Configurable parameters const VariableTransformsParameters &options_; + + /// Observation and geoval data + ObsFilterData data_; /// Observation space - ioda::ObsSpace &obsdb_; + ioda::ObsSpace &obsdb_ = data_.obsspace(); ioda::ObsDataVector &flags_; - const std::vector &apply_; /// Missing value (int) const int missingValueInt = util::missingValue(1); /// Missing value (float) @@ -155,8 +162,8 @@ class TransformFactory { public: static std::unique_ptr create( const std::string &, const VariableTransformsParameters &, - ioda::ObsSpace &, const std::shared_ptr> &, - const std::vector &); + const ObsFilterData &, + const std::shared_ptr> &); virtual ~TransformFactory() = default; protected: @@ -164,9 +171,9 @@ class TransformFactory { private: virtual std::unique_ptr make( - const VariableTransformsParameters &, ioda::ObsSpace &, - const std::shared_ptr> &, - const std::vector &) = 0; + const VariableTransformsParameters &, + const ObsFilterData &, + const std::shared_ptr> &) = 0; static std::map &getMakers() { static std::map makers_; return makers_; @@ -177,10 +184,10 @@ class TransformFactory { template class TransformMaker : public TransformFactory { virtual std::unique_ptr make( - const VariableTransformsParameters &options, ioda::ObsSpace &os, - const std::shared_ptr> &flags, - const std::vector &apply) { - return std::unique_ptr(new T(options, os, flags, apply)); + const VariableTransformsParameters &options, + const ObsFilterData& data, + const std::shared_ptr> &flags) { + return std::unique_ptr(new T(options, data, flags)); } public: From 94cba9c08c823ce315661e33d1034f029d7cf96c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Thu, 12 Aug 2021 15:24:59 +0100 Subject: [PATCH 82/98] Fix bugs in ObsFilterData (#1432) * Fix retrieval of variables stored in ObsDataVectors associated with ObsFilterData. * Fix ObsFilterData::has() and dtype() for variables stored in associated ObsDataVectors. * Test the recent fixes. * Test the overload of get() taking an ObsDataVector for ObsSpace variables and components of associated ObsVectors. * Reduced code duplication. Co-authored-by: Ryan Honeyager --- src/ufo/filters/ObsFilterData.cc | 17 +++- test/testinput/obsfilterdata.yaml | 16 ++++ test/ufo/ObsFilterData.h | 124 ++++++++++++++++++++++-------- 3 files changed, 124 insertions(+), 33 deletions(-) diff --git a/src/ufo/filters/ObsFilterData.cc b/src/ufo/filters/ObsFilterData.cc index 22a224899..3fb1870bd 100644 --- a/src/ufo/filters/ObsFilterData.cc +++ b/src/ufo/filters/ObsFilterData.cc @@ -90,7 +90,10 @@ bool ObsFilterData::has(const Variable & varname) const { } else if (grp == "ObsDiag" || grp == "ObsBiasTerm") { return (diags_ && diags_->has(var)); } else { - return this->hasVector(grp, var) || this->hasDataVector(grp, var) || obsdb_.has(grp, var); + return this->hasVector(grp, var) || + this->hasDataVector(grp, var) || + this->hasDataVectorInt(grp, var) || + obsdb_.has(grp, var); } return false; } @@ -227,7 +230,10 @@ void ObsFilterData::get(const Variable & varname, ioda::ObsDataVector & v } else if (this->hasDataVector(grp, var)) { std::map *>::const_iterator jv = dvecsf_.find(grp); - values = *jv->second; + for (size_t ivar = 0; ivar < varname.size(); ++ivar) { + const std::string currvar = varname.variable(ivar); + values[currvar] = (*jv->second)[currvar]; + } /// Produce a clear error message for incompatible ObsFunction types } else if (eckit::StringTools::endsWith(grp, "ObsFunction")) { throw eckit::BadParameter("ObsFilterData::get(): " + varname.fullName() + @@ -249,7 +255,10 @@ void ObsFilterData::get(const Variable & varname, ioda::ObsDataVector & val /// For ObsDataVector } else if (this->hasDataVectorInt(grp, var)) { std::map *>::const_iterator jv = dvecsi_.find(grp); - values = *jv->second; + for (size_t ivar = 0; ivar < varname.size(); ++ivar) { + const std::string currvar = varname.variable(ivar); + values[currvar] = (*jv->second)[currvar]; + } /// Produce a clear error message for incompatible ObsFunction types } else if (eckit::StringTools::endsWith(grp, "ObsFunction")) { throw eckit::BadParameter("ObsFilterData::get(): " + varname.fullName() + @@ -352,6 +361,8 @@ ioda::ObsDtype ObsFilterData::dtype(const Variable & varname) const { ioda::ObsDtype res = ioda::ObsDtype::Float; if (obsdb_.has(grp, var)) { res = obsdb_.dtype(grp, var); + } else if (hasDataVectorInt(varname.group(), varname.variable())) { + res = ioda::ObsDtype::Integer; } else if (!this->has(varname)) { oops::Log::error() << "ObsFilterData::dtype unable to find provided variable." << std::endl; diff --git a/test/testinput/obsfilterdata.yaml b/test/testinput/obsfilterdata.yaml index 6315cd005..c69a08586 100644 --- a/test/testinput/obsfilterdata.yaml +++ b/test/testinput/obsfilterdata.yaml @@ -33,6 +33,14 @@ obs filter data: variables: - name: eastward_wind@HofX - name: northward_wind@HofX + ObsError: + variables: + - name: eastward_wind@ObsErrorData + - name: northward_wind@ObsErrorData + QCFlags: + variables: + - name: eastward_wind@QCFlagsData + - name: northward_wind@QCFlagsData - obs space: name: amsua_n19 obsdatain: @@ -70,3 +78,11 @@ obs filter data: variables: - name: brightness_temperature@HofX channels: 1-10,15 + ObsError: + variables: + - name: brightness_temperature@ObsErrorData + channels: 1-10,15 + QCFlags: + variables: + - name: brightness_temperature@QCFlagsData + channels: 1-10,15 diff --git a/test/ufo/ObsFilterData.h b/test/ufo/ObsFilterData.h index 0c4aeb092..1b4706b8b 100644 --- a/test/ufo/ObsFilterData.h +++ b/test/ufo/ObsFilterData.h @@ -28,6 +28,30 @@ namespace ufo { namespace test { +// ----------------------------------------------------------------------------- + +template +void testHasDtypeAndGet(const ufo::ObsFilterData& data, ioda::ObsSpace &ospace, + const ufo::Variable &var, + ioda::ObsDtype expectedDtype, const std::vector &expectedValues) { + EXPECT(data.has(var)); + EXPECT(data.dtype(var) == expectedDtype); + + // Extract variable into an std::vector + { + std::vector vec; + data.get(var, vec); + EXPECT(vec == expectedValues); + } + + // Extract variable into an ObsDataVector + { + ioda::ObsDataVector vec(ospace, var.variable()); + data.get(var, vec); + EXPECT_EQUAL(vec.nvars(), 1); + EXPECT(vec[0] == expectedValues); + } +} // ----------------------------------------------------------------------------- @@ -62,6 +86,24 @@ void testObsFilterData() { const std::string hofxgroup = hofxconf.getString("group"); ioda::ObsVector hofx(ospace, hofxgroup); +/// Setup ObsErrors + const eckit::LocalConfiguration obserrorconf(confs[jconf], "ObsError"); + varconfs.clear(); + obserrorconf.get("variables", varconfs); + const Variables obserrorvars(varconfs); + ioda::ObsDataVector obserrors(ospace, obserrorvars.toOopsVariables()); + for (size_t i = 0; i < obserrors.nvars(); ++i) + std::fill(obserrors[i].begin(), obserrors[i].end(), 0.1f * i); + +/// Setup QCFlags + const eckit::LocalConfiguration qcflagsconf(confs[jconf], "QCFlags"); + varconfs.clear(); + qcflagsconf.get("variables", varconfs); + const Variables qcflagsvars(varconfs); + ioda::ObsDataVector qcflags(ospace, qcflagsvars.toOopsVariables()); + for (size_t i = 0; i < qcflags.nvars(); ++i) + std::fill(qcflags[i].begin(), qcflags[i].end(), i); + /// Setup ObsFilterData and test nlocs ObsFilterData data(ospace); EXPECT(data.nlocs() == ospace.nlocs()); @@ -72,54 +114,51 @@ void testObsFilterData() { dataconf.get("float variables", varconfs); ufo::Variables obsvars(varconfs); for (size_t jvar = 0; jvar < obsvars.nvars(); ++jvar) { - EXPECT(data.has(obsvars.variable(jvar))); - - EXPECT(data.dtype(obsvars.variable(jvar)) == ioda::ObsDtype::Float); + const ufo::Variable &var = obsvars.variable(jvar); - std::vector vec; - data.get(obsvars.variable(jvar), vec); std::vector ref(ospace.nlocs()); - ospace.get_db(obsvars.variable(jvar).group(), obsvars.variable(jvar).variable(), ref); - EXPECT(vec == ref); + ospace.get_db(var.group(), var.variable(), ref); + + testHasDtypeAndGet(data, ospace, var, ioda::ObsDtype::Float, ref); } + /// Check that has(), get() and dtype() work on integer variables in ObsSpace: varconfs.clear(); dataconf.get("integer variables", varconfs); ufo::Variables intvars(varconfs); for (size_t jvar = 0; jvar < intvars.nvars(); ++jvar) { - EXPECT(data.has(intvars.variable(jvar))); + const ufo::Variable &var = intvars.variable(jvar); - EXPECT(data.dtype(intvars.variable(jvar)) == ioda::ObsDtype::Integer); - - std::vector vec; - data.get(intvars.variable(jvar), vec); std::vector ref(ospace.nlocs()); - ospace.get_db(intvars.variable(jvar).group(), intvars.variable(jvar).variable(), ref); - EXPECT(vec == ref); + ospace.get_db(var.group(), var.variable(), ref); + + testHasDtypeAndGet(data, ospace, var, ioda::ObsDtype::Integer, ref); } - /// Check that get() works on string variables in ObsSpace: +/// Check that has(), get() and dtype() work on string variables in ObsSpace: varconfs.clear(); dataconf.get("string variables", varconfs); ufo::Variables strvars(varconfs); for (size_t jvar = 0; jvar < strvars.nvars(); ++jvar) { - std::vector vec; - data.get(strvars.variable(jvar), vec); + const ufo::Variable &var = strvars.variable(jvar); + std::vector ref(ospace.nlocs()); - ospace.get_db(strvars.variable(jvar).group(), strvars.variable(jvar).variable(), ref); - EXPECT(vec == ref); + ospace.get_db(var.group(), var.variable(), ref); + + testHasDtypeAndGet(data, ospace, var, ioda::ObsDtype::String, ref); } - /// Check that get() works on datetime variables in ObsSpace: +/// Check that has(), get() and dtype() work on datetime variables in ObsSpace: varconfs.clear(); dataconf.get("datetime variables", varconfs); ufo::Variables dtvars(varconfs); for (size_t jvar = 0; jvar < dtvars.nvars(); ++jvar) { - std::vector vec; - data.get(dtvars.variable(jvar), vec); + const ufo::Variable &var = dtvars.variable(jvar); + std::vector ref(ospace.nlocs()); - ospace.get_db(dtvars.variable(jvar).group(), dtvars.variable(jvar).variable(), ref); - EXPECT(vec == ref); + ospace.get_db(var.group(), var.variable(), ref); + + testHasDtypeAndGet(data, ospace, var, ioda::ObsDtype::DateTime, ref); } /// Check that associate(), has(), get() and dtype() work on ObsVector: @@ -135,17 +174,42 @@ void testObsFilterData() { data.associate(hofx, "HofX"); /// H(x) associated now for (size_t jvar = 0; jvar < hofxvars.nvars(); ++jvar) { - EXPECT(data.has(hofxvars.variable(jvar))); - - EXPECT(data.dtype(hofxvars.variable(jvar)) == ioda::ObsDtype::Float); + const ufo::Variable &var = hofxvars.variable(jvar); - std::vector vec; - data.get(hofxvars.variable(jvar), vec); std::vector ref(hofx.nlocs()); for (size_t jloc = 0; jloc < hofx.nlocs(); jloc++) { ref[jloc] = hofx[hofxvars.nvars() * jloc + jvar]; } - EXPECT(vec == ref); + + testHasDtypeAndGet(data, ospace, var, ioda::ObsDtype::Float, ref); + } + +/// Check that associate(), has(), get() and dtype() work on ObsDataVector: + for (size_t jvar = 0; jvar < obserrorvars.size(); ++jvar) { + EXPECT(!data.has(obserrorvars[jvar])); + } + data.associate(obserrors, "ObsErrorData"); +/// ObsErrorData associated now + for (size_t jvar = 0; jvar < obserrorvars.nvars(); ++jvar) { + const ufo::Variable &var = obserrorvars.variable(jvar); + + const std::vector &ref = obserrors[var.variable()]; + + testHasDtypeAndGet(data, ospace, var, ioda::ObsDtype::Float, ref); + } + +/// Check that associate(), has(), get() and dtype() work on ObsDataVector: + for (size_t jvar = 0; jvar < qcflagsvars.size(); ++jvar) { + EXPECT(!data.has(qcflagsvars[jvar])); + } + data.associate(qcflags, "QCFlagsData"); +/// QCFlags associated now + for (size_t jvar = 0; jvar < qcflagsvars.nvars(); ++jvar) { + const ufo::Variable &var = qcflagsvars.variable(jvar); + + const std::vector &ref = qcflags[var.variable()]; + + testHasDtypeAndGet(data, ospace, var, ioda::ObsDtype::Integer, ref); } /// Check that associate(), has() and get() work on GeoVaLs: From 110cf27cc213a7cc9cf7b058c45f1e1cc7ae1a5d Mon Sep 17 00:00:00 2001 From: Brett Candy <67381785+brettcandy@users.noreply.github.com> Date: Thu, 12 Aug 2021 21:05:57 +0100 Subject: [PATCH 83/98] Update surface filter to handle SSMIS surface flag (#1403) * updated surface filter for ssmis report type * updated surface type code that passes ctests * update branch to include ctest for this modification * update * rename variable for reported surface type so it follows value@group * Restart CI Co-authored-by: Ryan Honeyager --- .../filters/obsfunctions/SetSurfaceType.cc | 23 +++++++++++-------- src/ufo/filters/obsfunctions/SetSurfaceType.h | 6 +++++ test/CMakeLists.txt | 8 +++++++ .../function_setsurfacetype_ssmis.yaml | 22 ++++++++++++++++++ 4 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 test/testinput/function_setsurfacetype_ssmis.yaml diff --git a/src/ufo/filters/obsfunctions/SetSurfaceType.cc b/src/ufo/filters/obsfunctions/SetSurfaceType.cc index 4ab412588..abf8b1443 100644 --- a/src/ufo/filters/obsfunctions/SetSurfaceType.cc +++ b/src/ufo/filters/obsfunctions/SetSurfaceType.cc @@ -37,7 +37,7 @@ namespace ufo { invars_ += Variable("latitude@MetaData"); if (options_.UseReportSurface.value()) { - invars_ += Variable("land_sea@MetaData"); + invars_ += Variable(options_.SurfaceMetaDataName.value()); } if (options_.UseReportElevation.value()) { @@ -94,20 +94,23 @@ namespace ufo { } // if available and requested, set closest appropriate surface type using reported surface - if (options_.UseReportSurface.value() && in.has(Variable("land_sea@MetaData"))) { - std::vector reported_land_sea(nlocs); - in.get(Variable("land_sea@MetaData"), reported_land_sea); + if (options_.UseReportSurface.value()) { + std::vector reported_surftype(nlocs, surftype_invalid); + + // load reported surf type takinginto account variable name from the yaml config file + in.get(Variable(options_.SurfaceMetaDataName.value()), reported_surftype); for (size_t iloc = 0; iloc < nlocs; ++iloc) { - if ( reported_land_sea[iloc] == AAPP_surftype::sea || - reported_land_sea[iloc] == BUFR_surftype::posice ) { + if ( reported_surftype[iloc] == AAPP_surftype::sea || + reported_surftype[iloc] == BUFR_surftype::ocean || + reported_surftype[iloc] == BUFR_surftype::posice ) { surftype[iloc] = surftype_sea_; - } else if ( reported_land_sea[iloc] == BUFR_surftype::land ) { + } else if ( reported_surftype[iloc] == BUFR_surftype::land ) { surftype[iloc] = surftype_land_; - } else if ( reported_land_sea[iloc] == BUFR_surftype::ice ) { + } else if ( reported_surftype[iloc] == BUFR_surftype::ice ) { surftype[iloc] = surftype_seaice_; - } else if ( reported_land_sea[iloc] == BUFR_surftype::coast || - reported_land_sea[iloc] == BUFR_surftype::nrcoast) { + } else if ( reported_surftype[iloc] == BUFR_surftype::coast || + reported_surftype[iloc] == BUFR_surftype::nrcoast) { surftype[iloc] = land_sea[iloc]; } } diff --git a/src/ufo/filters/obsfunctions/SetSurfaceType.h b/src/ufo/filters/obsfunctions/SetSurfaceType.h index 92041e61e..c3bc198da 100644 --- a/src/ufo/filters/obsfunctions/SetSurfaceType.h +++ b/src/ufo/filters/obsfunctions/SetSurfaceType.h @@ -44,6 +44,12 @@ class SetSurfaceTypeParameters : public oops::Parameters { /// UseReportSurface: true oops::Parameter UseReportSurface{"UseReportSurface", false, this}; + /// Change name of variable storing the reported surface type + /// Default is 'land_sea@MetaData' + /// Example: to set to SSMIS report type (which is surface_flag@MetaData) set + /// SurfaceReport Name: surface_flag@MetaData + oops::Parameter SurfaceMetaDataName{"SurfaceReport Name", "land_sea@MetaData", this}; + /// Use reported Surface Elevation (default false) /// Example: To use the AAPPreported elevation set /// UseReportElevation: true diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0de0154c4..9cda21466 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -84,6 +84,7 @@ list( APPEND ufo_test_input testinput/function_obserrmean_atms.yaml testinput/function_reperr.yaml testinput/function_setsurfacetype.yaml + testinput/function_setsurfacetype_ssmis.yaml testinput/function_satwind_indiv_errors.yaml testinput/function_scatret.yaml testinput/function_scatret_atms.yaml @@ -2912,6 +2913,13 @@ ecbuild_add_test( TARGET test_ufo_function_setsurfacetype DEPENDS test_ObsFunction.x TEST_DEPENDS ufo_get_ioda_test_data ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_function_setsurfacetype_ssmis + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x + ARGS "testinput/function_setsurfacetype_ssmis.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFunction.x + TEST_DEPENDS ufo_get_ufo_test_data ) + ecbuild_add_test( TARGET test_ufo_function_modelsurfaceheightadjust_windvector COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x ARGS "testinput/unit_tests/function_ModelSurfaceHeightAdjustWindVector.yaml" diff --git a/test/testinput/function_setsurfacetype_ssmis.yaml b/test/testinput/function_setsurfacetype_ssmis.yaml new file mode 100644 index 000000000..fd7f73f84 --- /dev/null +++ b/test/testinput/function_setsurfacetype_ssmis.yaml @@ -0,0 +1,22 @@ +window begin: 2021-05-21T08:59:59Z +window end: 2021-05-21T15:00:00Z + +observations: +- obs space: + name: ssmis_f17 + obsdatain: + obsfile: Data/ufo/testinput_tier_1/ssmis_f17_20210521T1200Z_obfn_ref.nc4 + obsdataout: + obsfile: Data/ssmis_f17_obs_20210521T1200Z_obfn_out.nc4 + simulated variables: [brightness_temperature] + channels: &channels 1-24 + geovals: + filename: + Data/ufo/testinput_tier_1/geovals_ssmis_f17_20210521T1200Z_obfn_ref.nc4 + obs function: + name: SetSurfaceType@ObsFunction + options: + UseReportSurface: true + SurfaceReport Name: surface_flag@MetaData + variables: [surface_type] + tolerance: 1.0e-7 From 16219ec3d07875648a3839de5f3b4ab2bd81fdb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Mon, 16 Aug 2021 19:11:04 +0100 Subject: [PATCH 84/98] Pass ioda::ObsTopLevelParameters rather than Configuration to the constructor of ioda::ObsSpace. (#1406) Co-authored-by: Ryan Honeyager Co-authored-by: Stephen Herbener <32968781+srherbener@users.noreply.github.com> --- src/ufo/Locations.cc | 4 +++- src/ufo/filters/HistoryCheckParameters.h | 4 +++- test/ufo/ConventionalProfileProcessing.h | 4 +++- test/ufo/GaussianThinning.h | 4 +++- test/ufo/GeoVaLs.h | 4 +++- test/ufo/HistoryCheck.h | 4 +++- test/ufo/Locations.h | 5 ++++- test/ufo/MetOfficeBuddyCheck.h | 4 +++- test/ufo/MetOfficeBuddyPairFinder.h | 4 +++- test/ufo/ObsBias.h | 5 +++-- test/ufo/ObsBiasCovarianceDetails.h | 5 +++-- test/ufo/ObsDiagnostics.h | 4 +++- test/ufo/ObsErrorAssign.h | 4 +++- test/ufo/ObsFilterData.h | 4 +++- test/ufo/ObsFilters.h | 3 +-- test/ufo/OperatorUtils.h | 5 ++++- test/ufo/ParallelPoissonDiskThinning.h | 4 +++- test/ufo/PoissonDiskThinning.h | 4 +++- test/ufo/PrimitiveVariables.h | 5 ++++- test/ufo/ProcessWhere.h | 5 +++-- test/ufo/StuckCheck.h | 4 +++- test/ufo/TemporalThinning.h | 4 +++- test/ufo/TrackCheck.h | 4 +++- test/ufo/TrackCheckShip.h | 4 +++- test/ufo/TrackCheckShipMainLoop.h | 4 +++- 25 files changed, 76 insertions(+), 29 deletions(-) diff --git a/src/ufo/Locations.cc b/src/ufo/Locations.cc index 0cd88f931..763b5c5b3 100644 --- a/src/ufo/Locations.cc +++ b/src/ufo/Locations.cc @@ -57,8 +57,10 @@ Locations::Locations(const eckit::Configuration & conf, const eckit::LocalConfiguration obsconf(conf, "obs space"); const util::DateTime bgn = util::DateTime(conf.getString("window begin")); const util::DateTime end = util::DateTime(conf.getString("window end")); + ioda::ObsTopLevelParameters obsparams; + obsparams.validateAndDeserialize(obsconf); - ioda::ObsSpace obspace(obsconf, comm, bgn, end, oops::mpi::myself()); + ioda::ObsSpace obspace(obsparams, comm, bgn, end, oops::mpi::myself()); const size_t nlocs = obspace.nlocs(); dist_ = obspace.distribution(); diff --git a/src/ufo/filters/HistoryCheckParameters.h b/src/ufo/filters/HistoryCheckParameters.h index e3ce62c5a..abfd45612 100644 --- a/src/ufo/filters/HistoryCheckParameters.h +++ b/src/ufo/filters/HistoryCheckParameters.h @@ -9,6 +9,8 @@ #include +#include "ioda/ObsSpaceParameters.h" + #include "oops/util/Duration.h" #include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/Parameters.h" @@ -52,7 +54,7 @@ class HistoryCheckParameters : public TrackCheckUtilsParameters { /// Creates a new obs space with the wider window that is determined by the observation subtype. /// Needs: name (can be set with setValue), simulated variables, obsdatain.obsfile. - oops::RequiredParameter largerObsSpace { + oops::RequiredParameter largerObsSpace { "obs space", this }; diff --git a/test/ufo/ConventionalProfileProcessing.h b/test/ufo/ConventionalProfileProcessing.h index e7181bb96..8a591ccae 100644 --- a/test/ufo/ConventionalProfileProcessing.h +++ b/test/ufo/ConventionalProfileProcessing.h @@ -60,7 +60,9 @@ void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); - ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsParams; + obsParams.validateAndDeserialize(obsSpaceConf); + ioda::ObsSpace obsspace(obsParams, oops::mpi::world(), bgn, end, oops::mpi::myself()); const Variables filtervars = Variables(obsspace.obsvariables()); diff --git a/test/ufo/GaussianThinning.h b/test/ufo/GaussianThinning.h index 2fb0444e4..9495a1151 100644 --- a/test/ufo/GaussianThinning.h +++ b/test/ufo/GaussianThinning.h @@ -34,7 +34,9 @@ void testGaussianThinning(const eckit::LocalConfiguration &conf) { util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); - ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsParams; + obsParams.validateAndDeserialize(obsSpaceConf); + ioda::ObsSpace obsspace(obsParams, oops::mpi::world(), bgn, end, oops::mpi::myself()); if (conf.has("air_pressures")) { const std::vector air_pressures = conf.getFloatVector("air_pressures"); diff --git a/test/ufo/GeoVaLs.h b/test/ufo/GeoVaLs.h index 75b9c1152..513d28239 100644 --- a/test/ufo/GeoVaLs.h +++ b/test/ufo/GeoVaLs.h @@ -44,7 +44,9 @@ void testGeoVaLs() { for (size_t jconf = 0; jconf < confs.size(); ++jconf) { /// Setup ObsSpace const eckit::LocalConfiguration obsconf(confs[jconf], "obs space"); - ioda::ObsSpace ospace(obsconf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsparams; + obsparams.validateAndDeserialize(obsconf); + ioda::ObsSpace ospace(obsparams, oops::mpi::world(), bgn, end, oops::mpi::myself()); /// Setup GeoVaLs const eckit::LocalConfiguration gconf(confs[jconf], "geovals"); diff --git a/test/ufo/HistoryCheck.h b/test/ufo/HistoryCheck.h index 0b59ebf48..ee63d1d3e 100644 --- a/test/ufo/HistoryCheck.h +++ b/test/ufo/HistoryCheck.h @@ -31,7 +31,9 @@ void testHistoryCheck(const eckit::LocalConfiguration &conf) { util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); - ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsParams; + obsParams.validateAndDeserialize(obsSpaceConf); + ioda::ObsSpace obsspace(obsParams, oops::mpi::world(), bgn, end, oops::mpi::myself()); eckit::LocalConfiguration filterConf(conf, "History Check"); ufo::HistoryCheckParameters filterParameters; diff --git a/test/ufo/Locations.h b/test/ufo/Locations.h index 3f7655bfb..6b9d6912e 100644 --- a/test/ufo/Locations.h +++ b/test/ufo/Locations.h @@ -72,7 +72,10 @@ class LocationsTestFixture : private boost::noncopyable { util::DateTime bgn(conf.getString("window begin")); util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsconf(conf, "obs space"); - obsspace_.reset(new ioda::ObsSpace(obsconf, oops::mpi::world(), bgn, end, oops::mpi::myself())); + ioda::ObsTopLevelParameters obsparams; + obsparams.validateAndDeserialize(obsconf); + obsspace_.reset(new ioda::ObsSpace(obsparams, oops::mpi::world(), bgn, end, + oops::mpi::myself())); testconfig_ = conf.getSubConfiguration("locations test"); testparams_.validateAndDeserialize(testconfig_); } diff --git a/test/ufo/MetOfficeBuddyCheck.h b/test/ufo/MetOfficeBuddyCheck.h index 399647b29..779b83a0e 100644 --- a/test/ufo/MetOfficeBuddyCheck.h +++ b/test/ufo/MetOfficeBuddyCheck.h @@ -37,7 +37,9 @@ void testMetOfficeBuddyCheck(const eckit::LocalConfiguration &conf) { util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); - ioda::ObsSpace obsSpace(obsSpaceConf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsParams; + obsParams.validateAndDeserialize(obsSpaceConf); + ioda::ObsSpace obsSpace(obsParams, oops::mpi::world(), bgn, end, oops::mpi::myself()); const eckit::LocalConfiguration floatVarInitConf(conf, "FloatVariables"); for (const std::string & varNameGroup : floatVarInitConf.keys()) { diff --git a/test/ufo/MetOfficeBuddyPairFinder.h b/test/ufo/MetOfficeBuddyPairFinder.h index f2c45916f..4c321d97e 100644 --- a/test/ufo/MetOfficeBuddyPairFinder.h +++ b/test/ufo/MetOfficeBuddyPairFinder.h @@ -96,7 +96,9 @@ void testDuplicatesAndBuddyCountConstraints(const eckit::LocalConfiguration &con util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); - ioda::ObsSpace obsSpace(obsSpaceConf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsParams; + obsParams.validateAndDeserialize(obsSpaceConf); + ioda::ObsSpace obsSpace(obsParams, oops::mpi::world(), bgn, end, oops::mpi::myself()); boost::optional> airPressures; if (obsSpace.has("MetaData", "air_pressure")) { diff --git a/test/ufo/ObsBias.h b/test/ufo/ObsBias.h index f41ba878f..f4a392eca 100644 --- a/test/ufo/ObsBias.h +++ b/test/ufo/ObsBias.h @@ -40,8 +40,9 @@ void testObsBiasReadWrite() { = conf.getSubConfigurations("observations"); for (auto & oconf : obsconfs) { - ioda::ObsSpace odb(oconf.getSubConfiguration("obs space"), oops::mpi::world(), - bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsparams; + obsparams.validateAndDeserialize(oconf.getSubConfiguration("obs space")); + ioda::ObsSpace odb(obsparams, oops::mpi::world(), bgn, end, oops::mpi::myself()); // setup ObsBias parameters eckit::LocalConfiguration biasconf = oconf.getSubConfiguration("obs bias"); diff --git a/test/ufo/ObsBiasCovarianceDetails.h b/test/ufo/ObsBiasCovarianceDetails.h index 5efe25f70..f5f18abef 100644 --- a/test/ufo/ObsBiasCovarianceDetails.h +++ b/test/ufo/ObsBiasCovarianceDetails.h @@ -40,8 +40,9 @@ void testObsBiasCovarianceDetails() { = conf.getSubConfigurations("observations"); for (auto & oconf : obsconfs) { - ioda::ObsSpace odb(oconf.getSubConfiguration("obs space"), oops::mpi::world(), - bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsparams; + obsparams.validateAndDeserialize(oconf.getSubConfiguration("obs space")); + ioda::ObsSpace odb(obsparams, oops::mpi::world(), bgn, end, oops::mpi::myself()); // Setup ObsBias eckit::LocalConfiguration biasconf = oconf.getSubConfiguration("obs bias"); diff --git a/test/ufo/ObsDiagnostics.h b/test/ufo/ObsDiagnostics.h index 36b8449dd..e492224f0 100644 --- a/test/ufo/ObsDiagnostics.h +++ b/test/ufo/ObsDiagnostics.h @@ -48,7 +48,9 @@ void testObsDiagnostics() { util::DateTime bgn(conf.getString("window begin")); util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsconf(conf, "obs space"); - ioda::ObsSpace ospace(obsconf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsparams; + obsparams.validateAndDeserialize(obsconf); + ioda::ObsSpace ospace(obsparams, oops::mpi::world(), bgn, end, oops::mpi::myself()); const size_t nlocs = ospace.nlocs(); // initialize observation operator (set variables requested from the model, diff --git a/test/ufo/ObsErrorAssign.h b/test/ufo/ObsErrorAssign.h index 6319c21e7..325b7e133 100644 --- a/test/ufo/ObsErrorAssign.h +++ b/test/ufo/ObsErrorAssign.h @@ -34,7 +34,9 @@ void testObsErrorAssign(const eckit::LocalConfiguration &conf) { util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); - ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsParams; + obsParams.validateAndDeserialize(obsSpaceConf); + ioda::ObsSpace obsspace(obsParams, oops::mpi::world(), bgn, end, oops::mpi::myself()); std::vector varnames {"air_pressure", "station_id", "observation_type", "latitude_band", "channel_number", "processing_center", diff --git a/test/ufo/ObsFilterData.h b/test/ufo/ObsFilterData.h index 1b4706b8b..7a3ada56d 100644 --- a/test/ufo/ObsFilterData.h +++ b/test/ufo/ObsFilterData.h @@ -65,7 +65,9 @@ void testObsFilterData() { for (size_t jconf = 0; jconf < confs.size(); ++jconf) { /// Setup ObsSpace const eckit::LocalConfiguration obsconf(confs[jconf], "obs space"); - ioda::ObsSpace ospace(obsconf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsparams; + obsparams.validateAndDeserialize(obsconf); + ioda::ObsSpace ospace(obsparams, oops::mpi::world(), bgn, end, oops::mpi::myself()); /// Setup GeoVaLs const eckit::LocalConfiguration gconf(confs[jconf], "geovals"); diff --git a/test/ufo/ObsFilters.h b/test/ufo/ObsFilters.h index 51c1b57d2..4ac5f9df4 100644 --- a/test/ufo/ObsFilters.h +++ b/test/ufo/ObsFilters.h @@ -91,8 +91,7 @@ class ObsTypeParameters : public oops::Parameters { public: /// Options used to configure the observation space. - oops::Parameter obsSpace{ - "obs space", eckit::LocalConfiguration(), this}; + oops::Parameter obsSpace{"obs space", {}, this}; /// Options used to configure observation filters. oops::Parameter>> obsFilters{ diff --git a/test/ufo/OperatorUtils.h b/test/ufo/OperatorUtils.h index e429b6b7d..9749c92fa 100644 --- a/test/ufo/OperatorUtils.h +++ b/test/ufo/OperatorUtils.h @@ -44,7 +44,10 @@ class TestFixture : private boost::noncopyable { util::DateTime bgn(conf.getString("window begin")); util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsconf(conf, "obs space"); - obsspace_.reset(new ioda::ObsSpace(obsconf, oops::mpi::world(), bgn, end, oops::mpi::myself())); + ioda::ObsTopLevelParameters obsparams; + obsparams.validateAndDeserialize(obsconf); + obsspace_.reset(new ioda::ObsSpace(obsparams, oops::mpi::world(), bgn, end, + oops::mpi::myself())); config_ = conf.getSubConfiguration("cases"); } diff --git a/test/ufo/ParallelPoissonDiskThinning.h b/test/ufo/ParallelPoissonDiskThinning.h index c7a468ce9..6202ec366 100644 --- a/test/ufo/ParallelPoissonDiskThinning.h +++ b/test/ufo/ParallelPoissonDiskThinning.h @@ -47,7 +47,9 @@ void testPoissonDiskThinning(const eckit::LocalConfiguration &conf, util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); - ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsParams; + obsParams.validateAndDeserialize(obsSpaceConf); + ioda::ObsSpace obsspace(obsParams, oops::mpi::world(), bgn, end, oops::mpi::myself()); std::shared_ptr> obserr(new ioda::ObsDataVector( obsspace, obsspace.obsvariables(), "ObsError")); diff --git a/test/ufo/PoissonDiskThinning.h b/test/ufo/PoissonDiskThinning.h index 1f30fd967..c163fe6df 100644 --- a/test/ufo/PoissonDiskThinning.h +++ b/test/ufo/PoissonDiskThinning.h @@ -33,7 +33,9 @@ void testPoissonDiskThinning(const eckit::LocalConfiguration &conf, util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); - ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsParams; + obsParams.validateAndDeserialize(obsSpaceConf); + ioda::ObsSpace obsspace(obsParams, oops::mpi::world(), bgn, end, oops::mpi::myself()); if (conf.has("air_pressures")) { const std::vector air_pressures = conf.getFloatVector("air_pressures"); diff --git a/test/ufo/PrimitiveVariables.h b/test/ufo/PrimitiveVariables.h index 4a1868b99..16b16a2ac 100644 --- a/test/ufo/PrimitiveVariables.h +++ b/test/ufo/PrimitiveVariables.h @@ -66,7 +66,10 @@ class TestFixture : private boost::noncopyable { util::DateTime bgn(conf.getString("window begin")); util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsconf(conf, "obs space"); - obsspace_.reset(new ioda::ObsSpace(obsconf, oops::mpi::world(), bgn, end, oops::mpi::myself())); + ioda::ObsTopLevelParameters obsparams; + obsparams.validateAndDeserialize(obsconf); + obsspace_.reset(new ioda::ObsSpace(obsparams, oops::mpi::world(), bgn, end, + oops::mpi::myself())); data_.reset(new ObsFilterData(*obsspace_)); } diff --git a/test/ufo/ProcessWhere.h b/test/ufo/ProcessWhere.h index 4cfb90778..713a9e164 100644 --- a/test/ufo/ProcessWhere.h +++ b/test/ufo/ProcessWhere.h @@ -46,8 +46,9 @@ void testProcessWhere(const eckit::LocalConfiguration &conf, util::DateTime end(conf.getString("window end")); eckit::LocalConfiguration obsconf(conf, "obs space"); - - ioda::ObsSpace ospace(obsconf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsparams; + obsparams.validateAndDeserialize(obsconf); + ioda::ObsSpace ospace(obsparams, oops::mpi::world(), bgn, end, oops::mpi::myself()); ObsFilterData data(ospace); const int nlocs = conf.getInt("nlocs"); diff --git a/test/ufo/StuckCheck.h b/test/ufo/StuckCheck.h index 33edcaeab..f415c7bbd 100644 --- a/test/ufo/StuckCheck.h +++ b/test/ufo/StuckCheck.h @@ -34,7 +34,9 @@ void testStuckCheck(const eckit::LocalConfiguration &conf) { util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); - ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsParams; + obsParams.validateAndDeserialize(obsSpaceConf); + ioda::ObsSpace obsspace(obsParams, oops::mpi::world(), bgn, end, oops::mpi::myself()); if (conf.has("air_temperatures")) { const std::vector airTemperatures = conf.getFloatVector("air_temperatures"); diff --git a/test/ufo/TemporalThinning.h b/test/ufo/TemporalThinning.h index 1a0a2d149..8e7585f49 100644 --- a/test/ufo/TemporalThinning.h +++ b/test/ufo/TemporalThinning.h @@ -34,7 +34,9 @@ void testTemporalThinning(const eckit::LocalConfiguration &conf) { util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); - ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsParams; + obsParams.validateAndDeserialize(obsSpaceConf); + ioda::ObsSpace obsspace(obsParams, oops::mpi::world(), bgn, end, oops::mpi::myself()); if (conf.has("category")) { const std::vector categories = conf.getIntVector("category"); diff --git a/test/ufo/TrackCheck.h b/test/ufo/TrackCheck.h index d23889151..41a9a5c71 100644 --- a/test/ufo/TrackCheck.h +++ b/test/ufo/TrackCheck.h @@ -34,7 +34,9 @@ void testTrackCheck(const eckit::LocalConfiguration &conf) { util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); - ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsParams; + obsParams.validateAndDeserialize(obsSpaceConf); + ioda::ObsSpace obsspace(obsParams, oops::mpi::world(), bgn, end, oops::mpi::myself()); if (conf.has("air_pressures")) { const std::vector airPressures = conf.getFloatVector("air_pressures"); diff --git a/test/ufo/TrackCheckShip.h b/test/ufo/TrackCheckShip.h index 4ce397b05..a33ac80bc 100644 --- a/test/ufo/TrackCheckShip.h +++ b/test/ufo/TrackCheckShip.h @@ -40,7 +40,9 @@ const boost::optional setupRunFilter( util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); - ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsParams; + obsParams.validateAndDeserialize(obsSpaceConf); + ioda::ObsSpace obsspace(obsParams, oops::mpi::world(), bgn, end, oops::mpi::myself()); if (conf.has("station_ids")) { const std::vector stationIds = conf.getIntVector("station_ids"); diff --git a/test/ufo/TrackCheckShipMainLoop.h b/test/ufo/TrackCheckShipMainLoop.h index 1fc1a2bbb..3dc53d25d 100644 --- a/test/ufo/TrackCheckShipMainLoop.h +++ b/test/ufo/TrackCheckShipMainLoop.h @@ -35,7 +35,9 @@ void testFirstRejectionSimultaneousIncluded(const eckit::LocalConfiguration &con util::DateTime end(conf.getString("window end")); const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); - ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::world(), bgn, end, oops::mpi::myself()); + ioda::ObsTopLevelParameters obsParams; + obsParams.validateAndDeserialize(obsSpaceConf); + ioda::ObsSpace obsspace(obsParams, oops::mpi::world(), bgn, end, oops::mpi::myself()); if (conf.has("station_ids")) { const std::vector stationIds = conf.getIntVector("station_ids"); From 0b80085e0561534d64dd0b05f4c545bdaf2596b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Mon, 16 Aug 2021 21:20:05 +0100 Subject: [PATCH 85/98] Obs operator parameters (#1430) * Define a Parameters subclass for ObsOperator and LinearObsOperator. * For operators already defining a Parameters subclass, derive this subclass from ObsOperatorParametersBase and pass it to the constructor. * Add extra documentation. * Fixed a typo in a comment. Co-authored-by: Anna Shlyaeva --- src/mains/RunCRTM.h | 5 +- src/ufo/CMakeLists.txt | 1 + src/ufo/LinearObsOperator.cc | 4 +- src/ufo/LinearObsOperator.h | 4 +- src/ufo/LinearObsOperatorBase.cc | 24 +++-- src/ufo/LinearObsOperatorBase.h | 96 +++++++++++++++++-- src/ufo/ObsOperator.cc | 4 +- src/ufo/ObsOperator.h | 4 +- src/ufo/ObsOperatorBase.cc | 18 +++- src/ufo/ObsOperatorBase.h | 94 ++++++++++++++++-- src/ufo/ObsOperatorParametersBase.h | 62 ++++++++++++ .../ObsBackgroundErrorIdentity.cc | 8 +- .../ObsBackgroundErrorIdentity.h | 24 ++--- .../ObsBackgroundErrorVertInterp.cc | 8 +- .../ObsBackgroundErrorVertInterp.h | 24 ++--- src/ufo/categoricaloper/ObsCategorical.cc | 6 +- src/ufo/categoricaloper/ObsCategorical.h | 12 +-- src/ufo/categoricaloper/ObsCategoricalData.h | 41 ++++++-- .../ObsCategoricalParameters.h | 9 +- src/ufo/categoricaloper/ObsCategoricalTLAD.cc | 4 +- src/ufo/categoricaloper/ObsCategoricalTLAD.h | 12 +-- src/ufo/compositeoper/ObsComposite.cc | 12 +-- src/ufo/compositeoper/ObsComposite.h | 11 ++- .../compositeoper/ObsCompositeParameters.h | 8 +- src/ufo/compositeoper/ObsCompositeTLAD.cc | 9 +- src/ufo/compositeoper/ObsCompositeTLAD.h | 11 ++- .../RefMetOffice/ObsGnssroRefMetOffice.cc | 6 +- .../RefMetOffice/ObsGnssroRefMetOffice.h | 12 +-- .../ObsGnssroRefMetOfficeParameters.h | 7 +- .../RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc | 6 +- .../RefMetOffice/ObsGnssroRefMetOfficeTLAD.h | 12 +-- src/ufo/timeoper/ObsTimeOper.cc | 4 +- src/ufo/timeoper/ObsTimeOperTLAD.cc | 4 +- test/ufo/ObsDiagnostics.h | 4 +- test/ufo/ObsFilters.h | 2 +- 35 files changed, 415 insertions(+), 157 deletions(-) create mode 100644 src/ufo/ObsOperatorParametersBase.h diff --git a/src/mains/RunCRTM.h b/src/mains/RunCRTM.h index 045e1b7d3..1728b90b8 100644 --- a/src/mains/RunCRTM.h +++ b/src/mains/RunCRTM.h @@ -57,8 +57,9 @@ template class RunCRTM : public oops::Application { for (std::size_t jj = 0; jj < obsdb.size(); ++jj) { eckit::LocalConfiguration obsopconf(conf[jj], "obs operator"); - - ObsOperator_ hop(obsdb[jj], obsopconf); + typename ObsOperator_::Parameters_ obsopparams; + obsopparams.validateAndDeserialize(obsopconf); + ObsOperator_ hop(obsdb[jj], obsopparams); const eckit::LocalConfiguration gconf(conf[jj], "geovals"); const GeoVaLs_ gval(gconf, obsdb[jj], hop.requiredVars()); diff --git a/src/ufo/CMakeLists.txt b/src/ufo/CMakeLists.txt index af3573b02..993f95b11 100644 --- a/src/ufo/CMakeLists.txt +++ b/src/ufo/CMakeLists.txt @@ -37,6 +37,7 @@ list( APPEND ufo_src_files ObsOperator.h ObsOperatorBase.cc ObsOperatorBase.h + ObsOperatorParametersBase.h ObsTraits.h locations_f.cc locations_f.h diff --git a/src/ufo/LinearObsOperator.cc b/src/ufo/LinearObsOperator.cc index 50228dcf9..e1ce118f6 100644 --- a/src/ufo/LinearObsOperator.cc +++ b/src/ufo/LinearObsOperator.cc @@ -20,8 +20,8 @@ namespace ufo { // ----------------------------------------------------------------------------- -LinearObsOperator::LinearObsOperator(ioda::ObsSpace & os, const eckit::Configuration & conf) - : oper_(LinearObsOperatorFactory::create(os, conf)), odb_(os) +LinearObsOperator::LinearObsOperator(ioda::ObsSpace & os, const Parameters_ & params) + : oper_(LinearObsOperatorFactory::create(os, params.operatorParameters)), odb_(os) { // We use += rather than = to make sure the Variables objects contain no duplicate entries // and the variables are sorted alphabetically. diff --git a/src/ufo/LinearObsOperator.h b/src/ufo/LinearObsOperator.h index 7557f1ef8..2107e6771 100644 --- a/src/ufo/LinearObsOperator.h +++ b/src/ufo/LinearObsOperator.h @@ -40,7 +40,9 @@ namespace ufo { class LinearObsOperator : public util::Printable, private boost::noncopyable { public: - LinearObsOperator(ioda::ObsSpace &, const eckit::Configuration &); + typedef LinearObsOperatorParametersWrapper Parameters_; + + LinearObsOperator(ioda::ObsSpace &, const Parameters_ &); /// Obs Operator void setTrajectory(const GeoVaLs &, const ObsBias &); diff --git a/src/ufo/LinearObsOperatorBase.cc b/src/ufo/LinearObsOperatorBase.cc index ef9cad5db..20d9b52d2 100644 --- a/src/ufo/LinearObsOperatorBase.cc +++ b/src/ufo/LinearObsOperatorBase.cc @@ -7,7 +7,8 @@ #include "ufo/LinearObsOperatorBase.h" -#include "eckit/config/Configuration.h" +#include + #include "ioda/ObsSpace.h" #include "oops/util/abor1_cpp.h" #include "oops/util/Logger.h" @@ -33,24 +34,33 @@ LinearObsOperatorFactory::LinearObsOperatorFactory(const std::string & name) { // ----------------------------------------------------------------------------- -LinearObsOperatorBase * LinearObsOperatorFactory::create(const ioda::ObsSpace & odb, - const eckit::Configuration & conf) { +LinearObsOperatorBase * LinearObsOperatorFactory::create( + const ioda::ObsSpace & odb, const ObsOperatorParametersBase & params) { oops::Log::trace() << "LinearObsOperatorBase::create starting" << std::endl; - std::string id; - - id = conf.getString("name"); + const std::string &id = params.name.value().value(); typename std::map::iterator jloc = getMakers().find(id); if (jloc == getMakers().end()) { oops::Log::error() << id << " does not exist in ufo::LinearObsOperatorFactory." << std::endl; ABORT("Element does not exist in ufo::LinearObsOperatorFactory."); } - LinearObsOperatorBase * ptr = jloc->second->make(odb, conf); + LinearObsOperatorBase * ptr = jloc->second->make(odb, params); oops::Log::trace() << "LinearObsOperatorBase::create done" << std::endl; return ptr; } // ----------------------------------------------------------------------------- +std::unique_ptr +LinearObsOperatorFactory::createParameters(const std::string &name) { + typename std::map::iterator it = + getMakers().find(name); + if (it == getMakers().end()) { + throw std::runtime_error(name + " does not exist in ufo::LinearObsOperatorFactory"); + } + return it->second->makeParameters(); +} +// ----------------------------------------------------------------------------- + } // namespace ufo diff --git a/src/ufo/LinearObsOperatorBase.h b/src/ufo/LinearObsOperatorBase.h index 9961268ba..1e8a36b31 100644 --- a/src/ufo/LinearObsOperatorBase.h +++ b/src/ufo/LinearObsOperatorBase.h @@ -9,15 +9,23 @@ #define UFO_LINEAROBSOPERATORBASE_H_ #include +#include #include +#include #include #include "eckit/config/Configuration.h" #include "ioda/ObsSpace.h" #include "oops/base/Variables.h" -#include "oops/util/abor1_cpp.h" +#include "oops/util/AssociativeContainers.h" +#include "oops/util/parameters/ConfigurationParameter.h" +#include "oops/util/parameters/HasParameters_.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/ParametersOrConfiguration.h" +#include "oops/util/parameters/RequiredPolymorphicParameter.h" #include "oops/util/Printable.h" +#include "ufo/ObsOperatorParametersBase.h" namespace ioda { class ObsVector; @@ -28,8 +36,21 @@ class GeoVaLs; class ObsDiagnostics; // ----------------------------------------------------------------------------- -/// Base class for observation operators - +/// Base class for linear observation operators +/// +/// Note: subclasses can opt to extract their settings either from +/// a Configuration object or from a subclass of ObsOperatorParametersBase. +/// +/// In the former case, they should provide a constructor with the following signature: +/// +/// SubclassName(const ioda::ObsSpace &, const eckit::Configuration &); +/// +/// In the latter case, the implementer should first define a subclass of ObsOperatorParametersBase +/// holding the settings of the operator in question. The LinearObsOperatorBase subclass should +/// then typedef `Parameters_` to the name of the ObsOperatorParametersBase subclass and provide a +/// constructor with the following signature: +/// +/// SubclassName(const ioda::ObsSpace &, const Parameters_ &); class LinearObsOperatorBase : public util::Printable, private boost::noncopyable { public: @@ -62,15 +83,57 @@ class LinearObsOperatorBase : public util::Printable, // ----------------------------------------------------------------------------- -/// Obs Operator Factory +class LinearObsOperatorFactory; + +// ----------------------------------------------------------------------------- + +/// \brief Contains a polymorphic parameter holding an instance of a subclass of +/// ObsOperatorParametersBase. +class LinearObsOperatorParametersWrapper : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(LinearObsOperatorParametersWrapper, Parameters) + public: + /// After deserialization, holds an instance of a subclass of ObsOperatorParametersBase + /// controlling the behavior of a linear observation operator. The type of the subclass is + /// determined by the value of the "name" key in the Configuration object from which this object + /// is deserialized. + oops::RequiredPolymorphicParameter + operatorParameters{"name", this}; +}; + +// ----------------------------------------------------------------------------- + +/// Linear obs operator factory class LinearObsOperatorFactory { public: - static LinearObsOperatorBase * create(const ioda::ObsSpace &, const eckit::Configuration &); + /// \brief Create and return a new linear observation operator. + /// + /// The type of the operator is determined by the `name` attribute of \p params. \p params must + /// be an instance of the subclass of ObsOperatorParametersBase associated with that operator, + /// otherwise an exception will be thrown. + static LinearObsOperatorBase * create(const ioda::ObsSpace &, const ObsOperatorParametersBase &); + + /// \brief Create and return an instance of the subclass of ObsOperatorParametersBase + /// storing parameters of linear observation operators of the specified type. + static std::unique_ptr createParameters( + const std::string &name); + + /// \brief Return the names of all operators that can be created by one of the registered makers. + static std::vector getMakerNames() { + return oops::keys(getMakers()); + } + virtual ~LinearObsOperatorFactory() = default; + protected: - explicit LinearObsOperatorFactory(const std::string &); + /// \brief Register a maker able to create linear observation operators of type \p name. + explicit LinearObsOperatorFactory(const std::string &name); + private: - virtual LinearObsOperatorBase * make(const ioda::ObsSpace &, const eckit::Configuration &) = 0; + virtual LinearObsOperatorBase * make(const ioda::ObsSpace &, + const ObsOperatorParametersBase &) = 0; + + virtual std::unique_ptr makeParameters() const = 0; + static std::map < std::string, LinearObsOperatorFactory * > & getMakers() { static std::map < std::string, LinearObsOperatorFactory * > makers_; return makers_; @@ -81,10 +144,23 @@ class LinearObsOperatorFactory { template class LinearObsOperatorMaker : public LinearObsOperatorFactory { - virtual LinearObsOperatorBase * make(const ioda::ObsSpace & odb, - const eckit::Configuration & conf) { - return new T(odb, conf); + /// Defined as T::Parameters_ if T defines a Parameters_ type; otherwise as + /// GenericObsOperatorParameters. + typedef oops::TParameters_IfAvailableElseFallbackType_t + Parameters_; + + LinearObsOperatorBase * make(const ioda::ObsSpace & odb, + const ObsOperatorParametersBase & params) override { + const auto &stronglyTypedParams = dynamic_cast(params); + return new T(odb, + oops::parametersOrConfiguration::value>( + stronglyTypedParams)); } + + std::unique_ptr makeParameters() const override { + return std::make_unique(); + } + public: explicit LinearObsOperatorMaker(const std::string & name) : LinearObsOperatorFactory(name) {} }; diff --git a/src/ufo/ObsOperator.cc b/src/ufo/ObsOperator.cc index 5adfdd7e6..6f3d71080 100644 --- a/src/ufo/ObsOperator.cc +++ b/src/ufo/ObsOperator.cc @@ -27,8 +27,8 @@ namespace ufo { // ----------------------------------------------------------------------------- -ObsOperator::ObsOperator(ioda::ObsSpace & os, const eckit::Configuration & conf) - : oper_(ObsOperatorFactory::create(os, conf)), odb_(os) +ObsOperator::ObsOperator(ioda::ObsSpace & os, const Parameters_ & params) + : oper_(ObsOperatorFactory::create(os, params.operatorParameters)), odb_(os) { // We use += rather than = to make sure the Variables objects contain no duplicate entries // and the variables are sorted alphabetically. diff --git a/src/ufo/ObsOperator.h b/src/ufo/ObsOperator.h index 33f5e99c2..e4add5465 100644 --- a/src/ufo/ObsOperator.h +++ b/src/ufo/ObsOperator.h @@ -42,7 +42,9 @@ namespace ufo { class ObsOperator : public util::Printable, private boost::noncopyable { public: - ObsOperator(ioda::ObsSpace &, const eckit::Configuration &); + typedef ObsOperatorParametersWrapper Parameters_; + + ObsOperator(ioda::ObsSpace &, const Parameters_ &); /// Obs Operator void simulateObs(const GeoVaLs &, ioda::ObsVector &, const ObsBias &, ObsDiagnostics &) const; diff --git a/src/ufo/ObsOperatorBase.cc b/src/ufo/ObsOperatorBase.cc index 895402704..ff24d9f45 100644 --- a/src/ufo/ObsOperatorBase.cc +++ b/src/ufo/ObsOperatorBase.cc @@ -48,19 +48,31 @@ ObsOperatorFactory::ObsOperatorFactory(const std::string & name) { // ----------------------------------------------------------------------------- ObsOperatorBase * ObsOperatorFactory::create(const ioda::ObsSpace & odb, - const eckit::Configuration & conf) { + const ObsOperatorParametersBase & params) { oops::Log::trace() << "ObsOperatorBase::create starting" << std::endl; - const std::string id = conf.getString("name"); + const std::string &id = params.name.value().value(); typename std::map::iterator jloc = getMakers().find(id); if (jloc == getMakers().end()) { oops::Log::error() << id << " does not exist in ufo::ObsOperatorFactory." << std::endl; ABORT("Element does not exist in ufo::ObsOperatorFactory."); } - ObsOperatorBase * ptr = jloc->second->make(odb, conf); + ObsOperatorBase * ptr = jloc->second->make(odb, params); oops::Log::trace() << "ObsOperatorBase::create done" << std::endl; return ptr; } // ----------------------------------------------------------------------------- +std::unique_ptr +ObsOperatorFactory::createParameters(const std::string &name) { + typename std::map::iterator it = + getMakers().find(name); + if (it == getMakers().end()) { + throw std::runtime_error(name + " does not exist in ufo::ObsOperatorFactory"); + } + return it->second->makeParameters(); +} + +// ----------------------------------------------------------------------------- + } // namespace ufo diff --git a/src/ufo/ObsOperatorBase.h b/src/ufo/ObsOperatorBase.h index 3b7bda8f4..a9fea7e14 100644 --- a/src/ufo/ObsOperatorBase.h +++ b/src/ufo/ObsOperatorBase.h @@ -11,14 +11,21 @@ #include #include #include +#include #include #include "eckit/config/Configuration.h" #include "ioda/ObsSpace.h" #include "oops/base/Variables.h" -#include "oops/util/abor1_cpp.h" +#include "oops/util/AssociativeContainers.h" +#include "oops/util/parameters/ConfigurationParameter.h" +#include "oops/util/parameters/HasParameters_.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/ParametersOrConfiguration.h" +#include "oops/util/parameters/RequiredPolymorphicParameter.h" #include "oops/util/Printable.h" +#include "ufo/ObsOperatorParametersBase.h" namespace ioda { class ObsVector; @@ -31,11 +38,26 @@ namespace ufo { // ----------------------------------------------------------------------------- /// Base class for observation operators - +/// +/// Note: subclasses can opt to extract their settings either from +/// a Configuration object or from a subclass of ObsOperatorParametersBase. +/// +/// In the former case, they should provide a constructor with the following signature: +/// +/// SubclassName(const ioda::ObsSpace &, const eckit::Configuration &); +/// +/// In the latter case, the implementer should first define a subclass of ObsOperatorParametersBase +/// holding the settings of the operator in question. The ObsOperatorBase subclass should then +/// typedef `Parameters_` to the name of the ObsOperatorParametersBase subclass and provide a +/// constructor with the following signature: +/// +/// SubclassName(const ioda::ObsSpace &, const Parameters_ &); class ObsOperatorBase : public util::Printable, private boost::noncopyable { public: - ObsOperatorBase(const ioda::ObsSpace & odb, const eckit::Configuration &) + // The second parameter is unused and at some point will be removed. + explicit ObsOperatorBase(const ioda::ObsSpace & odb, + const eckit::Configuration & = eckit::LocalConfiguration()) : odb_(odb) {} virtual ~ObsOperatorBase() {} @@ -60,15 +82,56 @@ class ObsOperatorBase : public util::Printable, // ----------------------------------------------------------------------------- +class ObsOperatorFactory; + +// ----------------------------------------------------------------------------- + +/// \brief Contains a polymorphic parameter holding an instance of a subclass of +/// ObsOperatorParametersBase. +class ObsOperatorParametersWrapper : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsOperatorParametersWrapper, Parameters) + public: + /// After deserialization, holds an instance of a subclass of ObsOperatorParametersBase + /// controlling the behavior of an observation operator. The type of the subclass is determined + /// by the value of the "name" key in the Configuration object from which this object + /// is deserialized. + oops::RequiredPolymorphicParameter + operatorParameters{"name", this}; +}; + +// ----------------------------------------------------------------------------- + /// Obs Operator Factory class ObsOperatorFactory { public: - static ObsOperatorBase * create(const ioda::ObsSpace &, const eckit::Configuration &); + /// \brief Create and return a new observation operator. + /// + /// The type of the operator is determined by the `name` attribute of \p params. \p params must + /// be an instance of the subclass of ObsOperatorParametersBase associated with that operator, + /// otherwise an exception will be thrown. + static ObsOperatorBase * create(const ioda::ObsSpace &, const ObsOperatorParametersBase ¶ms); + + /// \brief Create and return an instance of the subclass of ObsOperatorParametersBase + /// storing parameters of observation operators of the specified type. + static std::unique_ptr createParameters( + const std::string &name); + + /// \brief Return the names of all operators that can be created by one of the registered makers. + static std::vector getMakerNames() { + return oops::keys(getMakers()); + } + virtual ~ObsOperatorFactory() = default; + protected: - explicit ObsOperatorFactory(const std::string &); + /// \brief Register a maker able to create observation operators of type \p name. + explicit ObsOperatorFactory(const std::string &name); + private: - virtual ObsOperatorBase * make(const ioda::ObsSpace &, const eckit::Configuration &) = 0; + virtual ObsOperatorBase * make(const ioda::ObsSpace &, const ObsOperatorParametersBase &) = 0; + + virtual std::unique_ptr makeParameters() const = 0; + static std::map < std::string, ObsOperatorFactory * > & getMakers() { static std::map < std::string, ObsOperatorFactory * > makers_; return makers_; @@ -79,8 +142,23 @@ class ObsOperatorFactory { template class ObsOperatorMaker : public ObsOperatorFactory { - virtual ObsOperatorBase * make(const ioda::ObsSpace & odb, const eckit::Configuration & conf) - { return new T(odb, conf); } + /// Defined as T::Parameters_ if T defines a Parameters_ type; otherwise as + /// GenericObsOperatorParameters. + typedef oops::TParameters_IfAvailableElseFallbackType_t + Parameters_; + + ObsOperatorBase * make(const ioda::ObsSpace & odb, + const ObsOperatorParametersBase & params) override { + const auto &stronglyTypedParams = dynamic_cast(params); + return new T(odb, + oops::parametersOrConfiguration::value>( + stronglyTypedParams)); + } + + std::unique_ptr makeParameters() const override { + return std::make_unique(); + } + public: explicit ObsOperatorMaker(const std::string & name) : ObsOperatorFactory(name) {} }; diff --git a/src/ufo/ObsOperatorParametersBase.h b/src/ufo/ObsOperatorParametersBase.h new file mode 100644 index 000000000..05dd15bd7 --- /dev/null +++ b/src/ufo/ObsOperatorParametersBase.h @@ -0,0 +1,62 @@ +/* + * (C) Crown Copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_OBSOPERATORPARAMETERSBASE_H_ +#define UFO_OBSOPERATORPARAMETERSBASE_H_ + +#include + +#include "oops/util/parameters/ConfigurationParameter.h" +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameters.h" + +namespace ioda { + class ObsVector; +} + +namespace ufo { + +// ----------------------------------------------------------------------------- +/// \brief Base class of classes storing configuration parameters of specific observation operators +/// and linear observation operators. +class ObsOperatorParametersBase : public oops::Parameters { + OOPS_ABSTRACT_PARAMETERS(ObsOperatorParametersBase, Parameters) + + public: + /// \brief Observation operator type. + /// + /// \note This parameter is marked as optional because it is only required in certain + /// circumstances (e.g. when observation operator parameters are deserialized into an + /// ObsOperatorParametersWrapper and used by OperatorFactory to instantiate a operator whose + /// type is determined at runtime), but not others (e.g. in tests written with a particular + /// operator in mind). ObsOperatorParametersWrapper will throw an exception if this parameter + /// is not provided. + oops::OptionalParameter name{"name", this}; +}; + +// ----------------------------------------------------------------------------- + +/// \brief A subclass of ObsOperatorParametersBase storing the values of all options in a +/// single Configuration object. +/// +/// This object can be accessed by calling the value() method of the \p config member variable. +/// +/// The ConfigurationParameter class does not perform any parameter validation; operators using +/// GenericObsOperatorParameters should therefore ideally be refactored, replacing this +/// class with a dedicated subclass of ObsOperatorParametersBase storing each parameter in +/// a separate (Optional/Required)Parameter object. +class GenericObsOperatorParameters : public ObsOperatorParametersBase { + OOPS_CONCRETE_PARAMETERS(GenericObsOperatorParameters, ObsOperatorParametersBase) + public: + oops::ConfigurationParameter config{this}; +}; + +// ----------------------------------------------------------------------------- + +} // namespace ufo + +#endif // UFO_OBSOPERATORPARAMETERSBASE_H_ diff --git a/src/ufo/backgrounderroridentity/ObsBackgroundErrorIdentity.cc b/src/ufo/backgrounderroridentity/ObsBackgroundErrorIdentity.cc index b454945a6..b2fd124aa 100644 --- a/src/ufo/backgrounderroridentity/ObsBackgroundErrorIdentity.cc +++ b/src/ufo/backgrounderroridentity/ObsBackgroundErrorIdentity.cc @@ -27,14 +27,12 @@ namespace ufo { static ObsOperatorMaker maker("BackgroundErrorIdentity"); ObsBackgroundErrorIdentity::ObsBackgroundErrorIdentity(const ioda::ObsSpace & odb, - const eckit::Configuration & config) - : ObsOperatorBase(odb, config), - odb_(odb) + const Parameters_ & parameters) + : ObsOperatorBase(odb), + odb_(odb), parameters_(parameters) { oops::Log::trace() << "ObsBackgroundErrorIdentity constructor entered" << std::endl; - parameters_.validateAndDeserialize(config); - // simulateObs() may be asked to interpolate the background errors of any simulated variables. // We need to assume the worst, i.e. that we'll need to interpolate all of them. const oops::Variables &obsvars = odb.obsvariables(); diff --git a/src/ufo/backgrounderroridentity/ObsBackgroundErrorIdentity.h b/src/ufo/backgrounderroridentity/ObsBackgroundErrorIdentity.h index 4e9441dd4..39ff0a7da 100644 --- a/src/ufo/backgrounderroridentity/ObsBackgroundErrorIdentity.h +++ b/src/ufo/backgrounderroridentity/ObsBackgroundErrorIdentity.h @@ -17,17 +17,13 @@ #include "oops/util/ObjectCounter.h" #include "oops/util/parameters/OptionalParameter.h" #include "oops/util/parameters/Parameter.h" -#include "oops/util/parameters/Parameters.h" #include "ufo/filters/Variable.h" #include "ufo/ObsOperatorBase.h" +#include "ufo/ObsOperatorParametersBase.h" #include "ufo/utils/parameters/ParameterTraitsVariable.h" /// Forward declarations -namespace eckit { - class Configuration; -} - namespace ioda { class ObsSpace; class ObsVector; @@ -39,16 +35,10 @@ namespace ufo { class ObsDiagnostics; /// \brief Options controlling the ObsBackgroundErrorIdentity observation operator. -class ObsBackgroundErrorIdentityParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(ObsBackgroundErrorIdentityParameters, Parameters) +class ObsBackgroundErrorIdentityParameters : public ObsOperatorParametersBase { + OOPS_CONCRETE_PARAMETERS(ObsBackgroundErrorIdentityParameters, ObsOperatorParametersBase) public: - /// Name of the ObsOperator. Must be BackgroundErrorIdentity. - /// - /// TODO(wsmigaj): create an ObsOperatorParametersBase class, move this parameter there and - /// derive ObsBackgroundErrorIdentityParameters from that class. - oops::Parameter name{"name", "", this}; - /// Simulated variables whose background errors may be calculated by this operator. /// If not specified, defaults to the list of all simulated variables in the ObsSpace. oops::OptionalParameter> variables{"variables", this}; @@ -82,9 +72,13 @@ class ObsBackgroundErrorIdentityParameters : public oops::Parameters { class ObsBackgroundErrorIdentity : public ObsOperatorBase, private util::ObjectCounter { public: + /// The type of parameters accepted by the constructor of this operator. + /// This typedef is used by the ObsOperatorFactory. + typedef ObsBackgroundErrorIdentityParameters Parameters_; + static const std::string classname() {return "ufo::ObsBackgroundErrorIdentity";} - ObsBackgroundErrorIdentity(const ioda::ObsSpace &, const eckit::Configuration &); + ObsBackgroundErrorIdentity(const ioda::ObsSpace &, const Parameters_ &); virtual ~ObsBackgroundErrorIdentity(); @@ -99,7 +93,7 @@ class ObsBackgroundErrorIdentity : public ObsOperatorBase, private: const ioda::ObsSpace& odb_; - ObsBackgroundErrorIdentityParameters parameters_; + Parameters_ parameters_; oops::Variables requiredVars_; }; diff --git a/src/ufo/backgrounderrorvertinterp/ObsBackgroundErrorVertInterp.cc b/src/ufo/backgrounderrorvertinterp/ObsBackgroundErrorVertInterp.cc index 4fbe25e0f..9922eee1a 100644 --- a/src/ufo/backgrounderrorvertinterp/ObsBackgroundErrorVertInterp.cc +++ b/src/ufo/backgrounderrorvertinterp/ObsBackgroundErrorVertInterp.cc @@ -27,14 +27,12 @@ namespace ufo { static ObsOperatorMaker maker("BackgroundErrorVertInterp"); ObsBackgroundErrorVertInterp::ObsBackgroundErrorVertInterp(const ioda::ObsSpace & odb, - const eckit::Configuration & config) - : ObsOperatorBase(odb, config), - odb_(odb) + const Parameters_ & parameters) + : ObsOperatorBase(odb), + odb_(odb), parameters_(parameters) { oops::Log::trace() << "ObsBackgroundErrorVertInterp constructor entered" << std::endl; - parameters_.validateAndDeserialize(config); - requiredVars_.push_back(parameters_.verticalCoordinate); // simulateObs() may be asked to interpolate the background errors of any simulated variables. // We need to assume the worst, i.e. that we'll need to interpolate all of them. diff --git a/src/ufo/backgrounderrorvertinterp/ObsBackgroundErrorVertInterp.h b/src/ufo/backgrounderrorvertinterp/ObsBackgroundErrorVertInterp.h index 3bbb43545..dd4f96b7d 100644 --- a/src/ufo/backgrounderrorvertinterp/ObsBackgroundErrorVertInterp.h +++ b/src/ufo/backgrounderrorvertinterp/ObsBackgroundErrorVertInterp.h @@ -17,18 +17,14 @@ #include "oops/util/ObjectCounter.h" #include "oops/util/parameters/OptionalParameter.h" #include "oops/util/parameters/Parameter.h" -#include "oops/util/parameters/Parameters.h" #include "oops/util/parameters/RequiredParameter.h" #include "ufo/filters/Variable.h" #include "ufo/ObsOperatorBase.h" +#include "ufo/ObsOperatorParametersBase.h" #include "ufo/utils/parameters/ParameterTraitsVariable.h" /// Forward declarations -namespace eckit { - class Configuration; -} - namespace ioda { class ObsSpace; class ObsVector; @@ -40,16 +36,10 @@ namespace ufo { class ObsDiagnostics; /// \brief Options controlling the ObsBackgroundErrorVertInterp observation operator. -class ObsBackgroundErrorVertInterpParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(ObsBackgroundErrorVertInterpParameters, Parameters) +class ObsBackgroundErrorVertInterpParameters : public ObsOperatorParametersBase { + OOPS_CONCRETE_PARAMETERS(ObsBackgroundErrorVertInterpParameters, ObsOperatorParametersBase) public: - /// Name of the ObsOperator. Must be BackgroundErrorVertInterp. - /// - /// TODO(wsmigaj): create an ObsOperatorParametersBase class, move this parameter there and - /// derive ObsBackgroundErrorVertInterpParameters from that class. - oops::Parameter name{"name", "", this}; - /// Simulated variables whose background errors may be calculated by this operator. /// If not specified, defaults to the list of all simulated variables in the ObsSpace. oops::OptionalParameter> variables{"variables", this}; @@ -99,9 +89,13 @@ class ObsBackgroundErrorVertInterpParameters : public oops::Parameters { class ObsBackgroundErrorVertInterp : public ObsOperatorBase, private util::ObjectCounter { public: + /// The type of parameters accepted by the constructor of this operator. + /// This typedef is used by the ObsOperatorFactory. + typedef ObsBackgroundErrorVertInterpParameters Parameters_; + static const std::string classname() {return "ufo::ObsBackgroundErrorVertInterp";} - ObsBackgroundErrorVertInterp(const ioda::ObsSpace &, const eckit::Configuration &); + ObsBackgroundErrorVertInterp(const ioda::ObsSpace &, const Parameters_ &); virtual ~ObsBackgroundErrorVertInterp(); @@ -116,7 +110,7 @@ class ObsBackgroundErrorVertInterp : public ObsOperatorBase, private: const ioda::ObsSpace& odb_; - ObsBackgroundErrorVertInterpParameters parameters_; + Parameters_ parameters_; oops::Variables requiredVars_; }; diff --git a/src/ufo/categoricaloper/ObsCategorical.cc b/src/ufo/categoricaloper/ObsCategorical.cc index 27fe67dd4..911d16e7c 100644 --- a/src/ufo/categoricaloper/ObsCategorical.cc +++ b/src/ufo/categoricaloper/ObsCategorical.cc @@ -27,12 +27,12 @@ static ObsOperatorMaker obsCategoricalMaker_("Categorical"); // ----------------------------------------------------------------------------- ObsCategorical::ObsCategorical(const ioda::ObsSpace & odb, - const eckit::Configuration & config) - : ObsOperatorBase(odb, config), odb_(odb) + const Parameters_ & params) + : ObsOperatorBase(odb), odb_(odb) { oops::Log::trace() << "ObsCategorical constructor starting" << std::endl; - data_.configure(odb, config); + data_.configure(odb, params); oops::Log::trace() << "ObsCategorical constructor finished" << std::endl; } diff --git a/src/ufo/categoricaloper/ObsCategorical.h b/src/ufo/categoricaloper/ObsCategorical.h index 3ddbbddb3..fa857f510 100644 --- a/src/ufo/categoricaloper/ObsCategorical.h +++ b/src/ufo/categoricaloper/ObsCategorical.h @@ -21,10 +21,6 @@ #include "ufo/ObsOperatorBase.h" /// Forward declarations -namespace eckit { - class Configuration; -} - namespace ioda { class ObsSpace; class ObsVector; @@ -85,9 +81,13 @@ namespace ufo { class ObsCategorical : public ObsOperatorBase, private util::ObjectCounter { public: + /// The type of parameters accepted by the constructor of this operator. + /// This typedef is used by the ObsOperatorFactory. + typedef ObsCategoricalParameters Parameters_; + static const std::string classname() {return "ufo::ObsCategorical";} - ObsCategorical(const ioda::ObsSpace &, const eckit::Configuration &); + ObsCategorical(const ioda::ObsSpace &, const Parameters_ &); ~ObsCategorical() override; void simulateObs(const GeoVaLs &, ioda::ObsVector &, ObsDiagnostics &) const override; @@ -102,7 +102,7 @@ class ObsCategorical : public ObsOperatorBase, const ioda::ObsSpace& odb_; /// Data handler for the Categorical operator and TL/AD code. - ObsCategoricalData data_; + ObsCategoricalData data_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/categoricaloper/ObsCategoricalData.h b/src/ufo/categoricaloper/ObsCategoricalData.h index 8bde78843..8aa013ee8 100644 --- a/src/ufo/categoricaloper/ObsCategoricalData.h +++ b/src/ufo/categoricaloper/ObsCategoricalData.h @@ -34,17 +34,41 @@ namespace ioda { } namespace ufo { + +// Define some traits that enable ObsCategoricalData to work with both linear and nonlinear +// operators. + +class LinearObsOperatorBase; +class LinearObsOperatorFactory; +class LinearObsOperatorParametersWrapper; + +class ObsOperatorBase; +class ObsOperatorFactory; +class ObsOperatorParametersWrapper; + +template +struct ObsOperatorTraits {}; + +template <> +struct ObsOperatorTraits { + typedef ObsOperatorFactory Factory_; + typedef ObsOperatorParametersWrapper ParametersWrapper_; +}; + +template <> +struct ObsOperatorTraits { + typedef LinearObsOperatorFactory Factory_; + typedef LinearObsOperatorParametersWrapper ParametersWrapper_; +}; + /// \brief Data handler for the Categorical observation operator and TL/AD code. -template +template class ObsCategoricalData { public: /// Get all information related to the configuration of the Categorical operator and TL/AD code. void configure(const ioda::ObsSpace & odb, - const eckit::Configuration & config) + const ObsCategoricalParameters & parameters) { - ObsCategoricalParameters parameters; - parameters.validateAndDeserialize(config); - // Get categorical variable from ObsSpace (and throw an exception if it is not present). // In the ObsSpace, the categorical variable can be either a vector of strings or // a vector of integers; if the latter, it is converted to a vector of strings here. @@ -89,7 +113,12 @@ class ObsCategoricalData { // Create list of component operators. for (const eckit::LocalConfiguration &operatorConfig : parameters.operatorConfigurations.value()) { - std::unique_ptr op(OPFACTORY::create(odb, operatorConfig)); + typedef typename ObsOperatorTraits::Factory_ Factory_; + typedef typename ObsOperatorTraits::ParametersWrapper_ ParametersWrapper_; + + ParametersWrapper_ operatorParams; + operatorParams.validateAndDeserialize(operatorConfig); + std::unique_ptr op(Factory_::create(odb, operatorParams.operatorParameters)); requiredVars_ += op->requiredVars(); components_.emplace(std::make_pair(operatorConfig.getString("name"), std::move(op))); } diff --git a/src/ufo/categoricaloper/ObsCategoricalParameters.h b/src/ufo/categoricaloper/ObsCategoricalParameters.h index d484c3d5f..1f356fc98 100644 --- a/src/ufo/categoricaloper/ObsCategoricalParameters.h +++ b/src/ufo/categoricaloper/ObsCategoricalParameters.h @@ -13,19 +13,16 @@ #include #include "oops/util/parameters/OptionalParameter.h" -#include "oops/util/parameters/Parameters.h" #include "oops/util/parameters/RequiredParameter.h" +#include "ufo/ObsOperatorParametersBase.h" namespace ufo { /// Configuration options recognized by the Categorical operator. -class ObsCategoricalParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(ObsCategoricalParameters, Parameters) +class ObsCategoricalParameters : public ObsOperatorParametersBase { + OOPS_CONCRETE_PARAMETERS(ObsCategoricalParameters, ObsOperatorParametersBase) public: - /// Operator name. In future will be moved to a base class for parameters of all ObsOperators. - oops::OptionalParameter name{"name", this}; - /// Categorical variable used to divide H(x) into sections. oops::RequiredParameter categoricalVariable{"categorical variable", this}; diff --git a/src/ufo/categoricaloper/ObsCategoricalTLAD.cc b/src/ufo/categoricaloper/ObsCategoricalTLAD.cc index fc2fa2397..6f8109632 100644 --- a/src/ufo/categoricaloper/ObsCategoricalTLAD.cc +++ b/src/ufo/categoricaloper/ObsCategoricalTLAD.cc @@ -26,12 +26,12 @@ static LinearObsOperatorMaker makerCategoricalTL_("Categoric // ----------------------------------------------------------------------------- ObsCategoricalTLAD::ObsCategoricalTLAD(const ioda::ObsSpace & odb, - const eckit::Configuration & config) + const Parameters_ & params) : LinearObsOperatorBase(odb), odb_(odb) { oops::Log::trace() << "ObsCategoricalTLAD constructor starting" << std::endl; - data_.configure(odb, config); + data_.configure(odb, params); oops::Log::trace() << "ObsCategoricalTLAD created." << std::endl; } diff --git a/src/ufo/categoricaloper/ObsCategoricalTLAD.h b/src/ufo/categoricaloper/ObsCategoricalTLAD.h index b0e44a2ba..7c34b6ebf 100644 --- a/src/ufo/categoricaloper/ObsCategoricalTLAD.h +++ b/src/ufo/categoricaloper/ObsCategoricalTLAD.h @@ -21,10 +21,6 @@ #include "ufo/LinearObsOperatorBase.h" // Forward declarations -namespace eckit { - class Configuration; -} - namespace ioda { class ObsSpace; class ObsVector; @@ -39,9 +35,13 @@ namespace ufo { class ObsCategoricalTLAD : public LinearObsOperatorBase, private util::ObjectCounter { public: + /// The type of parameters accepted by the constructor of this operator. + /// This typedef is used by the LinearObsOperatorFactory. + typedef ObsCategoricalParameters Parameters_; + static const std::string classname() { return "ufo::ObsCategoricalTLAD"; } - ObsCategoricalTLAD(const ioda::ObsSpace &, const eckit::Configuration &); + ObsCategoricalTLAD(const ioda::ObsSpace &, const Parameters_ &); ~ObsCategoricalTLAD() override; void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; @@ -58,7 +58,7 @@ class ObsCategoricalTLAD : public LinearObsOperatorBase, const ioda::ObsSpace& odb_; /// Data handler for the Categorical operator and TL/AD code. - ObsCategoricalData data_; + ObsCategoricalData data_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/compositeoper/ObsComposite.cc b/src/ufo/compositeoper/ObsComposite.cc index 3bbdbd63d..8f131674e 100644 --- a/src/ufo/compositeoper/ObsComposite.cc +++ b/src/ufo/compositeoper/ObsComposite.cc @@ -28,16 +28,16 @@ namespace ufo { static ObsOperatorMaker obsCompositeMaker_("Composite"); // ----------------------------------------------------------------------------- -ObsComposite::ObsComposite(const ioda::ObsSpace & odb, - const eckit::Configuration & config) - : ObsOperatorBase(odb, config), odb_(odb) +ObsComposite::ObsComposite(const ioda::ObsSpace & odb, const Parameters_ & parameters) + : ObsOperatorBase(odb), odb_(odb) { oops::Log::trace() << "ObsComposite constructor starting" << std::endl; - ObsCompositeParameters parameters; - parameters.validateAndDeserialize(config); for (const eckit::LocalConfiguration &operatorConfig : parameters.components.value()) { - std::unique_ptr op(ObsOperatorFactory::create(odb, operatorConfig)); + ObsOperatorParametersWrapper operatorParams; + operatorParams.validateAndDeserialize(operatorConfig); + std::unique_ptr op( + ObsOperatorFactory::create(odb, operatorParams.operatorParameters)); requiredVars_ += op->requiredVars(); components_.push_back(std::move(op)); } diff --git a/src/ufo/compositeoper/ObsComposite.h b/src/ufo/compositeoper/ObsComposite.h index 20f5dd368..cc1aa8aaf 100644 --- a/src/ufo/compositeoper/ObsComposite.h +++ b/src/ufo/compositeoper/ObsComposite.h @@ -16,13 +16,10 @@ #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" +#include "ufo/compositeoper/ObsCompositeParameters.h" #include "ufo/ObsOperatorBase.h" /// Forward declarations -namespace eckit { - class Configuration; -} - namespace ioda { class ObsSpace; class ObsVector; @@ -58,9 +55,13 @@ namespace ufo { class ObsComposite : public ObsOperatorBase, private util::ObjectCounter { public: + /// The type of parameters accepted by the constructor of this operator. + /// This typedef is used by the ObsOperatorFactory. + typedef ObsCompositeParameters Parameters_; + static const std::string classname() {return "ufo::ObsComposite";} - ObsComposite(const ioda::ObsSpace &, const eckit::Configuration &); + ObsComposite(const ioda::ObsSpace &, const Parameters_ &); ~ObsComposite() override; void simulateObs(const GeoVaLs &, ioda::ObsVector &, ObsDiagnostics &) const override; diff --git a/src/ufo/compositeoper/ObsCompositeParameters.h b/src/ufo/compositeoper/ObsCompositeParameters.h index 8e4e76ef6..c1c0d1a95 100644 --- a/src/ufo/compositeoper/ObsCompositeParameters.h +++ b/src/ufo/compositeoper/ObsCompositeParameters.h @@ -12,18 +12,16 @@ #include #include "oops/util/parameters/OptionalParameter.h" -#include "oops/util/parameters/Parameters.h" #include "oops/util/parameters/RequiredParameter.h" +#include "ufo/ObsOperatorParametersBase.h" namespace ufo { /// Configuration options recognized by the Composite operator. -class ObsCompositeParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(ObsCompositeParameters, Parameters) +class ObsCompositeParameters : public ObsOperatorParametersBase { + OOPS_CONCRETE_PARAMETERS(ObsCompositeParameters, ObsOperatorParametersBase) public: - /// Operator name. In future will be moved to a base class for parameters of all ObsOperators. - oops::OptionalParameter name{"name", this}; /// A list of configuration options for each operator used to simulate a subset of variables. oops::RequiredParameter> components{"components", this}; }; diff --git a/src/ufo/compositeoper/ObsCompositeTLAD.cc b/src/ufo/compositeoper/ObsCompositeTLAD.cc index 3823d1cdb..fd5c54e04 100644 --- a/src/ufo/compositeoper/ObsCompositeTLAD.cc +++ b/src/ufo/compositeoper/ObsCompositeTLAD.cc @@ -25,17 +25,16 @@ namespace ufo { static LinearObsOperatorMaker makerCompositeTL_("Composite"); // ----------------------------------------------------------------------------- -ObsCompositeTLAD::ObsCompositeTLAD(const ioda::ObsSpace & odb, - const eckit::Configuration & config) +ObsCompositeTLAD::ObsCompositeTLAD(const ioda::ObsSpace & odb, const Parameters_ & parameters) : LinearObsOperatorBase(odb) { oops::Log::trace() << "ObsCompositeTLAD constructor starting" << std::endl; - ObsCompositeParameters parameters; - parameters.validateAndDeserialize(config); for (const eckit::LocalConfiguration &operatorConfig : parameters.components.value()) { + LinearObsOperatorParametersWrapper operatorParams; + operatorParams.validateAndDeserialize(operatorConfig); std::unique_ptr op( - LinearObsOperatorFactory::create(odb, operatorConfig)); + LinearObsOperatorFactory::create(odb, operatorParams.operatorParameters)); requiredVars_ += op->requiredVars(); components_.push_back(std::move(op)); } diff --git a/src/ufo/compositeoper/ObsCompositeTLAD.h b/src/ufo/compositeoper/ObsCompositeTLAD.h index 372e72d53..4405a6ca0 100644 --- a/src/ufo/compositeoper/ObsCompositeTLAD.h +++ b/src/ufo/compositeoper/ObsCompositeTLAD.h @@ -16,13 +16,10 @@ #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" +#include "ufo/compositeoper/ObsCompositeParameters.h" #include "ufo/LinearObsOperatorBase.h" // Forward declarations -namespace eckit { - class Configuration; -} - namespace ioda { class ObsSpace; class ObsVector; @@ -37,9 +34,13 @@ namespace ufo { class ObsCompositeTLAD : public LinearObsOperatorBase, private util::ObjectCounter { public: + /// The type of parameters accepted by the constructor of this operator. + /// This typedef is used by the LinearObsOperatorFactory. + typedef ObsCompositeParameters Parameters_; + static const std::string classname() { return "ufo::ObsCompositeTLAD"; } - ObsCompositeTLAD(const ioda::ObsSpace &, const eckit::Configuration &); + ObsCompositeTLAD(const ioda::ObsSpace &, const Parameters_ &); ~ObsCompositeTLAD() override; void setTrajectory(const GeoVaLs &, ObsDiagnostics &) override; diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc index c75fc3daa..3d8df5174 100644 --- a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.cc @@ -26,10 +26,10 @@ static ObsOperatorMaker makerGnssroRefMetOffice_("GnssroR // ----------------------------------------------------------------------------- ObsGnssroRefMetOffice::ObsGnssroRefMetOffice(const ioda::ObsSpace & odb, - const eckit::Configuration & config) - : ObsOperatorBase(odb, config), keyOperGnssroRefMetOffice_(0), odb_(odb), varin_() + const Parameters_ & parameters) + : ObsOperatorBase(odb), keyOperGnssroRefMetOffice_(0), odb_(odb), varin_(), + parameters_(parameters) { - parameters_.validateAndDeserialize(config); ObsGnssroRefMetOfficeOptions obsOptions = parameters_.obsOptions.value(); const std::vector vv{"air_pressure_levels", "specific_humidity", diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h index 023adfe91..bb0e08b82 100644 --- a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOffice.h @@ -18,10 +18,6 @@ #include "ufo/ObsOperatorBase.h" #include "ObsGnssroRefMetOfficeParameters.h" -namespace eckit { - class Configuration; -} - namespace ioda { class ObsSpace; class ObsVector; @@ -37,9 +33,13 @@ namespace ufo { class ObsGnssroRefMetOffice : public ObsOperatorBase, private util::ObjectCounter { public: + /// The type of parameters accepted by the constructor of this operator. + /// This typedef is used by the ObsOperatorFactory. + typedef ObsGnssroRefMetOfficeParameters Parameters_; + static const std::string classname() {return "ufo::ObsGnssroRefMetOffice";} - ObsGnssroRefMetOffice(const ioda::ObsSpace &, const eckit::Configuration &); + ObsGnssroRefMetOffice(const ioda::ObsSpace &, const Parameters_ &); virtual ~ObsGnssroRefMetOffice(); // Obs Operator @@ -56,7 +56,7 @@ class ObsGnssroRefMetOffice : public ObsOperatorBase, F90hop keyOperGnssroRefMetOffice_; const ioda::ObsSpace& odb_; std::unique_ptr varin_; - ObsGnssroRefMetOfficeParameters parameters_; + Parameters_ parameters_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeParameters.h b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeParameters.h index b1927db37..c52079633 100644 --- a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeParameters.h +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeParameters.h @@ -14,6 +14,7 @@ #include "oops/util/parameters/OptionalParameter.h" #include "oops/util/parameters/Parameters.h" #include "oops/util/parameters/RequiredParameter.h" +#include "ufo/ObsOperatorParametersBase.h" namespace ufo { @@ -34,12 +35,10 @@ class ObsGnssroRefMetOfficeOptions : public oops::Parameters { }; /// Configuration options recognized by the refractivity operator. -class ObsGnssroRefMetOfficeParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(ObsGnssroRefMetOfficeParameters, Parameters) +class ObsGnssroRefMetOfficeParameters : public ObsOperatorParametersBase { + OOPS_CONCRETE_PARAMETERS(ObsGnssroRefMetOfficeParameters, ObsOperatorParametersBase) public: - /// Operator name. In future will be moved to a base class for parameters of all ObsOperators. - oops::OptionalParameter name{"name", this}; /// Obs Options - settings for the observation operator oops::Parameter obsOptions{"obs options", ObsGnssroRefMetOfficeOptions(), this}; diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc index 1630d0155..a2c7aa04f 100644 --- a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.cc @@ -27,10 +27,10 @@ static LinearObsOperatorMaker // ----------------------------------------------------------------------------- ObsGnssroRefMetOfficeTLAD::ObsGnssroRefMetOfficeTLAD(const ioda::ObsSpace & odb, - const eckit::Configuration & config) - : LinearObsOperatorBase(odb), keyOperGnssroRefMetOffice_(0), varin_() + const Parameters_ & parameters) + : LinearObsOperatorBase(odb), keyOperGnssroRefMetOffice_(0), varin_(), + parameters_(parameters) { - parameters_.validateAndDeserialize(config); ObsGnssroRefMetOfficeOptions obsOptions = parameters_.obsOptions.value(); ufo_gnssro_refmetoffice_tlad_setup_f90(keyOperGnssroRefMetOffice_, diff --git a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.h b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.h index b482b6053..21bbf1d23 100644 --- a/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.h +++ b/src/ufo/gnssro/RefMetOffice/ObsGnssroRefMetOfficeTLAD.h @@ -19,10 +19,6 @@ #include "ObsGnssroRefMetOfficeParameters.h" // Forward declarations -namespace eckit { - class Configuration; -} - namespace ioda { class ObsSpace; class ObsVector; @@ -38,9 +34,13 @@ namespace ufo { class ObsGnssroRefMetOfficeTLAD : public LinearObsOperatorBase, private util::ObjectCounter { public: + /// The type of parameters accepted by the constructor of this operator. + /// This typedef is used by the LinearObsOperatorFactory. + typedef ObsGnssroRefMetOfficeParameters Parameters_; + static const std::string classname() {return "ufo::ObsGnssroRefMetOfficeTLAD";} - ObsGnssroRefMetOfficeTLAD(const ioda::ObsSpace &, const eckit::Configuration &); + ObsGnssroRefMetOfficeTLAD(const ioda::ObsSpace &, const Parameters_ &); virtual ~ObsGnssroRefMetOfficeTLAD(); // Obs Operators @@ -58,7 +58,7 @@ class ObsGnssroRefMetOfficeTLAD : public LinearObsOperatorBase, void print(std::ostream &) const override; F90hop keyOperGnssroRefMetOffice_; std::unique_ptr varin_; - ObsGnssroRefMetOfficeParameters parameters_; + Parameters_ parameters_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/timeoper/ObsTimeOper.cc b/src/ufo/timeoper/ObsTimeOper.cc index 980f98c02..ae6a615c5 100644 --- a/src/ufo/timeoper/ObsTimeOper.cc +++ b/src/ufo/timeoper/ObsTimeOper.cc @@ -34,7 +34,9 @@ ObsTimeOper::ObsTimeOper(const ioda::ObsSpace & odb, const eckit::Configuration & config) : ObsOperatorBase(odb, config), actualoperator_(ObsOperatorFactory::create( - odb, eckit::LocalConfiguration(config, "obs operator"))), + odb, + oops::validateAndDeserialize( + eckit::LocalConfiguration(config, "obs operator")).operatorParameters)), odb_(odb), timeWeights_(timeWeightCreate(odb, config)) { oops::Log::trace() << "ObsTimeOper creating" << std::endl; diff --git a/src/ufo/timeoper/ObsTimeOperTLAD.cc b/src/ufo/timeoper/ObsTimeOperTLAD.cc index 3e4130c7c..e6820776a 100644 --- a/src/ufo/timeoper/ObsTimeOperTLAD.cc +++ b/src/ufo/timeoper/ObsTimeOperTLAD.cc @@ -32,7 +32,9 @@ ObsTimeOperTLAD::ObsTimeOperTLAD(const ioda::ObsSpace & odb, const eckit::Configuration & config) : LinearObsOperatorBase(odb), actualoperator_(LinearObsOperatorFactory::create( - odb, eckit::LocalConfiguration(config, "obs operator"))), + odb, + oops::validateAndDeserialize( + eckit::LocalConfiguration(config, "obs operator")).operatorParameters)), timeWeights_(timeWeightCreate(odb, config)) { oops::Log::trace() << "ObsTimeOperTLAD created" << std::endl; diff --git a/test/ufo/ObsDiagnostics.h b/test/ufo/ObsDiagnostics.h index e492224f0..76cf07cce 100644 --- a/test/ufo/ObsDiagnostics.h +++ b/test/ufo/ObsDiagnostics.h @@ -56,7 +56,9 @@ void testObsDiagnostics() { // initialize observation operator (set variables requested from the model, // variables simulated by the observation operator, other init) eckit::LocalConfiguration obsopconf(conf, "obs operator"); - ObsOperator hop(ospace, obsopconf); + ObsOperatorParametersWrapper obsopparams; + obsopparams.validateAndDeserialize(obsopconf); + ObsOperator hop(ospace, obsopparams); // read geovals from the file eckit::LocalConfiguration gconf(conf, "geovals"); diff --git a/test/ufo/ObsFilters.h b/test/ufo/ObsFilters.h index 4ac5f9df4..5e74839eb 100644 --- a/test/ufo/ObsFilters.h +++ b/test/ufo/ObsFilters.h @@ -103,7 +103,7 @@ class ObsTypeParameters : public oops::Parameters { /// precalculated and stored in the IODA file used to initialize the ObsSpace. In that case the /// `obs operator` keyword should be omitted and instead the `HofX` option should be set to the /// name of the group of ObsSpace variables containing the precalculated model equivalents. - oops::OptionalParameter obsOperator{"obs operator", this}; + oops::OptionalParameter obsOperator{"obs operator", this}; /// Group of variables storing precalculated model equivalents of observations. See the /// description of the `obs operator` option for more information. From dd10870ee3130ae2d1f1d4d6e07560a711ce9d5c Mon Sep 17 00:00:00 2001 From: JJ Guerrette Date: Tue, 17 Aug 2021 17:09:49 -0600 Subject: [PATCH 86/98] Pass bias as argument to simulateObs and postFilter (#1404) * Implementation part * Test application part * Pass bias by reference instead of pointer * Make bias an ObsVector instead of an ObsDataVector * Add bias argument for postFilter in auto-generated filter --- src/mains/RunCRTM.h | 4 +++- src/ufo/ObsOperator.cc | 9 ++++----- src/ufo/ObsOperator.h | 3 ++- src/ufo/filters/BackgroundCheck.cc | 14 ++++++++------ src/ufo/filters/ObsDiagnosticsWriter.h | 3 ++- src/ufo/filters/ObsFilterData.cc | 6 +++--- src/ufo/filters/ObsProcessorBase.cc | 8 ++++++-- src/ufo/filters/ObsProcessorBase.h | 3 ++- src/ufo/filters/QCmanager.cc | 4 +++- src/ufo/filters/QCmanager.h | 3 ++- src/ufo/filters/obsfunctions/CLWRetMW.h | 4 ++-- .../obsfunctions/CloudDetectMinResidualAVHRR.h | 4 ++-- .../filters/obsfunctions/HydrometeorCheckAMSUA.h | 5 +++-- .../filters/obsfunctions/HydrometeorCheckATMS.h | 5 +++-- src/ufo/filters/obsfunctions/SCATRetMW.h | 4 ++-- src/ufo/filters/obsfunctions/SIRetMW.h | 4 ++-- src/ufo/filters/obsfunctions/SymmCldImpactIR.cc | 4 ++-- test/ufo/ConventionalProfileProcessing.h | 8 ++++++-- test/ufo/MetOfficeBuddyCheck.h | 7 +++++-- test/ufo/ObsDiagnostics.h | 6 +++++- test/ufo/ObsFilters.h | 11 ++++++++--- test/ufo/ObsFunction.h | 5 +++++ tools/new_qc/example/Example.cc | 5 +++-- tools/new_qc/example/Example.h | 3 ++- tools/new_qc/example/Example.interface.F90 | 5 +++-- tools/new_qc/example/Example.interface.h | 2 +- tools/new_qc/example/ufo_example_mod.F90 | 3 ++- 27 files changed, 91 insertions(+), 51 deletions(-) diff --git a/src/mains/RunCRTM.h b/src/mains/RunCRTM.h index 1728b90b8..a66a2f263 100644 --- a/src/mains/RunCRTM.h +++ b/src/mains/RunCRTM.h @@ -70,9 +70,11 @@ template class RunCRTM : public oops::Application { const ObsAuxCtrl_ ybias(obsdb[jj], biasparams); ObsVector_ hofx(obsdb[jj]); + ObsVector_ bias(obsdb[jj]); + bias.zero(); ObsDiags_ diag(obsdb[jj], hop.locations(), diagvars); - hop.simulateObs(gval, hofx, ybias, diag); + hop.simulateObs(gval, hofx, ybias, bias, diag); const double zz = hofx.rms(); const double xx = conf[jj].getDouble("rms ref"); diff --git a/src/ufo/ObsOperator.cc b/src/ufo/ObsOperator.cc index 6f3d71080..797688a39 100644 --- a/src/ufo/ObsOperator.cc +++ b/src/ufo/ObsOperator.cc @@ -45,15 +45,14 @@ ObsOperator::ObsOperator(ioda::ObsSpace & os, const Parameters_ & params) // ----------------------------------------------------------------------------- void ObsOperator::simulateObs(const GeoVaLs & gvals, ioda::ObsVector & yy, - const ObsBias & bias, ObsDiagnostics & ydiags) const { + const ObsBias & biascoeff, ioda::ObsVector & ybias, + ObsDiagnostics & ydiags) const { oper_->simulateObs(gvals, yy, ydiags); - if (bias) { - ioda::ObsVector ybias(odb_); + if (biascoeff) { ObsBiasOperator biasoper(odb_); - biasoper.computeObsBias(gvals, ybias, bias, ydiags); + biasoper.computeObsBias(gvals, ybias, biascoeff, ydiags); // update H(x) with bias correction yy += ybias; - ybias.save("ObsBias"); } } diff --git a/src/ufo/ObsOperator.h b/src/ufo/ObsOperator.h index e4add5465..9199345ac 100644 --- a/src/ufo/ObsOperator.h +++ b/src/ufo/ObsOperator.h @@ -47,7 +47,8 @@ class ObsOperator : public util::Printable, ObsOperator(ioda::ObsSpace &, const Parameters_ &); /// Obs Operator - void simulateObs(const GeoVaLs &, ioda::ObsVector &, const ObsBias &, ObsDiagnostics &) const; + void simulateObs(const GeoVaLs &, ioda::ObsVector &, const ObsBias &, ioda::ObsVector &, + ObsDiagnostics &) const; /// Operator input required from Model const oops::Variables & requiredVars() const; diff --git a/src/ufo/filters/BackgroundCheck.cc b/src/ufo/filters/BackgroundCheck.cc index cba4216fc..8924b16a7 100644 --- a/src/ufo/filters/BackgroundCheck.cc +++ b/src/ufo/filters/BackgroundCheck.cc @@ -90,18 +90,17 @@ void BackgroundCheck::applyFilter(const std::vector & apply, oops::Log::debug() << "BackgroundCheck obserr: " << *obserr_ << std::endl; ioda::ObsDataVector obs(obsdb_, filtervars.toOopsVariables(), "ObsValue"); - ioda::ObsDataVector obsbias(obsdb_, filtervars.toOopsVariables(), "ObsBias", false); + std::string test_hofx = parameters_.test_hofx.value(); + Variables varhofx(filtervars_, test_hofx); // Get function absolute threshold - if (parameters_.functionAbsoluteThreshold.value()) { // Get function absolute threshold info from configuration const Variable &rtvar = parameters_.functionAbsoluteThreshold.value()->front(); ioda::ObsDataVector function_abs_threshold(obsdb_, rtvar.toOopsVariables()); data_.get(rtvar, function_abs_threshold); - Variables varhofx(filtervars_, test_hofx); for (size_t jv = 0; jv < filtervars.nvars(); ++jv) { size_t iv = observed.find(filtervars.variable(jv).variable()); // H(x) @@ -123,7 +122,7 @@ void BackgroundCheck::applyFilter(const std::vector & apply, } } } else { - Variables varhofx(filtervars_, test_hofx); + Variables varbias(filtervars_, "ObsBiasData"); for (size_t jv = 0; jv < filtervars.nvars(); ++jv) { size_t iv = observed.find(filtervars.variable(jv).variable()); // H(x) @@ -135,6 +134,9 @@ void BackgroundCheck::applyFilter(const std::vector & apply, if (thresholdWrtBGerror) { data_.get(backgrErrVariable(filtervars[jv]), hofxerr); } +// H(x) bias correction + std::vector bias; + data_.get(varbias.variable(jv), bias); // Threshold for current variable std::vector abs_thr(obsdb_.nlocs(), std::numeric_limits::max()); @@ -155,8 +157,8 @@ void BackgroundCheck::applyFilter(const std::vector & apply, (*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])) { ASSERT(obs[jv][jobs] != util::missingValue(obs[jv][jobs])); if (parameters_.BiasCorrectionFactor.value()) { - ASSERT(obsbias[jv][jobs] != util::missingValue(obsbias[jv][jobs])); - bc_factor[jobs] = bc_factor[jobs]*obsbias[jv][jobs]; + ASSERT(bias[jobs] != util::missingValue(bias[jobs])); + bc_factor[jobs] = bc_factor[jobs]*bias[jobs]; } ASSERT(hofx[jobs] != util::missingValue(hofx[jobs])); diff --git a/src/ufo/filters/ObsDiagnosticsWriter.h b/src/ufo/filters/ObsDiagnosticsWriter.h index 92c60f134..92385ee0a 100644 --- a/src/ufo/filters/ObsDiagnosticsWriter.h +++ b/src/ufo/filters/ObsDiagnosticsWriter.h @@ -43,7 +43,8 @@ class ObsDiagnosticsWriter : public oops::interface::ObsFilterBase { void preProcess() override {} void priorFilter(const GeoVaLs &) override {} - void postFilter(const ioda::ObsVector &, const ObsDiagnostics & diags) override { + void postFilter(const ioda::ObsVector &, const ioda::ObsVector &, + const ObsDiagnostics & diags) override { diags.write(config_); } diff --git a/src/ufo/filters/ObsFilterData.cc b/src/ufo/filters/ObsFilterData.cc index 3fb1870bd..bb4422291 100644 --- a/src/ufo/filters/ObsFilterData.cc +++ b/src/ufo/filters/ObsFilterData.cc @@ -39,9 +39,9 @@ void ObsFilterData::associate(const GeoVaLs & gvals) { } // ----------------------------------------------------------------------------- -/*! Associates H(x) ObsVector with this ObsFilterData */ -void ObsFilterData::associate(const ioda::ObsVector & hofx, const std::string & name) { - ovecs_[name] = &hofx; +/*! Associates H(x)-like ObsVector with this ObsFilterData */ +void ObsFilterData::associate(const ioda::ObsVector & data, const std::string & name) { + ovecs_[name] = &data; } // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/ObsProcessorBase.cc b/src/ufo/filters/ObsProcessorBase.cc index 7e4943d0e..a39bc2298 100644 --- a/src/ufo/filters/ObsProcessorBase.cc +++ b/src/ufo/filters/ObsProcessorBase.cc @@ -54,7 +54,8 @@ void ObsProcessorBase::preProcess() { oops::Log::trace() << "ObsProcessorBase preProcess begin" << std::endl; // Cannot determine earlier when to apply filter because subclass // constructors add to allvars - if (allvars_.hasGroup("HofX") || allvars_.hasGroup("ObsDiag") || deferToPost_) { + if (allvars_.hasGroup("HofX") || allvars_.hasGroup("ObsDiag") || + allvars_.hasGroup("ObsBiasData") || deferToPost_) { post_ = true; } else { if (allvars_.hasGroup("GeoVaLs")) { @@ -77,10 +78,13 @@ void ObsProcessorBase::priorFilter(const GeoVaLs & gv) { // ----------------------------------------------------------------------------- -void ObsProcessorBase::postFilter(const ioda::ObsVector & hofx, const ObsDiagnostics & diags) { +void ObsProcessorBase::postFilter(const ioda::ObsVector & hofx, + const ioda::ObsVector & bias, + const ObsDiagnostics & diags) { oops::Log::trace() << "ObsProcessorBase postFilter begin" << std::endl; if (post_) { data_.associate(hofx, "HofX"); + data_.associate(bias, "ObsBiasData"); data_.associate(diags); this->doFilter(); } diff --git a/src/ufo/filters/ObsProcessorBase.h b/src/ufo/filters/ObsProcessorBase.h index cdf824f78..a9ceeb3c7 100644 --- a/src/ufo/filters/ObsProcessorBase.h +++ b/src/ufo/filters/ObsProcessorBase.h @@ -45,7 +45,8 @@ class ObsProcessorBase : public oops::interface::ObsFilterBase { void preProcess() override; void priorFilter(const GeoVaLs &) override; - void postFilter(const ioda::ObsVector &, const ObsDiagnostics &) override; + void postFilter(const ioda::ObsVector &, const ioda::ObsVector &, + const ObsDiagnostics &) override; oops::Variables requiredVars() const override { return allvars_.allFromGroup("GeoVaLs").toOopsVariables();} diff --git a/src/ufo/filters/QCmanager.cc b/src/ufo/filters/QCmanager.cc index 6ee70757f..6a2a207a5 100644 --- a/src/ufo/filters/QCmanager.cc +++ b/src/ufo/filters/QCmanager.cc @@ -94,7 +94,9 @@ QCmanager::QCmanager(ioda::ObsSpace & obsdb, const eckit::Configuration & /*conf // ----------------------------------------------------------------------------- -void QCmanager::postFilter(const ioda::ObsVector & hofx, const ObsDiagnostics &) { +void QCmanager::postFilter(const ioda::ObsVector & hofx, + const ioda::ObsVector & /*bias*/, + const ObsDiagnostics & /*diags*/) { oops::Log::trace() << "QCmanager postFilter" << std::endl; const double missing = util::missingValue(missing); diff --git a/src/ufo/filters/QCmanager.h b/src/ufo/filters/QCmanager.h index 4d3f887d8..d56070e3b 100644 --- a/src/ufo/filters/QCmanager.h +++ b/src/ufo/filters/QCmanager.h @@ -42,7 +42,8 @@ class QCmanager : public oops::interface::ObsFilterBase { void preProcess() override {} void priorFilter(const GeoVaLs &) override {} - void postFilter(const ioda::ObsVector &, const ObsDiagnostics &) override; + void postFilter(const ioda::ObsVector &, const ioda::ObsVector &, + const ObsDiagnostics &) override; oops::Variables requiredVars() const override {return nogeovals_;} oops::Variables requiredHdiagnostics() const override {return nodiags_;} diff --git a/src/ufo/filters/obsfunctions/CLWRetMW.h b/src/ufo/filters/obsfunctions/CLWRetMW.h index 0e2e76b92..49b8174ba 100755 --- a/src/ufo/filters/obsfunctions/CLWRetMW.h +++ b/src/ufo/filters/obsfunctions/CLWRetMW.h @@ -57,10 +57,10 @@ class CLWRetMWParameters : public oops::Parameters { /// bias_application: ObsValue oops::Parameter addBias{"bias_application", "HofX", this}; - /// Name of the bias correction group used to replace the default group (default is ObsBias) + /// Name of the bias correction group used to replace the default group (default is ObsBiasData) /// Example: use observation bias correction values from GSI /// test_bias: GsiObsBias - oops::Parameter testBias{"test_bias", "ObsBias", this}; + oops::Parameter testBias{"test_bias", "ObsBiasData", this}; /// Cloud index CIret_37v37h_diff: /// 1.0 - (Tb_37v - Tb_37h)/(Tb_37v_clr - Tb_37h_clr), which is used in diff --git a/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.h b/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.h index 5bea6c56f..90b46a647 100755 --- a/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.h +++ b/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.h @@ -48,8 +48,8 @@ class CloudDetectMinResidualAVHRRParameters : public oops::Parameters { /// Name of the HofX group used to replace the default group (default is HofX) oops::Parameter testHofX{"test_hofx", "HofX", this}; - /// Name of the bias correction group used to replace the default group (default is ObsBias) - oops::Parameter testBias{"test_bias", "ObsBias", this}; + /// Name of the bias correction group used to replace the default group (default is ObsBiasData) + oops::Parameter testBias{"test_bias", "ObsBiasData", this}; /// Name of the data group to which the QC flag is applied (default is QCflagsData) oops::Parameter testQCflag{"test_qcflag", "QCflagsData", this}; diff --git a/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.h b/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.h index 7ae58a97d..37aed2e37 100755 --- a/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.h +++ b/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.h @@ -48,10 +48,11 @@ class HydrometeorCheckAMSUAParameters : public oops::Parameters { /// Name of the HofX group used to replace the default group (default is HofX) oops::Parameter testHofX{"test_hofx", "HofX", this}; - /// Name of the group for bias correction used to replace the default group (default is ObsBias) + /// Name of the group for bias correction used to replace the default group (default is + /// ObsBiasData) /// Example: use observation bias correction values from GSI /// test_bias: GsiObsBias - oops::Parameter testBias{"test_bias", "ObsBias", this}; + oops::Parameter testBias{"test_bias", "ObsBiasData", this}; /// Name of the group for bias correction terms used to replace the default group /// (default is ObsBiasTerm) diff --git a/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.h b/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.h index 041f29f73..c1935e544 100755 --- a/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.h +++ b/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.h @@ -48,10 +48,11 @@ class HydrometeorCheckATMSParameters : public oops::Parameters { /// Name of the HofX group used to replace the default group (default is HofX) oops::Parameter testHofX{"test_hofx", "HofX", this}; - /// Name of the group for bias correction used to replace the default group (default is ObsBias) + /// Name of the group for bias correction used to replace the default group (default is + /// ObsBiasData) /// Example: use observation bias correction values from GSI /// test_bias: GsiObsBias - oops::Parameter testBias{"test_bias", "ObsBias", this}; + oops::Parameter testBias{"test_bias", "ObsBiasData", this}; /// Name of the group for bias correction terms used to replace the default group /// (default is ObsBiasTerm) diff --git a/src/ufo/filters/obsfunctions/SCATRetMW.h b/src/ufo/filters/obsfunctions/SCATRetMW.h index 09541c700..52ad006c0 100755 --- a/src/ufo/filters/obsfunctions/SCATRetMW.h +++ b/src/ufo/filters/obsfunctions/SCATRetMW.h @@ -64,10 +64,10 @@ class SCATRetMWParameters : public oops::Parameters { /// bias_application: ObsValue oops::Parameter addBias{"bias_application", "HofX", this}; - /// Name of the bias correction group used to replace the default group (default is ObsBias) + /// Name of the bias correction group used to replace the default group (default is ObsBiasData) /// Example: use observation bias correction values from GSI /// test_bias: GsiObsBias - oops::Parameter testBias{"test_bias", "ObsBias", this}; + oops::Parameter testBias{"test_bias", "ObsBiasData", this}; }; /// diff --git a/src/ufo/filters/obsfunctions/SIRetMW.h b/src/ufo/filters/obsfunctions/SIRetMW.h index 54735436a..495f68939 100755 --- a/src/ufo/filters/obsfunctions/SIRetMW.h +++ b/src/ufo/filters/obsfunctions/SIRetMW.h @@ -56,10 +56,10 @@ class SIRetMWParameters : public oops::Parameters { /// bias_application: ObsValue oops::Parameter addBias{"bias_application", "HofX", this}; - /// Name of the bias correction group used to replace the default group (default is ObsBias) + /// Name of the bias correction group used to replace the default group (default is ObsBiasData) /// Example: use observation bias correction values from GSI /// test_bias: GsiObsBias - oops::Parameter testBias{"test_bias", "ObsBias", this}; + oops::Parameter testBias{"test_bias", "ObsBiasData", this}; }; /// diff --git a/src/ufo/filters/obsfunctions/SymmCldImpactIR.cc b/src/ufo/filters/obsfunctions/SymmCldImpactIR.cc index 65970b7e3..7910af940 100755 --- a/src/ufo/filters/obsfunctions/SymmCldImpactIR.cc +++ b/src/ufo/filters/obsfunctions/SymmCldImpactIR.cc @@ -65,8 +65,8 @@ void SymmCldImpactIR::compute(const ObsFilterData & in, in.get(Variable("brightness_temperature_assuming_clear_sky@ObsDiag", channels_)[ich], clr); in.get(Variable("brightness_temperature@HofX", channels_)[ich], bak); in.get(Variable("brightness_temperature@ObsValue", channels_)[ich], obs); - if (in.has(Variable("brightness_temperature@ObsBias", channels_)[ich])) { - in.get(Variable("brightness_temperature@ObsBias", channels_)[ich], bias); + if (in.has(Variable("brightness_temperature@ObsBiasData", channels_)[ich])) { + in.get(Variable("brightness_temperature@ObsBiasData", channels_)[ich], bias); } else { std::fill(bias.begin(), bias.end(), 0.0f); } diff --git a/test/ufo/ConventionalProfileProcessing.h b/test/ufo/ConventionalProfileProcessing.h index 8a591ccae..f06e44b17 100644 --- a/test/ufo/ConventionalProfileProcessing.h +++ b/test/ufo/ConventionalProfileProcessing.h @@ -19,6 +19,7 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" +#include "ioda/ObsDataVector.h" #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" #include "oops/mpi/mpi.h" @@ -70,6 +71,9 @@ void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { ioda::ObsVector hofx(obsspace); + ioda::ObsVector bias(obsspace); + bias.zero(); + const eckit::LocalConfiguration obsdiagconf(conf, "obs diagnostics"); std::vector varconfs; obsdiagconf.get("variables", varconfs); @@ -117,9 +121,9 @@ void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { filter.preProcess(); filter.priorFilter(*geovals); if (expectThrowDuringOperation) - EXPECT_THROWS(filter.postFilter(hofx, obsdiags)); + EXPECT_THROWS(filter.postFilter(hofx, bias, obsdiags)); else - filter.postFilter(hofx, obsdiags); + filter.postFilter(hofx, bias, obsdiags); // Determine whether the mismatch check should be bypassed or not. // It might be necessary to disable the mismatch check in tests which are diff --git a/test/ufo/MetOfficeBuddyCheck.h b/test/ufo/MetOfficeBuddyCheck.h index 779b83a0e..9a5fc3dab 100644 --- a/test/ufo/MetOfficeBuddyCheck.h +++ b/test/ufo/MetOfficeBuddyCheck.h @@ -69,8 +69,11 @@ void testMetOfficeBuddyCheck(const eckit::LocalConfiguration &conf) { filter.preProcess(); ioda::ObsVector hofx(obsSpace, "HofX"); - ufo::Locations locations(conf, obsSpace.comm()); + ioda::ObsVector bias(obsSpace); + bias.zero(); + + ufo::Locations locations(conf, obsSpace.comm()); const eckit::LocalConfiguration diagConf = conf.getSubConfiguration("obs diagnostics"); oops::Variables diagVars; for (const std::string &name : diagConf.keys()) @@ -82,7 +85,7 @@ void testMetOfficeBuddyCheck(const eckit::LocalConfiguration &conf) { obsDiags.save(diag, name, 0); } - filter.postFilter(hofx, obsDiags); + filter.postFilter(hofx, bias, obsDiags); const eckit::LocalConfiguration pgeConf(conf, "ExpectedGrossErrorProbabilities"); for (const std::string & varNameGroup : pgeConf.keys()) { diff --git a/test/ufo/ObsDiagnostics.h b/test/ufo/ObsDiagnostics.h index 76cf07cce..091817eef 100644 --- a/test/ufo/ObsDiagnostics.h +++ b/test/ufo/ObsDiagnostics.h @@ -73,6 +73,10 @@ void testObsDiagnostics() { // create obsvector to hold H(x) ioda::ObsVector hofx(ospace); + // create obsvector to hold bias + ioda::ObsVector bias(ospace); + bias.zero(); + // create diagnostics to hold HofX diags eckit::LocalConfiguration diagconf(conf, "obs diagnostics"); oops::Variables diagvars(diagconf, "variables"); @@ -81,7 +85,7 @@ void testObsDiagnostics() { ObsDiagnostics diags(ospace, *(locs.get()), diagvars); // call H(x) to compute diagnostics - hop.simulateObs(gval, hofx, ybias, diags); + hop.simulateObs(gval, hofx, ybias, bias, diags); // read tolerance and reference Diagnostics const double tol = conf.getDouble("tolerance"); diff --git a/test/ufo/ObsFilters.h b/test/ufo/ObsFilters.h index 5e74839eb..58205c6ae 100644 --- a/test/ufo/ObsFilters.h +++ b/test/ufo/ObsFilters.h @@ -403,6 +403,11 @@ void testFilters(size_t obsSpaceIndex, oops::ObsSpace &obspace, /// call priorFilter and postFilter if hofx is available oops::Variables geovars = filters.requiredVars(); oops::Variables diagvars = filters.requiredHdiagnostics(); + +/// initialize zero bias + ObsVector_ bias(obspace); + bias.zero(); + if (params.hofx.value() != boost::none) { /// read GeoVaLs from file if required std::unique_ptr gval; @@ -430,7 +435,7 @@ void testFilters(size_t obsSpaceIndex, oops::ObsSpace &obspace, << std::endl; } const ObsDiags_ diags(obsdiagconf, obspace, diagvars); - filters.postFilter(hofx, diags); + filters.postFilter(hofx, bias, diags); } else if (params.obsOperator.value() != boost::none) { /// read GeoVaLs, compute H(x) and ObsDiags oops::Log::info() << "ObsOperator section specified, computing HofX" << std::endl; @@ -450,9 +455,9 @@ void testFilters(size_t obsSpaceIndex, oops::ObsSpace &obspace, diagvars += ybias.requiredHdiagnostics(); ObsDiags_ diags(obspace, hop.locations(), diagvars); filters.priorFilter(gval); - hop.simulateObs(gval, hofx, ybias, diags); + hop.simulateObs(gval, hofx, ybias, bias, diags); hofx.save("hofx"); - filters.postFilter(hofx, diags); + filters.postFilter(hofx, bias, diags); } else if (geovars.size() > 0) { /// Only call priorFilter if (params.geovals.value() == boost::none) diff --git a/test/ufo/ObsFunction.h b/test/ufo/ObsFunction.h index 5f12ca86c..cf6f16ece 100644 --- a/test/ufo/ObsFunction.h +++ b/test/ufo/ObsFunction.h @@ -161,6 +161,11 @@ void doTestFunction(ioda::ObsSpace &ospace, const eckit::Configuration &conf) { inputs.associate(*gval); } +/// Setup zero ObsBias + ioda::ObsVector bias(ospace); + bias.zero(); + inputs.associate(bias, "ObsBiasData"); + /// Setup ObsDiags const oops::Variables diagvars = allfuncvars.allFromGroup("ObsDiag").toOopsVariables(); std::unique_ptr diags; diff --git a/tools/new_qc/example/Example.cc b/tools/new_qc/example/Example.cc index f0198d3c3..d719babbd 100644 --- a/tools/new_qc/example/Example.cc +++ b/tools/new_qc/example/Example.cc @@ -47,10 +47,11 @@ void Example::priorFilter(const GeoVaLs & gv) { // ----------------------------------------------------------------------------- -void Example::postFilter(const ioda::ObsVector & hofxb, const ObsDiagnostics & diags) { +void Example::postFilter(const ioda::ObsVector & hofxb, const ioda::ObsVector & bias, + const ObsDiagnostics & diags) { oops::Log::trace() << "Example postFilter" << std::endl; ufo_example_post_f90(key_, obsdb_, hofxb.nvars(), hofxb.nlocs(), hofxb.toFortran(), - diags.toFortran()); + bias.toFortran(), diags.toFortran()); } // ----------------------------------------------------------------------------- diff --git a/tools/new_qc/example/Example.h b/tools/new_qc/example/Example.h index 00acc944d..da7c03e74 100644 --- a/tools/new_qc/example/Example.h +++ b/tools/new_qc/example/Example.h @@ -46,7 +46,8 @@ class Example : public oops::interface::ObsFilterBase, void preProcess() override {} void priorFilter(const GeoVaLs &) override; - void postFilter(const ioda::ObsVector &, const ObsDiagnostics &) override; + void postFilter(const ioda::ObsVector &, const ioda::ObsVector &, + const ObsDiagnostics &) override; oops::Variables requiredVars() const override {return geovars_;} oops::Variables requiredHdiagnostics() const override {return diagnostics_;} diff --git a/tools/new_qc/example/Example.interface.F90 b/tools/new_qc/example/Example.interface.F90 index 719d286e3..9b4d719d7 100644 --- a/tools/new_qc/example/Example.interface.F90 +++ b/tools/new_qc/example/Example.interface.F90 @@ -88,12 +88,13 @@ end subroutine ufo_example_prior_c ! ------------------------------------------------------------------------------ -subroutine ufo_example_post_c(c_self, c_obspace, c_nvars, c_nlocs, c_hofx, c_key_hofxdiags) bind(c,name='ufo_example_post_f90') +subroutine ufo_example_post_c(c_self, c_obspace, c_nvars, c_nlocs, c_hofx, c_hofxbias, c_key_hofxdiags) bind(c,name='ufo_example_post_f90') implicit none integer(c_int), intent(in) :: c_self type(c_ptr), value, intent(in) :: c_obspace integer(c_int), intent(in) :: c_nvars, c_nlocs real(c_double), intent(in) :: c_hofx(c_nvars, c_nlocs) +real(c_double), intent(in) :: c_hofxbias(c_nvars, c_nlocs) integer(c_int), intent(in) :: c_key_hofxdiags type(ufo_example), pointer :: self @@ -102,7 +103,7 @@ subroutine ufo_example_post_c(c_self, c_obspace, c_nvars, c_nlocs, c_hofx, c_key call ufo_example_registry%get(c_self, self) call ufo_geovals_registry%get(c_key_hofxdiags, hofxdiags) -call ufo_example_post(self, c_obspace, c_nvars, c_nlocs, c_hofx, hofxdiags) +call ufo_example_post(self, c_obspace, c_nvars, c_nlocs, c_hofx, c_hofxbias, hofxdiags) end subroutine ufo_example_post_c diff --git a/tools/new_qc/example/Example.interface.h b/tools/new_qc/example/Example.interface.h index d79e1dcd0..a46221261 100644 --- a/tools/new_qc/example/Example.interface.h +++ b/tools/new_qc/example/Example.interface.h @@ -34,7 +34,7 @@ extern "C" { void ufo_example_prior_f90(const F90check &, const ioda::ObsSpace &, const F90goms &); void ufo_example_post_f90(const F90check &, const ioda::ObsSpace &, const int &, - const int &, const double &, const F90goms &); + const int &, const double &, const double &, const F90goms &); } // extern C } // namespace ufo diff --git a/tools/new_qc/example/ufo_example_mod.F90 b/tools/new_qc/example/ufo_example_mod.F90 index 8f151079a..db5ab50e1 100644 --- a/tools/new_qc/example/ufo_example_mod.F90 +++ b/tools/new_qc/example/ufo_example_mod.F90 @@ -61,12 +61,13 @@ end subroutine ufo_example_prior ! ------------------------------------------------------------------------------ -subroutine ufo_example_post(self, obspace, nvars, nlocs, hofx, hofxdiags) +subroutine ufo_example_post(self, obspace, nvars, nlocs, hofx, hofxbias, hofxdiags) implicit none type(ufo_example), intent(in) :: self type(c_ptr), value, intent(in) :: obspace integer, intent(in) :: nvars, nlocs real(c_double), intent(in) :: hofx(nvars, nlocs) +real(c_double), intent(in) :: hofxbias(nvars, nlocs) type(ufo_geovals), intent(in) :: hofxdiags end subroutine ufo_example_post From da0195611fac5ae96d0fe4d57f41135cbcec7ffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Thu, 19 Aug 2021 13:45:56 +0100 Subject: [PATCH 87/98] Construct ObsError implementations from Parameters rather than a Configuration. (#1407) --- src/ufo/errors/ObsErrorCrossVarCov.cc | 5 +---- src/ufo/errors/ObsErrorCrossVarCov.h | 10 +++++++--- src/ufo/errors/ObsErrorDiagonal.cc | 6 +++--- src/ufo/errors/ObsErrorDiagonal.h | 12 ++++++++---- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/ufo/errors/ObsErrorCrossVarCov.cc b/src/ufo/errors/ObsErrorCrossVarCov.cc index 738fd6c0e..316a59c9f 100644 --- a/src/ufo/errors/ObsErrorCrossVarCov.cc +++ b/src/ufo/errors/ObsErrorCrossVarCov.cc @@ -18,16 +18,13 @@ namespace ufo { // ----------------------------------------------------------------------------- -ObsErrorCrossVarCov::ObsErrorCrossVarCov(const eckit::Configuration & conf, +ObsErrorCrossVarCov::ObsErrorCrossVarCov(const Parameters_ & options, ioda::ObsSpace & obspace, const eckit::mpi::Comm &timeComm) : ObsErrorBase(timeComm), stddev_(obspace, "ObsError"), varcorrelations_(Eigen::MatrixXd::Identity(stddev_.nvars(), stddev_.nvars())) { - ObsErrorCrossVarCovParameters options; - options.deserialize(conf); - // Open and read error correlations from the hdf5 file ioda::Engines::BackendNames backendName = ioda::Engines::BackendNames::Hdf5File; ioda::Engines::BackendCreationParameters backendParams; diff --git a/src/ufo/errors/ObsErrorCrossVarCov.h b/src/ufo/errors/ObsErrorCrossVarCov.h index 32c48e8ae..75dde554f 100644 --- a/src/ufo/errors/ObsErrorCrossVarCov.h +++ b/src/ufo/errors/ObsErrorCrossVarCov.h @@ -32,8 +32,8 @@ namespace ioda { namespace ufo { /// \brief Parameters for obs errors with cross-variable correlations -class ObsErrorCrossVarCovParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(ObsErrorCrossVarCovParameters, Parameters) +class ObsErrorCrossVarCovParameters : public oops::ObsErrorParametersBase { + OOPS_CONCRETE_PARAMETERS(ObsErrorCrossVarCovParameters, ObsErrorParametersBase) public: /// Input file containing correlations oops::RequiredParameter inputFile{"input file", this}; @@ -50,8 +50,12 @@ class ObsErrorCrossVarCovParameters : public oops::Parameters { /// on the diagonal, and C is the correlation matrix. class ObsErrorCrossVarCov : public oops::interface::ObsErrorBase { public: + /// The type of parameters passed to the constructor. + /// This typedef is used by the ObsErrorFactory. + typedef ObsErrorCrossVarCovParameters Parameters_; + /// Initialize observation errors - ObsErrorCrossVarCov(const eckit::Configuration &, ioda::ObsSpace &, + ObsErrorCrossVarCov(const Parameters_ &, ioda::ObsSpace &, const eckit::mpi::Comm &timeComm); /// Update obs error standard deviations to be equal to \p stddev diff --git a/src/ufo/errors/ObsErrorDiagonal.cc b/src/ufo/errors/ObsErrorDiagonal.cc index eb8cb9416..77e1cd640 100644 --- a/src/ufo/errors/ObsErrorDiagonal.cc +++ b/src/ufo/errors/ObsErrorDiagonal.cc @@ -16,11 +16,11 @@ namespace ufo { // ----------------------------------------------------------------------------- -ObsErrorDiagonal::ObsErrorDiagonal(const eckit::Configuration & conf, ioda::ObsSpace & obsgeom, +ObsErrorDiagonal::ObsErrorDiagonal(const Parameters_ & options, ioda::ObsSpace & obsgeom, const eckit::mpi::Comm &timeComm) - : ObsErrorBase(timeComm), stddev_(obsgeom, "ObsError"), inverseVariance_(obsgeom) + : ObsErrorBase(timeComm), + stddev_(obsgeom, "ObsError"), inverseVariance_(obsgeom), options_(options) { - options_.deserialize(conf); inverseVariance_ = stddev_; inverseVariance_ *= stddev_; inverseVariance_.invert(); diff --git a/src/ufo/errors/ObsErrorDiagonal.h b/src/ufo/errors/ObsErrorDiagonal.h index 521c83210..7bec16348 100644 --- a/src/ufo/errors/ObsErrorDiagonal.h +++ b/src/ufo/errors/ObsErrorDiagonal.h @@ -32,8 +32,8 @@ namespace ioda { namespace ufo { /// \brief Parameters for diagonal obs errors -class ObsErrorDiagonalParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(ObsErrorDiagonalParameters, Parameters) +class ObsErrorDiagonalParameters : public oops::ObsErrorParametersBase { + OOPS_CONCRETE_PARAMETERS(ObsErrorDiagonalParameters, ObsErrorParametersBase) public: /// perturbation amplitude multiplier oops::Parameter pert{"random amplitude", 1.0, this}; @@ -43,9 +43,13 @@ class ObsErrorDiagonalParameters : public oops::Parameters { /// \brief Diagonal observation error covariance matrix. class ObsErrorDiagonal : public oops::interface::ObsErrorBase { public: + /// The type of parameters passed to the constructor. + /// This typedef is used by the ObsErrorFactory. + typedef ObsErrorDiagonalParameters Parameters_; + static const std::string classname() {return "ufo::ObsErrorDiagonal";} - ObsErrorDiagonal(const eckit::Configuration &, ioda::ObsSpace &, + ObsErrorDiagonal(const Parameters_ &, ioda::ObsSpace &, const eckit::mpi::Comm &timeComm); /// Update after obs errors potentially changed @@ -76,7 +80,7 @@ class ObsErrorDiagonal : public oops::interface::ObsErrorBase { void print(std::ostream &) const override; ioda::ObsVector stddev_; ioda::ObsVector inverseVariance_; - ObsErrorDiagonalParameters options_; + Parameters_ options_; }; // ----------------------------------------------------------------------------- From 7147100bb555e59e1b7605210b1f9cd08a693f31 Mon Sep 17 00:00:00 2001 From: Carwyn Pelley Date: Thu, 19 Aug 2021 17:08:47 +0100 Subject: [PATCH 88/98] DataExtractor linear and bilinear interpolation 3D array support and NetCDF backend 3D payload gen. support (#1411) * Minor changes * ENH: First pass UI for the bilinear interpolator * MAINT: Comb. parameters for bilinear UI fail * ENH: Working basic bilinear TODO: - Support range (i.e. extraction). - Test different data type combinations. - Add end-to-end test (yaml). - Extend everything to support >2D arrays? * TEST: Mixed type tests added. - Fixed exception msg. TODO: - Alternate dimension mapping. - range support. - 3D arrays. * ENH: Outside bounds assign missing handling. Next: - Test missing data handling (assign nearest neighbour). * ENH: Range support and tested. * TEST: End-to-end DrawValueFromFile with bilinear * MAINT: Coding norms * BUG: Inline function * ENH: Partial changes * MAINT: Review changes * ENH: Working 3D arrays except for the bilinear method * ENH: Fully working 3D arrays TODO: - Test various mappings 3D arrays. * TEST: Tested 3D array support bilinear TODO: - End-to-end linear, bilinear, nearest/exact on 3D * MAINT: Review changes bugfix closest missing * ENH: Split Linear and Bilinear methods * DOC: Neglected to update the doc to reflect previous commit * MAINT: Split payloads into separate files * MAINT: Minor changes for Chris * TEST: 3D end-to-end testing * DOC: Minor changes * Resolving differences with develop * ENH: Using templated alias * Finalised review changes * DOC: Minor doc change * MAINT: Review changes * MAINT: Fixed minor spelling errors Co-authored-by: Ryan Honeyager --- src/ufo/utils/dataextractor/DataExtractor.cc | 180 +++++------ src/ufo/utils/dataextractor/DataExtractor.h | 295 +++++++++++++----- .../dataextractor/DataExtractorCSVBackend.cc | 16 +- .../utils/dataextractor/DataExtractorInput.h | 2 +- .../DataExtractorNetCDFBackend.cc | 24 +- .../DataExtractorNetCDFBackend.h | 2 +- test/testinput/obserror_assign_unittests.yaml | 2 + .../function_drawvaluefromfile.yaml | 131 +++++++- test/ufo/DataExtractor.h | 154 ++++++--- test/ufo/ObsErrorAssign.h | 3 +- 10 files changed, 546 insertions(+), 263 deletions(-) diff --git a/src/ufo/utils/dataextractor/DataExtractor.cc b/src/ufo/utils/dataextractor/DataExtractor.cc index 85b42638b..2fcabfc63 100644 --- a/src/ufo/utils/dataextractor/DataExtractor.cc +++ b/src/ufo/utils/dataextractor/DataExtractor.cc @@ -27,8 +27,10 @@ namespace ufo { + namespace { + /// \brief Boost visitor which allows us to sort a vector. class SortUpdateVisitor : public boost::static_visitor { public: @@ -329,16 +331,15 @@ void match(InterpMethod method, } -/// \brief Perform piecewise linear interpolation along axis `dimIndex` at the 'location' `obVal`. +/// \brief Perform piecewise linear interpolation of the provided array `varValues` at 'location' +/// `obVal`. /// -/// \details It is assumed that previous calls to extract() have extracted a single 1D slice of -/// the interpolated array parallel to the axis `varName`. This function returns the value -/// produced by piecewise linear interpolation of this slice at the point `obVal`. +/// \details It is assumed that the provided 1D array is described by coordinate `varName`. +/// This function returns the value produced by piecewise linear interpolation of this array at +/// the point `obVal`. /// -/// \param[in] dimIndex -/// Axis of `interpolatedArray` along which to interpolate. -/// \param[in] ranges -/// Ranges of slices of `interpolatedArray` matching all constraints considered so far. +/// \param[in] range +/// Defines how to constrain (slice) `varValues` along with `interpolatedArray`. /// \param[in] varName /// Name of the coordinate along which to interpolate. /// \param[in] varValues @@ -348,23 +349,12 @@ void match(InterpMethod method, /// \param[in] interpolatedArray /// Interpolated array. template -float linearInterpolation(size_t dimIndex, - const std::array &ranges, - const std::string &varName, - const std::vector &varValues, - const CoordinateValue &obVal, - const boost::multi_array &interpolatedArray) { - // Sanity check constraint - int sizeDim0 = ranges[0].size(); - int sizeDim1 = ranges[1].size(); - if ((dimIndex == 1 && !(sizeDim1 > 1 && sizeDim0 == 1)) || - (dimIndex == 0 && !(sizeDim0 > 1 && sizeDim1 == 1))) { - throw eckit::Exception("Linear interpolation failed - data must be 1D.", Here()); - } - - // Constrain our index range in the relevant dimension. - const ConstrainedRange &range = ranges[static_cast(dimIndex)]; - +float linearInterpolation( + const std::string &varName, + const std::vector &varValues, + const CoordinateValue &obVal, + const ConstrainedRange &range, + const DataExtractorPayload::const_array_view<1>::type &interpolatedArray) { if ((obVal > varValues[range.end() - 1]) || (obVal < varValues[range.begin()])) { throw eckit::Exception("Linear interpolation failed, value is beyond grid extent." "No extrapolation supported.", @@ -375,27 +365,13 @@ float linearInterpolation(size_t dimIndex, varValues.begin() + range.end(), obVal) - varValues.begin(); - // Determine upper or lower indices from this - if (varValues[nnIndex] == obVal) { - // No interpolation required (is equal) - float res; - if (dimIndex == 1) { - res = interpolatedArray[ranges[0].begin()][nnIndex]; - } else { - res = interpolatedArray[nnIndex][ranges[1].begin()]; - } - return res; - } + // No interpolation required (is equal) + if (varValues[nnIndex] == obVal) + return interpolatedArray[nnIndex]; + // Linearly interpolate between these two indices. - float zUpper; - float zLower; - if (dimIndex == 1) { - zUpper = interpolatedArray[ranges[0].begin()][nnIndex]; - zLower = interpolatedArray[ranges[0].begin()][nnIndex-1]; - } else { - zUpper = interpolatedArray[nnIndex][ranges[1].begin()]; - zLower = interpolatedArray[nnIndex-1][ranges[1].begin()]; - } + const float zUpper = interpolatedArray[nnIndex]; + const float zLower = interpolatedArray[nnIndex-1]; float res = ((static_cast(obVal - varValues[nnIndex-1]) / static_cast(varValues[nnIndex] - varValues[nnIndex-1])) * (zUpper - zLower)) + zLower; @@ -403,12 +379,12 @@ float linearInterpolation(size_t dimIndex, } -float linearInterpolation(size_t dimIndex, - const std::array &ranges, - const std::string &varName, - const std::vector &varValues, - const std::string &obVal, - const boost::multi_array &interpolatedArray) { +float linearInterpolation( + const std::string &varName, + const std::vector &varValues, + const std::string &obVal, + const ConstrainedRange &range, + const DataExtractorPayload::const_array_view<1>::type &interpolatedArray) { throw eckit::UserError("Linear interpolation cannot be performed along coordinate axes indexed " "by string variables such as " + varName + ".", Here()); } @@ -423,9 +399,10 @@ DataExtractor::DataExtractor(const std::string &filepath, load(filepath, group); // Start by constraining to the full range of our data resetExtract(); - // Initialise splitter for both dimensions - splitter_.emplace_back(ufo::RecursiveSplitter(interpolatedArray2D_.shape()[0])); - splitter_.emplace_back(ufo::RecursiveSplitter(interpolatedArray2D_.shape()[1])); + // Initialise splitter for each dimension + splitter_.emplace_back(ufo::RecursiveSplitter(interpolatedArray_.shape()[0])); + splitter_.emplace_back(ufo::RecursiveSplitter(interpolatedArray_.shape()[1])); + splitter_.emplace_back(ufo::RecursiveSplitter(interpolatedArray_.shape()[2])); } @@ -437,9 +414,10 @@ void DataExtractor::load(const std::string &filepath, coord2DimMapping_ = std::move(input.coord2DimMapping); dim2CoordMapping_ = std::move(input.dim2CoordMapping); coordsVals_ = std::move(input.coordsVals); - interpolatedArray2D_.resize(boost::extents[input.payloadArray.shape()[0]] - [input.payloadArray.shape()[1]]); - interpolatedArray2D_ = std::move(input.payloadArray); + interpolatedArray_.resize(boost::extents[input.payloadArray.shape()[0]] + [input.payloadArray.shape()[1]] + [input.payloadArray.shape()[2]]); + interpolatedArray_ = std::move(input.payloadArray); // Set the unconstrained size of matching ranges along both axes of the payload array. for (size_t i = 0; i < constrainedRanges_.size(); ++i) constrainedRanges_[i] = ConstrainedRange(input.payloadArray.shape()[i]); @@ -463,49 +441,55 @@ DataExtractor::createBackendFor(const std::string &filepath) { template void DataExtractor::sort() { - boost::multi_array sortedArray = interpolatedArray2D_; + DataExtractorPayload sortedArray = interpolatedArray_; nextCoordToExtractBy_ = coordsToExtractBy_.begin(); for (size_t dim = 0; dim < dim2CoordMapping_.size(); ++dim) { + if (interpolatedArray_.shape()[dim] == 1) // Avoid sorting scalar coordinates + continue; + // Reorder coordinates for (auto &coord : dim2CoordMapping_[dim]) { auto &coordVal = coordsVals_[coord]; SortVisitor visitor(splitter_[dim]); boost::apply_visitor(visitor, coordVal); } + // Reorder the array to be interpolated - if (dim == 0) { - int ind = -1; - for (const auto &group : splitter_[dim].groups()) { - for (const auto &index : group) { - ind++; - oops::Log::debug() << "Sort index dim0; index-from: " << ind << " index-to: " << - index << std::endl; - for (size_t j = 0; j < interpolatedArray2D_.shape()[1]; j++) { - sortedArray[ind][j] = interpolatedArray2D_[index][j]; - } - } + int ind = 0; + std::array otherDims; + for (size_t odim = 0; odim < interpolatedArray_.dimensionality; ++odim) { + if (odim != dim) { + otherDims[ind] = odim; + ind++; } - // Replace the unsorted array with the sorted one. - interpolatedArray2D_ = sortedArray; - } else if (dim == 1) { - int ind = -1; - for (const auto &group : splitter_[dim].groups()) { - for (const auto &index : group) { - ind++; - oops::Log::debug() << "Sort index dim1; index-from: " << ind << " index-to: " << - index << std::endl; - for (size_t i = 0; i < interpolatedArray2D_.shape()[0]; i++) { - sortedArray[i][ind] = interpolatedArray2D_[i][index]; + } + + ind = 0; + for (const auto &group : splitter_[dim].groups()) { + for (const auto &index : group) { + oops::Log::debug() << "Sort index dim" << dim << "; index-from: " << ind << + " index-to: " << index << std::endl; + for (size_t j = 0; j < interpolatedArray_.shape()[otherDims[0]]; j++) { + for (size_t k = 0; k < interpolatedArray_.shape()[otherDims[1]]; k++) { + if (dim == 0) { + sortedArray[ind][j][k] = interpolatedArray_[index][j][k]; + } else if (dim == 1) { + sortedArray[j][ind][k] = interpolatedArray_[j][index][k]; + } else if (dim == 2) { + sortedArray[j][k][ind] = interpolatedArray_[j][k][index]; + } else { + // We shouldn't ever end up here (exception should be thrown eariler). + throw eckit::Exception("Unable to reorder the array to be interpolated: " + "it has more than 3 dimensions.", Here()); + } } } + ind++; } - // Replace the unsorted array with the sorted one. - interpolatedArray2D_ = sortedArray; - } else { - throw eckit::Exception("Unable to reorder the array to be interpolated: " - "it has more than 2 dimensions.", Here()); } + // Replace the unsorted array with the sorted one. + interpolatedArray_ = sortedArray; } } @@ -590,12 +574,13 @@ void DataExtractor::maybeExtractByLinearInterpolation(const T &o template <> template void DataExtractor::maybeExtractByLinearInterpolation(const T &obVal) { - result_ = linearInterpolation(nextCoordToExtractBy_->payloadDim, - constrainedRanges_, - nextCoordToExtractBy_->name, + int dimIndex = nextCoordToExtractBy_->payloadDim; + const auto &interpolatedArray = get1DSlice(interpolatedArray_, + dimIndex, + constrainedRanges_); + result_ = linearInterpolation(nextCoordToExtractBy_->name, boost::get>(nextCoordToExtractBy_->values), - obVal, - interpolatedArray2D_); + obVal, constrainedRanges_[dimIndex], interpolatedArray); resultSet_ = true; } @@ -632,14 +617,13 @@ ExtractedValue DataExtractor::getUniqueMatch() const { // extraction process. ASSERT(!resultSet_); - int sizeDim0 = constrainedRanges_[0].size(); - int sizeDim1 = constrainedRanges_[1].size(); - if (sizeDim0 != 1 || sizeDim1 != 1) { - throw eckit::Exception("Previous calls to extract() have failed to identify " - "a single value to return.", Here()); - } - return interpolatedArray2D_[constrainedRanges_[0].begin()] - [constrainedRanges_[1].begin()]; + for (size_t dim=0; dim < constrainedRanges_.size(); dim++) + if (constrainedRanges_[dim].size() != 1) + throw eckit::Exception("Previous calls to extract() have failed to identify " + "a single value to return.", Here()); + return interpolatedArray_[constrainedRanges_[0].begin()] + [constrainedRanges_[1].begin()] + [constrainedRanges_[2].begin()]; } diff --git a/src/ufo/utils/dataextractor/DataExtractor.h b/src/ufo/utils/dataextractor/DataExtractor.h index 04405daca..dd07dbdc2 100644 --- a/src/ufo/utils/dataextractor/DataExtractor.h +++ b/src/ufo/utils/dataextractor/DataExtractor.h @@ -35,9 +35,142 @@ struct Named_Variable; namespace ufo { +template +using DataExtractorPayload = boost::multi_array; + + +/// \brief Fetch a 1D sliced view of a boost multi_array object. +/// +/// \details Given a set of constraints (ranges), return a 1D sliced view of the array. +/// This will be along the dimension specified. The constraints must constrain all but the +/// dimension that the 1D varies. Note that the 1D slice returned is not itself constrained or +/// reordered. + +/// \param[in] array +/// Some boost array for us to return its sliced view. +/// \param[in] dimIndex +/// Dimension index corresponding to this 1D slice. Though this argument might be considered +/// superfluous with also passing 'ranges', it does serve to ensure that the 1D slice returned +/// corresponds to the dimension index you were expecting. +/// \param[in] ranges +/// Constraints for the array provided, used for extracting a 1D slice. +/// +/// Example: +/// \code +/// // Define some 3D boost array +/// boost::multi_array array (boost::extents[2][3][2]); +/// for (int i=0; i<2; i++) +/// for (int j=0; j<3; j++) +/// for (int k=0; k<2; j++) +/// array[i][j][k] = ... +/// +/// // Let's fetch a 1D slice, constraining our first and last dimensions. +/// std::array ranges {ConstrainedRange(2), ConstrainedRange(3), +/// ConstrainedRange(2)}; +/// ranges[0].constrain(0, 1); // Let's pick the very first index... +/// ranges[2].constrain(0, 1); // Let's pick the very first index... +/// +/// auto arraySliceView = get1DSlice(array, 1, ranges); +/// \endcode +template +const typename DataExtractorPayload::template const_array_view<1>::type get1DSlice( + const DataExtractorPayload &array, const size_t dimIndex, + const std::array &ranges) { + + // Sanity check constraints + for (size_t dim=0; dim < ranges.size(); dim++) + if ((dim != dimIndex) && (ranges[dim].size() > 1)) + throw eckit::Exception( + "Unable to fetch a 1D array slice with the provided constraints.", Here()); + + typedef boost::multi_array_types::index_range range_t; + typedef typename DataExtractorPayload::template const_array_view<1>::type view1D; + typename DataExtractorPayload::index_gen indices; + + if (dimIndex == 0) { + return array[indices[range_t()][ranges[1].begin()][ranges[2].begin()]]; + } else if (dimIndex == 1) { + return array[indices[ranges[0].begin()][range_t()][ranges[2].begin()]]; + } else if (dimIndex == 2) { + return array[indices[ranges[0].begin()][ranges[1].begin()][range_t()]]; + } else { + // We shouldn't ever end up here (exception should be thrown earlier). + throw eckit::UserError("Invalid dimension index, expecting value mappings corresponding to 1 " + "of 3 axes.", Here()); + } +} -/// \brief Perform bilinear interpolation along axis `dimIndex0` and `dimIndex1` at 'location' -/// `obVal0` and `obVal1` corresponding to those axes. + +/// \brief Fetch a 2D sliced view of a boost multi_array object. +/// +/// \details Given a set of constraints (ranges), return a 2D sliced view of the array. +/// This will be along the two dimensions specified. The constraints must constrain all +/// but the two dimensions specified. Note that the 2D slice returned is not itself constrained +/// or reordered. +/// +/// \param[in] array +/// Some boost array for us to return its sliced view. +/// \param[in] dimIndex0 +/// First of two 'array' dimension indices corresponding to our 2D slice. +/// Though this argument might be considered superfluous with also passing 'ranges', it does +/// serve to ensure that the 2D slice returned corresponds to the dimension index you were +/// expecting. +/// \param[in] dimIndex1 +/// Second of two 'array' dimension indices corresponding to our 2D slice. +/// Though this argument might be considered superfluous with also passing 'ranges', it does +/// serve to ensure that the 2D slice returned corresponds to the dimension index you were +/// expecting. +/// \param[in] ranges +/// Constraints for the array provided, used for extracting a 2D slice. +/// +/// Example: +/// \code +/// // Define some 3D boost array +/// boost::multi_array array (boost::extents[2][3][2]); +/// for (int i=0; i<2; i++) +/// for (int j=0; j<3; j++) +/// for (int k=0; k<2; j++) +/// array[i][j][k] = ... +/// +/// // Let's fetch a 2D slice, constraining our first dimension to a specific index. +/// std::array ranges {ConstrainedRange(2), ConstrainedRange(3), +/// ConstrainedRange(2)}; +/// ranges[0].constrain(0, 1); // Let's pick the very first index... +/// +/// auto arraySliceView = get1DSlice(array, 1, 2, ranges); +/// \endcode +template +const typename DataExtractorPayload::template const_array_view<2>::type get2DSlice( + const DataExtractorPayload &array, const size_t dimIndex0, const size_t dimIndex1, + const std::array &ranges) { + + // Sanity check constraints + for (size_t dim=0; dim < ranges.size(); dim++) + if ((dim != dimIndex0) && (dim != dimIndex1) && (ranges[dim].size() > 1)) + throw eckit::Exception( + "Unable to fetch a 2D array slice with the provided constraints.", Here()); + + typedef boost::multi_array_types::index_range range_t; + typedef typename DataExtractorPayload::template const_array_view<2>::type view2D; + typename DataExtractorPayload::index_gen indices; + + size_t sumIndex = dimIndex0 + dimIndex1; + if (sumIndex == 1) { + return array[indices[range_t()][range_t()][ranges[2].begin()]]; + } else if (sumIndex == 2) { + return array[indices[range_t()][ranges[1].begin()][range_t()]]; + } else if (sumIndex == 3) { + return array[indices[ranges[0].begin()][range_t()][range_t()]]; + } else { + // We shouldn't ever end up here (exception should be thrown earlier). + throw eckit::UserError("Invalid dimension index, expecting value mappings corresponding to two " + "of 3 axes.", Here()); + } +} + + +/// \brief Perform bilinear interpolation at 'location' at location `obVal0` and `obVal1`, +/// corresponding to the first and second axes. /// /// \details This function returns the value produced by a bilinear interpolation at point /// `obVal0, obVal1`. Where any of the neighbouring points used in the calculation are missing, @@ -46,45 +179,41 @@ namespace ufo { /// This must be a final call in the sequence calls to extract. This template handles all types /// except string (see below overloads). /// -/// \param[in] dimIndex0 -/// One of two axes of `interpolatedArray` along which to interpolate. This axis corresponds to -/// that which `varName0`, `varValues0` describes and to which `obVal0` is to be interpolated. -/// \param[in] dimIndex1 -/// One of two axes of `interpolatedArray` along which to interpolate. This axis corresponds to -/// that which `varName1`, `varValues1` describes and to which `obVal1` is to be interpolated. -/// \param[in] ranges -/// Ranges of slices of `interpolatedArray` matching all constraints considered so far. /// \param[in] varName0 -/// Name of the coordinate along one axis of two over which to interpolate. -/// \param[in] varName1 -/// Name of the coordinate along one axis of two over which to interpolate. +/// Name of the coordinate describing the first dimension of 'interpolatedArray' over which we +/// are to interpolate. /// \param[in] varValues0 /// Vector of values of the `varName0` coordinate. -/// \param[in] varValues1 -/// Vector of values of the `varName1` coordinate. /// \param[in] obVal0 /// Interpolation location along the axis corresponding to `varName0`. +/// \param[in] range0 +/// Defines how to constrain (slice) `varValues0` and the corresponding first dimension of +/// 'interpolatedArray'. +/// \param[in] varName1 +/// Name of the coordinate describing the second dimension of 'interpolatedArray' over which we +/// are to interpolate. +/// \param[in] varValues1 +/// Vector of values of the `varName1` coordinate. /// \param[in] obVal1 /// Interpolation location along the axis corresponding to `varName1`. +/// \param[in] range1 +/// Defines how to constrain (slice) `varValues1` and the corresponding second dimension of +/// 'interpolatedArray'. /// \param[in] interpolatedArray -/// Interpolated array. +/// 2D interpolated array view. template -float bilinearInterpolation(const size_t dimIndex0, - const std::string &varName0, - const std::vector &varValues0, - const T &obVal0, - const size_t dimIndex1, - const std::string &varName1, - const std::vector &varValues1, - const R &obVal1, - const std::array &ranges, - const boost::multi_array &interpolatedArray) { +float bilinearInterpolation( + const std::string &varName0, + const std::vector &varValues0, + const T &obVal0, + const ConstrainedRange &range0, + const std::string &varName1, + const std::vector &varValues1, + const R &obVal1, + const ConstrainedRange &range1, + const DataExtractorPayload::const_array_view<2>::type &interpolatedArray) { const float missing = util::missingValue(missing); - // Constrain our index range in the relevant dimension. - const ConstrainedRange &range0 = ranges[dimIndex0]; - const ConstrainedRange &range1 = ranges[dimIndex1]; - const int nnIndex0 = std::lower_bound(varValues0.begin() + range0.begin(), varValues0.begin() + range0.end(), obVal0) - varValues0.begin(); @@ -92,18 +221,9 @@ float bilinearInterpolation(const size_t dimIndex0, varValues1.begin() + range1.end(), obVal1) - varValues1.begin(); - // Convenience function to extract specified index from the multi-array - auto getMultiArrayVal = [&] (int index0, int index1) { - if (dimIndex0 == 0) { - return interpolatedArray[index0][index1]; - } else { - return interpolatedArray[index1][index0]; - } - }; - if ((varValues0[nnIndex0] == obVal0) && (varValues1[nnIndex1] == obVal1)) { // No interpolation required. - return getMultiArrayVal(nnIndex0, nnIndex1); + return interpolatedArray[nnIndex0][nnIndex1]; } // Setup points @@ -125,8 +245,8 @@ float bilinearInterpolation(const size_t dimIndex0, return missing; // Z values at these locations - const float q11 = getMultiArrayVal(ix1, iy1), q12 = getMultiArrayVal(ix1, iy2), - q22 = getMultiArrayVal(ix2, iy2), q21 = getMultiArrayVal(ix2, iy1); + const float q11 = interpolatedArray[ix1][iy1], q12 = interpolatedArray[ix1][iy2], + q22 = interpolatedArray[ix2][iy2], q21 = interpolatedArray[ix2][iy1]; // Missing data handling // - Pick nearest non-missing neighbour of the moore neighbourhood (no diagonals) if any of these @@ -164,49 +284,49 @@ float bilinearInterpolation(const size_t dimIndex0, /// \brief Bilinear interpolation template, templated coord1, string coord2. template -float bilinearInterpolation(const size_t dimIndex0, - const std::string &varName0, - const std::vector &varValues0, - const T &obVal0, - const size_t dimIndex1, - const std::string &varName1, - const std::vector &varValues1, - const std::string &obVal1, - const std::array &ranges, - const boost::multi_array &interpolatedArray) { +float bilinearInterpolation( + const std::string &varName0, + const std::vector &varValues0, + const T &obVal0, + const ConstrainedRange &range0, + const std::string &varName1, + const std::vector &varValues1, + const std::string &obVal1, + const ConstrainedRange &range1, + const DataExtractorPayload::const_array_view<2>::type &interpolatedArray) { throw eckit::UserError("Bilinear interpolation cannot be performed along coordinate axes indexed " "by string variables such as " + varName1 + ".", Here()); } /// \brief Bilinear interpolation template, string coord1, templated coord2. -template -float bilinearInterpolation(const size_t dimIndex0, - const std::string &varName0, - const std::vector &varValues0, - const std::string &obVal0, - const size_t dimIndex1, - const std::string &varName1, - const std::vector &varValues1, - const R &obVal1, - const std::array &ranges, - const boost::multi_array &interpolatedArray) { +template +float bilinearInterpolation( + const std::string &varName0, + const std::vector &varValues0, + const std::string &obVal0, + const ConstrainedRange &range0, + const std::string &varName1, + const std::vector &varValues1, + const T &obVal1, + const ConstrainedRange &range1, + const DataExtractorPayload::const_array_view<2>::type &interpolatedArray) { throw eckit::UserError("Bilinear interpolation cannot be performed along coordinate axes indexed " "by string variables such as " + varName0 + ".", Here()); } -// Bilinear interpolation template, string coord1, string coord2. -inline float bilinearInterpolation(const size_t dimIndex0, - const std::string &varName0, - const std::vector &varValues0, - const std::string &obVal0, - const size_t dimIndex1, - const std::string &varName1, - const std::vector &varValues1, - const std::string &obVal1, - const std::array &ranges, - const boost::multi_array &interpolatedArray) { +/// Bilinear interpolation template, string coord1, string coord2. +inline float bilinearInterpolation( + const std::string &varName0, + const std::vector &varValues0, + const std::string &obVal0, + const ConstrainedRange &range0, + const std::string &varName1, + const std::vector &varValues1, + const std::string &obVal1, + const ConstrainedRange &range1, + const DataExtractorPayload::const_array_view<2>::type &interpolatedArray) { throw eckit::UserError("Bilinear interpolation cannot be performed along coordinate axes indexed " "by string variables such as " + varName0 + " or " + varName1 + ".", Here()); @@ -349,7 +469,7 @@ class DataExtractor /// \brief Perform extract, given the observation values associated with the current extract /// iteration and the next. - /// \details Calls the relevant extract methood (linear), corresponding to the coordinate + /// \details Calls the relevant extract method (linear), corresponding to the coordinate /// associated with this extract iteration and the next (along with the associated interpolation /// method). This method actually functions as two iterations, passing the current iteration /// coordinate and the next iteration coordinate. These are passed to the underlying binary @@ -373,7 +493,7 @@ class DataExtractor } /// \brief Fetch the final interpolated value. - /// \details This will only be succesful if previous calls to extract() have produced a single + /// \details This will only be successful if previous calls to extract() have produced a single /// value to return. ExtractedValue getResult(); @@ -419,7 +539,7 @@ class DataExtractor const std::string &filepath); // Object represent the extraction range in both dimensions. - std::array constrainedRanges_; + std::array constrainedRanges_; // Container holding coordinate arrays (of all supported types) loaded from the input file. typedef boost::variant, @@ -427,7 +547,7 @@ class DataExtractor std::vector> CoordinateValues; std::unordered_map coordsVals_; // The array to be interpolated (the payload array). - boost::multi_array interpolatedArray2D_; + DataExtractorPayload interpolatedArray_; // Linear interpolation result. Only used when ExtractedValue = float. interpolation. float result_; // Set to true if result_ is a valid value. @@ -463,8 +583,7 @@ template <> template void DataExtractor::maybeExtractByBiLinearInterpolation( const T &obValDim0, const R &obValDim1) { - auto &range = constrainedRanges_; - + auto &ranges = constrainedRanges_; const size_t dimIndex0 = nextCoordToExtractBy_->payloadDim; const std::string &varName0 = nextCoordToExtractBy_->name; const std::vector &varValues0 = boost::get>(nextCoordToExtractBy_->values); @@ -475,11 +594,19 @@ void DataExtractor::maybeExtractByBiLinearInterpolation( "method 'bilinear'.", Here()); const size_t dimIndex1 = nextCoordToExtractBy_->payloadDim; const std::string &varName1 = nextCoordToExtractBy_->name; - const std::vector &varValues1 = boost::get>(nextCoordToExtractBy_->values); - result_ = bilinearInterpolation(dimIndex0, varName0, varValues0, obValDim0, - dimIndex1, varName1, varValues1, obValDim1, - range, interpolatedArray2D_); + + auto interpolatedArray = get2DSlice(interpolatedArray_, dimIndex0, dimIndex1, + ranges); + if (dimIndex1 > dimIndex0) { + result_ = bilinearInterpolation(varName0, varValues0, obValDim0, ranges[dimIndex0], + varName1, varValues1, obValDim1, ranges[dimIndex1], + interpolatedArray); + } else { + result_ = bilinearInterpolation(varName1, varValues1, obValDim1, ranges[dimIndex1], + varName0, varValues0, obValDim0, ranges[dimIndex0], + interpolatedArray); + } resultSet_ = true; } diff --git a/src/ufo/utils/dataextractor/DataExtractorCSVBackend.cc b/src/ufo/utils/dataextractor/DataExtractorCSVBackend.cc index 5706f9484..bc80edce9 100644 --- a/src/ufo/utils/dataextractor/DataExtractorCSVBackend.cc +++ b/src/ufo/utils/dataextractor/DataExtractorCSVBackend.cc @@ -76,22 +76,22 @@ class AppendValueVisitor : public boost::static_visitor { template void convertVectorToColumnArray(const std::vector &source, - boost::multi_array &destination) { + boost::multi_array &destination) { const Source missingSource = util::missingValue(Source()); const Destination missingDestination = util::missingValue(Destination()); - destination.resize(boost::extents[source.size()][1]); + destination.resize(boost::extents[source.size()][1][1]); for (size_t i = 0; i < source.size(); ++i) if (source[i] != missingSource) - destination[i][0] = static_cast(source[i]); + destination[i][0][0] = static_cast(source[i]); else - destination[i][0] = missingDestination; + destination[i][0][0] = missingDestination; } /// Visitor that converts an std::vector to a boost::multi_array with one column. template class ConvertToBoostMultiArrayVisitor : public boost::static_visitor { public: - explicit ConvertToBoostMultiArrayVisitor(boost::multi_array &output) : + explicit ConvertToBoostMultiArrayVisitor(boost::multi_array &output) : output_(output) {} @@ -99,9 +99,9 @@ class ConvertToBoostMultiArrayVisitor : public boost::static_visitor { typename std::enable_if::value, bool>::type = true> void operator()(const std::vector &values) { - output_.resize(boost::extents[values.size()][1]); + output_.resize(boost::extents[values.size()][1][1]); for (size_t i = 0; i < values.size(); ++i) - output_[i][0] = values[i]; + output_[i][0][0] = values[i]; } template { } private: - boost::multi_array &output_; + boost::multi_array &output_; }; /// \brief Find the index of the column whose name ends with `@` followed by `payloadGroup` diff --git a/src/ufo/utils/dataextractor/DataExtractorInput.h b/src/ufo/utils/dataextractor/DataExtractorInput.h index 19f71c9d2..ef4a28df9 100644 --- a/src/ufo/utils/dataextractor/DataExtractorInput.h +++ b/src/ufo/utils/dataextractor/DataExtractorInput.h @@ -52,7 +52,7 @@ struct DataExtractorInputBase { template struct DataExtractorInput : public DataExtractorInputBase { /// Array from which values will be extracted - boost::multi_array payloadArray; + boost::multi_array payloadArray; }; } // namespace ufo diff --git a/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.cc b/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.cc index f03f85731..203c329a4 100644 --- a/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.cc +++ b/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.cc @@ -227,11 +227,6 @@ DataExtractorInput DataExtractorNetCDFBackend::l // Load the array to be interpolated // -------------------------------- - // NOTE: - // - We might want to eventually put together a more generalised approach (a collapse - // method taking the dimension as argument) - // - We might also want to not assume that the array is ordered? - by using - // coordinate info. though I'm not sure why it wouldn't be... if (isCovariant == "true") { // This is a full matrix or a stack of full matrices - pull out the diagonals // -------------------------------- @@ -246,10 +241,10 @@ DataExtractorInput DataExtractorNetCDFBackend::l fullMatrices.num_elements())); // Extract their diagonals into the payload array - result.payloadArray.resize(boost::extents[dimdim.dimsCur[1]][dimdim.dimsCur[2]]); + result.payloadArray.resize(boost::extents[dimdim.dimsCur[1]][dimdim.dimsCur[2]][1]); for (int i = 0; i < dimdim.dimsCur[1]; i++) { for (int k = 0; k < dimdim.dimsCur[2]; k++) { - result.payloadArray[i][k] = fullMatrices[i][i][k]; + result.payloadArray[i][k][0] = fullMatrices[i][i][k]; } } } else if (dimdim.dimensionality == 2) { @@ -260,9 +255,9 @@ DataExtractorInput DataExtractorNetCDFBackend::l fullMatrix.num_elements())); // Extract its diagonal into the payload array - result.payloadArray.resize(boost::extents[dimdim.dimsCur[1]][1]); + result.payloadArray.resize(boost::extents[dimdim.dimsCur[1]][1][1]); for (int i = 0; i < dimdim.dimsCur[1]; i++) { - result.payloadArray[i][0] = fullMatrix[i][i]; + result.payloadArray[i][0][0] = fullMatrix[i][i]; } } else { throw eckit::Exception("Expecting 3D or 2D array for error covariance.", Here()); @@ -283,14 +278,17 @@ DataExtractorInput DataExtractorNetCDFBackend::l dim2CoordMapping[dim].emplace_back(coord.first); // update our reverse lookup } } + } else if (dimdim.dimensionality == 3) { + result.payloadArray.resize( + boost::extents[dimdim.dimsCur[0]][dimdim.dimsCur[1]][dimdim.dimsCur[2]]); + interpolatedArrayCoord.read(gsl::span(result.payloadArray.data(), + result.payloadArray.num_elements())); } else if (dimdim.dimensionality == 2) { - // This is just the diagonals - read directly into our container - result.payloadArray.resize(boost::extents[dimdim.dimsCur[0]][dimdim.dimsCur[1]]); + result.payloadArray.resize(boost::extents[dimdim.dimsCur[0]][dimdim.dimsCur[1]][1]); interpolatedArrayCoord.read(gsl::span(result.payloadArray.data(), result.payloadArray.num_elements())); } else if (dimdim.dimensionality == 1) { - // This is just the diagonals - read directly into our container - result.payloadArray.resize(boost::extents[dimdim.dimsCur[0]][1]); + result.payloadArray.resize(boost::extents[dimdim.dimsCur[0]][1][1]); interpolatedArrayCoord.read(gsl::span(result.payloadArray.data(), result.payloadArray.num_elements())); } else { diff --git a/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h b/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h index 5e1e220c2..b66ee7a93 100644 --- a/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h +++ b/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h @@ -24,7 +24,7 @@ namespace ufo /// This is the variable containing the values to be extracted (also known as the _payload_). /// Its type needs to match the template parameter `ExtractedValue`, which must be /// set to either `float`, `int` or `std::string`. -/// - For most types of data, the above array should be 1D or 2D. As a special case, +/// - For most types of data, the above array should be 1D, 2D or 3D. As a special case, /// if this class is used to extract variances, the file may contain a full 2D covariance /// matrix or a stack of such matrices stored as a 3D array, with variances located on the /// diagonal of each matrix. In that case, the array should be equipped with a `full` attribute diff --git a/test/testinput/obserror_assign_unittests.yaml b/test/testinput/obserror_assign_unittests.yaml index 10b8b1047..96939abbb 100644 --- a/test/testinput/obserror_assign_unittests.yaml +++ b/test/testinput/obserror_assign_unittests.yaml @@ -367,6 +367,8 @@ Assign observation error from NetCDF variance; attempt at using 'linear' for str options: file: Data/ufo/testinput_tier_1/obserror_multi_variant_extract_variance_alt.nc interpolation: + - name: air_pressure@MetaData + method: exact - name: StringVar@MetaData method: linear expectExceptionWithMessage: Linear interpolation cannot be performed along coordinate axes indexed by string variables such as StringVar@MetaData diff --git a/test/testinput/unit_tests/function_drawvaluefromfile.yaml b/test/testinput/unit_tests/function_drawvaluefromfile.yaml index 6d644e2a7..0bcc56798 100644 --- a/test/testinput/unit_tests/function_drawvaluefromfile.yaml +++ b/test/testinput/unit_tests/function_drawvaluefromfile.yaml @@ -301,6 +301,24 @@ observations: variables: [assigned_float_variable_bilin] expect compute to throw exception with message: Bilinear interpolation can be used when extracting floating-point values, but not integers or strings +- obs space: &ObsSpace + name: Payload array with scalar coordinates. + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@IntObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_advanced5.nc + group: IntResult2D + interpolation: + - name: longitude@MetaData + method: nearest + - name: latitude@MetaData + method: nearest + variables: [assigned_int_variable_3] + tolerance: 1.0e-6 + # 4. Tests of DrawValueFromFile@StringObsFunction using a NetCDF input file @@ -400,7 +418,7 @@ observations: method: bilinear - name: latitude@MetaData method: bilinear - variables: [assigned_float_variable_bilin] + variables: [assigned_float_variable_bilin_0] expect compute to throw exception with message: Bilinear interpolation can be used when extracting floating-point values, but not integers or strings # 5. Tests of DrawValueFromFile@ObsFunction (float) using a NetCDF input file @@ -420,7 +438,7 @@ observations: method: bilinear - name: latitude@MetaData method: bilinear - variables: [assigned_float_variable_bilin] + variables: [assigned_float_variable_bilin_0] tolerance: 1.0e-6 - obs space: &ObsSpace @@ -438,7 +456,7 @@ observations: method: bilinear - name: latitude@MetaData method: nearest - variables: [assigned_float_variable_bilin] + variables: [assigned_float_variable_bilin_0] expect constructor to throw exception with message: Bilinear interpolation can only be supplied as the final two arguments - obs space: &ObsSpace @@ -456,7 +474,7 @@ observations: method: nearest - name: latitude@MetaData method: bilinear - variables: [assigned_float_variable_bilin] + variables: [assigned_float_variable_bilin_0] expect constructor to throw exception with message: Bilinear interpolation requires two variables - obs space: &ObsSpace @@ -474,7 +492,7 @@ observations: method: linear - name: latitude@MetaData method: nearest - variables: [assigned_float_variable_bilin] + variables: [assigned_float_variable_bilin_0] expect constructor to throw exception with message: Linear interpolation can only be supplied as the very last argument - obs space: &ObsSpace @@ -492,6 +510,105 @@ observations: method: linear - name: latitude@MetaData method: linear - variables: [assigned_float_variable_bilin] - tolerance: 1.0e-6 + variables: [assigned_float_variable_bilin_0] expect constructor to throw exception with message: Linear interpolation can only be supplied as the very last argument + +- obs space: &ObsSpace + name: 3D payload array with bilinear interpolation + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_advanced4.nc + group: FloatResult2D + interpolation: + - name: int_variable_1@MetaData + method: nearest + - name: longitude@MetaData + method: bilinear + - name: latitude@MetaData + method: bilinear + variables: [assigned_float_variable_bilin_0] + tolerance: 1.0e-6 + +- obs space: &ObsSpace + name: 3D payload array with bilinear interpolation alt. mapping + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_advanced4.nc + group: FloatResult2D + interpolation: + - name: int_variable_1@MetaData + method: nearest + - name: latitude@MetaData + method: bilinear + - name: longitude@MetaData + method: bilinear + variables: [assigned_float_variable_bilin_0] + tolerance: 1.0e-6 + +- obs space: &ObsSpace + name: 3D payload array with linear interpolation + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_advanced4.nc + group: FloatResult2D + interpolation: + - name: int_variable_1@MetaData + method: nearest + - name: longitude@MetaData + method: nearest + - name: latitude@MetaData + method: linear + variables: [assigned_float_variable_lin_0] + tolerance: 1.0e-6 + +- obs space: &ObsSpace + name: 3D payload array with linear interpolation coordinate mapped to the final dimension + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_advanced4.nc + group: FloatResult2D + interpolation: + - name: latitude@MetaData + method: nearest + - name: longitude@MetaData + method: nearest + - name: int_variable_1@MetaData + method: linear + variables: [assigned_float_variable_lin_1] + tolerance: 1.0e-6 + +- obs space: &ObsSpace + name: 3D payload array with bilinear interpolation coords mapped to the final two dimensions + obsdatain: + obsfile: Data/ufo/testinput_tier_1/variable_assignment_testdata.nc + simulated variables: [air_temperature] + obs function: + name: DrawValueFromFile@ObsFunction + options: + file: Data/ufo/testinput_tier_1/drawvaluefromfile_advanced4.nc + group: FloatResult2D + interpolation: + - name: latitude@MetaData + method: nearest + - name: int_variable_1@MetaData + method: bilinear + - name: longitude@MetaData + method: bilinear + variables: [assigned_float_variable_bilin_1] + tolerance: 1.0e-6 diff --git a/test/ufo/DataExtractor.h b/test/ufo/DataExtractor.h index ae13981c8..64ed1b649 100644 --- a/test/ufo/DataExtractor.h +++ b/test/ufo/DataExtractor.h @@ -34,26 +34,28 @@ float run_basic(const T obVal0, const R obVal1, const std::vector &varValues0 const int dimIndex1 = 1; const std::string &varName0 = "var0"; const std::string &varName1 = "var1"; - const std::array ranges { + const std::array ranges { ConstrainedRange(varValues0.size()), - ConstrainedRange(varValues1.size())}; + ConstrainedRange(varValues1.size()), + ConstrainedRange(1)}; assert(varValues0.size() == 5); assert(varValues1.size() == 3); - boost::multi_array interpolatedArray(boost::extents[5][3]); + boost::multi_array interpolatedArray(boost::extents[5][3][1]); std::vector tmp = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; size_t ind = 0; for (int j=0; j < interpolatedArray.shape()[1]; j++) { for (int i=0; i < interpolatedArray.shape()[0]; i++) { - interpolatedArray[i][j] = tmp[ind]; + interpolatedArray[i][j][0] = tmp[ind]; ind++; } } - return bilinearInterpolation(dimIndex0, varName0, varValues0, obVal0, - dimIndex1, varName1, varValues1, obVal1, - ranges, interpolatedArray); + auto array = get2DSlice(interpolatedArray, dimIndex0, dimIndex1, ranges); + return bilinearInterpolation(varName0, varValues0, obVal0, ranges[dimIndex0], + varName1, varValues1, obVal1, ranges[dimIndex1], + array); } @@ -68,7 +70,7 @@ float run_basic(const T obVal0, const R obVal1) { CASE("ufo/DataExtractor/bilinearinterp/float_linear") { // Effectively becomes linear interpolation along dim1. - const float res = run_basic(4.0, 4.2); + const float res = run_basic(4.0f, 4.2f); EXPECT(oops::is_close_absolute(res, 7.5f, 1e-5f, 0, oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); } @@ -139,6 +141,7 @@ CASE("ufo/DataExtractor/bilinearinterp/int_float_dtype") { oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); } + CASE("ufo/DataExtractor/bilinearinterp/float_int_dtype") { const float res = run_basic(3.0, 3); EXPECT(oops::is_close_absolute(res, 4.0f, 1e-5f, 0, @@ -166,21 +169,23 @@ CASE("ufo/DataExtractor/bilinearinterp/string_dtype") { float run_missing(const float obVal0, const float obVal1, const std::vector data) { const std::vector varValues0 {2, 4, 6, 8, 10}; const std::vector varValues1 {2, 4, 6}; - const std::array ranges { + const std::array ranges { ConstrainedRange(varValues0.size()), - ConstrainedRange(varValues1.size())}; + ConstrainedRange(varValues1.size()), + ConstrainedRange(1)}; - boost::multi_array interpolatedArray(boost::extents[5][3]); + boost::multi_array interpolatedArray(boost::extents[5][3][1]); size_t ind = 0; for (int j=0; j < interpolatedArray.shape()[1]; j++) { for (int i=0; i < interpolatedArray.shape()[0]; i++) { - interpolatedArray[i][j] = data[ind]; + interpolatedArray[i][j][0] = data[ind]; ind++; } } - return bilinearInterpolation(0, "var0", varValues0, obVal0, - 1, "var1", varValues1, obVal1, - ranges, interpolatedArray); + auto array = get2DSlice(interpolatedArray, 0, 1, ranges); + return bilinearInterpolation("var0", varValues0, obVal0, ranges[0], + "var1", varValues1, obVal1, ranges[1], + array); } @@ -216,36 +221,45 @@ CASE("ufo/DataExtractor/bilinearinterp/all_missing") { } -CASE("ufo/DataExtractor/bilinearinterp/range_constrain") { - // Let's constrain the range and change the dimension mapping. - const float obVal0 = 3.0, obVal1 = 3.0; +float run_range_constrained(const float obVal0, const float obVal1, + const std::array &ranges) { // Coordinates containing values that would change the answer if not excluded // via the "range" specified. const std::vector varValues0 {2, 4, 2, 4, 6}; const std::vector varValues1 {3, 2, 4}; - ConstrainedRange con0 = ConstrainedRange(varValues0.size()); - con0.constrain(2, varValues0.size()); // range will ignore 1st two. - ConstrainedRange con1 = ConstrainedRange(varValues1.size()); - con1.constrain(1, varValues1.size()); // range will ignore 1st. - const std::array ranges {con1, con0}; - const std::vector data = {1, 6, 11, 2, 7, 12, 3, 8, 13, 4, 9, 14, 5, 10, 15}; - boost::multi_array interpolatedArray(boost::extents[3][5]); + boost::multi_array interpolatedArray(boost::extents[3][5][1]); size_t ind = 0; for (int j=0; j < interpolatedArray.shape()[1]; j++) { for (int i=0; i < interpolatedArray.shape()[0]; i++) { - interpolatedArray[i][j] = data[ind]; + interpolatedArray[i][j][0] = data[ind]; ind++; } } - const float res = bilinearInterpolation(1, "var0", varValues0, obVal0, - 0, "var1", varValues1, obVal1, - ranges, interpolatedArray); + auto array = get2DSlice(interpolatedArray, 1, 0, ranges); + return bilinearInterpolation("var1", varValues1, obVal1, ranges[0], + "var0", varValues0, obVal0, ranges[1], + array); +} + + +CASE("ufo/DataExtractor/bilinearinterp/range_constrain") { + // Let's constrain the range and change the dimension mapping. + const float obVal0 = 3.0, obVal1 = 3.0; + + ConstrainedRange con0 = ConstrainedRange(5); + con0.constrain(2, 5); // range will ignore 1st two. + ConstrainedRange con1 = ConstrainedRange(3); + con1.constrain(1, 3); // range will ignore 1st. + ConstrainedRange con2 = ConstrainedRange(1); + const std::array ranges {con1, con0, con2}; + + const float res = run_range_constrained(obVal0, obVal1, ranges); EXPECT(oops::is_close_absolute(res, 11.0f, 1e-5f, 0, oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); } @@ -255,34 +269,74 @@ CASE("ufo/DataExtractor/bilinearinterp/range_constrain_extrapolation") { // Constraining the range, such that, what would otherwise be within bounds is now out of bounds // so returns missing. const float obVal0 = 5.0, obVal1 = 3.0; - // Coordinates containing values that would change the answer if not excluded - // via the "range" specified. - const std::vector varValues0 {2, 4, 2, 4, 6}; - const std::vector varValues1 {3, 2, 4}; - ConstrainedRange con0 = ConstrainedRange(varValues0.size()); - con0.constrain(2, varValues0.size()-1); // range will ignore 1st two and the final element. - ConstrainedRange con1 = ConstrainedRange(varValues1.size()); - con1.constrain(1, varValues1.size()); // range will ignore 1st. - const std::array ranges {con1, con0}; + ConstrainedRange con0 = ConstrainedRange(5); + con0.constrain(2, 4); // range will ignore 1st two and the final element. + ConstrainedRange con1 = ConstrainedRange(3); + con1.constrain(1, 3); // range will ignore 1st. + ConstrainedRange con2 = ConstrainedRange(1); + const std::array ranges {con1, con0, con2}; - const std::vector data = {1, 6, 11, - 2, 7, 12, - 3, 8, 13, - 4, 9, 14, - 5, 10, 15}; - boost::multi_array interpolatedArray(boost::extents[3][5]); + const float res = run_range_constrained(obVal0, obVal1, ranges); + EXPECT_EQUAL(res, missing); +} + + +CASE("ufo/DataExtractor/bilinearinterp/range_constrain_3D_array") { + // Constraining the second dimension. array shape: (5, 6, 3); 2D slice: (:, 2, :) + const std::vector varValues0 {2, 4, 6, 8, 10}; + const std::vector varValues1 {2, 4, 6}; + const int dimIndex0 = 0; + const int dimIndex1 = 2; + const std::string &varName0 = "var0"; + const std::string &varName1 = "var1"; + std::array ranges { + ConstrainedRange(varValues0.size()), + ConstrainedRange(6), + ConstrainedRange(varValues1.size())}; + ranges[1].constrain(2, 3); + + boost::multi_array interpolatedArray(boost::extents[5][6][3]); + const std::vector tmp = {1, 2, 3, 4, 5, + 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15}; size_t ind = 0; - for (int j=0; j < interpolatedArray.shape()[1]; j++) { + for (int j=0; j < interpolatedArray.shape()[2]; j++) { for (int i=0; i < interpolatedArray.shape()[0]; i++) { - interpolatedArray[i][j] = data[ind]; + interpolatedArray[i][2][j] = tmp[ind]; ind++; } } - const float res = bilinearInterpolation(1, "var0", varValues0, obVal0, - 0, "var1", varValues1, obVal1, - ranges, interpolatedArray); - EXPECT_EQUAL(res, missing); + auto array = get2DSlice(interpolatedArray, dimIndex0, dimIndex1, ranges); + const float res = bilinearInterpolation(varName0, varValues0, 4.2f, ranges[dimIndex0], + varName1, varValues1, 4.2f, ranges[dimIndex1], + array); + EXPECT(oops::is_close_absolute(res, 7.6f, 1e-5f, 0, + oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); +} + + +CASE("ufo/DataExtractor/get2DSlice/not_2d_slice") { + // The range specified means that we don't have a single 2D slice of the array + // All dimensions here are unconstrained. + const std::array ranges {ConstrainedRange(2), ConstrainedRange(2), + ConstrainedRange(2)}; + boost::multi_array interpolatedArray(boost::extents[2][2][2]); + + const std::string msg = "Unable to fetch a 2D array slice with the provided constraints."; + EXPECT_THROWS_MSG(get2DSlice(interpolatedArray, 0, 1, ranges), msg.c_str()); +} + + +CASE("ufo/DataExtractor/get1DSlice/not_1d_slice") { + // The range specified means that we don't have a single 1D slice of the array + // All dimensions here are unconstrained. + const std::array ranges {ConstrainedRange(2), ConstrainedRange(2), + ConstrainedRange(2)}; + boost::multi_array interpolatedArray(boost::extents[2][2][2]); + + const std::string msg = "Unable to fetch a 1D array slice with the provided constraints."; + EXPECT_THROWS_MSG(get1DSlice(interpolatedArray, 0, ranges), msg.c_str()); } diff --git a/test/ufo/ObsErrorAssign.h b/test/ufo/ObsErrorAssign.h index 325b7e133..af99e40b9 100644 --- a/test/ufo/ObsErrorAssign.h +++ b/test/ufo/ObsErrorAssign.h @@ -82,7 +82,8 @@ void testObsErrorAssign(const eckit::LocalConfiguration &conf) { int ind = 0; for (size_t varn = 0; varn < obserr->nvars(); ++varn) { for (size_t locn = 0; locn < obserr->nlocs(); ++locn) { - EXPECT(std::abs((*obserr)[varn][locn] - expectedObsError[ind]) < 1e-4); + EXPECT(oops::is_close_absolute((*obserr)[varn][locn], expectedObsError[ind], 1e-4f, 0, + oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); ind++; } } From c80f13a4561da2413c188e0ab89763fa025a3f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Fri, 20 Aug 2021 14:36:52 +0100 Subject: [PATCH 89/98] Suppress potentially verbose output from a number of filters. (#1458) Co-authored-by: Fabio L R Diniz <45880035+fabiolrdiniz@users.noreply.github.com> --- src/ufo/filters/Gaussian_Thinning.cc | 20 -------------------- src/ufo/filters/MetOfficeBuddyCheck.cc | 13 ------------- src/ufo/filters/PoissonDiskThinning.cc | 4 ---- src/ufo/filters/StuckCheck.cc | 2 -- src/ufo/filters/TemporalThinning.cc | 4 ---- src/ufo/filters/TrackCheck.cc | 4 ---- 6 files changed, 47 deletions(-) diff --git a/src/ufo/filters/Gaussian_Thinning.cc b/src/ufo/filters/Gaussian_Thinning.cc index 53703a93e..0902aefab 100644 --- a/src/ufo/filters/Gaussian_Thinning.cc +++ b/src/ufo/filters/Gaussian_Thinning.cc @@ -77,10 +77,6 @@ void Gaussian_Thinning::applyFilter(const std::vector & apply, const std::vector isThinned = identifyThinnedObservations( validObsIds, obsAccessor, splitter, distancesToBinCenter); obsAccessor.flagRejectedObservations(isThinned, flagged); - - if (filtervars.size() != 0) { - oops::Log::trace() << "Gaussian_Thinning: flagged? = " << flagged[0] << std::endl; - } } // ----------------------------------------------------------------------------- @@ -151,11 +147,6 @@ void Gaussian_Thinning::groupObservationsBySpatialLocation( splitter.groupBy(latBins); splitter.groupBy(lonBins); - oops::Log::debug() << "Gaussian_Thinning: latitudes = " << lat << std::endl; - oops::Log::debug() << "Gaussian_Thinning: longitudes = " << lon << std::endl; - oops::Log::debug() << "Gaussian_Thinning: lat bins = " << latBins << std::endl; - oops::Log::debug() << "Gaussian_Thinning: lon bins = " << lonBins << std::endl; - for (size_t validObsIndex = 0; validObsIndex < validObsIds.size(); ++validObsIndex) { const size_t obsId = validObsIds[validObsIndex]; float component = distanceCalculator.spatialDistanceComponent( @@ -229,9 +220,6 @@ void Gaussian_Thinning::groupObservationsByVerticalCoordinate( } splitter.groupBy(bins); - oops::Log::debug() << "Gaussian_Thinning: vertical coords = " << vcoord << std::endl; - oops::Log::debug() << "Gaussian_Thinning: vertical coord bins = " << bins << std::endl; - for (size_t validObsIndex = 0; validObsIndex < validObsIds.size(); ++validObsIndex) { const size_t obsId = validObsIds[validObsIndex]; const float component = distanceCalculator.nonspatialDistanceComponent( @@ -297,11 +285,6 @@ void Gaussian_Thinning::groupObservationsByTime( } splitter.groupBy(bins); - oops::Log::debug() << "Gaussian_Thinning: times = "; - eckit::__print_list(oops::Log::debug(), times, eckit::VectorPrintSimple()); - oops::Log::debug() << std::endl; - oops::Log::debug() << "Gaussian_Thinning: time bins = " << bins << std::endl; - for (size_t validObsIndex = 0; validObsIndex < validObsIds.size(); ++validObsIndex) { const size_t obsId = validObsIds[validObsIndex]; const float component = distanceCalculator.nonspatialDistanceComponent( @@ -392,7 +375,6 @@ std::function Gaussian_Thinning::makeObservationComparator const ObsAccessor &obsAccessor) const { if (options_.priorityVariable.value() == boost::none) { - oops::Log::debug() << "priority_variable not found" << std::endl; return [&distancesToBinCenter](size_t validObsIndexA, size_t validObsIndexB) { return distancesToBinCenter[validObsIndexA] < distancesToBinCenter[validObsIndexB]; }; @@ -403,8 +385,6 @@ std::function Gaussian_Thinning::makeObservationComparator std::vector priorities = obsAccessor.getIntVariableFromObsSpace( priorityVariable.group(), priorityVariable.variable()); - oops::Log::debug() << "priorities = " << priorities << std::endl; - // TODO(wsmigaj): In C++14, use move capture for 'priorities'. return [priorities, &validObsIds, &distancesToBinCenter] (size_t validObsIndexA, size_t validObsIndexB) { diff --git a/src/ufo/filters/MetOfficeBuddyCheck.cc b/src/ufo/filters/MetOfficeBuddyCheck.cc index ad906a064..cd5f13f8d 100644 --- a/src/ufo/filters/MetOfficeBuddyCheck.cc +++ b/src/ufo/filters/MetOfficeBuddyCheck.cc @@ -104,9 +104,6 @@ Eigen::ArrayXXi deriveIndices(const ioda::ObsSpace & obsdb, for (size_t ilocs = 0; ilocs < rSort.size(); ++ilocs) { if (extended_obs_space && ((*extended_obs_space)[rSort[ilocs]] != 1)) continue; - oops::Log::debug() << "MetOfficeBuddyCheck: profile index (record_number, location_number)"; - oops::Log::debug() << " = original obs index; (" << recnum << "," << ilocs << ") = "; - oops::Log::debug() << rSort[ilocs] << std::endl; profileIndex(recnum, levnum) = rSort[ilocs]; levnum++; } @@ -332,10 +329,6 @@ void MetOfficeBuddyCheck::applyFilter(const std::vector & apply, varNameAndGrossErrProbs.second); flagRejectedObservations(filtervars, calculatedGrossErrProbsByVarName, flagged); - - if (filtervars.size() != 0) { - oops::Log::trace() << "MetOfficeBuddyCheck: flagged? = " << flagged[0] << std::endl; - } } Variable MetOfficeBuddyCheck::backgroundErrorVariable(const Variable &filterVariable) const { @@ -485,9 +478,6 @@ void MetOfficeBuddyCheck::checkScalarData(const std::vector const bool isMaster = obsdb_.comm().rank() == 0; if (isMaster) { - oops::Log::trace() << __func__ << " " - << " dampingFactor1 = " << options_.dampingFactor1 - << ", dampingFactor2 = " << options_.dampingFactor2 << '\n'; oops::Log::trace() << "ObsA ObsB StatIdA StatIdB lev DiffA DiffB " "Dist Corr Agree PgeA PgeB Mult\n"; } @@ -593,9 +583,6 @@ void MetOfficeBuddyCheck::checkVectorData(const std::vector const bool isMaster = obsdb_.comm().rank() == 0; if (isMaster) { - oops::Log::trace() << __func__ << " " - << " dampingFactor1 = " << options_.dampingFactor1 - << ", dampingFactor2 = " << options_.dampingFactor2 << '\n'; oops::Log::trace() << "ObsA ObsB StatIdA StatIdB lev LDiffA LDiffB TDiffA TDiffB " "Dist Corr Agree PgeA PgeB Mult\n"; } diff --git a/src/ufo/filters/PoissonDiskThinning.cc b/src/ufo/filters/PoissonDiskThinning.cc index 0883c3574..c7dbf72aa 100644 --- a/src/ufo/filters/PoissonDiskThinning.cc +++ b/src/ufo/filters/PoissonDiskThinning.cc @@ -208,10 +208,6 @@ void PoissonDiskThinning::applyFilter(const std::vector & apply, } obsAccessor.flagRejectedObservations(isThinned, flagged); - - if (filtervars.size() != 0) { - oops::Log::trace() << "PoissonDiskThinning: flagged? = " << flagged[0] << std::endl; - } } ObsAccessor PoissonDiskThinning::createObsAccessor() const { diff --git a/src/ufo/filters/StuckCheck.cc b/src/ufo/filters/StuckCheck.cc index b00c52d1c..424b358a5 100644 --- a/src/ufo/filters/StuckCheck.cc +++ b/src/ufo/filters/StuckCheck.cc @@ -157,8 +157,6 @@ void StuckCheck::potentiallyRejectStreak( size_t observationIndex) { const size_t obsIndex = validObsIds.at(*(stationIndicesBegin + observationIndex)); isRejected[obsIndex] = true; - oops::Log::trace() << "StuckCheck: Observation " << observationIndex << - " rejected from station " << stationId << std::endl; }; size_t streakLength = endOfStreakIndex - startOfStreakIndex + 1; diff --git a/src/ufo/filters/TemporalThinning.cc b/src/ufo/filters/TemporalThinning.cc index d58f529a1..ce40c65f2 100644 --- a/src/ufo/filters/TemporalThinning.cc +++ b/src/ufo/filters/TemporalThinning.cc @@ -323,10 +323,6 @@ void TemporalThinning::applyFilter(const std::vector & apply, const std::vector isThinned = identifyThinnedObservations(apply, filtervars, obsAccessor); obsAccessor.flagRejectedObservations(isThinned, flagged); - - if (filtervars.size() != 0) { - oops::Log::trace() << "TemporalThinning: flagged? = " << flagged[0] << std::endl; - } } ObsAccessor TemporalThinning::createObsAccessor() const { diff --git a/src/ufo/filters/TrackCheck.cc b/src/ufo/filters/TrackCheck.cc index 5964219ae..300301982 100644 --- a/src/ufo/filters/TrackCheck.cc +++ b/src/ufo/filters/TrackCheck.cc @@ -126,10 +126,6 @@ void TrackCheck::applyFilter(const std::vector & apply, obsPressureLoc, maxSpeedByPressure, isRejected); } obsAccessor.flagRejectedObservations(isRejected, flagged); - - if (filtervars.size() != 0) { - oops::Log::trace() << "TrackCheck: flagged? = " << flagged[0] << std::endl; - } } TrackCheck::ObsGroupPressureLocationTime TrackCheck::collectObsPressuresLocationsTimes( From 4ffb92013c69dec8641da81961ddf4e3c89091cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Fri, 20 Aug 2021 21:41:01 +0100 Subject: [PATCH 90/98] Calculate solar zenith angle (#1442) * Allow ObsFunction tests to save output to ioda files. * Allow ObsFunction tests to require missing values in the output to be matched exactly. * Add a function calculating the solar zenith angle at each location. Co-authored-by: Ryan Honeyager --- src/ufo/filters/CMakeLists.txt | 2 + src/ufo/filters/obsfunctions/SolarZenith.cc | 222 ++++++++++++++++++ src/ufo/filters/obsfunctions/SolarZenith.h | 50 ++++ test/CMakeLists.txt | 16 ++ .../unit_tests/function_solarzenith.yaml | 14 ++ .../function_solarzenith_skiprejected.yaml | 27 +++ test/ufo/ObsFunction.h | 27 ++- 7 files changed, 353 insertions(+), 5 deletions(-) create mode 100644 src/ufo/filters/obsfunctions/SolarZenith.cc create mode 100644 src/ufo/filters/obsfunctions/SolarZenith.h create mode 100644 test/testinput/unit_tests/function_solarzenith.yaml create mode 100644 test/testinput/unit_tests/function_solarzenith_skiprejected.yaml diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index a5fc2a4b3..43ec42657 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -233,6 +233,8 @@ set ( filters_files obsfunctions/SCATRetMW.h obsfunctions/SymmCldImpactIR.cc obsfunctions/SymmCldImpactIR.h + obsfunctions/SolarZenith.cc + obsfunctions/SolarZenith.h obsfunctions/SunGlintAngle.cc obsfunctions/SunGlintAngle.h obsfunctions/TropopauseEstimate.cc diff --git a/src/ufo/filters/obsfunctions/SolarZenith.cc b/src/ufo/filters/obsfunctions/SolarZenith.cc new file mode 100644 index 000000000..b87bb1cb1 --- /dev/null +++ b/src/ufo/filters/obsfunctions/SolarZenith.cc @@ -0,0 +1,222 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/filters/obsfunctions/SolarZenith.h" + +#include +#include +#include +#include + +#include "ioda/ObsDataVector.h" +#include "ufo/filters/QCflags.h" +#include "ufo/filters/Variable.h" +#include "ufo/utils/Constants.h" + +namespace ufo { + +namespace { + +/// Return a vector whose ith element is set to true if and only if observations of all simulated +/// variables at the ith location have been rejected. +std::vector identifyRejectedObservations(const ObsFilterData &data) { + const size_t nlocs = data.nlocs(); + const oops::Variables &simulatedVars = data.obsspace().obsvariables(); + std::vector rejected(nlocs, true); + + std::vector qcflags(nlocs); + for (size_t iv = 0; iv < simulatedVars.size(); ++iv) { + data.get(Variable(simulatedVars[iv] + "@QCflagsData"), qcflags); + for (size_t iloc = 0; iloc < nlocs; ++iloc) + if (qcflags[iloc] == QCflags::pass) + rejected[iloc] = false; + } + + return rejected; +} + +} // namespace + +static ObsFunctionMaker maker_("SolarZenith"); + +SolarZenith::SolarZenith(const eckit::LocalConfiguration & conf) { + options_.validateAndDeserialize(conf); + + // List of required ObsSpace variables + invars_ += Variable("latitude@MetaData"); + invars_ += Variable("longitude@MetaData"); + invars_ += Variable("datetime@MetaData"); +} + +void SolarZenith::compute(const ObsFilterData & in, ioda::ObsDataVector & out) const { + const float missingFloat = util::missingValue(float()); + const util::DateTime missingDateTime = util::missingValue(util::DateTime()); + + const int secondsPerDay = 60 * 60 * 24; + const double centuriesPerDay = 1.0 / 36525.0; + const double hoursPerSecond = 1.0 / 3600.0; + const double degreesLongitudePerHour = 15.0; + const double hoursPerDegreeLongitude = 1.0 / degreesLongitudePerHour; + const double one_over_360 = 1.0 / 360.0; + + const util::DateTime startOfLastDayOf19thCentury(1899, 12, 31, 0, 0, 0); + + const size_t nlocs = in.nlocs(); + + // Inputs + std::vector lats(nlocs), lons(nlocs); + std::vector datetimes(nlocs); + in.get(Variable("latitude@MetaData"), lats); + in.get(Variable("longitude@MetaData"), lons); + in.get(Variable("datetime@MetaData"), datetimes); + + std::vector rejected; + const bool skipRejected = options_.skipRejected; + if (skipRejected) + rejected = identifyRejectedObservations(in); + + // Output + std::vector &zenith = out[0]; + std::fill(zenith.begin(), zenith.end(), missingFloat); + + // Statistics + size_t numRejected = 0; + size_t numMissingLats = 0; + size_t numMissingLons = 0; + size_t numMissingDatetimes = 0; + size_t numOutOfRangeLats = 0; + size_t numOutOfRangeDatetimes = 0; + + // Values dependent on day only (reused for all consecutive datetimes from the same day) + util::DateTime dayStart = missingDateTime; + util::DateTime dayEnd = missingDateTime; + // Equation of time (more details below) + double eqnt; + // Sine and cosine of declination + double sinDecl, cosDecl; + + for (size_t loc = 0; loc < nlocs; ++loc) { + if (skipRejected && rejected[loc]) { + ++numRejected; + oops::Log::debug() << "SolarZenith: ob " << loc << " has already been rejected" + << ". Output set to missing data\n"; + continue; + } + if (lats[loc] == missingFloat) { + ++numMissingLats; + oops::Log::debug() << "SolarZenith: missing latitude encountered for ob " << loc + << ". Output set to missing data\n"; + continue; + } + if (lons[loc] == missingFloat) { + ++numMissingLons; + oops::Log::debug() << "SolarZenith: missing longitude encountered for ob " << loc + << ". Output set to missing data\n"; + continue; + } + if (datetimes[loc] == missingDateTime) { + ++numMissingDatetimes; + oops::Log::debug() << "SolarZenith: missing datetime encountered for ob " << loc + << ". Output set to missing data\n"; + continue; + } + if (lats[loc] < -90 || lats[loc] > 90) { + ++numOutOfRangeLats; + oops::Log::debug() << "SolarZenith: latitude " << lats[loc] << " of ob " << loc + << " is out of range. Output set to missing data\n"; + continue; + } + + const util::DateTime &datetime = datetimes[loc]; + + if (datetime < dayStart || datetime >= dayEnd) { + // Calculate quantities dependent only on the date (not time). + int year, month, day, hour, minute, second; + datetime.toYYYYMMDDhhmmss(year, month, day, hour, minute, second); + if (year > 1950 && year <= 2200) { + dayStart = util::DateTime(year, month, day, 0, 0, 0); + dayEnd = dayStart + util::Duration(secondsPerDay); + + // Day since 31 Dec 1899 ("0 Jan 1900") + // (Used instead of 1 Jan 1900 since 2000 was a leap year.) + const std::size_t centuryDay = + (dayStart - startOfLastDayOf19thCentury).toSeconds() / secondsPerDay; + const double rcd = centuryDay * centuriesPerDay; // Fraction of days elapsed this century + const double rcd2 = rcd * rcd; + double ydeg = (rcd * 36000.769 + 279.697) * one_over_360; + ydeg = std::fmod(ydeg, 1.0) * 360.0; + const double yrad = ydeg * Constants::deg2rad; + + // Compute equation of time (in seconds) for this day + // (No reference for this but it gives the correct answers + // when compared with table in Norton's Star Atlas.) + // The linter protests about extra spaces used for alignment, so is disabled. + eqnt = - (( 93.0 + 14.23 * rcd - 0.0144 * rcd2) * std::sin(yrad)) // NOLINT + - ((432.5 - 3.71 * rcd - 0.2063 * rcd2) * std::cos(yrad)) // NOLINT + + ((596.9 - 0.81 * rcd - 0.0096 * rcd2) * std::sin(2.0 * yrad)) // NOLINT + - (( 1.4 + 0.28 * rcd) * std::cos(2.0 * yrad)) // NOLINT + + (( 3.8 + 0.6 * rcd) * std::sin(3.0 * yrad)) // NOLINT + + (( 19.5 - 0.21 * rcd - 0.0103 * rcd2) * std::cos(3.0 * yrad)) // NOLINT + - (( 12.8 - 0.03 * rcd) * std::sin(4.0 * yrad)); // NOLINT + + // Get solar declination for given day (radians) + const double sinalp = std::sin((ydeg - eqnt / 240.0) * Constants::deg2rad); + const double taneqn = 0.43382 - 0.00027 * rcd; + const double decl = std::atan(taneqn * sinalp); + eqnt *= hoursPerSecond; // Convert to hours + + // Sine and cosine of declination + sinDecl = std::sin(decl); + cosDecl = std::cos(decl); + } else { + ++numOutOfRangeDatetimes; + oops::Log::debug() << "SolarZenith: date/time " << datetime << " of ob " << loc + << "is out of range. Output set to missing data\n"; + continue; + } + } + + const double lat = lats[loc]; + const double lon = lons[loc]; + + const double latInRadians = lat * Constants::deg2rad; + const double sinLat = std::sin(latInRadians); + const double cosLat = std::cos(latInRadians); + + const std::int64_t secondsSinceDayStart = (datetime - dayStart).toSeconds(); + const double hoursSinceDayStart = secondsSinceDayStart * hoursPerSecond; + const double localSolarTimeInHours = lon * hoursPerDegreeLongitude + eqnt + hoursSinceDayStart; + // Local hour angle (when longitude is 0, this is the Greenwich hour angle given in the + // Air Almanac) + const double hourAngleInRadians = + (localSolarTimeInHours * degreesLongitudePerHour + 180.0) * Constants::deg2rad; + + const double sinEv = sinDecl * sinLat + cosDecl * cosLat * std::cos(hourAngleInRadians); + zenith[loc] = (M_PI / 2 - std::asin(sinEv)) * Constants::rad2deg; + } + + // Notify about "bad" observations + if (numRejected != 0) + oops::Log::trace() << "SolarZenith: " << numMissingLats << " obs had already been rejected\n"; + if (numMissingLats != 0) + oops::Log::trace() << "SolarZenith: " << numMissingLats << " obs had missing latitude\n"; + if (numMissingLons != 0) + oops::Log::trace() << "SolarZenith: " << numMissingLats << " obs had missing longitude\n"; + if (numOutOfRangeLats != 0) + oops::Log::trace() << "SolarZenith: " << numOutOfRangeLats + << " obs had out-of-range latitude\n"; + if (numOutOfRangeDatetimes != 0) + oops::Log::trace() << "SolarZenith: " << numOutOfRangeDatetimes + << " obs had out-of-range datetime\n"; + oops::Log::trace().flush(); +} + +const ufo::Variables & SolarZenith::requiredVariables() const { + return invars_; +} + +} // namespace ufo diff --git a/src/ufo/filters/obsfunctions/SolarZenith.h b/src/ufo/filters/obsfunctions/SolarZenith.h new file mode 100644 index 000000000..c9dc19b4a --- /dev/null +++ b/src/ufo/filters/obsfunctions/SolarZenith.h @@ -0,0 +1,50 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_FILTERS_OBSFUNCTIONS_SOLARZENITH_H_ +#define UFO_FILTERS_OBSFUNCTIONS_SOLARZENITH_H_ + +#include "ufo/filters/obsfunctions/ObsFunctionBase.h" +#include "ufo/filters/Variables.h" + +namespace ufo { + +/// \brief Configuration options of SolarZenith. +class SolarZenithParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(SolarZenithParameters, Parameters) + + public: + /// Set this option to `true` to skip calculations and produce missing values at locations + /// where all simulated variables have been rejected. Default: `false`. + oops::Parameter skipRejected{"skip rejected", false, this}; +}; + +/// \brief Compute the solar zenith angle of observations (in degrees) as a function of their time +/// and location. +/// +/// References: +/// * `Ops_Solar_Zenith` (subroutine in the Met Office OPS system): original source code +/// * Air Almanac: useful for checking GHA and DECL +/// * Norton's Star Atlas: for equation of time +/// * Robinson N., Solar Radiation, Ch. 2: for useful introduction to theory/terminology. +class SolarZenith : public ObsFunctionBase { + public: + explicit SolarZenith(const eckit::LocalConfiguration &conf); + + void compute(const ObsFilterData &, ioda::ObsDataVector &) const override; + const ufo::Variables & requiredVariables() const override; + + private: + SolarZenithParameters options_; + ufo::Variables invars_; +}; + +// ----------------------------------------------------------------------------- + +} // namespace ufo + +#endif // UFO_FILTERS_OBSFUNCTIONS_SOLARZENITH_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9cda21466..1d3081707 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -295,6 +295,8 @@ list( APPEND ufo_test_input testinput/unit_tests/function_obserrorfactorquotient.yaml testinput/unit_tests/function_satwindsLNVDcheck.yaml testinput/unit_tests/function_satwindsSPDBcheck.yaml + testinput/unit_tests/function_solarzenith.yaml + testinput/unit_tests/function_solarzenith_skiprejected.yaml testinput/unit_tests/function_wdirdiff.yaml testinput/unit_tests/utils_metoffice_sort.yaml testinput/unit_tests/variabletransforms_rhumidity.yaml @@ -2609,6 +2611,13 @@ ecbuild_add_test( TARGET test_ufo_variabletransforms_profilehorizontaldrift DEPENDS test_ObsFilters.x TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_solarzenith_skiprejected + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x + ARGS "testinput/unit_tests/function_solarzenith_skiprejected.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFilters.x + TEST_DEPENDS ufo_get_ufo_test_data ) + # Test Functions ecbuild_add_test( TARGET test_ufo_function_bgddepartureanomaly @@ -2949,6 +2958,13 @@ ecbuild_add_test( TARGET test_ufo_function_modelsurfaceheightadjust_marinewind DEPENDS test_ObsFunction.x TEST_DEPENDS ufo_get_ufo_test_data) +ecbuild_add_test( TARGET test_ufo_function_solarzenith + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFunction.x + ARGS "testinput/unit_tests/function_solarzenith.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFunction.x + TEST_DEPENDS ufo_get_ufo_test_data) + # Test Diagnostics ecbuild_add_test( TARGET test_ufo_obsdiag_background_error_vert_interp_air_pressure diff --git a/test/testinput/unit_tests/function_solarzenith.yaml b/test/testinput/unit_tests/function_solarzenith.yaml new file mode 100644 index 000000000..6f60147d0 --- /dev/null +++ b/test/testinput/unit_tests/function_solarzenith.yaml @@ -0,0 +1,14 @@ +window begin: 1900-01-01T00:00:00Z +window end: 9999-01-01T00:00:00Z + +observations: +- obs space: &ObsSpace + name: Zenith 1 + obsdatain: + obsfile: Data/ufo/testinput_tier_1/solar_zenith.nc4 + simulated variables: [air_temperature, surface_pressure] + obs function: &ObsFunction + name: SolarZenith@ObsFunction + variables: [zenith] + tolerance: 1.0e-7 + expect missing value locations to match: true diff --git a/test/testinput/unit_tests/function_solarzenith_skiprejected.yaml b/test/testinput/unit_tests/function_solarzenith_skiprejected.yaml new file mode 100644 index 000000000..b58e295cb --- /dev/null +++ b/test/testinput/unit_tests/function_solarzenith_skiprejected.yaml @@ -0,0 +1,27 @@ +# Tests the "skip rejected" option of the SolarZenith ObsFunction, which makes the ObsFunction +# produce missing values at locations where all simulated variables have been rejected. + +window begin: 1900-01-01T00:00:00Z +window end: 9999-01-01T00:00:00Z + +observations: +- obs space: &ObsSpace + name: Zenith 1 + obsdatain: + obsfile: Data/ufo/testinput_tier_1/solar_zenith.nc4 + simulated variables: [air_temperature, surface_pressure] + obs filters: + - filter: Variable Assignment + assignments: + - name: zenith@MetaData + type: float + function: + name: SolarZenith@ObsFunction + options: + skip rejected: true + compareVariables: + - test: + name: zenith@MetaData + reference: + name: zenith_skip_rejected@TestReference + relTol: 1e-6 diff --git a/test/ufo/ObsFunction.h b/test/ufo/ObsFunction.h index cf6f16ece..ad362251d 100644 --- a/test/ufo/ObsFunction.h +++ b/test/ufo/ObsFunction.h @@ -52,9 +52,14 @@ const char *expectConstructorToThrow = "expect constructor to throw exception wi // ----------------------------------------------------------------------------- +/// \param[out] num_missing_mismatches +/// Number of locations containing a missing value in `vals` but not in `ref`, or in `ref` but not +/// in `vals`. void dataVectorDiff(const ioda::ObsSpace & ospace, ioda::ObsDataVector & vals, - const ioda::ObsDataVector & ref, std::vector & rms_out) { - float missing = util::missingValue(missing); + const ioda::ObsDataVector & ref, std::vector & rms_out, + size_t &num_missing_mismatches) { + const float missing = util::missingValue(missing); + num_missing_mismatches = 0; /// Loop through variables and calculate rms for each variable for (size_t ivar = 0; ivar < vals.nvars() ; ++ivar) { for (size_t jj = 0; jj < ref.nlocs() ; ++jj) { @@ -63,6 +68,9 @@ void dataVectorDiff(const ioda::ObsSpace & ospace, ioda::ObsDataVector & } else { vals[ivar][jj] = missing; } + if ((vals[ivar][jj] != missing) ^ (ref[ivar][jj] != missing)) { + ++num_missing_mismatches; + } } int nobs = globalNumNonMissingObs(*ospace.distribution(), 1, vals[ivar]); float rms = dotProduct(*ospace.distribution(), 1, vals[ivar], vals[ivar]); @@ -107,12 +115,15 @@ void checkResults(const ioda::ObsSpace &ospace, ioda::ObsDataVector & vals_ofd, const ioda::ObsDataVector & ref) { const double tol = obsfuncconf.getDouble("tolerance"); + const bool expectMissingToMatch = + obsfuncconf.getBool("expect missing value locations to match", false); const size_t nvars = ref.nvars(); EXPECT_EQUAL(vals.nvars(), nvars); /// Calculate rms(f(x) - ref) and compare to tolerance std::vector rms_out(nvars); - dataVectorDiff(ospace, vals, ref, rms_out); + size_t numMissingMismatches; + dataVectorDiff(ospace, vals, ref, rms_out, numMissingMismatches); oops::Log::info() << "Vector difference between reference and computed: " << std::endl; oops::Log::info() << vals << std::endl; @@ -120,8 +131,10 @@ void checkResults(const ioda::ObsSpace &ospace, // FIXME(someone): whatever this does, it certainly doesn't convert percentages to fractions EXPECT(rms_out[ivar] < 100*tol); // change tol from percent to actual value. } + if (expectMissingToMatch) + EXPECT_EQUAL(numMissingMismatches, 0); - dataVectorDiff(ospace, vals_ofd, ref, rms_out); + dataVectorDiff(ospace, vals_ofd, ref, rms_out, numMissingMismatches); oops::Log::info() << "Vector difference between reference and computed via ObsFilterData: " << std::endl; oops::Log::info() << vals_ofd << std::endl; @@ -129,6 +142,8 @@ void checkResults(const ioda::ObsSpace &ospace, // FIXME(someone): whatever this does, it certainly doesn't convert percentages to fractions EXPECT(rms_out[ivar] < 100*tol); // change tol from percent to actual value. } + if (expectMissingToMatch) + EXPECT_EQUAL(numMissingMismatches, 0); } // ----------------------------------------------------------------------------- @@ -244,7 +259,9 @@ class ObsFunction : public oops::Test { { testFunction(); }); } - void clear() const override {} + void clear() const override { + ::test::ObsTestsFixture::reset(); + } }; // ----------------------------------------------------------------------------- From 2c8888cdb7ced665cdaef42dacc592e56a07133b Mon Sep 17 00:00:00 2001 From: guillaumevernieres Date: Mon, 23 Aug 2021 16:38:07 -0400 Subject: [PATCH 91/98] passify action (#1460) * wip * passive obs action * wip * test passify action * reverted adt.yaml * code/comment tidy and better testing * passify to passivate --- src/ufo/filters/CMakeLists.txt | 2 + src/ufo/filters/QCmanager.cc | 1 + src/ufo/filters/actions/PassivateObs.cc | 45 ++++++++++++++++++++++ src/ufo/filters/actions/PassivateObs.h | 50 +++++++++++++++++++++++++ test/testinput/qc_actions.yaml | 20 ++++++++++ 5 files changed, 118 insertions(+) create mode 100644 src/ufo/filters/actions/PassivateObs.cc create mode 100644 src/ufo/filters/actions/PassivateObs.h diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index 43ec42657..429e5f4b5 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -135,6 +135,8 @@ set ( filters_files actions/FilterActionBase.h actions/InflateError.cc actions/InflateError.h + actions/PassivateObs.cc + actions/PassivateObs.h actions/RejectObs.cc actions/RejectObs.h obsfunctions/BennartzScatIndex.cc diff --git a/src/ufo/filters/QCmanager.cc b/src/ufo/filters/QCmanager.cc index 6a2a207a5..a098a5594 100644 --- a/src/ufo/filters/QCmanager.cc +++ b/src/ufo/filters/QCmanager.cc @@ -131,6 +131,7 @@ void QCmanager::print(std::ostream & os) const { {77, nullptr}, // } will be added up and reported together // "Normal" cases reported in a uniform way + {QCflags::passive, "passive observations"}, {QCflags::missing, "missing values"}, {QCflags::preQC, "rejected by pre QC"}, {QCflags::bounds, "out of bounds"}, diff --git a/src/ufo/filters/actions/PassivateObs.cc b/src/ufo/filters/actions/PassivateObs.cc new file mode 100644 index 000000000..753e0c58c --- /dev/null +++ b/src/ufo/filters/actions/PassivateObs.cc @@ -0,0 +1,45 @@ +/* + * (C) Copyright 2021-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/filters/actions/PassivateObs.h" + +#include "ioda/ObsDataVector.h" +#include "ufo/filters/ObsFilterData.h" +#include "ufo/filters/QCflags.h" + +namespace ufo { + +// ----------------------------------------------------------------------------- + +static FilterActionMaker makerPassivateObs_("passivate"); + +// ----------------------------------------------------------------------------- + + PassivateObs::PassivateObs(const PassivateObsParameters ¶meters) + : allvars_() { + } + +// ----------------------------------------------------------------------------- + +void PassivateObs::apply(const Variables & vars, + const std::vector> & flagged, + const ObsFilterData &, + int, + ioda::ObsDataVector & flags, + ioda::ObsDataVector &) const { + for (size_t ifiltervar = 0; ifiltervar < vars.nvars(); ++ifiltervar) { + size_t iallvar = flags.varnames().find(vars.variable(ifiltervar).variable()); + for (size_t jobs = 0; jobs < flags.nlocs(); ++jobs) { + if (flagged[ifiltervar][jobs] && flags[iallvar][jobs] == QCflags::pass) + flags[iallvar][jobs] = QCflags::passive; + } + } +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/filters/actions/PassivateObs.h b/src/ufo/filters/actions/PassivateObs.h new file mode 100644 index 000000000..bb3a01f9e --- /dev/null +++ b/src/ufo/filters/actions/PassivateObs.h @@ -0,0 +1,50 @@ +/* + * (C) Copyright 2021-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_FILTERS_ACTIONS_PASSIVATEOBS_H_ +#define UFO_FILTERS_ACTIONS_PASSIVATEOBS_H_ + +#include + +#include "ufo/filters/actions/FilterActionBase.h" +#include "ufo/filters/Variables.h" + +namespace ufo { + +class ObsFilterData; + +// ----------------------------------------------------------------------------- + +class PassivateObsParameters : public FilterActionParametersBase { + OOPS_CONCRETE_PARAMETERS(PassivateObsParameters, FilterActionParametersBase); +}; + +// ----------------------------------------------------------------------------- + +/// Flag observations as passive. +class PassivateObs : public FilterActionBase { + public: + /// The type of parameters accepted by the constructor of this action. + /// This typedef is used by the FilterActionFactory. + typedef PassivateObsParameters Parameters_; + + explicit PassivateObs(const Parameters_ &); + ~PassivateObs() {} + + void apply(const Variables &, const std::vector> &, + const ObsFilterData &, int, + ioda::ObsDataVector &, ioda::ObsDataVector &) const override; + const ufo::Variables & requiredVariables() const override {return allvars_;} + private: + Variables allvars_; +}; + +// ----------------------------------------------------------------------------- + +} // namespace ufo + +#endif // UFO_FILTERS_ACTIONS_PASSIVATEOBS_H_ diff --git a/test/testinput/qc_actions.yaml b/test/testinput/qc_actions.yaml index 153b57e14..71b6b38fe 100644 --- a/test/testinput/qc_actions.yaml +++ b/test/testinput/qc_actions.yaml @@ -146,3 +146,23 @@ observations: maxvalue: 7 # all observations of variable1 with var1 <= 5 and of variable2 with var1 <= 7 should be accepted passedBenchmark: 12 + +# Test "passivate" +- obs space: + name: test data + obsdatain: + obsfile: Data/ufo/testinput_tier_1/filters_testdata.nc4 + simulated variables: [variable1, variable2] + HofX: HofX + obs filters: + # passivate observations with var1 >= 5 + - filter: BlackList + action: + name: passivate + where: + - variable: + name: var1@MetaData # = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 + minvalue: 5 + benchmarkFlag: 1 # passive + flaggedBenchmark: 12 # expect 12 observations to be assigned the "passive" flag + passedBenchmark: 8 # expect 8 observations to be assigned the "passed" flag From 7f17f20f9b46fe78208338fd6ab0350a6266ae49 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Tue, 24 Aug 2021 11:25:42 -0600 Subject: [PATCH 92/98] BUGFIX: only output bias correction coefficients on task 0 (#1475) * only output bias correction coeffs on task 0 * fix for warning messages + comments --- src/ufo/ObsBias.cc | 9 +++++++-- src/ufo/ObsBias.h | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ufo/ObsBias.cc b/src/ufo/ObsBias.cc index 863461f44..8fc33b458 100644 --- a/src/ufo/ObsBias.cc +++ b/src/ufo/ObsBias.cc @@ -32,7 +32,8 @@ namespace ufo { // ----------------------------------------------------------------------------- ObsBias::ObsBias(ioda::ObsSpace & odb, const ObsBiasParameters & params) - : numStaticPredictors_(0), numVariablePredictors_(0), vars_(odb.obsvariables()) { + : numStaticPredictors_(0), numVariablePredictors_(0), vars_(odb.obsvariables()), + rank_(odb.distribution()->rank()) { oops::Log::trace() << "ObsBias::create starting." << std::endl; // Predictor factory @@ -66,7 +67,7 @@ ObsBias::ObsBias(const ObsBias & other, const bool copy) numStaticPredictors_(other.numStaticPredictors_), numVariablePredictors_(other.numVariablePredictors_), vars_(other.vars_), - geovars_(other.geovars_), hdiags_(other.hdiags_) { + geovars_(other.geovars_), hdiags_(other.hdiags_), rank_(other.rank_) { oops::Log::trace() << "ObsBias::copy ctor starting." << std::endl; // Initialize the biascoeffs @@ -97,6 +98,7 @@ ObsBias & ObsBias::operator=(const ObsBias & rhs) { vars_ = rhs.vars_; geovars_ = rhs.geovars_; hdiags_ = rhs.hdiags_; + rank_ = rhs.rank_; } return *this; } @@ -190,6 +192,9 @@ ioda::ObsGroup saveBiasCoeffsWithChannels(ioda::Group & parent, // ----------------------------------------------------------------------------- void ObsBias::write(const Parameters_ & params) const { + // only write files out on the task with MPI rank 0 + if (rank_ != 0) return; + if (params.outputFile.value() != boost::none) { // FIXME: only implemented for channels currently if (vars_.channels().size() == 0) { diff --git a/src/ufo/ObsBias.h b/src/ufo/ObsBias.h index be4c8c476..ac4679bd0 100644 --- a/src/ufo/ObsBias.h +++ b/src/ufo/ObsBias.h @@ -119,6 +119,9 @@ class ObsBias : public util::Printable, oops::Variables geovars_; /// Diagnostics that need to be requested from the obs operator (for computation of predictors) oops::Variables hdiags_; + + /// MPI rank, used to determine whether the task should output bias coeffs to a file + size_t rank_; }; // ----------------------------------------------------------------------------- From f20cb04260d080fa8a05f8d238c89e9678437bf4 Mon Sep 17 00:00:00 2001 From: adammartins Date: Fri, 27 Aug 2021 15:49:50 +0100 Subject: [PATCH 93/98] Feature/amv qi processing (#1439) * working with filterbase * update to dofilter and add check for existing variable in database * use new data and test all variables * PR changes * change try to if --- src/ufo/filters/CMakeLists.txt | 2 + src/ufo/filters/ProcessAMVQI.cc | 163 ++++++++++++++++++++++++++ src/ufo/filters/ProcessAMVQI.h | 78 ++++++++++++ src/ufo/instantiateObsFilterFactory.h | 3 + test/CMakeLists.txt | 8 ++ test/testinput/qc_processamvqi.yaml | 47 ++++++++ 6 files changed, 301 insertions(+) create mode 100644 src/ufo/filters/ProcessAMVQI.cc create mode 100644 src/ufo/filters/ProcessAMVQI.h create mode 100644 test/testinput/qc_processamvqi.yaml diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index 429e5f4b5..ac91a64f8 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -66,6 +66,8 @@ set ( filters_files PreQC.h ProbabilityGrossErrorWholeReport.cc ProbabilityGrossErrorWholeReport.h + ProcessAMVQI.cc + ProcessAMVQI.h QCflags.h QCmanager.cc QCmanager.h diff --git a/src/ufo/filters/ProcessAMVQI.cc b/src/ufo/filters/ProcessAMVQI.cc new file mode 100644 index 000000000..4c81ac763 --- /dev/null +++ b/src/ufo/filters/ProcessAMVQI.cc @@ -0,0 +1,163 @@ +/* + * (C) Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ufo/filters/ProcessAMVQI.h" + +#include +#include +#include + +#include "ioda/ObsSpace.h" + +#include "oops/util/Logger.h" + +#include "ufo/filters/processWhere.h" + +namespace ufo { + +// ----------------------------------------------------------------------------- + +ProcessAMVQI::ProcessAMVQI(ioda::ObsSpace & obsdb, const Parameters_ & parameters, + std::shared_ptr> flags, + std::shared_ptr> obserr) + : ObsProcessorBase(obsdb, false /*deferToPost?*/, flags, obserr), + parameters_(parameters) +{ + oops::Log::trace() << "ProcessAMVQI contructor starting" << std::endl; +} + +// ----------------------------------------------------------------------------- + +ProcessAMVQI::~ProcessAMVQI() { + oops::Log::trace() << "ProcessAMVQI destructed" << std::endl; +} + +// ----------------------------------------------------------------------------- +/*! \brief A filter to convert new BUFR formatted data into variables with names + * corressponding to the wind generating application. + * + * Example: + * \code{.unparsed} + * obs filter: + * - filter: Process AMV QI + * number of generating apps: 4 + * \endcode + * + * \author A.Martins (Met Office) + * + * \date 02/08/2021: Created + */ +void ProcessAMVQI::doFilter() const { + oops::Log::trace() << "ProcessAMVQI doFilter" << std::endl; + + const float missing = util::missingValue(float()); + const int int_missing = util::missingValue(int()); + const size_t nlocs = obsdb_.nlocs(); + const size_t number_of_apps = parameters_.number_of_apps.value(); + + // vectors to store BUFR data + std::vector percent_confidence(nlocs); + std::vector wind_generating_application(nlocs); + + // Create vector of strings for percent_confidence_ to new variable + // Wind generating application number = QI type + // Table 1: + // 1 = Full weighted mixture of individual quality tests + // 2 = Weighted mixture of individual tests, but excluding forecast comparison + // 3 = Recursive filter function + // 4 = Common quality index (QI) without forecast + // 5 = QI without forecast + // 6 = QI with forecast + // 7 = Estimated Error (EE) in m/s converted to a percent confidence + std::vector new_variables = { + "QI_full_weighted_mixture", + "QI_weighted_mixture_exc_forecast_comparison", + "QI_recursive_filter_function", + "QI_common", + "QI_without_forecast", + "QI_with_forecast", + "QI_estimated_error" }; + const size_t total_variables = new_variables.size(); + + // create variable to write to obsdb + std::vector> new_percent_confidence(total_variables, + std::vector(nlocs, missing)); + + // Get BUFR data + for (size_t iapp = 0; iapp < number_of_apps; ++iapp) { + // names of variables + std::string percent_confidence_name = "percent_confidence_"; + std::string wind_generating_application_name = "wind_generating_application_"; + + obsdb_.get_db("MetaData", + percent_confidence_name.append(std::to_string(iapp + 1)), + percent_confidence); + + obsdb_.get_db("MetaData", + wind_generating_application_name.append(std::to_string(iapp + 1)), + wind_generating_application); + + for (size_t idata = 0; idata < nlocs; ++idata) { + if (wind_generating_application[idata] != int_missing) { + new_percent_confidence[wind_generating_application[idata] - 1][idata] = + percent_confidence[idata]; + } + } + } + + // Need to check database for the named variables and add to them if they exist. + for (size_t inum = 0; inum < total_variables; ++inum) { + if (std::any_of(new_percent_confidence[inum].begin(), + new_percent_confidence[inum].end(), + [missing] (float elem) {return elem != missing;})) { + // Check if new_variable already exists in obsdb + std::vector existing_new_percent_confidence(nlocs, missing); + if (obsdb_.has("MetaData", + new_variables[inum])) { + obsdb_.get_db("MetaData", + new_variables[inum], + existing_new_percent_confidence); + } else { + oops::Log::trace() << "ProcessAMIQI: New variable: " << new_variables[inum] << + " not present in input data" << std::endl; + } + + // Check if existing_new_percent_confidence has non-missing values + // and update new_percent_confidence. + + for (size_t idata = 0; idata < nlocs; ++idata) { + if (existing_new_percent_confidence[idata] != missing) { + if (new_percent_confidence[inum][idata] == missing) { + new_percent_confidence[inum][idata] = + existing_new_percent_confidence[idata]; + } else { + oops::Log::trace() << "[WARN] New variable (created from new BUFR data)" << "\n" + "[WARN] already has value in old BUFR format" << "\n" + "[WARN] new BUFR: " << + new_percent_confidence[inum][idata] << "\n" + "[WARN] old BUFR: " << + existing_new_percent_confidence[idata] << "\n"; + } + } + } + // write to new or existing QI vectors + obsdb_.put_db("MetaData", + new_variables[inum], + new_percent_confidence[inum]); + } + } +} + +// ----------------------------------------------------------------------------- + +void ProcessAMVQI::print(std::ostream & os) const { + os << "ProcessAMVQI filter" << parameters_ << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace ufo diff --git a/src/ufo/filters/ProcessAMVQI.h b/src/ufo/filters/ProcessAMVQI.h new file mode 100644 index 000000000..94eb2d9c8 --- /dev/null +++ b/src/ufo/filters/ProcessAMVQI.h @@ -0,0 +1,78 @@ +/* + * (C) Copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef UFO_FILTERS_PROCESSAMVQI_H_ +#define UFO_FILTERS_PROCESSAMVQI_H_ + +#include +#include +#include +#include + +#include "oops/generic/ObsFilterParametersBase.h" +#include "oops/util/ObjectCounter.h" +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/RequiredParameter.h" +#include "ufo/filters/ObsProcessorBase.h" + +namespace ufo { + +/// \brief Parameters controlling the operation of the ProcessAMVQI filter. +class ProcessAMVQIParameters : public oops::ObsFilterParametersBase { + OOPS_CONCRETE_PARAMETERS(ProcessAMVQIParameters, ObsFilterParametersBase) + + public: + /// How many generating applications to search for. + oops::RequiredParameter number_of_apps { + "number of generating apps", + this + }; +}; + +/// \brief A filter to convert new BUFR formatted data into variables with names +/// corressponding to the wind generating application. +/// +/// \details This filter will convert variables of "percent_confidence_" and +/// "wind_generating_application_" to variables named corresponding to the +/// wind_generating_application (see Table 1). +/// +/// Table 1: +/// Wind generating application number = QI type +/// 1 = Full weighted mixture of individual quality tests +/// 2 = Weighted mixture of individual tests, but excluding forecast comparison +/// 3 = Recursive filter function +/// 4 = Common quality index (QI) without forecast +/// 5 = QI without forecast +/// 6 = QI with forecast +/// 7 = Estimated Error (EE) in m/s converted to a percent confidence +/// +/// See ProcessAMVQIParameters for the documentation of +/// the parameters controlling this filter. +class ProcessAMVQI : public ObsProcessorBase, + private util::ObjectCounter { + public: + /// The type of parameters accepted by the constructor of this filter. + /// This typedef is used by the FilterFactory. + typedef ProcessAMVQIParameters Parameters_; + + static const std::string classname() {return "ufo::ProcessAMVQI";} + + ProcessAMVQI(ioda::ObsSpace & obsdb, const Parameters_ & parameters, + std::shared_ptr> flags, + std::shared_ptr> obserr); + ~ProcessAMVQI() override; + + private: + void print(std::ostream &) const override; + void doFilter() const override; + + Parameters_ parameters_; +}; + +} // namespace ufo + +#endif // UFO_FILTERS_PROCESSAMVQI_H_ diff --git a/src/ufo/instantiateObsFilterFactory.h b/src/ufo/instantiateObsFilterFactory.h index 8f9854a2d..b1107a526 100644 --- a/src/ufo/instantiateObsFilterFactory.h +++ b/src/ufo/instantiateObsFilterFactory.h @@ -35,6 +35,7 @@ #include "ufo/filters/PoissonDiskThinning.h" #include "ufo/filters/PreQC.h" #include "ufo/filters/ProbabilityGrossErrorWholeReport.h" +#include "ufo/filters/ProcessAMVQI.h" #include "ufo/filters/ProfileBackgroundCheck.h" #include "ufo/filters/ProfileFewObsCheck.h" #include "ufo/filters/QCmanager.h" @@ -137,6 +138,8 @@ template void instantiateObsFilterFactory() { ImpactHeightCheckMaker("GNSSRO Impact Height Check"); static oops::interface::FilterMaker SatwindInversionCorrectionMaker("Satwind Inversion Correction"); + static oops::interface::FilterMaker + ProcessAMVQIMaker("Process AMV QI"); // Only include this filter if rttov is present #if defined(RTTOV_FOUND) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1d3081707..86ab81f60 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -199,6 +199,7 @@ list( APPEND ufo_test_input testinput/qc_poisson_disk_thinning_parallel.yaml testinput/qc_preqc.yaml testinput/qc_preqc_halo.yaml + testinput/qc_processamvqi.yaml testinput/qc_profile_background_check.yaml testinput/qc_profile_fewobs_check.yaml testinput/qc_stuckcheck.yaml @@ -3416,6 +3417,13 @@ ecbuild_add_test( TARGET test_satellite_selector_predictor DEPENDS test_Predictor.x TEST_DEPENDS ufo_get_ufo_test_data ) +ecbuild_add_test( TARGET test_ufo_qc_processamvqi + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x + ARGS "testinput/qc_processamvqi.yaml" + ENVIRONMENT OOPS_TRAPFPE=1 + DEPENDS test_ObsFilters.x + TEST_DEPENDS ufo_get_ufo_test_data ) + ##################################################################### # Files for CRTM tests ##################################################################### diff --git a/test/testinput/qc_processamvqi.yaml b/test/testinput/qc_processamvqi.yaml new file mode 100644 index 000000000..4bb68eb06 --- /dev/null +++ b/test/testinput/qc_processamvqi.yaml @@ -0,0 +1,47 @@ +window begin: 2020-10-01T03:00:00Z +window end: 2020-10-01T09:00:00Z + +observations: +- obs space: + name: Satwind + obsdatain: + obsfile: Data/ufo/testinput_tier_1/satwind_obs_1d_2020100106_noinv_qiODB.nc4 + simulated variables: [wind_from_direction] + obs filters: + - filter: Variable Assignment + assignments: + - name: temporary_input_QI_recursive_filter_function@MetaData + type: float + source variable: QI_recursive_filter_function@MetaData + - filter: Process AMV QI + number of generating apps: 4 + # Test file contains GOES-16 data with generating applications=[2,4,5,6] + # Percent_confidence_1 => 6 => QI_with_forecast + # Percent_confidence_2 => 5 => QI_without_forecast + # QI_with_/without_forecast are also present in input data, + # dummy_input_... have been added to input to compare with modified + # QI_with/without_forecast variables. + # Percent_confidence_3 => 4 => QI_common + # Percent_confidence_4 => 2 => QI_weighted_mixture_exc_forecast_comparison + # QI_recursive_filter_function is unmodified from input data + compareVariables: + - reference: + name: temporary_input_QI_recursive_filter_function@MetaData + test: + name: QI_recursive_filter_function@MetaData + - reference: + name: dummy_input_QI_with_forecast@TestReference + test: + name: QI_with_forecast@MetaData + - reference: + name: dummy_input_QI_without_forecast@TestReference + test: + name: QI_without_forecast@MetaData + - reference: + name: percent_confidence_3@MetaData + test: + name: QI_common@MetaData + - reference: + name: percent_confidence_4@MetaData + test: + name: QI_weighted_mixture_exc_forecast_comparison@MetaData From 7abb04ae14aa846a9887f9d305423122441bb9de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Fri, 27 Aug 2021 17:21:14 +0100 Subject: [PATCH 94/98] Implement the OPS-compatible sort function without calling std::make_heap and std::sort_heap. (#1446) It turns out they produce STL implementation-dependent results. Co-authored-by: Stephen Herbener <32968781+srherbener@users.noreply.github.com> Co-authored-by: Hui Shao --- src/ufo/utils/metoffice/MetOfficeSort.h | 69 +++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/src/ufo/utils/metoffice/MetOfficeSort.h b/src/ufo/utils/metoffice/MetOfficeSort.h index 827b26b7a..516a86b64 100644 --- a/src/ufo/utils/metoffice/MetOfficeSort.h +++ b/src/ufo/utils/metoffice/MetOfficeSort.h @@ -8,10 +8,72 @@ #ifndef UFO_UTILS_METOFFICE_METOFFICESORT_H_ #define UFO_UTILS_METOFFICE_METOFFICESORT_H_ -#include +#include namespace ufo { +namespace metofficesortdetail { + +/// Sink element `parentIndex` of the heap starting at `heapStart` with `heapLength` elements +/// until its correct position. +/// +/// If the parent's key is smaller than the key of any of its children, the parent is swapped with +/// its largest child. This is repeated until the parent's key is at least as large as the keys of +/// its children. +template +void sinkParent(typename RandomIt::difference_type parentIndex, + RandomIt heapStart, typename RandomIt::difference_type heapLength, + const UnaryOperation &key) { + typedef typename RandomIt::difference_type Index; + + const auto parentKey = key(*(heapStart + parentIndex)); + Index childIndex = 2 * parentIndex + 1; // left child + while (childIndex < heapLength) { + // Identify the largest child (preferring the left child if they're equal) + auto childKey = key(*(heapStart + childIndex)); + Index rightChildIndex = childIndex + 1; + if (rightChildIndex < heapLength) { + auto rightChildKey = key(*(heapStart + rightChildIndex)); + if (rightChildKey > childKey) { + childIndex = rightChildIndex; + childKey = std::move(rightChildKey); + } + } + // Is the parent at least as large as its largest child? + if (parentKey >= childKey) + break; // Yes. The parent is now at the correct position; we're done + // No. Swap the parent with its largest child + std::swap(*(heapStart + parentIndex), *(heapStart + childIndex)); + // Continue sinking the parent. + parentIndex = childIndex; + childIndex = 2 * parentIndex + 1; + } +} + +/// Arrange the range `[first, last)` into a max-heap ordered by key \p key. +template +void makeHeap(RandomIt first, RandomIt last, const UnaryOperation &key) { + const auto len = last - first; + // Arrange all trees with more than one level into heaps + for (auto parentIndex = len / 2 - 1; parentIndex >= 0; --parentIndex) { + sinkParent(parentIndex, first, len, key); + } +} + +/// Sort the heap `[first, last)` in the order of ascending keys +template +void sortHeap(RandomIt first, RandomIt last, const UnaryOperation &key) { + auto len = last - first; + while (len > 1) { + // Move the largest element to the end of the heap and exclude it from the heap. + std::swap(*first, *(first + --len)); + // Arrange the remaining elements into a heap again. + sinkParent(0, first, len, key); + } +} + +} // namespace metofficesortdetail + /// \brief Sort the range `[first, last)` in the order of ascending keys using the same algorithm as /// the `Ops_Integer/Char/RealSort` subroutines from the Met Office OPS system. /// @@ -22,9 +84,8 @@ namespace ufo { /// and returning a key used as the sorting criterion. template void metOfficeSort(RandomIt first, RandomIt last, const UnaryOperation &key) { - auto compare = [&key] (const auto &a, const auto &b) { return key(a) <= key(b); }; - std::make_heap(first, last, compare); - std::sort_heap(first, last, compare); + metofficesortdetail::makeHeap(first, last, key); + metofficesortdetail::sortHeap(first, last, key); } /// \brief Sort the range `[first, last)` in ascending order using the same algorithm as From e34a2db44377b87e00aa4302986b3e27e3af15a0 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 1 Sep 2021 12:34:25 -0600 Subject: [PATCH 95/98] change bias correction parameter in BackgroundCheck to be true/false (#1489) --- ewok/jedi-gdas/ssmis_f17.yaml | 2 +- ewok/jedi-geos/ssmis_f17.yaml | 2 +- src/ufo/filters/BackgroundCheck.cc | 29 +++++++++---------- src/ufo/filters/BackgroundCheck.h | 4 +-- .../ssmis/ssmis_f17_gfs_HofX_qc.yaml | 2 +- .../ssmis_f17_gfs_backgroundcheck_nbc.yaml | 2 +- 6 files changed, 19 insertions(+), 22 deletions(-) diff --git a/ewok/jedi-gdas/ssmis_f17.yaml b/ewok/jedi-gdas/ssmis_f17.yaml index 6cf87fa31..921642a8c 100644 --- a/ewok/jedi-gdas/ssmis_f17.yaml +++ b/ewok/jedi-gdas/ssmis_f17.yaml @@ -61,7 +61,7 @@ obs filters: - name: brightness_temperature channels: *ssmis_f17_channels absolute threshold: 3.5 - bias correction parameter: 1.0 + remove bias correction: true action: name: reject # #step2: clw check diff --git a/ewok/jedi-geos/ssmis_f17.yaml b/ewok/jedi-geos/ssmis_f17.yaml index 6cf87fa31..921642a8c 100644 --- a/ewok/jedi-geos/ssmis_f17.yaml +++ b/ewok/jedi-geos/ssmis_f17.yaml @@ -61,7 +61,7 @@ obs filters: - name: brightness_temperature channels: *ssmis_f17_channels absolute threshold: 3.5 - bias correction parameter: 1.0 + remove bias correction: true action: name: reject # #step2: clw check diff --git a/src/ufo/filters/BackgroundCheck.cc b/src/ufo/filters/BackgroundCheck.cc index 8924b16a7..c9c52a4c3 100644 --- a/src/ufo/filters/BackgroundCheck.cc +++ b/src/ufo/filters/BackgroundCheck.cc @@ -125,7 +125,7 @@ void BackgroundCheck::applyFilter(const std::vector & apply, Variables varbias(filtervars_, "ObsBiasData"); for (size_t jv = 0; jv < filtervars.nvars(); ++jv) { size_t iv = observed.find(filtervars.variable(jv).variable()); -// H(x) +// H(x) (including bias correction) std::vector hofx; data_.get(varhofx.variable(jv), hofx); // H(x) error @@ -134,33 +134,27 @@ void BackgroundCheck::applyFilter(const std::vector & apply, if (thresholdWrtBGerror) { data_.get(backgrErrVariable(filtervars[jv]), hofxerr); } -// H(x) bias correction - std::vector bias; - data_.get(varbias.variable(jv), bias); +// Bias correction (only read in if removeBiasCorrection is set to true, otherwise +// set to zero). + std::vector bias(obsdb_.nlocs(), 0.0); + if (parameters_.removeBiasCorrection) { + data_.get(varbias.variable(jv), bias); + } // Threshold for current variable std::vector abs_thr(obsdb_.nlocs(), std::numeric_limits::max()); std::vector thr(obsdb_.nlocs(), std::numeric_limits::max()); - std::vector bc_factor(obsdb_.nlocs(), 0.0); - if (parameters_.absoluteThreshold.value()) abs_thr = getScalarOrFilterData(*parameters_.absoluteThreshold.value(), data_); if (parameters_.threshold.value()) thr = getScalarOrFilterData(*parameters_.threshold.value(), data_); -// Bias Correction parameter - if (parameters_.BiasCorrectionFactor.value()) - bc_factor = getScalarOrFilterData(*parameters_.BiasCorrectionFactor.value(), data_); - for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass && (*obserr_)[iv][jobs] != util::missingValue((*obserr_)[iv][jobs])) { ASSERT(obs[jv][jobs] != util::missingValue(obs[jv][jobs])); - if (parameters_.BiasCorrectionFactor.value()) { - ASSERT(bias[jobs] != util::missingValue(bias[jobs])); - bc_factor[jobs] = bc_factor[jobs]*bias[jobs]; - } ASSERT(hofx[jobs] != util::missingValue(hofx[jobs])); + ASSERT(bias[jobs] != util::missingValue(bias[jobs])); const std::vector &errorMultiplier = thresholdWrtBGerror ? hofxerr : (*obserr_)[iv]; @@ -170,8 +164,11 @@ void BackgroundCheck::applyFilter(const std::vector & apply, ASSERT(zz < std::numeric_limits::max() && zz > 0.0); -// Check distance from background - if (std::abs(static_cast(hofx[jobs]) - obs[jv][jobs] - bc_factor[jobs]) > zz) { +// Check distance from background. hofx includes bias correction. +// If removeBiasCorrection is set to true, `bias` contains bias correction, and +// it is removed from hofx. +// Otherwise, `bias` is set to zero, and bias correction is not removed from hofx. + if (std::abs(hofx[jobs] - obs[jv][jobs] - bias[jobs]) > zz) { flagged[jv][jobs] = true; } } diff --git a/src/ufo/filters/BackgroundCheck.h b/src/ufo/filters/BackgroundCheck.h index d224d0c3a..8a104465b 100644 --- a/src/ufo/filters/BackgroundCheck.h +++ b/src/ufo/filters/BackgroundCheck.h @@ -72,8 +72,8 @@ class BackgroundCheckParameters : public FilterParametersBase { oops::OptionalParameter> functionAbsoluteThreshold{ "function absolute threshold", this}; - /// add option parameter accounting for with or without bias correction (GSI ssmis case) - oops::OptionalParameter BiasCorrectionFactor{"bias correction parameter", this}; + /// The filter uses bias-corrected H(x) unless `remove bias correction` is set to true. + oops::Parameter removeBiasCorrection{"remove bias correction", false, this}; /// Name of the HofX group used to replace the default group (default is HofX) oops::Parameter test_hofx{"test_hofx", "HofX", this}; diff --git a/test/testinput/instrumentTests/ssmis/ssmis_f17_gfs_HofX_qc.yaml b/test/testinput/instrumentTests/ssmis/ssmis_f17_gfs_HofX_qc.yaml index 54d8d0f91..eede69484 100644 --- a/test/testinput/instrumentTests/ssmis/ssmis_f17_gfs_HofX_qc.yaml +++ b/test/testinput/instrumentTests/ssmis/ssmis_f17_gfs_HofX_qc.yaml @@ -67,7 +67,7 @@ observations: - name: brightness_temperature channels: *channels absolute threshold: 3.5 - bias correction parameter: 1.0 + remove bias correction: true action: name: reject # #step2: clw check diff --git a/test/testinput/ssmis_f17_gfs_backgroundcheck_nbc.yaml b/test/testinput/ssmis_f17_gfs_backgroundcheck_nbc.yaml index 07796045b..cf2928ad6 100644 --- a/test/testinput/ssmis_f17_gfs_backgroundcheck_nbc.yaml +++ b/test/testinput/ssmis_f17_gfs_backgroundcheck_nbc.yaml @@ -59,7 +59,7 @@ observations: - name: brightness_temperature channels: *channels absolute threshold: 3.5 - bias correction parameter: 1.0 + remove bias correction: true action: name: reject passedBenchmark: 1555 From b4d8c218cad65aed4213b251d8c6a16c0783bafd Mon Sep 17 00:00:00 2001 From: Mark Miesch Date: Thu, 9 Sep 2021 07:50:52 -0600 Subject: [PATCH 96/98] clean up doxygen for mpas-jedi release (#1500) --- LICENSE => LICENSE.md | 2 +- README.md | 2 +- docs/Doxyfile.in | 9 +-------- docs/mainpage.h | 12 ++++++++++++ 4 files changed, 15 insertions(+), 10 deletions(-) rename LICENSE => LICENSE.md (99%) create mode 100644 docs/mainpage.h diff --git a/LICENSE b/LICENSE.md similarity index 99% rename from LICENSE rename to LICENSE.md index 261eeb9e9..04d9ae185 100644 --- a/LICENSE +++ b/LICENSE.md @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2017-2021 UCAR Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index fdbf6715a..118d3dfe1 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Unified Forward Operators for Joint Effort for Data assimilation Integration (JEDI) project. -(C) Copyright 2017-2019 UCAR. +(C) Copyright 2017-2021 UCAR. This software is licensed under the terms of the Apache Licence Version 2.0 which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 2ef22ee98..b16e0f2d1 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -909,7 +909,7 @@ RECURSIVE = YES # Note that relative paths are relative to the directory from which doxygen is # run. -#EXCLUDE = @CMAKE_CURRENT_SOURCE_DIR@/../src/util +EXCLUDE = @CMAKE_CURRENT_SOURCE_DIR@/../test/testinput/reference/README.md # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded @@ -1126,13 +1126,6 @@ VERBATIM_HEADERS = YES ALPHABETICAL_INDEX = YES -# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in -# which the alphabetical index list will be split. -# Minimum value: 1, maximum value: 20, default value: 5. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -COLS_IN_ALPHA_INDEX = 5 - # In case all classes in a project start with a common prefix, all classes will # be put under the same header in the alphabetical index. The IGNORE_PREFIX tag # can be used to specify a prefix (or a list of prefixes) that should be ignored diff --git a/docs/mainpage.h b/docs/mainpage.h new file mode 100644 index 000000000..1fbafb9cb --- /dev/null +++ b/docs/mainpage.h @@ -0,0 +1,12 @@ +#pragma once + +// This file defines what appears on the Main Page of the documentation +// generated by doxygen. The file contains no code, and does not appear +// in any cpp include statement. +// +/*! + * \mainpage Unified Forward Operator (UFO) + * + * UFO provides the observational operators needed to compute departures and innovations. In other words, it enables the comparison between model forecasts and observations that lies at the heart of the data assimilation process. UFO also provides related functionality related to observations such as quality control (QC) filters and variational bias correction. + * + */ From a7b079a52796fc5f448dd63e2c5215c76659ec7d Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Tue, 21 Sep 2021 10:04:20 -0600 Subject: [PATCH 97/98] update dash url for mpas release (#1516) * update url * update url --- test/CMakeLists.txt | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 86ab81f60..476ade0dc 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -394,32 +394,22 @@ function(ADD_DOWNLOAD_TEST repo test_files_dirname output_data_path) if( DEFINED ENV{ECBUILD_DOWNLOAD_BASE_URL} ) set( ECBUILD_DOWNLOAD_BASE_URL "$ENV{ECBUILD_DOWNLOAD_BASE_URL}/${repo}/${GIT_TAG_FUNC}" ) else() - set( ECBUILD_DOWNLOAD_BASE_URL https://dashrepo.ucar.edu/api/v1/dataset/147_miesch/version/1.1.0/file ) + set( ECBUILD_DOWNLOAD_BASE_URL https://dashrepo.ucar.edu/api/v1/dataset/147b_jcsda/version/1.0.0/file ) endif() - set( DIRNAME ${GIT_TAG_FUNC} ) set( checksum "0" ) - set( TESTFILE_DIR_REPO "${CMAKE_SOURCE_DIR}/test-data-release/${repo}/${DIRNAME}" ) - + set( TESTFILE_DIR_REPO "${CMAKE_SOURCE_DIR}/test-data-release/${repo}/${GIT_TAG_FUNC}" ) # Create test-data-release in source directory - file( MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/test-data-release ) - # temporary solution for release 1.1.0 - if ( ${repo} STREQUAL "crtm" ) - list( APPEND REPO_DATA_DOWNLOADER_ARGS - ${ECBUILD_DOWNLOAD_BASE_URL} - ${TESTFILE_DIR_REPO} - ${test_files_dirname} - ${checksum} ) - file( MAKE_DIRECTORY ${TESTFILE_DIR_REPO} ) - else() - list( APPEND REPO_DATA_DOWNLOADER_ARGS - ${ECBUILD_DOWNLOAD_BASE_URL} - ${CMAKE_SOURCE_DIR}/test-data-release - ${test_files_dirname} - ${checksum} ) - endif() - message( STATUS "ECBUILD_DOWNLOAD_BASE_URL is set, ${repo} test files will - be downloaded from ${ECBUILD_DOWNLOAD_BASE_URL}/test-data-release") + file( MAKE_DIRECTORY ${TESTFILE_DIR_REPO} ) + list( APPEND REPO_DATA_DOWNLOADER_ARGS + ${ECBUILD_DOWNLOAD_BASE_URL} + ${CMAKE_SOURCE_DIR}/test-data-release + ${test_files_dirname} + ${checksum} ) + + message( STATUS "Test files will be downloaded from DASH and + saved to ${TESTFILE_DIR_REPO} for ${repo}") + set ( REPO_DATA_DOWNLOADER ufo_data_downloader.py ) # Any branch of REPO is being built. From 2b7c01061dc27ab67a629d422bd8117b7958ae03 Mon Sep 17 00:00:00 2001 From: Maryam Abdi-Oskouei Date: Tue, 21 Sep 2021 17:05:10 -0600 Subject: [PATCH 98/98] update version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f5b0de3c..e4955510c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ cmake_minimum_required( VERSION 3.12 ) -project( ufo VERSION 1.1.0 LANGUAGES C CXX Fortran ) +project( ufo VERSION 1.2.0 LANGUAGES C CXX Fortran ) ## Ecbuild integration find_package( ecbuild 3.3.2 REQUIRED )