Skip to content

Commit

Permalink
initial dense support in ies
Browse files Browse the repository at this point in the history
  • Loading branch information
@jtwhite79
jtwhite79 committed Dec 8, 2024
1 parent f7e62b1 commit d7fda95
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 24 deletions.
104 changes: 87 additions & 17 deletions src/libs/pestpp_common/EnsembleMethodUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2405,6 +2405,7 @@ void L2PhiHandler::save_residual_cov(ObservationEnsemble& oe, int iter)
pair<Covariance,Covariance> rcovs = res.get_empirical_cov_matrices(file_manager);
stringstream ss;
ss << file_manager->get_base_filename() << "." << iter << ".res.";

if (pest_scenario->get_pestpp_options().get_save_binary())
{
ss << "jcb";
Expand Down Expand Up @@ -4546,7 +4547,12 @@ pair<string,string> EnsembleMethod::save_ensembles(string tag, int cycle, Parame
ss << "." << "mean" << ".obs";
else
ss << "." << iter << ".obs";
if (pest_scenario.get_pestpp_options().get_save_binary())
if (pest_scenario.get_pestpp_options().get_save_dense())
{
ss << ".bin";
_oe.to_dense(ss.str());
}
else if (pest_scenario.get_pestpp_options().get_save_binary())
{
ss << ".jcb";
_oe.to_binary(ss.str());
Expand All @@ -4569,7 +4575,12 @@ pair<string,string> EnsembleMethod::save_ensembles(string tag, int cycle, Parame
ss << "." << "mean" << ".par";
else
ss << "." << iter << ".par";
if (pest_scenario.get_pestpp_options().get_save_binary())
if (pest_scenario.get_pestpp_options().get_save_dense())
{
ss << ".bin";
_pe.to_dense(ss.str());
}
else if (pest_scenario.get_pestpp_options().get_save_binary())
{
ss << ".jcb";
_pe.to_binary(ss.str());
Expand Down Expand Up @@ -5076,7 +5087,15 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing)

initialize_weights();
ss.str("");
if (pest_scenario.get_pestpp_options().get_save_binary())
if (pest_scenario.get_pestpp_options().get_save_dense())
{
ss << file_manager.get_base_filename();
if (cycle != NetPackage::NULL_DA_CYCLE)
ss << "." << cycle;
ss << ".weights.bin";
weights.to_dense(ss.str());
}
else if (pest_scenario.get_pestpp_options().get_save_binary())
{
ss << file_manager.get_base_filename();
if (cycle != NetPackage::NULL_DA_CYCLE)
Expand Down Expand Up @@ -5148,7 +5167,7 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing)
}

int num_par_elements = pest_scenario.get_ctl_parameters().size();
if ((pp_args.find("IES_ORDERED_BINARY") == pp_args.end()) && (pp_args.find("DA_ORDERED_BINARY") == pp_args.end()))
if ((!pest_scenario.get_pestpp_options().get_save_dense()) && (pp_args.find("IES_ORDERED_BINARY") == pp_args.end()) && (pp_args.find("DA_ORDERED_BINARY") == pp_args.end()))
{

//if ((pe.shape().second > 100000) && (pest_scenario.get_pestpp_options().get_save_binary()))
Expand All @@ -5172,7 +5191,16 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing)
message(2, "checking for denormal values in pe");
pe.check_for_normal("initial transformed parameter ensemble");
ss.str("");
if (pest_scenario.get_pestpp_options().get_save_binary())

if (pest_scenario.get_pestpp_options().get_save_dense())
{
ss << file_manager.get_base_filename();
if (cycle != NetPackage::NULL_DA_CYCLE)
ss << "." << cycle;
ss << ".0.par.bin";
pe.to_dense_unordered(ss.str());
}
else if (pest_scenario.get_pestpp_options().get_save_binary())
{
ss << file_manager.get_base_filename();
if (cycle != NetPackage::NULL_DA_CYCLE)
Expand All @@ -5192,7 +5220,15 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing)
message(2, "checking for denormal values in obs + noise ensemble");
oe_base.check_for_normal("obs+noise observation ensemble");
ss.str("");
if (pest_scenario.get_pestpp_options().get_save_binary())
if (pest_scenario.get_pestpp_options().get_save_dense())
{
ss << file_manager.get_base_filename();
if (cycle != NetPackage::NULL_DA_CYCLE)
ss << "." << cycle;
ss << ".obs+noise.bin";
oe_base.to_dense(ss.str());
}
else if (pest_scenario.get_pestpp_options().get_save_binary())
{
ss << file_manager.get_base_filename();
if (cycle != NetPackage::NULL_DA_CYCLE)
Expand Down Expand Up @@ -5432,7 +5468,15 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing)
}

ss.str("");
if (pest_scenario.get_pestpp_options().get_save_binary())
if (pest_scenario.get_pestpp_options().get_save_dense())
{
ss << file_manager.get_base_filename();
if (cycle != NetPackage::NULL_DA_CYCLE)
ss << "." << cycle;
ss << ".0.obs.bin";
oe.to_dense(ss.str());
}
else if (pest_scenario.get_pestpp_options().get_save_binary())
{
ss << file_manager.get_base_filename();
if (cycle != NetPackage::NULL_DA_CYCLE)
Expand Down Expand Up @@ -5641,7 +5685,13 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing)
(in_conflict.size() > 0))
{
ss.str("");
if (pest_scenario.get_pestpp_options().get_save_binary())
if (pest_scenario.get_pestpp_options().get_save_dense())
{
ss << file_manager.get_base_filename();
ss << ".adjusted.weights.bin";
weights.to_dense(ss.str());
}
else if (pest_scenario.get_pestpp_options().get_save_binary())
{
ss << file_manager.get_base_filename();
ss << ".adjusted.weights.jcb";
Expand Down Expand Up @@ -6891,7 +6941,11 @@ bool EnsembleMethod::solve(bool use_mda, vector<double> inflation_factors, vecto
if (!pest_scenario.get_pestpp_options().get_ies_save_lambda_en())
continue;

if (pest_scenario.get_pestpp_options().get_save_binary())
if (pest_scenario.get_pestpp_options().get_save_dense())
{
pe_lam_scale.to_dense_unordered(ss.str()+".bin");
}
else if (pest_scenario.get_pestpp_options().get_save_binary())
{
pe_lam_scale.to_binary(ss.str()+".jcb");
}
Expand Down Expand Up @@ -6992,7 +7046,12 @@ bool EnsembleMethod::solve(bool use_mda, vector<double> inflation_factors, vecto
ss.str("");
ss << file_manager.get_base_filename() << "." << iter << "." << lam_vals[i] << ".lambda." << scale_vals[i] << ".scale.obs";

if (pest_scenario.get_pestpp_options().get_save_binary())
if (pest_scenario.get_pestpp_options().get_save_dense())
{
ss << ".bin";
oe_lams[i].to_dense(ss.str());
}
else if (pest_scenario.get_pestpp_options().get_save_binary())
{
ss << ".jcb";
oe_lams[i].to_binary(ss.str());
Expand Down Expand Up @@ -7736,16 +7795,17 @@ bool EnsembleMethod::initialize_pe(Covariance& cov)
if (pest_scenario.get_pestpp_options().get_ies_verbose_level() > 1)
{


if (pest_scenario.get_pestpp_options().get_save_binary())
{
string filename = file_manager.get_base_filename() + ".prior.jcb";
message(1, "saving emprirical parameter covariance matrix to binary file: ", filename);
message(1, "saving empirical parameter covariance matrix to binary file: ", filename);
parcov.to_binary(filename);
}
else
{
string filename = file_manager.get_base_filename() + ".prior.cov";
message(1, "saving emprirical parameter covariance matrix to ASCII file: ", filename);
message(1, "saving empirical parameter covariance matrix to ASCII file: ", filename);
parcov.to_ascii(filename);
}
}
Expand Down Expand Up @@ -8297,7 +8357,12 @@ void EnsembleMethod::initialize_restart()
Observations obs = pest_scenario.get_ctl_observations();
oe_base.replace(base_par_idx, obs, BASE_REAL_NAME);
ss.str("");
if (pest_scenario.get_pestpp_options().get_save_binary())
if (pest_scenario.get_pestpp_options().get_save_dense())
{
ss << file_manager.get_base_filename() << ".obs+noise.bin";
oe_base.to_dense(ss.str());
}
else if (pest_scenario.get_pestpp_options().get_save_binary())
{
ss << file_manager.get_base_filename() << ".obs+noise.jcb";
oe_base.to_binary(ss.str());
Expand All @@ -8321,15 +8386,20 @@ void EnsembleMethod::initialize_restart()
}
weights.replace(base_par_idx, wobs, BASE_REAL_NAME);
ss.str("");
if (pest_scenario.get_pestpp_options().get_save_binary())
if (pest_scenario.get_pestpp_options().get_save_dense())
{
ss << file_manager.get_base_filename() << ".weights.bin";
weights.to_binary(ss.str());
}
else if (pest_scenario.get_pestpp_options().get_save_binary())
{
ss << file_manager.get_base_filename() << ".weights.jcb";
oe_base.to_binary(ss.str());
weights.to_binary(ss.str());
}
else
{
ss << file_manager.get_base_filename() << ".weights.csv";
oe_base.to_csv(ss.str());
weights.to_csv(ss.str());
}
message(1, "re-saved weight ensemble to ", ss.str());
}
Expand Down Expand Up @@ -8891,5 +8961,5 @@ vector<int> EnsembleMethod::get_subset_idxs(int size, int nreal_subset)

sort(subset_idxs.begin(), subset_idxs.end());
return subset_idxs;

}
8 changes: 4 additions & 4 deletions src/libs/pestpp_common/pest_data_structs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1194,9 +1194,9 @@ bool PestppOptions::assign_ies_value_by_key(const string& key, const string& val
return true;
}

else if (key == "IES_SAVE_DENSE")
else if (key == "SAVE_DENSE")
{
ies_save_dense = pest_utils::parse_string_arg_to_bool(value);
save_dense = pest_utils::parse_string_arg_to_bool(value);
return true;
}

Expand Down Expand Up @@ -1623,6 +1623,7 @@ void PestppOptions::summary(ostream& os) const
os << "random_seed: " << random_seed << endl;
os << "num_tpl_ins_threads: " << num_tpl_ins_threads << endl;
os << "save_binary: " << save_binary << endl;
os << "save_dense: " << save_dense << endl;
os << "ensemble_output_precision: " << ensemble_output_precision << endl;

os << "panther_echo: " << panther_echo << endl;
Expand Down Expand Up @@ -1838,7 +1839,6 @@ void PestppOptions::summary(ostream& os) const
os << "ies_phi_factors_by_real: " << ies_phi_factors_by_real << endl;
os << "ies_n_iter_reinflate: " << ies_n_iter_mean << endl;
os << "ies_updatebyreals: " << ies_updatebyreals << endl;
os << "ies_save_dense: " << ies_save_dense << endl;


os << endl << "pestpp-sen options: " << endl;
Expand Down Expand Up @@ -2026,7 +2026,7 @@ void PestppOptions::set_defaults()
set_ies_phi_factors_by_real(false);
set_ies_n_iter_mean(0);
set_ies_updatebyreals(false);
set_ies_save_dense(false);
set_save_dense(false);


// DA parameters
Expand Down
8 changes: 5 additions & 3 deletions src/libs/pestpp_common/pest_data_structs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,8 +494,9 @@ class PestppOptions {
void set_ies_phi_factors_by_real(bool _flag) {ies_phi_factors_by_real = _flag;}
bool get_ies_updatebyreals() const {return ies_updatebyreals;}
void set_ies_updatebyreals(bool _flag) {ies_updatebyreals = _flag;}
bool get_ies_save_dense() const {return ies_save_dense; }
void set_ies_save_dense(bool _flag) {ies_save_dense = _flag;}

bool get_save_dense() const {return save_dense; }
void set_save_dense(bool _flag) {save_dense = _flag;}



Expand Down Expand Up @@ -790,6 +791,7 @@ class PestppOptions {
bool ies_enforce_bounds;
double par_sigma_range;
bool save_binary;
bool save_dense;
string ies_localizer;
double ies_accept_phi_fac;
double ies_lambda_inc_fac;
Expand Down Expand Up @@ -828,7 +830,7 @@ class PestppOptions {
bool ies_phi_factors_by_real;
int ies_n_iter_mean;
bool ies_updatebyreals;
bool ies_save_dense;



// Data Assimilation parameters
Expand Down

0 comments on commit d7fda95

Please sign in to comment.