diff --git a/benchmarks/basic_tests.py b/benchmarks/basic_tests.py index 9f5453e0..bf5b2534 100644 --- a/benchmarks/basic_tests.py +++ b/benchmarks/basic_tests.py @@ -874,14 +874,31 @@ def mf6_v5_ies_test(): t_d = os.path.join(model_d,"template") m_d = os.path.join(model_d,"master_ies_glm_loc") - if os.path.exists(m_d): - shutil.rmtree(m_d) + #if os.path.exists(m_d): + # shutil.rmtree(m_d) pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_ies.pst")) pst.control_data.noptmax = 0 pst.write(os.path.join(t_d,"freyberg6_run_ies.pst")) pyemu.os_utils.run("{0} freyberg6_run_ies.pst".format(exe_path),cwd=t_d) pst.control_data.noptmax = 3 + par = pst.parameter_data + + eff_lb = (par.parlbnd + (np.abs(par.parlbnd.values)*.01)).to_dict() + eff_ub = (par.parubnd - (np.abs(par.parlbnd.values)*.01)).to_dict() + log_idx = par.partrans.apply(lambda x: x=="log").to_dict() + for p,log in log_idx.items(): + if log: + lb = np.log10(par.loc[p,"parlbnd"]) + eff_lb[p] = (lb + (np.abs(lb)*.01)) + ub = np.log10(par.loc[p,"parubnd"]) + eff_ub[p] = (ub - (np.abs(ub)*.01)) + + pargp_map = par.groupby(par.pargp).groups + print(pargp_map) + + + m_d = os.path.join(model_d, "master_ies_glm_noloc_standard") if os.path.exists(m_d): @@ -890,18 +907,50 @@ def mf6_v5_ies_test(): pst.pestpp_options.pop("ies_localizer",None) pst.pestpp_options.pop("ies_autoadaloc",None) pst.pestpp_options["ies_bad_phi_sigma"] = 2.5 + pst.pestpp_options["ies_num_reals"] = 30 + pst.pestpp_options["ensemble_output_precision"] = 40 pst.control_data.noptmax = 3 - pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_noloc_standard.pst")) - pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_noloc_standard.pst", num_workers=15, + pst_name = "freyberg6_run_ies_glm_noloc_standard.pst" + pst.write(os.path.join(t_d, pst_name)) + pyemu.os_utils.start_workers(t_d, exe_path, pst_name, num_workers=15, master_dir=m_d, worker_root=model_d, port=port) - - return + phidf = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".phi.actual.csv"))) + assert phidf.shape[0] == pst.control_data.noptmax + 1 + for i in range(1,pst.control_data.noptmax+1): + pcs = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".{0}.pcs.csv".format(i))),index_col=0) + #print(pcs) + pe = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".{0}.par.csv".format(i))),index_col=0) + print(pe.shape) + #print(pe) + groups = pcs.index.values.copy() + groups.sort() + for group in groups: + pnames = pargp_map[group].values + lb_count,ub_count = 0,0 + for pname in pnames: + lb,ub = eff_lb[pname],eff_ub[pname] + v = pe.loc[:,pname].values.copy() + if log_idx[pname]: + v = np.log10(v) + low = np.zeros_like(v,dtype=int) + low[v < lb] = 1 + high = np.zeros_like(v,dtype=int) + high[v > ub] = 1 + lb_count += low.sum() + ub_count += high.sum() + print(i,group,len(pnames),lb_count,pcs.loc[group,"num_at_near_lbound"],ub_count,pcs.loc[group,"num_at_near_ubound"]) + assert lb_count == pcs.loc[group,"num_at_near_lbound"] + assert ub_count == pcs.loc[group,"num_at_near_ubound"] + pst.write(os.path.join(t_d,"freyberg6_run_ies_glm_loc.pst")) m_d = os.path.join(model_d, "master_ies_glm_covloc") if os.path.exists(m_d): shutil.rmtree(m_d) pst.pestpp_options["ies_loc_type"] = "cov" + pst.pestpp_options["ies_lambda_mults"] = [1.0] + pst.pestpp_options["lambda_scale_fac"] = [1.0] + pst.control_data.noptmax = 2 #pst.pestpp_options.pop("ies_localizer",None) pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_covloc.pst")) pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_covloc.pst", num_workers=15, @@ -913,7 +962,9 @@ def mf6_v5_ies_test(): pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) pst.pestpp_options.pop("ies_localizer",None) pst.pestpp_options.pop("ies_autoadaloc",None) - pst.control_data.noptmax = 3 + pst.pestpp_options["ies_lambda_mults"] = [1.0] + pst.pestpp_options["lambda_scale_fac"] = [1.0] + pst.control_data.noptmax = 2 pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_noloc.pst")) pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_noloc.pst", num_workers=15, master_dir=m_d, worker_root=model_d, port=port) @@ -922,7 +973,9 @@ def mf6_v5_ies_test(): if os.path.exists(m_d): shutil.rmtree(m_d) pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) - pst.control_data.noptmax = 3 + pst.pestpp_options["ies_lambda_mults"] = [1.0] + pst.pestpp_options["lambda_scale_fac"] = [1.0] + pst.control_data.noptmax = 2 pst.pestpp_options["ies_use_mda"] = True pst.write(os.path.join(t_d, "freyberg6_run_ies_mda_loc.pst")) pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_mda_loc.pst", num_workers=15, @@ -943,7 +996,9 @@ def mf6_v5_ies_test(): if os.path.exists(m_d): shutil.rmtree(m_d) pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) - pst.control_data.noptmax = 3 + pst.control_data.noptmax = 2 + pst.pestpp_options["ies_lambda_mults"] = [1.0] + pst.pestpp_options["lambda_scale_fac"] = [1.0] pst.pestpp_options["ies_use_mda"] = True pst.pestpp_options.pop("ies_localizer", None) pst.pestpp_options.pop("ies_autoadaloc", None) @@ -955,7 +1010,9 @@ def mf6_v5_ies_test(): if os.path.exists(m_d): shutil.rmtree(m_d) pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) - pst.control_data.noptmax = 3 + pst.control_data.noptmax = 2 + pst.pestpp_options["ies_lambda_mults"] = [1.0] + pst.pestpp_options["lambda_scale_fac"] = [1.0] pst.pestpp_options["ies_num_threads"] = 1 pst.pestpp_options["ies_use_mda"] = False pst.pestpp_options.pop("ies_localizer", None) @@ -969,8 +1026,10 @@ def mf6_v5_ies_test(): if os.path.exists(m_d): shutil.rmtree(m_d) pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) - pst.control_data.noptmax = 3 + pst.control_data.noptmax = 2 pst.pestpp_options["ies_use_mda"] = False + pst.pestpp_options["ies_lambda_mults"] = [1.0] + pst.pestpp_options["lambda_scale_fac"] = [1.0] pst.pestpp_options.pop("ies_localizer", None) pst.pestpp_options.pop("ies_autoadaloc", None) pst.pestpp_options["ies_multimodal_alpha"] = 0.25 @@ -1229,6 +1288,38 @@ def fr_timeout_test(): print(oe.shape) assert oe.shape[0] == 5,oe.shape + with open(os.path.join(new_d,"run.py"),'w') as f: + f.write("import os\nimport time\nimport pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\n") + f.write("if not os.path.exists('run.info'):\n exit()\n") + f.write("lines = open('run.info','r').readlines()\nrnum = int(lines[-1].split()[-1].split(':')[-1])\n") + f.write("if rnum % 10 == 0:\n print(junk)\n") + pst.pestpp_options = {} + pst.pestpp_options["ies_num_reals"] = 20 # hard coded to conditional below + pst.pestpp_options["panther_agent_freeze_on_fail"] = True + #pst.pestpp_options["overdue_giveup_fac"] = 1.0e+10 + #pst.pestpp_options["overdue_giveup_minutes"] = 0.25 + pst.write(os.path.join(new_d, "pest.pst")) + pst.control_data.noptmax = 2 + + pst.write(os.path.join(new_d, "pest.pst")) + m_d = os.path.join(model_d,"fr_timeout_master_freeze") + #num workers hard coded with conditional below + pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=10,worker_root=model_d,master_dir=m_d) + #df = pyemu.helpers.parse_rmr_file(os.path.join(m_d,"pest.rmr")) + #print(df.action.to_list()) + oe = pd.read_csv(os.path.join(m_d,"pest.{0}.obs.csv".format(pst.control_data.noptmax)),index_col=0) + assert oe.shape[0] == 17 # hard coded to num reals + with open(os.path.join(m_d,"pest.rmr"),'r') as f: + for line in f: + if "timeout" in line.lower(): + raise Exception() + if line.strip().lower().endswith("agents connected"): + num = int(line.strip().split()[0]) + print(line.strip()) + assert num == 7 # hard coded above + + + def ins_missing_e_test(): import os import shutil @@ -1339,7 +1430,15 @@ def build_and_draw_prior(t_d="ends",num_reals=500): pe.to_binary(os.path.join(t_d,"prior.jcb")) +def run(): + model_d = "mf6_freyberg" + t_d = os.path.join(model_d,"template") + pst_name = "freyberg6_run_ies_glm_noloc_standard.pst" + pyemu.os_utils.start_workers(t_d, exe_path, pst_name, num_workers=15, + worker_root=model_d, port=4004) + if __name__ == "__main__": + #run() #mf6_v5_ies_test() #prep_ends() #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-glm.exe"),os.path.join("..","bin","win","pestpp-glm.exe")) @@ -1351,7 +1450,7 @@ def build_and_draw_prior(t_d="ends",num_reals=500): #glm_long_name_test() #sen_plusplus_test() #parchglim_test() - unc_file_test() + #unc_file_test() #cmdline_test() #secondary_marker_test() #basic_test("ies_10par_xsec") @@ -1378,6 +1477,8 @@ def build_and_draw_prior(t_d="ends",num_reals=500): #da_mf6_freyberg_test_2() #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-ies.exe"),os.path.join("..","bin","win","pestpp-ies.exe")) #tplins1_test() + + fr_timeout_test() #mf6_v5_ies_test() #mf6_v5_sen_test() diff --git a/src/libs/common/config_os.h b/src/libs/common/config_os.h index 92f4a831..e5edb080 100644 --- a/src/libs/common/config_os.h +++ b/src/libs/common/config_os.h @@ -2,7 +2,7 @@ #define CONFIG_OS_H_ -#define PESTPP_VERSION "5.2.6"; +#define PESTPP_VERSION "5.2.7"; #if defined(_WIN32) || defined(_WIN64) #define OS_WIN diff --git a/src/libs/pestpp_common/EnsembleMethodUtils.cpp b/src/libs/pestpp_common/EnsembleMethodUtils.cpp index 7acd5258..34cbe82c 100644 --- a/src/libs/pestpp_common/EnsembleMethodUtils.cpp +++ b/src/libs/pestpp_common/EnsembleMethodUtils.cpp @@ -3515,7 +3515,8 @@ void L2PhiHandler::report_group(bool echo) { { for (auto& oo : o.second) { - if (oo.second == 0.0) + //if (oo.second == 0.0) + if (snzgroups.find(oo.first) == snzgroups.end()) continue; len = max(len,(int)oo.first.size()); } @@ -4508,9 +4509,9 @@ void ParChangeSummarizer:: update(ParameterEnsemble& pe) for (int i = 0; i < num_reals; i++) { v = arr[i]; - if (v > (ub[par_name] * 1.01)) + if (v > (ub[par_name] - (abs(ub[par_name]) * 0.01))) num_out_u++; - else if (v < (lb[par_name] * 0.99)) + else if (v < (lb[par_name] + (abs(lb[par_name]) * 0.01))) num_out_l++; } value1 = init_moments.first[par_name]; diff --git a/src/libs/pestpp_common/Localizer.cpp b/src/libs/pestpp_common/Localizer.cpp index bd2a283b..06dc4b80 100644 --- a/src/libs/pestpp_common/Localizer.cpp +++ b/src/libs/pestpp_common/Localizer.cpp @@ -898,6 +898,8 @@ void AutoAdaLocThread::work(int thread_id) { if (list_obs.find(par_names[jpar]) != list_obs.end()) sobs = list_obs[par_names[jpar]]; // dont use at here - this way, any missing pars just get no obs + else + sobs.clear(); use_list_obs = true; } else diff --git a/src/libs/pestpp_common/Pest.cpp b/src/libs/pestpp_common/Pest.cpp index e63ab9f7..60859f96 100644 --- a/src/libs/pestpp_common/Pest.cpp +++ b/src/libs/pestpp_common/Pest.cpp @@ -968,6 +968,10 @@ int Pest::process_ctl_file(ifstream& fin, string _pst_filename) int Pest::process_ctl_file(ifstream& fin, string _pst_filename, ofstream& f_rec) { cout << "processing control file " << _pst_filename << endl; + if (!fin) + { + throw PestError("control file stream is not good"); + } string line; string line_upper; string section(""); diff --git a/src/libs/run_managers/abstract_base/RunStorage.cpp b/src/libs/run_managers/abstract_base/RunStorage.cpp index e3e32f79..3d38d3da 100644 --- a/src/libs/run_managers/abstract_base/RunStorage.cpp +++ b/src/libs/run_managers/abstract_base/RunStorage.cpp @@ -28,6 +28,7 @@ #include "Serialization.h" #include "Transformable.h" #include +#include "utilities.h" using std::numeric_limits; @@ -54,11 +55,27 @@ void RunStorage::reset(const vector &_par_names, const vector &_ { buf_stream.close(); } + + if ((pest_utils::check_exist_in(filename)) || (pest_utils::check_exist_out(filename))) + { + int flag = remove(filename.c_str()); + if (flag != 0) + { + throw runtime_error("RunStorage::reset(): error removing existing file '"+filename+"'"); + + } + } + buf_stream.open(filename.c_str(), ios_base::out | ios_base::binary); - buf_stream.close(); + if (!buf_stream) + { + throw runtime_error("RunStorage::reset() stream not good"); + } + buf_stream.close(); + buf_stream.open(filename.c_str(), ios_base::out | ios_base::in | ios_base::binary); //assert(buf_stream.good() == true); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::reset() stream not good"); } @@ -91,7 +108,7 @@ void RunStorage::reset(const vector &_par_names, const vector &_ buf_stream.seekp(get_stream_pos(end_of_runs), ios_base::beg); buf_stream.write(reinterpret_cast(&buf_status), sizeof(buf_status)); buf_stream.flush(); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::reset() stream not good"); } @@ -110,7 +127,7 @@ void RunStorage::init_restart(const std::string &_filename) } buf_stream.open(filename.c_str(), ios_base::out | ios_base::in | ios_base::binary | ios_base::ate); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::init_restart() stream not good"); } @@ -179,7 +196,7 @@ void RunStorage::init_restart(const std::string &_filename) buf_stream.write(reinterpret_cast(&buf_status), sizeof(buf_status)); buf_stream.flush(); } - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::init_restart() stream not good"); } @@ -187,20 +204,22 @@ void RunStorage::init_restart(const std::string &_filename) int RunStorage::get_nruns() { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::get_nruns() stream not good"); - } + streamoff init_pos = buf_stream.tellg(); buf_stream.seekg(0, ios_base::beg); std::int64_t n_runs_64; buf_stream.read((char*) &n_runs_64, sizeof(n_runs_64)); int n_runs = n_runs_64; buf_stream.seekg(init_pos); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::get_nruns() stream not good"); } + if (n_runs < 0) + { + cout << "RunStorage::get_nruns(): warning: nruns < 0: " << n_runs << endl; + } + return n_runs; } @@ -220,10 +239,6 @@ int RunStorage::get_num_good_runs() } int RunStorage::increment_nruns() { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::increment_nruns() stream not good"); - } buf_stream.seekg(0, ios_base::beg); std::int64_t n_runs_64; buf_stream.read((char*) &n_runs_64, sizeof(n_runs_64)); @@ -232,7 +247,7 @@ int RunStorage::increment_nruns() buf_stream.write((char*) &n_runs_64, sizeof(n_runs_64)); int n_runs = n_runs_64; buf_stream.flush(); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::increment_nruns() stream not good"); } @@ -256,10 +271,6 @@ streamoff RunStorage::get_stream_pos(int run_id) int RunStorage::add_run(const vector &model_pars, const string &info_txt, double info_value) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::add_run() stream not good"); - } std::int8_t r_status = 0; int run_id = increment_nruns() - 1; vector info_txt_buf; @@ -276,7 +287,7 @@ streamoff RunStorage::get_stream_pos(int run_id) buf_stream.seekp(get_stream_pos(end_of_runs), ios_base::beg); buf_stream.write(reinterpret_cast(&buf_status), sizeof(buf_status)); buf_stream.flush(); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::add_run() stream not good"); } @@ -285,10 +296,6 @@ streamoff RunStorage::get_stream_pos(int run_id) int RunStorage::add_run(const Eigen::VectorXd &model_pars, const string &info_txt, double info_value) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::add_run() stream not good"); - } std::int8_t r_status = 0; int run_id = increment_nruns() - 1; vector info_txt_buf; @@ -305,7 +312,7 @@ streamoff RunStorage::get_stream_pos(int run_id) buf_stream.seekp(get_stream_pos(end_of_runs), ios_base::beg); buf_stream.write(reinterpret_cast(&buf_status), sizeof(buf_status)); buf_stream.flush(); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::add_run() stream not good"); } @@ -331,9 +338,13 @@ void RunStorage::copy(const RunStorage &rhs_rs) // permission. So open it with write permission to create it, close // and then reopen it with read and write permisssion. buf_stream.open(filename.c_str(), ios_base::out | ios_base::binary | std::ofstream::trunc); + if (!buf_stream) + { + throw runtime_error("RunStorage::copy() stream not good"); + } buf_stream.close(); buf_stream.open(filename.c_str(), ios_base::out | ios_base::in | ios_base::binary); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::copy() stream not good"); } @@ -349,7 +360,7 @@ void RunStorage::copy(const RunStorage &rhs_rs) run_data_byte_size = rhs_rs.run_par_byte_size; par_names = rhs_rs.par_names; obs_names = rhs_rs.obs_names; - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::copy() stream not good"); } @@ -357,10 +368,7 @@ void RunStorage::copy(const RunStorage &rhs_rs) void RunStorage::update_run(int run_id, const Parameters &pars, const Observations &obs) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::update_run() stream not good"); - } + //set run status flage to complete std::int8_t r_status = 1; check_rec_id(run_id); @@ -393,7 +401,7 @@ void RunStorage::update_run(int run_id, const Parameters &pars, const Observatio buf_stream.seekp(get_stream_pos(end_of_runs), ios_base::beg); buf_stream.write(reinterpret_cast(&buf_status), sizeof(buf_status)); buf_stream.flush(); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::update_run() stream not good"); } @@ -402,10 +410,7 @@ void RunStorage::update_run(int run_id, const Parameters &pars, const Observatio void RunStorage::update_run(int run_id, const Observations &obs) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::udpate_run() stream not good"); - } + //set run status flage to complete std::int8_t r_status = 1; check_rec_id(run_id); @@ -442,7 +447,7 @@ void RunStorage::update_run(int run_id, const Observations &obs) buf_stream.seekp(get_stream_pos(end_of_runs), ios_base::beg); buf_stream.write(reinterpret_cast(&buf_status), sizeof(buf_status)); buf_stream.flush(); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::update_run() stream not good"); } @@ -450,10 +455,7 @@ void RunStorage::update_run(int run_id, const Observations &obs) void RunStorage::update_run(int run_id, const vector serial_data) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::update_run() stream not good"); - } + //set run status flage to complete std::int8_t r_status = 1; check_rec_size(serial_data); @@ -483,7 +485,7 @@ void RunStorage::update_run(int run_id, const vector serial_data) buf_stream.seekp(get_stream_pos(end_of_runs), ios_base::beg); buf_stream.write(reinterpret_cast(&buf_status), sizeof(buf_status)); buf_stream.flush(); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::update_run() stream not good"); } @@ -492,10 +494,7 @@ void RunStorage::update_run(int run_id, const vector serial_data) void RunStorage::update_run_failed(int run_id) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::update_run_failed() stream not good"); - } + std::int8_t r_status = get_run_status_native(run_id); if (r_status < 1) { @@ -506,7 +505,7 @@ void RunStorage::update_run_failed(int run_id) buf_stream.write(reinterpret_cast(&r_status), sizeof(r_status)); buf_stream.flush(); } - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::update_run_failed() stream not good"); } @@ -514,17 +513,14 @@ void RunStorage::update_run_failed(int run_id) void RunStorage::set_run_nfailed(int run_id, int nfail) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::set_run_nfailed() stream not good"); - } + std::int8_t r_status = -nfail; check_rec_id(run_id); //update run status flag buf_stream.seekp(get_stream_pos(run_id), ios_base::beg); buf_stream.write(reinterpret_cast(&r_status), sizeof(r_status)); buf_stream.flush(); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::set_run_nfailed() stream not good"); } @@ -532,15 +528,12 @@ void RunStorage::set_run_nfailed(int run_id, int nfail) std::int8_t RunStorage::get_run_status_native(int run_id) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::get_run_status_native() stream not good"); - } + std::int8_t r_status; check_rec_id(run_id); buf_stream.seekg(get_stream_pos(run_id), ios_base::beg); buf_stream.read(reinterpret_cast(&r_status), sizeof(r_status)); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::get_run_status_native() stream not good"); } @@ -555,10 +548,7 @@ int RunStorage::get_run_status(int run_id) void RunStorage::get_info(int run_id, int &run_status, string &info_txt, double &info_value) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::get_run_info() stream not good"); - } + std::int8_t r_status; vector info_txt_buf; info_txt_buf.resize(info_txt_length, '\0'); @@ -570,18 +560,16 @@ void RunStorage::get_info(int run_id, int &run_status, string &info_txt, double run_status = r_status; info_txt = info_txt_buf.data(); - if (!buf_stream.good()) + if (!buf_stream) { + cout << endl << endl << "-->get_info() bad. run_id:" << run_id << ", info_txt:" << info_txt << endl << endl; throw runtime_error("RunStorage::get_run_info() stream not good"); } } int RunStorage::get_run(int run_id, Parameters &pars, Observations &obs, string &info_txt, double &info_value, bool clear_old) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::get_run() stream not good"); - } + vector par_data; vector obs_data; @@ -596,7 +584,7 @@ int RunStorage::get_run(int run_id, Parameters &pars, Observations &obs, string pars.update_without_clear(par_names, par_data); obs.update_without_clear(obs_names, obs_data); } - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::get_run() stream not good"); } @@ -645,7 +633,7 @@ int RunStorage::get_run(int run_id, double *pars, size_t npars, double *obs, siz buf_stream.read(reinterpret_cast(obs), o_size * sizeof(double)); int status = r_status; info_txt = info_txt_buf.data(); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::get_run() stream not good"); } @@ -654,10 +642,6 @@ int RunStorage::get_run(int run_id, double *pars, size_t npars, double *obs, siz int RunStorage::get_run(int run_id, vector &pars_vec, vector &obs_vec, string &info_txt, double &info_value) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::get_run() stream not good"); - } std::int8_t r_status; vector info_txt_buf; info_txt_buf.resize(info_txt_length, '\0'); @@ -678,7 +662,7 @@ int RunStorage::get_run(int run_id, vector &pars_vec, vector &ob buf_stream.read(reinterpret_cast(&obs_vec[0]), n_obs * sizeof(double)); int status = r_status; info_txt = info_txt_buf.data(); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::get_run() stream not good"); } @@ -701,10 +685,7 @@ int RunStorage::get_run(int run_id, double *pars, size_t npars, double *obs, siz vector RunStorage::get_serial_pars(int run_id) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::get_serial_pars() stream not good"); - } + check_rec_id(run_id); std::int8_t r_status; @@ -713,7 +694,7 @@ vector RunStorage::get_serial_pars(int run_id) buf_stream.seekg(get_stream_pos(run_id), ios_base::beg); buf_stream.seekg(sizeof(r_status)+sizeof(char)*info_txt_length+sizeof(double), ios_base::cur); buf_stream.read(serial_data.data(), serial_data.size()); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::get_serial_pars() stream not good"); } @@ -722,10 +703,6 @@ vector RunStorage::get_serial_pars(int run_id) int RunStorage::get_parameters(int run_id, Parameters &pars) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::get_parameters() stream not good"); - } std::int8_t r_status; vector info_txt_buf; info_txt_buf.resize(info_txt_length, '\0'); @@ -744,7 +721,7 @@ int RunStorage::get_parameters(int run_id, Parameters &pars) buf_stream.read(reinterpret_cast(par_data.data()), n_par*sizeof(double)); pars.update(par_names, par_data); int status = r_status; - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::get_parameters() stream not good"); } @@ -754,10 +731,7 @@ int RunStorage::get_parameters(int run_id, Parameters &pars) int RunStorage::get_observations(int run_id, Observations &obs) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::get_observations() stream not good"); - } + std::int8_t r_status; vector info_txt_buf; info_txt_buf.resize(info_txt_length, '\0'); @@ -777,7 +751,7 @@ int RunStorage::get_observations(int run_id, Observations &obs) buf_stream.read(reinterpret_cast(obs_data.data()), n_obs*sizeof(double)); int status = r_status; obs.update(obs_names, obs_data); - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::get_observations() stream not good"); } @@ -787,10 +761,7 @@ int RunStorage::get_observations(int run_id, Observations &obs) int RunStorage::get_observations_vec(int run_id, vector &obs_data) { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::get_observations_vec() stream not good"); - } + std::int8_t r_status; vector info_txt_buf; info_txt_buf.resize(info_txt_length, '\0'); @@ -808,7 +779,7 @@ int RunStorage::get_observations_vec(int run_id, vector &obs_data) buf_stream.seekg(n_par*sizeof(double), ios_base::cur); buf_stream.read(reinterpret_cast(obs_data.data()), n_obs*sizeof(double)); int status = r_status; - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::get_observations_vec() stream not good"); } @@ -817,15 +788,12 @@ int RunStorage::get_observations_vec(int run_id, vector &obs_data) void RunStorage::free_memory() { - if (!buf_stream.good()) - { - throw runtime_error("RunStorage::free_memory() stream not good"); - } + if (buf_stream.is_open()) { buf_stream.close(); remove(filename.c_str()); } - if (!buf_stream.good()) + if (!buf_stream) { throw runtime_error("RunStorage::free_memory() stream not good"); } diff --git a/src/libs/run_managers/yamr/PantherAgent.cpp b/src/libs/run_managers/yamr/PantherAgent.cpp index 5137dba5..b77585bb 100644 --- a/src/libs/run_managers/yamr/PantherAgent.cpp +++ b/src/libs/run_managers/yamr/PantherAgent.cpp @@ -1133,13 +1133,13 @@ void PANTHERAgent::start_impl(const string &host, const string &port) ss.str(""); ss << "frozen"; report(ss.str(), true); - w_sleep(3000); - if (quit_file_found()) { - report("pest.stp file found, resetting panther_agent_freeze_on_fail and continuing...", - true); - pest_scenario.get_pestpp_options_ptr()->set_panther_debug_fail_freeze(false); - break; - } + w_sleep(10000); +// if (quit_file_found()) { +// report("pest.stp file found, resetting panther_agent_freeze_on_fail and continuing...", +// true); +// pest_scenario.get_pestpp_options_ptr()->set_panther_debug_fail_freeze(false); +// break; +// } } } } diff --git a/src/libs/run_managers/yamr/RunManagerPanther.cpp b/src/libs/run_managers/yamr/RunManagerPanther.cpp index 1ad2eef0..1151269b 100644 --- a/src/libs/run_managers/yamr/RunManagerPanther.cpp +++ b/src/libs/run_managers/yamr/RunManagerPanther.cpp @@ -1391,7 +1391,7 @@ void RunManagerPanther::process_message(int i_sock) if (run_finished(run_id)) { stringstream ss; - ss << "Prevoiusly completed run_id:" << run_id << " finished on:" << host_name << "$" << agent_info_iter->get_work_dir() << + ss << "Previously completed run_id:" << run_id << " finished on:" << host_name << "$" << agent_info_iter->get_work_dir() << " run time:" << agent_info_iter->get_runtime_minute() << " min group_id:" << group_id << " " << net_pack.get_info_txt() << " concurrent:" << get_n_concurrent(run_id); report(ss.str(), false); @@ -1448,6 +1448,10 @@ void RunManagerPanther::process_message(int i_sock) stringstream ss; ss << "Frozen agent:" << host_name << "$" << agent_info_iter->get_work_dir() << " is frozen because of panther_debug_freeze_on_fail = true..."; report(ss.str(), true); + ss.str(""); + ss <<"closing connection to frozen agent:" << host_name << "$" << agent_info_iter->get_work_dir() << ", note: the agent will continue in a frozen state"; + report(ss.str(), true); + close_agent(agent_info_iter); } else if (net_pack.get_type() == NetPackage::PackType::RUN_KILLED)