Feat timeouts

benchmarks/basic_tests.py

@@ -1049,14 +1049,16 @@ def mf6_v5_sen_test():
 
     t_d = os.path.join(model_d,"template")
     m_d = os.path.join(model_d,"master_sen")
-    #if os.path.exists(m_d):
-    #    shutil.rmtree(m_d)
+    if os.path.exists(m_d):
+       shutil.rmtree(m_d)
     pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_sen.pst"))
+    pst.pestpp_options["gsa_morris_p"] = 4
+    pst.pestpp_options["gsa_morris_r"] = 4
     pst.pestpp_options["panther_transfer_on_finish"] = ["freyberg6_freyberg.cbc","freyberg6.lst","ies_prior.jcb"]
     pst.write(os.path.join(t_d,"freyberg6_run_sen_trn.pst"))
     m_d = os.path.join(model_d,"master_sen")
     pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "freyberg6_run_sen_trn.pst",
-                                 num_workers=15, worker_root=model_d,
+                                 num_workers=50, worker_root=model_d,
                                  port=4004,verbose=True,master_dir=m_d)
 
     pst = pyemu.Pst(os.path.join(m_d,"freyberg6_run_sen_trn.pst"))
@@ -1798,7 +1800,8 @@ def tenpar_uniform_invest():
 
 
 if __name__ == "__main__":
-    tenpar_uniform_invest()
+    mf6_v5_sen_test()
+    #tenpar_uniform_invest()
     #tenpar_collapse_invest()
     #plot_collapse_invest()
 

documentation/pestpp_users_manual.md

@@ -1,7 +1,7 @@
 
  <img src="./media/image1.png" style="width:6.26806in;height:1.68194in" alt="A close up of a purple sign Description automatically generated" />
 
-# <a id='s1' />Version 5.2.14
+# <a id='s1' />Version 5.2.15
 
 <img src="./media/image2.png" style="width:6.26806in;height:3.05972in" />
 
@@ -70,7 +70,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 # Table of Contents
 
-- [Version 5.2.14](#s1)
+- [Version 5.2.15](#s1)
 - [Acknowledgements](#s2)
 - [Preface](#s3)
 - [License](#s4)

src/libs/common/config_os.h

@@ -2,7 +2,7 @@
 #define CONFIG_OS_H_
 
 
-#define PESTPP_VERSION "5.2.14";
+#define PESTPP_VERSION "5.2.15";
 
 #if defined(_WIN32) || defined(_WIN64)
 #define OS_WIN

src/libs/pestpp_common/EnsembleMethodUtils.cpp

@@ -1538,7 +1538,6 @@ void MmUpgradeThread::work(int thread_id, int iter, double cur_lam, bool use_glm
 
             if (verbose_level < 3)
                 return;
-            //cout << "thread: " << tid << ", " << t_count << ", " << prefix << " rows:cols" << mat.rows() << ":" << mat.cols() << endl;
             stringstream ss;
 
             ss << "thread_" << tid << ".count_ " << t_count << ".iter_" << iter << "." << prefix << ".dat";
@@ -3686,7 +3685,7 @@ void L2PhiHandler::apply_ineq_constraints(Eigen::MatrixXd &resid, Eigen::MatrixX
         for (int i = 0; i < resid.rows(); i++)
             col(i) = ((scol(i) > val) && (scol(i) < val2)) ? 0.0 : col(i);
         //cout << resid.col(idx) << endl;
-        cout << col << endl << endl;
+        //cout << col << endl << endl;
         resid.col(idx) = col;
         //cout << resid.col(idx) << endl;
     }
@@ -3715,7 +3714,7 @@ void L2PhiHandler::apply_ineq_constraints(Eigen::MatrixXd &resid, Eigen::MatrixX
         for (int i = 0; i < resid.rows(); i++)
             col(i) = (scol(i) < val) ? 0.0 : col(i);
         //cout << resid.col(idx) << endl;
-        cout << col << endl << endl;
+        //cout << col << endl << endl;
         resid.col(idx) = col;
         //cout << resid.col(idx) << endl;
     }
@@ -4319,7 +4318,7 @@ vector<int> run_ensemble_util(PerformanceLog* performance_log, ofstream& frec,Pa
 		}
 		if (failing_reals.size() > 0)
 		{
-			cout << "parameter ensemble consisteny check failed for " << failing_reals.size() << ", see .rec file for listing" << endl;
+			cout << "parameter ensemble consistency check failed for " << failing_reals.size() << ", see .rec file for listing" << endl;
 			frec << "ERROR: the following realizations failed consistency check:" << endl;
 			for (auto fr : failing_reals)
 				frec << fr << ",";
@@ -5042,14 +5041,17 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing)
         map<string,pair<double,double>> tt = ph.get_double_obs_bounds();
         if (!tt.empty())
         {
+            ss.str("");
+            ss << "double inequality defined through 'greater_than' and 'less_than' data for " << tt.size() << " observations" << endl;
             ss.str("");
             ss << "double inequality defined through 'greater_than' and 'less_than' data for observations:" << endl;
             for (const auto it : tt)
             {
                 ss << it.first << "," << it.second.first << " to " << it.second.second << endl;
             }
             ss << endl;
-            message(1,ss.str());
+            message(1,ss.str(),false);
+
         }
 
 

src/libs/run_managers/abstract_base/model_interface.cpp

@@ -639,7 +639,7 @@ void ModelInterface::remove_existing()
 		if (failed_file_op)
 		{
 			++n_tries;
-			w_sleep(1000);
+			w_sleep(500);
 			if (n_tries > 5)
 			{
 				ostringstream str;

src/libs/run_managers/yamr/PantherAgent.cpp

@@ -151,7 +151,7 @@ void PANTHERAgent::process_ctl_file(const string &ctl_filename)
 	mi.set_fill_tpl_zeros(pest_scenario.get_pestpp_options().get_fill_tpl_zeros());
 	mi.set_tpl_force_decimal(pest_scenario.get_pestpp_options().get_tpl_force_decimal());
 	mi.set_num_threads(pest_scenario.get_pestpp_options().get_num_tpl_ins_threads());
-    mi.set_sleep_ms(500);
+    mi.set_sleep_ms(100);
 	restart_on_error = pest_scenario.get_pestpp_options().get_panther_agent_restart_on_error();
 	max_time_without_master_ping_seconds = pest_scenario.get_pestpp_options().get_panther_agent_no_ping_timeout_secs();
 	FileManager fm("panther_agent");
@@ -538,7 +538,7 @@ std::pair<NetPackage::PackType,std::string> PANTHERAgent::run_model(Parameters &
 void PANTHERAgent::run_async(pest_utils::thread_flag* terminate, pest_utils::thread_flag* finished, exception_ptr& run_exception,
 	Parameters* pars, Observations* obs)
 {
-    mi.set_sleep_ms(300);
+    mi.set_sleep_ms(100);
 	mi.run(terminate,finished,run_exception, pars, obs);
 }
 

src/libs/run_managers/yamr/RunManagerPanther.cpp

@@ -560,7 +560,7 @@ RunManagerAbstract::RUN_UNTIL_COND RunManagerPanther::run_until(RUN_UNTIL_COND c
 		}
 
 	}
-    w_sleep(2000);
+    w_sleep(100);
 	n_no_ops = 0;
     while (true)
     {
@@ -726,7 +726,7 @@ void RunManagerPanther::run_idle_async()
 				idling.set(false);
 
 				// Sleep 1s to avoid spinlock
-				w_sleep(1000);
+				w_sleep(100);
 				continue;
 			}
 
@@ -2132,7 +2132,7 @@ RunManagerPanther::~RunManagerPanther(void)
 	err = w_close(listener);
 	FD_CLR(listener, &master);
 	// this is needed to ensure that the first slave closes properly
-	w_sleep(2000);
+	w_sleep(500);
 	for (int i = 0; i <= fdmax; i++)
 	{
 		if (FD_ISSET(i, &master))
@@ -2248,10 +2248,10 @@ void RunManagerYAMRCondor::cleanup(int cluster)
 	stringstream ss;
 	ss << "condor_rm " << cluster << " 1>cr_temp.stdout 2>cr_temp.stderr";
 	system(ss.str().c_str());
-	w_sleep(2000);
+	w_sleep(500);
 	ss.str(string());
 	ss << "condor_rm " << cluster << " -forcex 1>cr_temp.stdout 2>cr_temp.stderr";
-	w_sleep(2000);
+	w_sleep(500);
 	system(ss.str().c_str());
 	RunManagerPanther::close_agents();
 	cout << "   all agents freed " << endl << endl;