Merge remote-tracking branch 'origin/develop'

.github/workflows/ci.yml

@@ -0,0 +1,134 @@
+name: pestpp continuous integration
+
+on: 
+  schedule:
+    - cron: '0 8 * * *' # run at 8 AM UTC (12 am PST)
+  push:
+  pull_request:
+
+jobs:
+  pestppCI:
+    name: benchmarks
+    runs-on: ubuntu-latest #${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, ubuntu-latest] 
+        python-version: [3.8] # , 3.7, 3.6]
+        run-type: [std]
+        # test_repo: [""]
+        # test_dir: [""]
+        # test_script: [""]
+        # include:
+        #   - test_script: opt_test.py
+        #   - test_repo: "https://github.com/pestpp/pestpp-opt_benchmarks"
+        #   - test_dir: "pestpp-opt_benchmarks"
+
+        env:
+          - { test_dir: "benchmarks", test_repo: "https://github.com/usgs/pestpp", test_script: "basic_tests.py"}
+          - { test_dir: "pestpp-opt_benchmarks", test_repo: "https://github.com/pestpp/pestpp-opt_benchmarks", test_script: "opt_test.py"}
+          - { test_dir: "pestpp-ies_benchmarks", test_repo: "https://github.com/pestpp/pestpp-ies_benchmarks", test_script: "ies_test_base.py"}
+          - { test_dir: "pestpp-ies_benchmarks", test_repo: "https://github.com/pestpp/pestpp-ies_benchmarks", test_script: "ies_test_part1.py"}
+          - { test_dir: "pestpp-ies_benchmarks", test_repo: "https://github.com/pestpp/pestpp-ies_benchmarks", test_script: "ies_test_part2.py"}
+          - { test_dir: "pestpp-ies_benchmarks", test_repo: "https://github.com/pestpp/pestpp-ies_benchmarks", test_script: "ies_test_part3.py"}
+          - { test_dir: "pestpp-ies_benchmarks", test_repo: "https://github.com/pestpp/pestpp-ies_benchmarks", test_script: "ies_test_part4.py"}
+          - { test_dir: "pestpp-glm_benchmarks", test_repo: "https://github.com/pestpp/pestpp-glm_benchmarks", test_script: "glm_test_base.py"}
+    env: ${{ matrix.env }}
+
+    steps:
+
+    - uses:  actions/[email protected] 
+    # - name: Setup Ninja
+      # if: ${{ runner.os == 'Windows' }} 
+    - uses: seanmiddleditch/gha-setup-ninja@master
+
+    - name: Cache Miniconda
+      uses: actions/[email protected]
+      env:
+        # Increase this value to reset cache if environment.yml has changed
+        CACHE_NUMBER: 0
+      with:
+        path: ~/conda_pkgs_dir
+        key: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.run-type }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('etc/environment.yml') }}
+
+    - name: Set Windows ENV
+      if: runner.os == 'Windows'
+      uses: ilammy/msvc-dev-cmd@v1
+
+    # - name: Get specific version CMake, v3.19
+    #   if: ${{ runner.os == 'Windows' }} 
+    #   uses: lukka/get-cmake@latest
+
+    - name: clone test repo ${{ env.test_repo }}
+      shell: bash -l {0}
+      run: |
+        git clone ${{ env.test_repo }}
+
+    - name: PESTPP exe install
+      shell: bash -l {0}
+      run: |
+        mkdir build && cd build
+        if [[ "$RUNNER_OS" == "Windows" ]]; then
+            cmake -GNinja -DCMAKE_CXX_COMPILER=cl -DCMAKE_C_COMPILER=cl -DCMAKE_BUILD_TYPE=Release ..
+            ninja
+        else
+            cmake -DCMAKE_BUILD_TYPE=Release ..
+            make -j2
+        fi
+        cd ..
+
+   
+    # Standard python fails on windows without GDAL installation
+    # Using custom bash shell ("shell: bash -l {0}") with Miniconda
+    - name: Setup Miniconda
+      uses: conda-incubator/[email protected]
+      with:
+        python-version: ${{ matrix.python-version }}
+        mamba-version: "*"
+        channels: conda-forge
+        auto-update-conda: true
+        activate-environment: pyemu
+        use-only-tar-bz2: true
+
+    - name: Add packages to pyemu environment using mamba or conda
+      shell: bash -l {0}
+      run: |
+        if [ "$RUNNER_OS" == "Windows" ]; then
+          conda env update --name pyemu --file etc/environment.yml
+        else
+          mamba env update --name pyemu --file etc/environment.yml
+        fi
+    
+    - name: Install Flopy & pyemu?
+      shell: bash -l {0}
+      run: |
+        git clone -b develop --depth 1 https://github.com/modflowpy/flopy.git
+        cd flopy
+        python setup.py install
+        cd ..
+        pip install https://github.com/modflowpy/pymake/zipball/master
+        git clone -b develop --depth 1 https://github.com/pypest/pyemu.git
+        cd pyemu
+        python setup.py install
+        cd ..
+    
+    - name: aux bin path hackery
+      shell: bash -l {0}
+      run: |
+        if [[ "$RUNNER_OS" == "Windows" ]]; then
+            echo "..\..\test_bin\win" >> $GITHUB_PATH        
+        else
+            echo "../../test_bin/linux" >> $GITHUB_PATH
+        fi
+
+    - name: run test script ${{ env.test_script }} in ${{ env.test_dir }}
+      shell: bash -l {0}
+      run: |
+        ls -l
+        cd ${{ env.test_dir }}
+        nosetests -v ${{ env.test_script }}
+        cd
+
+        
+
+

etc/environment.yml

@@ -0,0 +1,12 @@
+name: pyemu
+channels:
+  - defaults
+dependencies:  
+  - numpy 
+  - pandas 
+  - pyshp
+  - nose-timer
+  - nbsphinx
+  - matplotlib
+  - coverage
+  - scipy

src/libs/common/config_os.h

@@ -2,7 +2,7 @@
 #define CONFIG_OS_H_
 
 
-#define PESTPP_VERSION "5.0.7";
+#define PESTPP_VERSION "5.0.8";
 
 #if defined(_WIN32) || defined(_WIN64)
 #define OS_WIN

src/libs/common/utilities.cpp

@@ -630,42 +630,63 @@ void  thread_exceptions::rethrow()
 	throw runtime_error(ss.str());
 }
 
-pair<string, string> parse_plusplus_line(const string& line)
+map<string, string> parse_plusplus_line(const string& _line)
 {
 	string key;
 	string value, org_value;
+	map<string, string> arg_map;
+	string line_copy = _line;
+	int i = 0;
+	while (true)
+	{
+		if (line_copy.size() <= 2)
+			break;
+		string tmp_line = line_copy;
+		pest_utils::strip_ip(tmp_line, "both");
+		vector<string> tmp_tokens;
+		pest_utils::tokenize(tmp_line, tmp_tokens, " \t", false);
+		if (tmp_tokens.size() > 1)
+			if (tmp_tokens[0] == "++")
+				if (tmp_tokens[1].substr(0, 1) == "#")
+				{
+					//return pair<string, string>("", "");
+					break;
+				}
+		if (tmp_line.substr(0, 3) == "++#")
+		{
+			//return pair<string, string>("", "");
+			break;
+		}
 
-	string tmp_line = line;
-	pest_utils::strip_ip(tmp_line, "both");
-	vector<string> tmp_tokens;
-	pest_utils::tokenize(tmp_line, tmp_tokens, " \t", false);
-	if (tmp_tokens.size() > 1)
-		if (tmp_tokens[0] == "++")
-			if (tmp_tokens[1].substr(0, 1) == "#")
-				return pair<string,string>("","");
-	if (tmp_line.substr(0, 3) == "++#")
-		return pair<string, string>("", "");
-
-	size_t found = line.find_first_of("#");
-	if (found == string::npos) {
-		found = line.length();
-	}
-	tmp_line = line.substr(0, found);
-	strip_ip(tmp_line, "both", "\t\n\r+ ");
-	tmp_line.erase(remove(tmp_line.begin(), tmp_line.end(), '\"'), tmp_line.end());
-	tmp_line.erase(remove(tmp_line.begin(), tmp_line.end(), '\''), tmp_line.end());
-	//upper_ip(tmp_line);
-
-	found = tmp_line.find_first_of("(");
-	if (found == string::npos)
-		throw runtime_error("incorrect format for '++' line (missing'('):" + line);
-	key = tmp_line.substr(0, found);
-	tmp_line = tmp_line.substr(found);
-	found = tmp_line.find_first_of(")");
-	if (found == string::npos)
-		throw runtime_error("incorrect format for '++' line (missing')'):" + line);
-	org_value = tmp_line.substr(1, found - 1);
-	return pair < string, string>(key, org_value);
+		size_t found = line_copy.find_first_of("#");
+		if (found == string::npos) {
+			found = line_copy.length();
+		}
+		tmp_line = line_copy.substr(0, found);
+		strip_ip(tmp_line, "both", "\t\n\r+ ");
+		tmp_line.erase(remove(tmp_line.begin(), tmp_line.end(), '\"'), tmp_line.end());
+		tmp_line.erase(remove(tmp_line.begin(), tmp_line.end(), '\''), tmp_line.end());
+		//upper_ip(tmp_line);
+
+		found = tmp_line.find_first_of("(");
+		if (found == string::npos)
+			throw runtime_error("incorrect format for '++' line (missing'('):" + _line);
+		key = tmp_line.substr(0, found);
+		tmp_line = tmp_line.substr(found);
+		found = tmp_line.find_first_of(")");
+		if (found == string::npos)
+			throw runtime_error("incorrect format for '++' line (missing')'):" + _line);
+		org_value = tmp_line.substr(1, found - 1);
+		found = line_copy.find_first_of(")");
+		if (found == string::npos)
+			throw runtime_error("error seeking ')' in line: " + line_copy);
+		line_copy = line_copy.substr(found+1);
+		arg_map[key] =  org_value;
+		i++;
+		if (i > _line.size())
+			throw runtime_error("parse_plusplus_line() error: infinite loop detected processing line: " + _line);
+	}
+	return arg_map;
 }
 
 

src/libs/common/utilities.h

@@ -231,7 +231,7 @@ bool check_exist_out(std::string filename);
 
 void try_clean_up_run_storage_files(const string& case_name);
 
-pair<string, string> parse_plusplus_line(const string& line);
+map<string, string> parse_plusplus_line(const string& line);
 
 //template <class dataType>
 //void read_twocol_ascii_to_map(std::map<std::string, dataType> &result,std::string filename, int header_lines=0, int data_col=1);

src/libs/pestpp_common/EnsembleSmoother.cpp

@@ -1248,20 +1248,7 @@ void IterEnsembleSmoother::initialize()
 	message(1, "saved obs+noise observation ensemble (obsval+noise) to ", ss.str());
 
 
-	if (center_on.size() > 0)
-	{
-		ss.str("");
-		ss << "centering on realization: '" << center_on << "' ";
-		message(1, ss.str());
-		vector<string> names = pe.get_real_names();
-		if (find(names.begin(), names.end(), center_on) == names.end())
-			throw_ies_error("'ies_center_on' realization not found in par en: " + center_on);
-		names = oe.get_real_names();
-		if (find(names.begin(), names.end(), center_on) == names.end())
-			throw_ies_error("'ies_center_on' realization not found in obs en: " + center_on);
-	}
-	else
-		message(1, "centering on ensemble mean vector");
+
 
 	if (pest_scenario.get_control_info().noptmax == -2)
 	{
@@ -1346,8 +1333,25 @@ void IterEnsembleSmoother::initialize()
 	if (obs_restart_csv.size() > 0)
 		initialize_restart();
 
-	//no restart
+	//check for center on 
+	if (center_on.size() > 0)
+	{
+		ss.str("");
+		ss << "centering on realization: '" << center_on << "' ";
+		message(1, ss.str());
+		vector<string> names = pe.get_real_names();
+		if (find(names.begin(), names.end(), center_on) == names.end())
+			throw_ies_error("'ies_center_on' realization not found in par en: " + center_on);
+		names = oe.get_real_names();
+		if (find(names.begin(), names.end(), center_on) == names.end())
+			throw_ies_error("'ies_center_on' realization not found in obs en: " + center_on);
+	}
 	else
+		message(1, "centering on ensemble mean vector");
+
+	//ok, now run the prior ensemble - after checking for center_on
+	//in case something is wrong with center_on
+	if (obs_restart_csv.size() == 0)
 	{
 		performance_log->log_event("running initial ensemble");
 		message(1, "running initial ensemble of size", oe.shape().first);
@@ -2804,6 +2808,28 @@ bool IterEnsembleSmoother::solve_new()
 			message(1, "abandoning current lambda ensembles, increasing lambda to ", new_lam);
 			message(1, "updating realizations with reduced phi");
 			update_reals_by_phi(pe_lams[best_idx], oe_lams[best_idx]);
+			ph.update(oe, pe);
+			//re-check phi
+			best_mean = ph.get_mean(L2PhiHandler::phiType::COMPOSITE);
+			best_std = ph.get_std(L2PhiHandler::phiType::COMPOSITE);
+			//replace the last entry in the best mean phi tracker
+			best_mean_phis[best_mean_phis.size() - 1] = best_mean;
+			message(1, "current best mean phi (after updating reduced-phi reals): ", best_mean);
+			if (best_mean < last_best_mean * acc_fac)
+			{
+				if (best_std < last_best_std * acc_fac)
+				{
+					double new_lam = lam_vals[best_idx] * lam_dec;
+					new_lam = (new_lam < lambda_min) ? lambda_min : new_lam;
+					message(0, "updating lambda to ", new_lam);
+					last_best_lam = new_lam;
+				}
+				else
+				{
+					message(0, "not updating lambda (standard deviation reduction criteria not met)");
+				}
+				last_best_std = best_std;
+			}
 			message(1, "returing to lambda calculations...");
 			return false;
 		}

src/libs/pestpp_common/Pest.cpp

@@ -1021,10 +1021,12 @@ int Pest::process_ctl_file(ifstream& fin, string _pst_filename, ofstream& f_rec)
 			else if (tokens.empty())
 			{
 				//skip blank line
+				lnum--;
 			}
 			else if (line[0] == '#')
 			{
-
+
+				lnum--;
 			}
 			else if (line_upper.substr(0, 2) == "++")
 			{

src/libs/pestpp_common/pest_data_structs.cpp

@@ -312,18 +312,26 @@ map<string,PestppOptions::ARG_STATUS> PestppOptions::parse_plusplus_line(const s
 {
 	map<string, ARG_STATUS> arg_map;
 	ARG_STATUS stat;
-
-	pair<string, string> spair = pest_utils::parse_plusplus_line(line);
-	if (spair.second.size() == 0)
-		return arg_map;
-	try
-	{
-		stat = assign_value_by_key(spair.first, spair.second);
-		arg_map[spair.first] = stat;
-	}
-	catch (...)
+
+	map<string, string> spairs = pest_utils::parse_plusplus_line(line);
+	for (auto& spair : spairs)
 	{
-		arg_map[spair.first] = ARG_STATUS::ARG_INVALID;
+		if (spair.second.size() == 0)
+		{
+			arg_map[spair.first] = ARG_STATUS::ARG_INVALID;
+			continue;
+		}
+
+		try
+		{
+			stat = assign_value_by_key(spair.first, spair.second);
+			arg_map[spair.first] = stat;
+		}
+		catch (...)
+		{
+			arg_map[spair.first] = ARG_STATUS::ARG_INVALID;
+		}
+
 	}
 	return arg_map;
 }