Development

.gitignore

@@ -31,6 +31,9 @@
 *.out
 *.app
 
+# Jupiter files
+*.ipynb_checkpoints
+
 # Other
 Scripts/symbolic_hermitians/*.pyc
 Scripts/symbolic_hermitians/__pycache__
@@ -39,4 +42,5 @@ Scripts/symbolic_hermitians/__pycache__
 .pydevproject
 .settings
 .vscode
-Source/generated_files
+Source/generated_files
+*~

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "amrex"]
-	path = amrex
-	url = https://github.com/dwillcox/amrex.git
+	path = submodules/amrex
+	url = https://github.com/AMReX-Codes/amrex.git
+[submodule "submodules/HighFive"]
+	path = submodules/HighFive
+	url = https://github.com/BlueBrain/HighFive.git

CHANGES.md

@@ -1,3 +1,16 @@
+# 1.4
+
+   * Restructured initial conditions to have python scripts generate particle distributions instead of doing so inside the C++ code. This improves the code's flexibility.
+
+   * Add option to output data in HDF5 format. Post-processing scripts only work with the original binary format, since yt only reads the binary format.
+
+   * Add realtime output of scalar quantities to make basic analysis many times faster than with the post-processing scripts.
+
+   * Include all of the basic post-processing scripts with Emu itself to avoid keeping multiple incompatible copies of them.
+
+# 1.3
+
+   * Incorporated various feature additions used for _Code Comparison for Fast Flavor Instability Simulations_ (https://doi.org/10.1103/PhysRevD.106.043011)
 
 # 1.2
 

Dockerfile

@@ -0,0 +1,6 @@
+FROM nvidia/cuda:11.4.3-devel-ubuntu20.04
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y python3 python3-pip gfortran build-essential libhdf5-openmpi-dev openmpi-bin pkg-config libopenmpi-dev openmpi-bin libblas-dev liblapack-dev libpnetcdf-dev git python-is-python3 gnuplot
+RUN pip3 install numpy matplotlib h5py scipy sympy yt
+ENV USER=jenkins
+ENV LOGNAME=jenkins

Exec/.gitignore

@@ -5,3 +5,6 @@ plt*
 tmp_build_dir
 Backtrace.*
 *.png
+*.dat
+*.h5
+*.pdf

Jenkinsfile

@@ -0,0 +1,154 @@
+pipeline {
+    triggers { pollSCM('') }  // Run tests whenever a new commit is detected.
+    agent { dockerfile {args '--gpus all -v /mnt/scratch/EOS:/EOS:ro -v /mnt/scratch/NuLib:/NuLib:ro'}} // Use the Dockerfile defined in the root Flash-X directory
+    environment {
+		// Get rid of Read -1, expected <someNumber>, errno =1 error
+    	// See https://github.com/open-mpi/ompi/issues/4948
+        OMPI_MCA_btl_vader_single_copy_mechanism = 'none'
+    }
+    stages {
+
+        //=============================//
+    	// Set up submodules and amrex //
+        //=============================//
+    	stage('Prerequisites'){ steps{
+				sh 'mpicc -v'
+				sh 'nvidia-smi'
+				sh 'nvcc -V'
+				sh 'git submodule update --init'
+				sh 'cp makefiles/GNUmakefile_jenkins_HDF5_CUDA Exec/GNUmakefile'
+				dir('Exec'){
+					sh 'make generate; make -j'
+				}	
+			}
+		}
+
+		stage('MSW'){ steps{
+				dir('Exec'){
+					sh 'python ../Scripts/initial_conditions/st0_msw_test.py'
+					sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_msw_test'
+					sh 'python ../Scripts/tests/msw_test.py'
+					sh 'rm -rf plt*'
+				}
+			}
+		}
+
+		stage('Bipolar'){ steps{
+				dir('Exec'){
+					sh 'python ../Scripts/initial_conditions/st1_bipolar_test.py'
+					sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_bipolar_test'
+					sh 'rm -rf plt*'
+				}
+			}
+		}
+
+		stage('Fast Flavor'){ steps{
+				dir('Exec'){
+				sh 'python ../Scripts/initial_conditions/st2_2beam_fast_flavor.py'
+					sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor'
+					sh 'python ../Scripts/tests/fast_flavor_test.py'
+					sh 'rm -rf plt*'
+				}
+			}
+		}
+
+		stage('Fast Flavor k'){ steps{
+				dir('Exec'){
+				sh 'python ../Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py'
+					sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor_nonzerok'
+					sh 'python ../Scripts/tests/fast_flavor_k_test.py'
+					sh 'rm -rf plt*'
+				}
+			}
+		}
+
+		stage('Fiducial 2F GPU Binary'){ steps{
+				dir('Exec'){
+					sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py'
+					sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_1d_fiducial'
+					sh 'python ../Scripts/data_reduction/reduce_data_fft.py'
+					sh 'python ../Scripts/data_reduction/reduce_data.py'
+					sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data.h5'
+					sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data_fft_power.h5'
+					sh 'python ../Scripts/babysitting/avgfee.py'
+					sh 'python ../Scripts/babysitting/power_spectrum.py'
+					sh 'python ../Scripts/data_reduction/convertToHDF5.py'
+					sh 'gnuplot ../Scripts/babysitting/avgfee_gnuplot.plt'
+					archiveArtifacts artifacts: '*.pdf'
+					sh 'rm -rf plt*'
+				}
+			}
+		}
+
+		stage('Fiducial 3F CPU HDF5'){ steps{
+				dir('Exec'){
+					sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5 GNUmakefile'
+					sh 'make realclean; make generate; make -j'
+					sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi_3F.py'
+					sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.ex ../sample_inputs/inputs_1d_fiducial'
+					/*sh 'python3 ../Scripts/babysitting/avgfee_HDF5.py'*/
+					sh 'rm -rf plt*'
+				}
+			}
+		}
+
+		stage('Collisions flavor instability'){ steps{
+				dir('Exec'){
+					sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile'
+					sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2'
+					sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py'
+					sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test'
+					sh 'python ../Scripts/data_reduction/reduce_data.py'
+					sh 'python ../Scripts/tests/coll_inst_test.py'
+					sh 'rm -rf plt* *pdf'
+				}
+			}
+		}
+		stage('BC periodic empty'){ steps{
+				dir('Exec'){
+					sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_bc_periodic_init'
+					sh 'python ../Scripts/collisions/writeparticleinfohdf5.py'
+					sh 'python ../Scripts/tests/bc_empty_init_test.py'
+					sh 'rm -rf plt* *pdf'
+				}
+			}
+		}
+		stage('Collisions to equilibrium'){ steps{
+				dir('Exec'){
+					sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile'
+					sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3'
+					sh 'python ../Scripts/initial_conditions/st7_empty_particles.py'
+					sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test'
+					sh 'python ../Scripts/tests/coll_equi_test.py'
+					sh 'rm -rf plt* *pdf'
+				}
+			}
+		}
+
+		stage('Fermi-Dirac test'){ steps{
+				dir('Exec'){
+					sh 'python ../Scripts/initial_conditions/st9_empty_particles_multi_energy.py'
+					sh 'python ../Scripts/collisions/nsm_constant_background_rho_Ye_T__writer.py'
+					sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fermi_dirac_test'
+					sh 'python ../Scripts/collisions/writeparticleinfohdf5.py'
+					sh 'python ../Scripts/tests/fermi_dirac_test.py'
+					sh 'rm -rf plt* *pdf rho_Ye_T.hdf5'
+				}
+			}
+		}
+
+    } // stages{
+
+    post {
+        always {
+			cleanWs(
+				cleanWhenNotBuilt: true,
+				deleteDirs: true,
+				disableDeferredWipeout: false,
+				notFailBuild: true,
+				patterns: [[pattern: 'submodules', type: 'EXCLUDE']] 
+			) // allow submodules to be cached
+		} 	
+    }
+
+} // pipeline{