From 97499f21cad18e1fc6aaf12cf49d1a94457090b0 Mon Sep 17 00:00:00 2001 From: Nicole F <73133869+nmford20@users.noreply.github.com> Date: Thu, 18 Mar 2021 19:49:41 -0400 Subject: [PATCH 001/276] Added descriptive amrex::Error messages in generate_code.py (#53) * Added descriptive amrex::Error messages in generate_code.py Swtiched amrex::Abort to amrex::Error with descriptive error message * added more to amrex::Error messages --- Scripts/symbolic_hermitians/generate_code.py | 21 +++++++++++++++++--- Source/FlavoredNeutrinoContainer.cpp | 2 ++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index a00e4636..5f551399 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -362,7 +362,12 @@ def sgn(t,var): for fii in fdlist: code.append("sumP += " + fii + ";") code.append("error = sumP-1.0;") - code.append("if( std::abs(error) > 100.*parms->maxError) amrex::Abort();") + code.append('if( std::abs(error) > 100.*parms->maxError) {') + code.append("std::ostringstream Convert;") + code.append('Convert << "Matrix trace (SumP) is not equal to 1, trace error exceeds 100*maxError: " << std::abs(error) << " > " << 100.*parms->maxError;') + code.append("std::string Trace_Error = Convert.str();") + code.append('amrex::Error(Trace_Error);') + code.append("}") code.append("if( std::abs(error) > parms->maxError ) {") for fii in fdlist: code.append(fii + " -= error/"+str(args.N)+";") @@ -371,7 +376,12 @@ def sgn(t,var): # make sure diagonals are positive for fii in fdlist: - code.append("if("+fii+"<-100.*parms->maxError) amrex::Abort();") + code.append('if('+fii+'<-100.*parms->maxError) {') + code.append("std::ostringstream Convert;") + code.append('Convert << "Diagonal element '+fii[14:20]+' is negative, less than -100*maxError: " << '+fii+' << " < " << -100.*parms->maxError;') + code.append("std::string Sign_Error = Convert.str();") + code.append('amrex::Error(Sign_Error);') + code.append("}") code.append("if("+fii+"<-parms->maxError) "+fii+"=0;") code.append("") @@ -381,7 +391,12 @@ def sgn(t,var): target_length = "p.rdata(PIdx::L"+t+")" code.append("length = "+sympy.cxxcode(sympy.simplify(length))+";") code.append("error = length-"+str(target_length)+";") - code.append("if( std::abs(error) > 100.*parms->maxError) amrex::Abort();") + code.append('if( std::abs(error) > 100.*parms->maxError) {') + code.append("std::ostringstream Convert;") + code.append('Convert << "flavor vector length differs from target length by more than 100*maxError: " << std::abs(error) << " > " << 100.*parms->maxError;') + code.append("std::string Length_Error = Convert.str();") + code.append('amrex::Error(Length_Error);') + code.append("}") code.append("if( std::abs(error) > parms->maxError) {") for fii in flist: code.append(fii+" /= length/"+str(target_length)+";") diff --git a/Source/FlavoredNeutrinoContainer.cpp b/Source/FlavoredNeutrinoContainer.cpp index e53c3386..ee050e2d 100644 --- a/Source/FlavoredNeutrinoContainer.cpp +++ b/Source/FlavoredNeutrinoContainer.cpp @@ -1,5 +1,7 @@ #include "FlavoredNeutrinoContainer.H" #include "Constants.H" +#include +#include using namespace amrex; From 4136a201b3e5dc92184eb558755d0eb9331e4c2d Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 19 Mar 2021 11:18:50 -0700 Subject: [PATCH 002/276] added grid variable to use in timestep calculation --- Source/Evolve.H | 2 +- Source/Evolve.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/Evolve.H b/Source/Evolve.H index 409a897e..fe9f1c26 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -16,7 +16,7 @@ namespace GIdx // with units of number density. Could add total number density later // too, but this helps with subtractive cancellation errors enum { - rho, T, Ye, // g/ccm, MeV, unitless + rho, T, Ye, requested_dt, // g/ccm, MeV, unitless #include "generated_files/Evolve.H_fill" ncomp }; diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index ae2c93b0..c631a7cf 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -15,6 +15,7 @@ namespace GIdx names.push_back("rho"); names.push_back("T"); names.push_back("Ye"); + names.push_back("requested_dt"); #include "generated_files/Evolve.cpp_grid_names_fill" } } From 802c851b398c17cebcc1cc4da76f20a76e485d11 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 19 Mar 2021 14:44:51 -0700 Subject: [PATCH 003/276] add ability to do scalar math with HermitianMatrix-es --- Scripts/symbolic_hermitians/HermitianUtils.py | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/Scripts/symbolic_hermitians/HermitianUtils.py b/Scripts/symbolic_hermitians/HermitianUtils.py index f0c47a36..6206e1aa 100644 --- a/Scripts/symbolic_hermitians/HermitianUtils.py +++ b/Scripts/symbolic_hermitians/HermitianUtils.py @@ -37,18 +37,41 @@ def __init__(self, size, entry_template = "H{}{}_{}"): self.construct() def __mul__(self, other): - result = copy.deepcopy(other) - result.H = self.H * other.H + result = copy.deepcopy(self) + if isinstance(other, self.__class__): + result.H = self.H * other.H + else: + for i in range(self.size): + for j in range(self.size): + result.H[i,j] *= other + return result + + def __truediv__(self, other): + result = copy.deepcopy(self) + if isinstance(other, self.__class__): + result.H = self.H * other.H + else: + for i in range(self.size): + for j in range(self.size): + result.H[i,j] /= other return result def __add__(self, other): - result = copy.deepcopy(other) - result.H = self.H + other.H + result = copy.deepcopy(self) + if isinstance(other, self.__class__): + result.H = self.H + other.H + else: + for i in range(self.size): + result.H[i,i] += other return result def __sub__(self, other): - result = copy.deepcopy(other) - result.H = self.H - other.H + result = copy.deepcopy(self) + if isinstance(other, self.__class__): + result.H = self.H - other.H + else: + for i in range(self.size): + result.H[i,i] -= other return result def construct(self): From e1f615d4699a13075a14a1598d0f274055ee33a2 Mon Sep 17 00:00:00 2001 From: Nicole F <73133869+nmford20@users.noreply.github.com> Date: Fri, 19 Mar 2021 18:24:18 -0400 Subject: [PATCH 004/276] added 3 flavor functionality to sample_inputs test simulations. Note: mass3_eV = 0 for these to run. (#56) --- Source/FlavoredNeutrinoContainerInit.cpp | 61 ++++++++++++++++++++++-- sample_inputs/inputs_bipolar_test | 3 +- sample_inputs/inputs_fast_flavor | 3 +- sample_inputs/inputs_msw_test | 3 +- 4 files changed, 63 insertions(+), 7 deletions(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 8117d11b..2eaa9b9f 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -248,7 +248,7 @@ InitParticles(const TestParams* parms) // set all particles to start in electron state (and anti-state) // Set N to be small enough that self-interaction is not important // Set all particle momenta to be such that one oscillation wavelength is 1cm - AMREX_ASSERT(NUM_FLAVORS==2); + AMREX_ASSERT(NUM_FLAVORS==3 or NUM_FLAVORS==2); // Set particle flavor p.rdata(PIdx::N) = 1.0; @@ -262,6 +262,19 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::f01_Imbar) = 0.0; p.rdata(PIdx::f11_Rebar) = 0.0; +#if (NUM_FLAVORS==3) + p.rdata(PIdx::f22_Re) = 0.0; + p.rdata(PIdx::f22_Rebar) = 0.0; + p.rdata(PIdx::f02_Re) = 0.0; + p.rdata(PIdx::f02_Im) = 0.0; + p.rdata(PIdx::f12_Re) = 0.0; + p.rdata(PIdx::f12_Im) = 0.0; + p.rdata(PIdx::f02_Rebar) = 0.0; + p.rdata(PIdx::f02_Imbar) = 0.0; + p.rdata(PIdx::f12_Rebar) = 0.0; + p.rdata(PIdx::f12_Imbar) = 0.0; +#endif + // set momentum so that a vacuum oscillation wavelength occurs over a distance of 1cm // Set particle velocity to c in a random direction Real dm2 = (parms->mass2-parms->mass1)*(parms->mass2-parms->mass1); //g^2 @@ -275,7 +288,7 @@ InitParticles(const TestParams* parms) // BIPOLAR OSCILLATION TEST // //==========================// else if(parms->simulation_type==1){ - AMREX_ASSERT(NUM_FLAVORS==2); + AMREX_ASSERT(NUM_FLAVORS==3 or NUM_FLAVORS==2); // Set particle flavor p.rdata(PIdx::f00_Re) = 1.0; @@ -287,6 +300,19 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::f01_Imbar) = 0.0; p.rdata(PIdx::f11_Rebar) = 0.0; +#if (NUM_FLAVORS==3) + p.rdata(PIdx::f22_Re) = 0.0; + p.rdata(PIdx::f22_Rebar) = 0.0; + p.rdata(PIdx::f02_Re) = 0.0; + p.rdata(PIdx::f02_Im) = 0.0; + p.rdata(PIdx::f12_Re) = 0.0; + p.rdata(PIdx::f12_Im) = 0.0; + p.rdata(PIdx::f02_Rebar) = 0.0; + p.rdata(PIdx::f02_Imbar) = 0.0; + p.rdata(PIdx::f12_Rebar) = 0.0; + p.rdata(PIdx::f12_Imbar) = 0.0; +#endif + // set energy to 50 MeV to match Richers+(2019) p.rdata(PIdx::pupt) = 50. * 1e6*CGSUnitsConst::eV; p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); @@ -307,7 +333,7 @@ InitParticles(const TestParams* parms) // 2-BEAM FAST FLAVOR TEST// //========================// else if(parms->simulation_type==2){ - AMREX_ASSERT(NUM_FLAVORS==2); + AMREX_ASSERT(NUM_FLAVORS==3 or NUM_FLAVORS==2); // Set particle flavor p.rdata(PIdx::f00_Re) = 1.0; @@ -319,6 +345,19 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::f01_Imbar) = 0.0; p.rdata(PIdx::f11_Rebar) = 0.0; +#if (NUM_FLAVORS==3) + p.rdata(PIdx::f22_Re) = 0.0; + p.rdata(PIdx::f22_Rebar) = 0.0; + p.rdata(PIdx::f02_Re) = 0.0; + p.rdata(PIdx::f02_Im) = 0.0; + p.rdata(PIdx::f12_Re) = 0.0; + p.rdata(PIdx::f12_Im) = 0.0; + p.rdata(PIdx::f02_Rebar) = 0.0; + p.rdata(PIdx::f02_Imbar) = 0.0; + p.rdata(PIdx::f12_Rebar) = 0.0; + p.rdata(PIdx::f12_Imbar) = 0.0; +#endif + // set energy to 50 MeV to match Richers+(2019) p.rdata(PIdx::pupt) = 50. * 1e6*CGSUnitsConst::eV; p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); @@ -340,7 +379,7 @@ InitParticles(const TestParams* parms) // 3- k!=0 BEAM FAST FLAVOR TEST // //===============================// else if(parms->simulation_type==3){ - AMREX_ASSERT(NUM_FLAVORS==2); + AMREX_ASSERT(NUM_FLAVORS==3 or NUM_FLAVORS==2); // perturbation parameters Real lambda = domain_length_z/(Real)parms->st3_wavelength_fraction_of_domain; @@ -356,6 +395,20 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::f01_Imbar) = 0.0; p.rdata(PIdx::f11_Rebar) = 0.0; +#if (NUM_FLAVORS==3) + //just perturbing the electron-muon flavor state, other terms can stay = 0.0 for simplicity + p.rdata(PIdx::f22_Re) = 0.0; + p.rdata(PIdx::f22_Rebar) = 0.0; + p.rdata(PIdx::f02_Re) = 0.0; + p.rdata(PIdx::f02_Im) = 0.0; + p.rdata(PIdx::f12_Re) = 0.0; + p.rdata(PIdx::f12_Im) = 0.0; + p.rdata(PIdx::f02_Rebar) = 0.0; + p.rdata(PIdx::f02_Imbar) = 0.0; + p.rdata(PIdx::f12_Rebar) = 0.0; + p.rdata(PIdx::f12_Imbar) = 0.0; +#endif + // set energy to 50 MeV to match Richers+(2019) p.rdata(PIdx::pupt) = 50. * 1e6*CGSUnitsConst::eV; p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index e8f043fb..7371ec7f 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -55,7 +55,8 @@ mass1_eV = -0.008596511 mass2_eV = 0 # mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] -mass3_eV = 0.049487372 +#mass3_eV = 0.049487372 +mass3_eV = 0 # 1-2 mixing angle in degrees [NO/IO:33.82] theta12_degrees = 33.82 diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index dd994cba..1e08cadf 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -55,7 +55,8 @@ mass1_eV = -0.008596511 mass2_eV = 0 # mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] -mass3_eV = 0.049487372 +#mass3_eV = 0.049487372 +mass3_eV = 0 # 1-2 mixing angle in degrees [NO/IO:33.82] theta12_degrees = 1e-6 diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index a8231fb5..15ee7794 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -55,7 +55,8 @@ mass1_eV = -0.008596511 mass2_eV = 0 # mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] -mass3_eV = 0.049487372 +# mass3_eV = 0.049487372 +mass3_eV = 0 # 1-2 mixing angle in degrees [NO/IO:33.82] theta12_degrees = 33.82 From 8c9bc48b53d98415ae6319e2063caac2facf92bf Mon Sep 17 00:00:00 2001 From: "Don E. Willcox" Date: Fri, 19 Mar 2021 15:25:47 -0700 Subject: [PATCH 005/276] update readme for code generation and fix commutator language (#55) --- Scripts/symbolic_hermitians/README.md | 45 +++++-------------- ...nb => Symbolic-Hermitian-Commutator.ipynb} | 10 ++--- 2 files changed, 16 insertions(+), 39 deletions(-) rename Scripts/symbolic_hermitians/{Symbolic-Hermitian-Anticommutator.ipynb => Symbolic-Hermitian-Commutator.ipynb} (97%) diff --git a/Scripts/symbolic_hermitians/README.md b/Scripts/symbolic_hermitians/README.md index 84cc6a6c..4b99b1f0 100644 --- a/Scripts/symbolic_hermitians/README.md +++ b/Scripts/symbolic_hermitians/README.md @@ -5,42 +5,19 @@ expresses a symbolic Hermitian matrix in terms of its independent real-valued Real and Imaginary components. This class also provides functions for calculating an -anticommutator scaled by I and generating code to implement this. +commutator scaled by I and generating code to implement this. -## Using HermitianUtils +For a self-contained demonstration, see the Jupyter notebook +`Symbolic-Hermitian-Commutator.ipynb`. -The python script `IAntiCommutator.py` gives an example -of how to use the `HermitianMatrix` class to calculate `C=i*[A,B]`. +# Using HermitianUtils to generate Emu code -The script generalizes to any desired matrix sizes and -determines symbol names with runtime arguments. +We generate all the computational kernels for Emu using the HermitianUtils +python module in the `generate_code.py` script. -The only required runtime argument is an integer matrix size. +The generated source files are placed in `Emu/Source/generated_files` and are +inserted into the Emu source code using compile-time `#include` statements. -To see all optional runtime arguments: - -``` -$ python3 IAntiCommutator.py -h -``` - -## Generating code with Particle data indexing - -To generate sample code with indexing into real Particle data: - -``` -$ python3 IAntiCommutator.py 2 -o code.cpp -a "p.rdata(PIdx::H{}{}_{})" -b "p.rdata(PIdx::f{}{}_{})" -c "p.rdata(PIdx::dfdt{}{}_{})" -``` - -And the file `code.cpp` will contain: - -``` -{ -p.rdata(PIdx::dfdt00_Re) = -2*p.rdata(PIdx::H01_Im)*p.rdata(PIdx::f01_Re) + 2*p.rdata(PIdx::H01_Re)*p.rdata(PIdx::f01_Im); - -p.rdata(PIdx::dfdt01_Re) = -p.rdata(PIdx::H00_Re)*p.rdata(PIdx::f01_Im) + p.rdata(PIdx::H01_Im)*p.rdata(PIdx::f00_Re) - p.rdata(PIdx::H01_Im)*p.rdata(PIdx::f11_Re) + p.rdata(PIdx::H11_Re)*p.rdata(PIdx::f01_Im); - -p.rdata(PIdx::dfdt01_Im) = p.rdata(PIdx::H00_Re)*p.rdata(PIdx::f01_Re) - p.rdata(PIdx::H01_Re)*p.rdata(PIdx::f00_Re) + p.rdata(PIdx::H01_Re)*p.rdata(PIdx::f11_Re) - p.rdata(PIdx::H11_Re)*p.rdata(PIdx::f01_Re); - -p.rdata(PIdx::dfdt11_Re) = 2*p.rdata(PIdx::H01_Im)*p.rdata(PIdx::f01_Re) - 2*p.rdata(PIdx::H01_Re)*p.rdata(PIdx::f01_Im); -} -``` \ No newline at end of file +To see options for `generate_code.py`, pass the `-h` option. You do not need to +run this script manually, when building Emu, use the `make generate +NUM_FLAVORS=N` command to generate these source files. diff --git a/Scripts/symbolic_hermitians/Symbolic-Hermitian-Anticommutator.ipynb b/Scripts/symbolic_hermitians/Symbolic-Hermitian-Commutator.ipynb similarity index 97% rename from Scripts/symbolic_hermitians/Symbolic-Hermitian-Anticommutator.ipynb rename to Scripts/symbolic_hermitians/Symbolic-Hermitian-Commutator.ipynb index ae39193a..1cd8f0db 100644 --- a/Scripts/symbolic_hermitians/Symbolic-Hermitian-Anticommutator.ipynb +++ b/Scripts/symbolic_hermitians/Symbolic-Hermitian-Commutator.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Symbolic Hermitian Anticommutator" + "# Symbolic Hermitian Commutator" ] }, { @@ -113,7 +113,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Calculate anticommutator $[H,f] = H \\cdot f - f \\cdot H$" + "Calculate commutator $[H,f] = H \\cdot f - f \\cdot H$" ] }, { @@ -122,7 +122,7 @@ "metadata": {}, "outputs": [], "source": [ - "AC = H*F - F*H" + "Commutator = H*F - F*H" ] }, { @@ -138,7 +138,7 @@ "metadata": {}, "outputs": [], "source": [ - "dFdt = sympy.I * AC" + "dFdt = sympy.I * Commutator" ] }, { @@ -263,7 +263,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.8.5" } }, "nbformat": 4, From 7da8c69543e740de03375215f7467e9c11948bd3 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 19 Mar 2021 15:41:10 -0700 Subject: [PATCH 006/276] rewrite how the magnitude of a complex number is written so sympy simplifies the result without extraneous Is in real numbers --- Scripts/symbolic_hermitians/HermitianUtils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Scripts/symbolic_hermitians/HermitianUtils.py b/Scripts/symbolic_hermitians/HermitianUtils.py index 6206e1aa..4854694c 100644 --- a/Scripts/symbolic_hermitians/HermitianUtils.py +++ b/Scripts/symbolic_hermitians/HermitianUtils.py @@ -102,7 +102,8 @@ def SU_vector_magnitude(self): mag2 = 0 for i in range(self.size): for j in range(i+1,self.size): - mag2 += self.H[i,j]*self.H[j,i] + re,im = self.H[i,j].as_real_imag() + mag2 += re**2 + im**2 # Now get the contribution from the diagonals # See wolfram page for generalization of Gell-Mann matrices From a5d0a63dd87c2627e825dc72faacf3adcd7a8b39 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 19 Mar 2021 16:01:12 -0700 Subject: [PATCH 007/276] use amrex reduction functions to evaluate and reduce Hamiltonian flavor vector length in place --- Scripts/symbolic_hermitians/generate_code.py | 25 +++++++++++++++----- Source/Evolve.H | 2 +- Source/Evolve.cpp | 25 +++++++++++++++----- 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 5f551399..10aa304c 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -7,6 +7,7 @@ from sympy.codegen.ast import Assignment from HermitianUtils import HermitianMatrix,SU_vector_ideal_magnitude import shutil +import math parser = argparse.ArgumentParser(description="Generates code for calculating C = i * [A,B] for symbolic NxN Hermitian matrices A, B, C, using real-valued Real and Imaginary components.") parser.add_argument("N", type=int, help="Size of NxN Hermitian matrices.") @@ -247,11 +248,23 @@ def delete_generated_files(): # Evolve.cpp_compute_dt_fill # #============================# code = [] - for t in tails: - for i in range(args.N): - line = "N_diag_max = max(N_diag_max, state.max(GIdx::N"+str(i)+str(i)+"_Re"+t+"));" - code.append(line) - code.append("N_diag_max *= 2*"+str(args.N)+";") # overestimate of net neutrino+antineutrino number density + length = sympy.symbols("length",real=True) + cell_volume = sympy.symbols("cell_volume",real=True) + rho = sympy.symbols("fab(i\,j\,k\,GIdx\:\:rho)",real=True) + Ye = sympy.symbols("fab(i\,j\,k\,GIdx\:\:Ye)",real=True) + mp = sympy.symbols("PhysConst\:\:Mp",real=True) + sqrt2GF = sympy.symbols("M_SQRT2*PhysConst\:\:GF",real=True) + + N = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::N{}{}_{})") + Nbar = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::N{}{}_{}bar)") + HSI = (N-Nbar) / cell_volume + HSI.H[0,0] += rho*Ye/mp + HSI *= sqrt2GF + + length, asdf = HSI.SU_vector_magnitude().as_real_imag() + code.append("length = "+sympy.cxxcode(sympy.simplify(length))+";") + + write_code(code, os.path.join(args.emu_home,"Source/generated_files","Evolve.cpp_compute_dt_fill")) #=======================================# @@ -415,4 +428,4 @@ def sgn(t,var): for t in tails: f = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") code.append("p.rdata(PIdx::L"+t+") = "+sympy.cxxcode(sympy.simplify(f.SU_vector_magnitude()))+";" ) - write_code(code, os.path.join(args.emu_home, "Source/generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length")) \ No newline at end of file + write_code(code, os.path.join(args.emu_home, "Source/generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length")) diff --git a/Source/Evolve.H b/Source/Evolve.H index fe9f1c26..409a897e 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -16,7 +16,7 @@ namespace GIdx // with units of number density. Could add total number density later // too, but this helps with subtractive cancellation errors enum { - rho, T, Ye, requested_dt, // g/ccm, MeV, unitless + rho, T, Ye, // g/ccm, MeV, unitless #include "generated_files/Evolve.H_fill" ncomp }; diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index c631a7cf..4d941de4 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -15,7 +15,6 @@ namespace GIdx names.push_back("rho"); names.push_back("T"); names.push_back("Ye"); - names.push_back("requested_dt"); #include "generated_files/Evolve.cpp_grid_names_fill" } } @@ -37,12 +36,26 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta Real dt_si_matter = 0.0; if (flavor_cfl_factor > 0.0) { // self-interaction and matter part of timestep limit - // NOTE: these currently over-estimate both potentials, but avoid reduction over all particles // NOTE: the vacuum potential is currently ignored. This requires a min reduction over particle energies - Real N_diag_max = 0; - #include "generated_files/Evolve.cpp_compute_dt_fill" - Real Vmax = std::sqrt(2.) * PhysConst::GF * std::max(N_diag_max/cell_volume, state.max(GIdx::rho)/PhysConst::Mp); - if(Vmax>0) dt_si_matter = PhysConst::hbar/Vmax*flavor_cfl_factor; + + ReduceOps reduce_op; + ReduceData reduce_data(reduce_op); + using ReduceTuple = typename decltype(reduce_data)::Type; + for (MFIter mfi(state); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.fabbox(); + auto const& fab = state.array(mfi); + reduce_op.eval(bx, reduce_data, + [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple + { + Real length = 0; + #include "generated_files/Evolve.cpp_compute_dt_fill" + return length; + }); + } + + Real Vmax = amrex::get<0>(reduce_data.value()); + dt_si_matter = PhysConst::hbar/Vmax*flavor_cfl_factor; } Real dt = 0.0; From a1f55b6387eddeabbcb6d6ea3ea822e65f5f1627 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 19 Mar 2021 16:02:33 -0700 Subject: [PATCH 008/276] reduce the flavor cfl factor in the input files to account for the fact that the timestep is more liberal now --- sample_inputs/inputs_1d_fiducial | 2 +- sample_inputs/inputs_bipolar_test | 2 +- sample_inputs/inputs_fast_flavor | 2 +- sample_inputs/inputs_fast_flavor_nonzerok | 2 +- sample_inputs/inputs_msw_test | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 1af1dc46..0d2508e3 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -10,7 +10,7 @@ st4_fluxfacbar = .333333333333333 st4_amplitude = 1e-6 cfl_factor = 0.5 -flavor_cfl_factor = 0.5 +flavor_cfl_factor = 0.05 maxError = 1e-6 integration.type = 1 diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index e8f043fb..de63afbd 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -1,6 +1,6 @@ simulation_type = 1 cfl_factor = 0.5 -flavor_cfl_factor = .5 +flavor_cfl_factor = .05 maxError = 1e-6 integration.type = 1 diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index dd994cba..1dcb388c 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -1,6 +1,6 @@ simulation_type = 2 cfl_factor = 0.5 -flavor_cfl_factor = .5 +flavor_cfl_factor = .05 maxError = 1e-6 integration.type = 1 diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 49c82619..97c94c40 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -3,7 +3,7 @@ st3_amplitude = 1e-6 st3_wavelength_fraction_of_domain = 1 cfl_factor = 0.5 -flavor_cfl_factor = 0.5 +flavor_cfl_factor = 0.05 maxError = 1e-6 integration.type = 1 diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index a8231fb5..b71b8f8f 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -1,6 +1,6 @@ simulation_type = 0 cfl_factor = 0.5 -flavor_cfl_factor = 0.1 +flavor_cfl_factor = 0.05 maxError = 1e-6 integration.type = 1 From fbcf466ce4753bb923c3f698e5bb2d308c35c0a9 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 19 Mar 2021 16:31:27 -0700 Subject: [PATCH 009/276] separate the SU vector magnitude function into one that returns the magnitude and one that returns the magnitude squared. This will allow us to add the flux components without repeatedly square rooting and squaring. --- Scripts/symbolic_hermitians/HermitianUtils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Scripts/symbolic_hermitians/HermitianUtils.py b/Scripts/symbolic_hermitians/HermitianUtils.py index 4854694c..9e71a68e 100644 --- a/Scripts/symbolic_hermitians/HermitianUtils.py +++ b/Scripts/symbolic_hermitians/HermitianUtils.py @@ -97,7 +97,7 @@ def conjugate(self): return self # return the length of the SU(n) vector - def SU_vector_magnitude(self): + def SU_vector_magnitude2(self): # first get the sum of the square of the off-diagonal elements mag2 = 0 for i in range(self.size): @@ -115,7 +115,10 @@ def SU_vector_magnitude(self): basis_coefficient *= sympy.sqrt(2./(l*(l+1.))) mag2 += (basis_coefficient/2.)**2 - return sympy.sqrt(mag2) + return mag2 + + def SU_vector_magnitude(self): + return sympy.sqrt(self.SU_vector_magnitude2()) def trace(self): result = 0 From ee3f4c3168107eea9d5d5069dd8f83377f79b830 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 19 Mar 2021 21:44:10 -0700 Subject: [PATCH 010/276] added flux term to max hamiltonian for timestep calculation --- Scripts/symbolic_hermitians/generate_code.py | 17 ++++++++++++----- Source/Evolve.cpp | 4 ++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 10aa304c..9e619599 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -254,16 +254,23 @@ def delete_generated_files(): Ye = sympy.symbols("fab(i\,j\,k\,GIdx\:\:Ye)",real=True) mp = sympy.symbols("PhysConst\:\:Mp",real=True) sqrt2GF = sympy.symbols("M_SQRT2*PhysConst\:\:GF",real=True) - + + # spherically symmetric part N = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::N{}{}_{})") Nbar = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::N{}{}_{}bar)") HSI = (N-Nbar) / cell_volume HSI.H[0,0] += rho*Ye/mp HSI *= sqrt2GF - - length, asdf = HSI.SU_vector_magnitude().as_real_imag() - code.append("length = "+sympy.cxxcode(sympy.simplify(length))+";") - + length2 = HSI.SU_vector_magnitude2() + code.append("length2 += "+sympy.cxxcode(sympy.simplify(length2))+";") + + # flux part + for component in ["x","y","z"]: + F = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::F"+component+"{}{}_{})") + Fbar = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::F"+component+"{}{}_{}bar)") + HSI = (F-Fbar) * sqrt2GF / cell_volume + length2 = HSI.SU_vector_magnitude2() + code.append("length2 += "+sympy.cxxcode(sympy.simplify(length2))+";") write_code(code, os.path.join(args.emu_home,"Source/generated_files","Evolve.cpp_compute_dt_fill")) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 4d941de4..733937c0 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -48,9 +48,9 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { - Real length = 0; + Real length2 = 0; #include "generated_files/Evolve.cpp_compute_dt_fill" - return length; + return sqrt(length2); }); } From 6c83fa1ba204c9a999b2ece0560320ef0b13be84 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sat, 20 Mar 2021 13:54:53 -0700 Subject: [PATCH 011/276] make the timestep account for the vacuum potential as well. Compute the minimum particle energy in initializing step to avoid doing a reduction over particles at every timestep under the assumption that particle energies do not change --- Scripts/symbolic_hermitians/generate_code.py | 16 +++++++++++++--- Source/Evolve.H | 2 +- Source/Evolve.cpp | 8 ++++---- Source/FlavoredNeutrinoContainer.H | 2 ++ Source/FlavoredNeutrinoContainerInit.cpp | 5 +++++ Source/main.cpp | 4 ++-- 6 files changed, 27 insertions(+), 10 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 9e619599..63a67fba 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -216,16 +216,26 @@ def delete_generated_files(): U = U23*U13*U12*P # create M2 matrix in Evolve.H - M2 = sympy.zeros(args.N,args.N) + M2_massbasis = sympy.zeros(args.N,args.N) for i in range(args.N): - M2[i,i] = sympy.symbols('parms->mass'+str(i+1),real=True)**2 - M2 = U*M2*Dagger(U) + M2_massbasis[i,i] = sympy.symbols('parms->mass'+str(i+1),real=True)**2 + M2 = U*M2_massbasis*Dagger(U) massmatrix = HermitianMatrix(args.N, "M2matrix{}{}_{}") massmatrix.H = M2 code = massmatrix.code() code = ["double "+code[i] for i in range(len(code))] write_code(code, os.path.join(args.emu_home, "Source/generated_files","Evolve.H_M2_fill")) + #=============================================# + # FlavoredNeutrinoContainerInit.cpp_Vvac_fill # + #=============================================# + code = [] + massmatrix_massbasis = HermitianMatrix(args.N, "M2massbasis{}{}_{}") + massmatrix_massbasis.H = M2_massbasis + M2length = massmatrix_massbasis.SU_vector_magnitude() + code.append("this->Vvac_max = "+sympy.cxxcode(sympy.simplify(M2length))+"/pupt_min;") + write_code(code, os.path.join(args.emu_home,"Source/generated_files","FlavoredNeutrinoContainerInit.cpp_Vvac_fill")) + #======================# # Evolve.cpp_Vvac_fill # #======================# diff --git a/Source/Evolve.H b/Source/Evolve.H index 409a897e..f3888c6b 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -26,7 +26,7 @@ namespace GIdx void Initialize(); }; -amrex::Real compute_dt(const amrex::Geometry& geom, const amrex::Real cfl_factor, const MultiFab& state, const Real flavor_cfl_factor); +amrex::Real compute_dt(const amrex::Geometry& geom, const amrex::Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor); void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 733937c0..2fd47aae 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -19,7 +19,7 @@ namespace GIdx } } -Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& state, const Real flavor_cfl_factor) +Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor) { AMREX_ASSERT(cfl_factor > 0.0 || flavor_cfl_factor > 0.0); @@ -35,8 +35,7 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta Real dt_si_matter = 0.0; if (flavor_cfl_factor > 0.0) { - // self-interaction and matter part of timestep limit - // NOTE: the vacuum potential is currently ignored. This requires a min reduction over particle energies + // self-interaction, matter, and vacuum part of timestep limit ReduceOps reduce_op; ReduceData reduce_data(reduce_op); @@ -55,7 +54,8 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta } Real Vmax = amrex::get<0>(reduce_data.value()); - dt_si_matter = PhysConst::hbar/Vmax*flavor_cfl_factor; + ParallelDescriptor::ReduceRealMax(Vmax); + dt_si_matter = PhysConst::hbar/(Vmax+neutrinos.Vvac_max)*flavor_cfl_factor; } Real dt = 0.0; diff --git a/Source/FlavoredNeutrinoContainer.H b/Source/FlavoredNeutrinoContainer.H index 43d4da6f..75a91065 100644 --- a/Source/FlavoredNeutrinoContainer.H +++ b/Source/FlavoredNeutrinoContainer.H @@ -68,6 +68,8 @@ class FlavoredNeutrinoContainer public: + Real Vvac_max; + FlavoredNeutrinoContainer(const amrex::Geometry & a_geom, const amrex::DistributionMapping & a_dmap, const amrex::BoxArray & a_ba); diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 8117d11b..53ad4283 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -450,4 +450,9 @@ InitParticles(const TestParams* parms) } }); } + + // get the minimum neutrino energy for calculating the timestep + Real pupt_min = amrex::ReduceMin(*this, [=] AMREX_GPU_DEVICE (const FlavoredNeutrinoContainer::ParticleType& p) -> Real { return p.rdata(PIdx::pupt); }); + ParallelDescriptor::ReduceRealMin(pupt_min); + #include "generated_files/FlavoredNeutrinoContainerInit.cpp_Vvac_fill" } diff --git a/Source/main.cpp b/Source/main.cpp index 309d5e63..68f59103 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -194,7 +194,7 @@ void evolve_flavor(const TestParams* parms) // Note: this won't be the same as the new-time grid data // because the last deposit_to_mesh call was at either the old time (forward Euler) // or the final RK stage, if using Runge-Kutta. - const Real dt = compute_dt(geom,parms->cfl_factor,state,parms->flavor_cfl_factor); + const Real dt = compute_dt(geom,parms->cfl_factor,state,neutrinos,parms->flavor_cfl_factor); integrator.set_timestep(dt); }; @@ -203,7 +203,7 @@ void evolve_flavor(const TestParams* parms) integrator.set_post_timestep(post_timestep_fun); // Get a starting timestep - const Real starting_dt = compute_dt(geom,parms->cfl_factor,state,parms->flavor_cfl_factor); + const Real starting_dt = compute_dt(geom,parms->cfl_factor,state,neutrinos_old,parms->flavor_cfl_factor); // Do all the science! amrex::Print() << "Starting timestepping loop... " << std::endl; From aa46ff8dc5cb6ab144d16eac4a9ef2c27bde867a Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sat, 20 Mar 2021 14:53:06 -0700 Subject: [PATCH 012/276] fixed unit issue with max vacuum potential --- Scripts/symbolic_hermitians/generate_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 63a67fba..642bb279 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -233,7 +233,7 @@ def delete_generated_files(): massmatrix_massbasis = HermitianMatrix(args.N, "M2massbasis{}{}_{}") massmatrix_massbasis.H = M2_massbasis M2length = massmatrix_massbasis.SU_vector_magnitude() - code.append("this->Vvac_max = "+sympy.cxxcode(sympy.simplify(M2length))+"/pupt_min;") + code.append("this->Vvac_max = "+sympy.cxxcode(sympy.simplify(M2length))+"*PhysConst::c4/pupt_min;") write_code(code, os.path.join(args.emu_home,"Source/generated_files","FlavoredNeutrinoContainerInit.cpp_Vvac_fill")) #======================# From ed34fd28a5aed79c71c5c2932626fc8e3a4bb149 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sat, 20 Mar 2021 14:54:08 -0700 Subject: [PATCH 013/276] override copyParticles function to also copy the new Vvac_max member variable --- Source/FlavoredNeutrinoContainer.H | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Source/FlavoredNeutrinoContainer.H b/Source/FlavoredNeutrinoContainer.H index 75a91065..c3456ea8 100644 --- a/Source/FlavoredNeutrinoContainer.H +++ b/Source/FlavoredNeutrinoContainer.H @@ -96,6 +96,11 @@ public: return attribute_names; } + void copyParticles(const FlavoredNeutrinoContainer& other, bool local=false){ + amrex::ParticleContainer::copyParticles(other, local); + this->Vvac_max = other.Vvac_max; + } + /* Public data members */ static ApplyFlavoredNeutrinoRHS particle_apply_rhs; }; From 43aa1d734972a338ebd0e7685b9be17094328aea Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sat, 20 Mar 2021 15:11:11 -0700 Subject: [PATCH 014/276] fixed broken implementation of conjugate --- Scripts/symbolic_hermitians/HermitianUtils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Scripts/symbolic_hermitians/HermitianUtils.py b/Scripts/symbolic_hermitians/HermitianUtils.py index 9e71a68e..3cd85803 100644 --- a/Scripts/symbolic_hermitians/HermitianUtils.py +++ b/Scripts/symbolic_hermitians/HermitianUtils.py @@ -93,7 +93,9 @@ def anticommutator(self, HermitianA, HermitianB): return self def conjugate(self): - self.H = Conjugate(self.H) + for i in range(self.size): + for j in range(i, self.size): + self.H[i,j] = conjugate(self.H[i,j]) return self # return the length of the SU(n) vector From 19be18d9c6efb56999ffa1d43390e032f7aa040e Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sat, 20 Mar 2021 15:11:51 -0700 Subject: [PATCH 015/276] fixed error in generate code - the length of the Hamiltonian vector needs antineutrinos to be conjugated --- Scripts/symbolic_hermitians/generate_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 642bb279..22895618 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -268,7 +268,7 @@ def delete_generated_files(): # spherically symmetric part N = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::N{}{}_{})") Nbar = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::N{}{}_{}bar)") - HSI = (N-Nbar) / cell_volume + HSI = (N-Nbar.conjugate()) / cell_volume HSI.H[0,0] += rho*Ye/mp HSI *= sqrt2GF length2 = HSI.SU_vector_magnitude2() From ee602e8428ecab4d9161a4b6bf25ee5b53294dcd Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sat, 20 Mar 2021 15:12:59 -0700 Subject: [PATCH 016/276] pushed flavor cfl factors back up as high as they can go without runing into the maxError limit --- sample_inputs/inputs_bipolar_test | 2 +- sample_inputs/inputs_fast_flavor | 2 +- sample_inputs/inputs_fast_flavor_nonzerok | 2 +- sample_inputs/inputs_msw_test | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index de63afbd..1c20a7d1 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -1,6 +1,6 @@ simulation_type = 1 cfl_factor = 0.5 -flavor_cfl_factor = .05 +flavor_cfl_factor = .1 maxError = 1e-6 integration.type = 1 diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index 1dcb388c..dd994cba 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -1,6 +1,6 @@ simulation_type = 2 cfl_factor = 0.5 -flavor_cfl_factor = .05 +flavor_cfl_factor = .5 maxError = 1e-6 integration.type = 1 diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 97c94c40..49c82619 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -3,7 +3,7 @@ st3_amplitude = 1e-6 st3_wavelength_fraction_of_domain = 1 cfl_factor = 0.5 -flavor_cfl_factor = 0.05 +flavor_cfl_factor = 0.5 maxError = 1e-6 integration.type = 1 diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index b71b8f8f..a8231fb5 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -1,6 +1,6 @@ simulation_type = 0 cfl_factor = 0.5 -flavor_cfl_factor = 0.05 +flavor_cfl_factor = 0.1 maxError = 1e-6 integration.type = 1 From 27c2dc716541761a5c25e452edb4e39eac217fac Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sat, 20 Mar 2021 16:22:46 -0700 Subject: [PATCH 017/276] set the reduction operator to return a second quantity. Will be used to limit timestep based on total lepton density --- Source/Evolve.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 2fd47aae..b2100cc5 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -37,8 +37,8 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta if (flavor_cfl_factor > 0.0) { // self-interaction, matter, and vacuum part of timestep limit - ReduceOps reduce_op; - ReduceData reduce_data(reduce_op); + ReduceOps reduce_op; + ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; for (MFIter mfi(state); mfi.isValid(); ++mfi) { @@ -47,14 +47,18 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { - Real length2 = 0; + Real length2=0, ndens_lepton_total=0; #include "generated_files/Evolve.cpp_compute_dt_fill" - return sqrt(length2); + return {sqrt(length2), ndens_lepton_total}; }); } - Real Vmax = amrex::get<0>(reduce_data.value()); + auto rv = reduce_data.value(); + Real Vmax = amrex::get<0>(rv); + Real ndens_lepton_max = amrex::get<1>(rv); ParallelDescriptor::ReduceRealMax(Vmax); + ParallelDescriptor::ReduceRealMax(ndens_lepton_max); + dt_si_matter = PhysConst::hbar/(Vmax+neutrinos.Vvac_max)*flavor_cfl_factor; } From 21527c14572c809af1d6a0d282ef0c6377e4f4aa Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sat, 20 Mar 2021 20:14:54 -0700 Subject: [PATCH 018/276] account for vacuum potential. Also cover case where total potential is 0 because of equal numbers of neutrinos and antineutrinos by introducing a parameter (max_adaptive_speedup). When set to 0 it behaves exactly as before, except for the inclusion of the vacuum potential in the timestep calculation. When set towards infinity, it always allows the "optimized" timestep, which may itself become infinite if terms happen to cancel by chance, but otherwise is a much better estimate of the required timestep --- Scripts/symbolic_hermitians/generate_code.py | 27 ++++++++---- Source/Evolve.H | 2 +- Source/Evolve.cpp | 45 +++++++++++++------- Source/Parameters.H | 2 + Source/main.cpp | 4 +- sample_inputs/inputs_bipolar_test | 3 +- sample_inputs/inputs_fast_flavor | 1 + sample_inputs/inputs_fast_flavor_nonzerok | 1 + sample_inputs/inputs_msw_test | 1 + 9 files changed, 57 insertions(+), 29 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 22895618..58599380 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -268,20 +268,29 @@ def delete_generated_files(): # spherically symmetric part N = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::N{}{}_{})") Nbar = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::N{}{}_{}bar)") - HSI = (N-Nbar.conjugate()) / cell_volume - HSI.H[0,0] += rho*Ye/mp - HSI *= sqrt2GF - length2 = HSI.SU_vector_magnitude2() - code.append("length2 += "+sympy.cxxcode(sympy.simplify(length2))+";") + HSI = (N-Nbar.conjugate()) + HSI.H[0,0] += rho*Ye/mp * cell_volume + V_adaptive2 = HSI.SU_vector_magnitude2() + code.append("V_adaptive2 += "+sympy.cxxcode(sympy.simplify(V_adaptive2))+";") # flux part for component in ["x","y","z"]: F = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::F"+component+"{}{}_{})") Fbar = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::F"+component+"{}{}_{}bar)") - HSI = (F-Fbar) * sqrt2GF / cell_volume - length2 = HSI.SU_vector_magnitude2() - code.append("length2 += "+sympy.cxxcode(sympy.simplify(length2))+";") - + HSI = (F-Fbar) + V_adaptive2 = HSI.SU_vector_magnitude2() + code.append("V_adaptive2 += "+sympy.cxxcode(sympy.simplify(V_adaptive2))+";") + + # put in the units + code.append("V_adaptive = sqrt(V_adaptive2)*"+sympy.cxxcode(sqrt2GF/cell_volume)+";") + + # old "stupid" way of computing the timestep. + # the factor of 2 accounts for potential worst-case effects of neutrinos and antineutrinos + for i in range(args.N): + code.append("V_stupid = max(V_stupid,"+sympy.cxxcode(N.H[i,i])+");") + code.append("V_stupid = max(V_stupid,"+sympy.cxxcode(Nbar.H[i,i])+");") + code.append("V_stupid = max(V_stupid,"+sympy.cxxcode(rho*Ye/mp*cell_volume)+");") + code.append("V_stupid *= "+sympy.cxxcode(2.0*args.N*sqrt2GF/cell_volume)+";") write_code(code, os.path.join(args.emu_home,"Source/generated_files","Evolve.cpp_compute_dt_fill")) #=======================================# diff --git a/Source/Evolve.H b/Source/Evolve.H index f3888c6b..ed387f1a 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -26,7 +26,7 @@ namespace GIdx void Initialize(); }; -amrex::Real compute_dt(const amrex::Geometry& geom, const amrex::Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor); +amrex::Real compute_dt(const amrex::Geometry& geom, const amrex::Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor, const Real max_adaptive_speedup); void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index b2100cc5..6f4c1234 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -19,7 +19,7 @@ namespace GIdx } } -Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor) +Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor, const Real max_adaptive_speedup) { AMREX_ASSERT(cfl_factor > 0.0 || flavor_cfl_factor > 0.0); @@ -33,10 +33,12 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta dt_translation = std::min(std::min(dxi[0],dxi[1]), dxi[2]) / PhysConst::c * cfl_factor; } - Real dt_si_matter = 0.0; + Real dt_flavor = 0.0; if (flavor_cfl_factor > 0.0) { - // self-interaction, matter, and vacuum part of timestep limit - + // define the reduction operator to get the max contribution to + // the potential from matter and neutrinos + // compute "effective" potential (ergs) that produces characteristic timescale + // when multiplied by hbar ReduceOps reduce_op; ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; @@ -47,28 +49,39 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { - Real length2=0, ndens_lepton_total=0; + Real V_adaptive=0, V_adaptive2=0, V_stupid=0; #include "generated_files/Evolve.cpp_compute_dt_fill" - return {sqrt(length2), ndens_lepton_total}; + return {V_adaptive, V_stupid}; }); } + // extract the reduced values from the combined reduced data structure auto rv = reduce_data.value(); - Real Vmax = amrex::get<0>(rv); - Real ndens_lepton_max = amrex::get<1>(rv); - ParallelDescriptor::ReduceRealMax(Vmax); - ParallelDescriptor::ReduceRealMax(ndens_lepton_max); - - dt_si_matter = PhysConst::hbar/(Vmax+neutrinos.Vvac_max)*flavor_cfl_factor; + Real Vmax_adaptive = amrex::get<0>(rv) + neutrinos.Vvac_max; + Real Vmax_stupid = amrex::get<1>(rv) + neutrinos.Vvac_max; + + // reduce across MPI ranks + ParallelDescriptor::ReduceRealMax(Vmax_adaptive); + ParallelDescriptor::ReduceRealMax(Vmax_stupid ); + + // define the dt associated with each method + Real dt_flavor_adaptive = PhysConst::hbar/Vmax_adaptive*flavor_cfl_factor; + Real dt_flavor_stupid = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; + + // pick the appropriate timestep + if(dt_flavor_adaptive*max_adaptive_speedup > dt_flavor_stupid) + dt_flavor = dt_flavor_adaptive; + else + dt_flavor = dt_flavor_stupid; } Real dt = 0.0; - if (dt_translation != 0.0 && dt_si_matter != 0.0) { - dt = std::min(dt_translation, dt_si_matter); + if (dt_translation != 0.0 && dt_flavor != 0.0) { + dt = std::min(dt_translation, dt_flavor); } else if (dt_translation != 0.0) { dt = dt_translation; - } else if (dt_si_matter != 0.0) { - dt = dt_si_matter; + } else if (dt_flavor != 0.0) { + dt = dt_flavor; } else { amrex::Error("Timestep selection failed, try using both cfl_factor and flavor_cfl_factor"); } diff --git a/Source/Parameters.H b/Source/Parameters.H index 26fbb946..53655f40 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -22,6 +22,7 @@ struct TestParams : public amrex::Gpu::Managed Real rho_in, Ye_in, T_in; // g/ccm, 1, MeV int simulation_type; Real cfl_factor, flavor_cfl_factor; + Real max_adaptive_speedup; bool do_restart; std::string restart_dir; Real maxError; @@ -58,6 +59,7 @@ struct TestParams : public amrex::Gpu::Managed pp.get("T_MeV", T_in); pp.get("cfl_factor", cfl_factor); pp.get("flavor_cfl_factor", flavor_cfl_factor); + pp.get("max_adaptive_speedup", max_adaptive_speedup); pp.get("write_plot_every", write_plot_every); pp.get("write_plot_particles_every", write_plot_particles_every); pp.get("do_restart", do_restart); diff --git a/Source/main.cpp b/Source/main.cpp index 68f59103..6e4eb3fa 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -194,7 +194,7 @@ void evolve_flavor(const TestParams* parms) // Note: this won't be the same as the new-time grid data // because the last deposit_to_mesh call was at either the old time (forward Euler) // or the final RK stage, if using Runge-Kutta. - const Real dt = compute_dt(geom,parms->cfl_factor,state,neutrinos,parms->flavor_cfl_factor); + const Real dt = compute_dt(geom,parms->cfl_factor,state,neutrinos,parms->flavor_cfl_factor,parms->max_adaptive_speedup); integrator.set_timestep(dt); }; @@ -203,7 +203,7 @@ void evolve_flavor(const TestParams* parms) integrator.set_post_timestep(post_timestep_fun); // Get a starting timestep - const Real starting_dt = compute_dt(geom,parms->cfl_factor,state,neutrinos_old,parms->flavor_cfl_factor); + const Real starting_dt = compute_dt(geom,parms->cfl_factor,state,neutrinos_old,parms->flavor_cfl_factor, parms->max_adaptive_speedup); // Do all the science! amrex::Print() << "Starting timestepping loop... " << std::endl; diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 1c20a7d1..668589ba 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -1,6 +1,7 @@ simulation_type = 1 cfl_factor = 0.5 -flavor_cfl_factor = .1 +max_adaptive_speedup = 0 +flavor_cfl_factor = .5 maxError = 1e-6 integration.type = 1 diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index dd994cba..965ed7db 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -1,6 +1,7 @@ simulation_type = 2 cfl_factor = 0.5 flavor_cfl_factor = .5 +max_adaptive_speedup = 0 maxError = 1e-6 integration.type = 1 diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 49c82619..32ccb2ae 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -4,6 +4,7 @@ st3_wavelength_fraction_of_domain = 1 cfl_factor = 0.5 flavor_cfl_factor = 0.5 +max_adaptive_speedup = 0 maxError = 1e-6 integration.type = 1 diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index a8231fb5..3a5445dd 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -1,6 +1,7 @@ simulation_type = 0 cfl_factor = 0.5 flavor_cfl_factor = 0.1 +max_adaptive_speedup = 0 maxError = 1e-6 integration.type = 1 From 549012770679294de0d3ae97b49e20d6cee5512b Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Mon, 22 Mar 2021 08:52:24 -0700 Subject: [PATCH 019/276] forgot to update the 1d_fiducial inputs file --- sample_inputs/inputs_1d_fiducial | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 0d2508e3..72ee3bfb 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -10,7 +10,8 @@ st4_fluxfacbar = .333333333333333 st4_amplitude = 1e-6 cfl_factor = 0.5 -flavor_cfl_factor = 0.05 +flavor_cfl_factor = 0.5 +max_adaptive_speedup = 0 maxError = 1e-6 integration.type = 1 From 044701d2763dbc9ea6caa828550d09861959dd0d Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Mon, 22 Mar 2021 19:31:49 -0700 Subject: [PATCH 020/276] addressing PR comments. Use static inline variable rather than overriding the ParticleContainer copy operator, fix mistake in conjugation of Hermitian matrices. --- Scripts/symbolic_hermitians/HermitianUtils.py | 4 ++-- Scripts/symbolic_hermitians/generate_code.py | 2 +- Source/Evolve.cpp | 4 ++-- Source/FlavoredNeutrinoContainer.H | 7 +------ 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Scripts/symbolic_hermitians/HermitianUtils.py b/Scripts/symbolic_hermitians/HermitianUtils.py index 3cd85803..434bb797 100644 --- a/Scripts/symbolic_hermitians/HermitianUtils.py +++ b/Scripts/symbolic_hermitians/HermitianUtils.py @@ -49,7 +49,7 @@ def __mul__(self, other): def __truediv__(self, other): result = copy.deepcopy(self) if isinstance(other, self.__class__): - result.H = self.H * other.H + raise ArithmeticError else: for i in range(self.size): for j in range(self.size): @@ -94,7 +94,7 @@ def anticommutator(self, HermitianA, HermitianB): def conjugate(self): for i in range(self.size): - for j in range(i, self.size): + for j in range(self.size): self.H[i,j] = conjugate(self.H[i,j]) return self diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 58599380..397f40f8 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -233,7 +233,7 @@ def delete_generated_files(): massmatrix_massbasis = HermitianMatrix(args.N, "M2massbasis{}{}_{}") massmatrix_massbasis.H = M2_massbasis M2length = massmatrix_massbasis.SU_vector_magnitude() - code.append("this->Vvac_max = "+sympy.cxxcode(sympy.simplify(M2length))+"*PhysConst::c4/pupt_min;") + code.append("Vvac_max = "+sympy.cxxcode(sympy.simplify(M2length))+"*PhysConst::c4/pupt_min;") write_code(code, os.path.join(args.emu_home,"Source/generated_files","FlavoredNeutrinoContainerInit.cpp_Vvac_fill")) #======================# diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 6f4c1234..e302d6a9 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -57,8 +57,8 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta // extract the reduced values from the combined reduced data structure auto rv = reduce_data.value(); - Real Vmax_adaptive = amrex::get<0>(rv) + neutrinos.Vvac_max; - Real Vmax_stupid = amrex::get<1>(rv) + neutrinos.Vvac_max; + Real Vmax_adaptive = amrex::get<0>(rv) + FlavoredNeutrinoContainer::Vvac_max; + Real Vmax_stupid = amrex::get<1>(rv) + FlavoredNeutrinoContainer::Vvac_max; // reduce across MPI ranks ParallelDescriptor::ReduceRealMax(Vmax_adaptive); diff --git a/Source/FlavoredNeutrinoContainer.H b/Source/FlavoredNeutrinoContainer.H index c3456ea8..cb6bcd68 100644 --- a/Source/FlavoredNeutrinoContainer.H +++ b/Source/FlavoredNeutrinoContainer.H @@ -68,7 +68,7 @@ class FlavoredNeutrinoContainer public: - Real Vvac_max; + inline static Real Vvac_max; FlavoredNeutrinoContainer(const amrex::Geometry & a_geom, const amrex::DistributionMapping & a_dmap, @@ -96,11 +96,6 @@ public: return attribute_names; } - void copyParticles(const FlavoredNeutrinoContainer& other, bool local=false){ - amrex::ParticleContainer::copyParticles(other, local); - this->Vvac_max = other.Vvac_max; - } - /* Public data members */ static ApplyFlavoredNeutrinoRHS particle_apply_rhs; }; From f2b6aa97be590df24117079bf788af420e6a8db6 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 27 Apr 2021 15:24:44 -0700 Subject: [PATCH 021/276] Minerbo initial conditions (#59) * added vscode and generated_files directories to the gitignore file * added Minerbo closure initial conditions * added references for the minerbo equations * added a tad more explanatory text * improved st5 code comments * fixed coefficient in front of minerbo closure to make it have an expectation value of 1. Was getting results too small by a factor of 4pi before. Co-authored-by: Sherwood Richers --- .gitignore | 4 +- Source/FlavoredNeutrinoContainerInit.cpp | 150 +++++++++++++++++++++++ Source/Parameters.H | 23 ++++ 3 files changed, 176 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 64aa844c..1c8c4b5f 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,6 @@ Scripts/symbolic_hermitians/__pycache__ .cproject .project .pydevproject -.settings \ No newline at end of file +.settings +.vscode +Source/generated_files \ No newline at end of file diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index d93580e3..3014b623 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -36,6 +36,49 @@ Gpu::ManagedVector > uniform_sphere_xyz(int nphi_at_equator){ return xyz; } +// residual for the root finder +// Z needs to be bigger if residual is positive +// Minerbo (1978) (unfortunately under Elsevier paywall) +// Can also see Richers (2020) https://ui.adsabs.harvard.edu/abs/2020PhRvD.102h3017R +// Eq.41 (where a is Z), but in the non-degenerate limit +// k->0, eta->0, N->Z/(4pi sinh(Z)) (just to make it integrate to 1) +// minerbo_residual is the "f" equation between eq.42 and 43 +Real minerbo_residual(const Real fluxfac, const Real Z){ + return fluxfac - 1.0/std::tanh(Z) + 1.0 / Z; +} +Real minerbo_residual_derivative(const Real fluxfac, const Real Z){ + return 1.0/(std::sinh(Z)*std::sinh(Z)) - 1.0/(Z*Z); +} +Real minerbo_Z(const Real fluxfac){ + // hard-code in these parameters because they are not + // really very important... + Real maxresidual = 1e-6; + Real maxcount = 20; + Real minfluxfac = 1e-3; + + // set the initial conditions + Real Z = 1.0; + + // catch the small flux factor case to prevent nans + if(fluxfac < minfluxfac) + Z = 3.*fluxfac; + else{ + Real residual = 1.0; + int count = 0; + while(std::abs(residual)>maxresidual and countmaxresidual) + amrex::Error("Failed to converge on a solution."); + } + + amrex::Print() << "fluxfac="< Date: Tue, 27 Apr 2021 15:29:02 -0700 Subject: [PATCH 022/276] Output build info (#63) * Copied Castro function for printing build information to screen * Copy castro function for outputting information into plotfiles, simply deleting lines that don't work in emu Co-authored-by: Sherwood Richers --- Make.Emu | 40 +++++++- Source/IO.H | 6 ++ Source/IO.cpp | 267 ++++++++++++++++++++++++++++++++++++++++++++++++ Source/main.cpp | 5 + 4 files changed, 317 insertions(+), 1 deletion(-) diff --git a/Make.Emu b/Make.Emu index 77e39393..24d9d145 100644 --- a/Make.Emu +++ b/Make.Emu @@ -24,10 +24,48 @@ include $(Ppack) DEFINES += -DNUM_FLAVORS=$(NUM_FLAVORS) -DSHAPE_FACTOR_ORDER=$(SHAPE_FACTOR_ORDER) -all: generate $(executable) +all: generate $(objEXETempDir)/AMReX_buildInfo.o $(executable) @echo SUCCESS generate: python3 $(EMU_HOME)/Scripts/symbolic_hermitians/generate_code.py $(NUM_FLAVORS) --emu_home $(EMU_HOME) +#------------------------------------------------------------------------------ +# build info (from Castro/Exec/Make.auto_source) +#------------------------------------------------------------------------------ +CEXE_headers += $(AMREX_HOME)/Tools/C_scripts/AMReX_buildInfo.H +INCLUDE_LOCATIONS += $(AMREX_HOME)/Tools/C_scripts + +# we make AMReX_buildInfo.cpp as we make the .o file, so we can delete +# it immediately. this way if the build is interrupted, we are +# guaranteed to remake it + +objForExecs += $(objEXETempDir)/AMReX_buildInfo.o + +.FORCE: +.PHONE: .FORCE + +# set BUILD_GIT_NAME and BUILD_GIT_DIR if you are building in a +# git-controlled dir not under Castro/ +EXTRA_BUILD_INFO := +ifdef BUILD_GIT_NAME + EXTRA_BUILD_INFO := --build_git_name "$(BUILD_GIT_NAME)" \ + --build_git_dir "$(BUILD_GIT_DIR)" +endif + +$(objEXETempDir)/AMReX_buildInfo.o: .FORCE + echo $(objEXETempDir) + $(AMREX_HOME)/Tools/C_scripts/makebuildinfo_C.py \ + --amrex_home "$(AMREX_HOME)" \ + --COMP "$(COMP)" --COMP_VERSION "$(COMP_VERSION)" \ + --CXX_comp_name "$(CXX)" --CXX_flags "$(CXXFLAGS) $(CPPFLAGS) $(includes)" \ + --F_comp_name "$(F90)" --F_flags "$(F90FLAGS)" \ + --link_flags "$(LDFLAGS)" --libraries "$(libraries)" \ + --MODULES "$(MNAMES)" $(EXTRA_BUILD_INFO) \ + --GIT "$(TOP) $(AMREX_HOME) $(MICROPHYSICS_HOME)" + $(SILENT) $(CCACHE) $(CXX) $(CXXFLAGS) $(CPPFLAGS) -c $(CXXEXEFLAGS) AMReX_buildInfo.cpp -o $(objEXETempDir)/AMReX_buildInfo.o + $(SILENT) $(RM) AMReX_buildInfo.cpp + + + include $(AMREX_HOME)/Tools/GNUMake/Make.rules diff --git a/Source/IO.H b/Source/IO.H index fd57f6cd..e851f0a2 100644 --- a/Source/IO.H +++ b/Source/IO.H @@ -17,4 +17,10 @@ RecoverParticles (const std::string& dir, FlavoredNeutrinoContainer& neutrinos, amrex::Real& time, int& step); +void +writeBuildInfo (); + +void +writeJobInfo (const std::string& dir, const amrex::Geometry& geom); + #endif diff --git a/Source/IO.cpp b/Source/IO.cpp index 7f291dff..a1b09cea 100644 --- a/Source/IO.cpp +++ b/Source/IO.cpp @@ -1,5 +1,6 @@ #include #include +#include #include "FlavoredNeutrinoContainer.H" #include "IO.H" @@ -31,6 +32,9 @@ WritePlotFile (const amrex::MultiFab& state, auto neutrino_varnames = neutrinos.get_attribute_names(); neutrinos.Checkpoint(plotfilename, "neutrinos", true, neutrino_varnames); } + + // write job information + writeJobInfo (plotfilename, geom); } void @@ -57,3 +61,266 @@ RecoverParticles (const std::string& dir, // print the step/time for the restart amrex::Print() << "Restarting after time step: " << step-1 << " t = " << time << " s. ct = " << PhysConst::c * time << " cm" << std::endl; } + + +// writeBuildInfo and writeJobInfo are copied from Castro/Source/driver/Castro_io.cpp +// and modified by Sherwood Richers +/* +SOURCE CODE LICENSE AGREEMENT +Castro, Copyright (c) 2015, +The Regents of the University of California, +through Lawrence Berkeley National Laboratory +(subject to receipt of any required approvals from the U.S. +Dept. of Energy). All rights reserved." + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +(1) Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +(2) Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +(3) Neither the name of the University of California, Lawrence +Berkeley National Laboratory, U.S. Dept. of Energy nor the names of +its contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You are under no obligation whatsoever to provide any bug fixes, +patches, or upgrades to the features, functionality or performance of +the source code ("Enhancements") to anyone; however, if you choose to +make your Enhancements available either publicly, or directly to +Lawrence Berkeley National Laboratory, without imposing a separate +written license agreement for such Enhancements, then you hereby grant +the following license: a non-exclusive, royalty-free perpetual +license to install, use, modify, prepare derivative works, incorporate +into other computer software, distribute, and sublicense such +enhancements or derivative works thereof, in binary and source code +form. +*/ +void +writeBuildInfo (){ + std::string PrettyLine = std::string(78, '=') + "\n"; + std::string OtherLine = std::string(78, '-') + "\n"; + std::string SkipSpace = std::string(8, ' '); + + // build information + std::cout << PrettyLine; + std::cout << " Emu Build Information\n"; + std::cout << PrettyLine; + + std::cout << "build date: " << buildInfoGetBuildDate() << "\n"; + std::cout << "build machine: " << buildInfoGetBuildMachine() << "\n"; + std::cout << "build dir: " << buildInfoGetBuildDir() << "\n"; + std::cout << "AMReX dir: " << buildInfoGetAMReXDir() << "\n"; + + std::cout << "\n"; + + std::cout << "COMP: " << buildInfoGetComp() << "\n"; + std::cout << "COMP version: " << buildInfoGetCompVersion() << "\n"; + + std::cout << "\n"; + + std::cout << "C++ compiler: " << buildInfoGetCXXName() << "\n"; + std::cout << "C++ flags: " << buildInfoGetCXXFlags() << "\n"; + + std::cout << "\n"; + + std::cout << "Link flags: " << buildInfoGetLinkFlags() << "\n"; + std::cout << "Libraries: " << buildInfoGetLibraries() << "\n"; + + std::cout << "\n"; + + for (int n = 1; n <= buildInfoGetNumModules(); n++) { + std::cout << buildInfoGetModuleName(n) << ": " << buildInfoGetModuleVal(n) << "\n"; + } + + std::cout << "\n"; + + const char* githash1 = buildInfoGetGitHash(1); + const char* githash2 = buildInfoGetGitHash(2); + if (strlen(githash1) > 0) { + std::cout << "Emu git describe: " << githash1 << "\n"; + } + if (strlen(githash2) > 0) { + std::cout << "AMReX git describe: " << githash2 << "\n"; + } + + const char* buildgithash = buildInfoGetBuildGitHash(); + const char* buildgitname = buildInfoGetBuildGitName(); + if (strlen(buildgithash) > 0){ + std::cout << buildgitname << " git describe: " << buildgithash << "\n"; + } + + std::cout << "\n"< 0) { + jobInfoFile << "Emu git describe: " << githash1 << "\n"; + } + if (strlen(githash2) > 0) { + jobInfoFile << "AMReX git describe: " << githash2 << "\n"; + } + + const char* buildgithash = buildInfoGetBuildGitHash(); + const char* buildgitname = buildInfoGetBuildGitName(); + if (strlen(buildgithash) > 0){ + jobInfoFile << buildgitname << " git describe: " << buildgithash << "\n"; + } + + jobInfoFile << "\n\n"; + + + // grid information + jobInfoFile << PrettyLine; + jobInfoFile << " Grid Information\n"; + jobInfoFile << PrettyLine; + + jobInfoFile << "geometry.is_periodic: "; + for (int idir = 0; idir < AMREX_SPACEDIM; idir++) { + jobInfoFile << geom.isPeriodic(idir) << " "; + } + jobInfoFile << "\n"; + + jobInfoFile << "geometry.coord_sys: " << geom.Coord() << "\n"; + + jobInfoFile << "geometry.prob_lo: "; + for (int idir = 0; idir < AMREX_SPACEDIM; idir++) { + jobInfoFile << geom.ProbLo(idir) << " "; + } + jobInfoFile << "\n"; + + jobInfoFile << "geometry.prob_hi: "; + for (int idir = 0; idir < AMREX_SPACEDIM; idir++) { + jobInfoFile << geom.ProbHi(idir) << " "; + } + jobInfoFile << "\n"; + + jobInfoFile << "amr.n_cell: "; + const int* domain_lo = geom.Domain().loVect(); + const int* domain_hi = geom.Domain().hiVect(); + for (int idir = 0; idir < AMREX_SPACEDIM; idir++) { + jobInfoFile << domain_hi[idir] - domain_lo[idir] + 1 << " "; + } + jobInfoFile << "\n\n"; + + jobInfoFile.close(); + + // now the external parameters + const int jobinfo_file_length = FullPathJobInfoFile.length(); + Vector jobinfo_file_name(jobinfo_file_length); + + for (int i = 0; i < jobinfo_file_length; i++) { + jobinfo_file_name[i] = FullPathJobInfoFile[i]; + } + +} diff --git a/Source/main.cpp b/Source/main.cpp index 6e4eb3fa..edd2d6c3 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -230,6 +230,11 @@ int main(int argc, char* argv[]) { amrex::Initialize(argc,argv); + // write build information to screen + if (ParallelDescriptor::IOProcessor()) { + writeBuildInfo(); + } + // by default amrex initializes rng deterministically // this uses the time for a different run each time amrex::InitRandom(ParallelDescriptor::MyProc()+time(NULL), ParallelDescriptor::NProcs()); From 8488c47101146ade05090060a95f321478840700 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 21 May 2021 17:41:48 -0700 Subject: [PATCH 023/276] im stoopid - fixing timestep logic --- Source/Evolve.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index e302d6a9..a1fbfe98 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -69,10 +69,9 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta Real dt_flavor_stupid = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; // pick the appropriate timestep - if(dt_flavor_adaptive*max_adaptive_speedup > dt_flavor_stupid) - dt_flavor = dt_flavor_adaptive; - else - dt_flavor = dt_flavor_stupid; + dt_flavor = dt_flavor_stupid; + if(max_adaptive_speedup>1) + dt_flavor = min(dt_flavor_stupid*max_adaptive_speedup, dt_flavor_adaptive); } Real dt = 0.0; From 9e793e3b33715e15d1e2793df6681fcbfe2714f7 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Thu, 16 Dec 2021 12:35:18 -0800 Subject: [PATCH 024/276] add average neutrino energy argument to simulation type 5 --- Source/FlavoredNeutrinoContainerInit.cpp | 2 +- Source/Parameters.H | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 3014b623..876de472 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -575,7 +575,7 @@ InitParticles(const TestParams* parms) AMREX_ASSERT(NUM_FLAVORS==3 or NUM_FLAVORS==2); // set energy to 50 MeV - p.rdata(PIdx::pupt) = 50. * 1e6*CGSUnitsConst::eV; + p.rdata(PIdx::pupt) = parms->st5_avgE_MeV * 1e6*CGSUnitsConst::eV; p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); p.rdata(PIdx::pupy) = u[1] * p.rdata(PIdx::pupt); p.rdata(PIdx::pupz) = u[2] * p.rdata(PIdx::pupt); diff --git a/Source/Parameters.H b/Source/Parameters.H index 498b6d91..170d0a8e 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -47,6 +47,7 @@ struct TestParams : public amrex::Gpu::Managed Real st5_fxnue, st5_fxnua, st5_fxnux; Real st5_fynue, st5_fynua, st5_fynux; Real st5_fznue, st5_fznua, st5_fznux; + Real st5_avgE_MeV; Real st5_amplitude; void Initialize(){ @@ -126,6 +127,7 @@ struct TestParams : public amrex::Gpu::Managed pp.get("st5_fznue",st5_fznue); pp.get("st5_fznua",st5_fznua); pp.get("st5_fznux",st5_fznux); + pp.get("st5_avgE_MeV",st5_avgE_MeV); pp.get("st5_amplitude",st5_amplitude); } } From debcf3b4a3abc185edbb726797fa5a3d6ba3ad7c Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 28 Sep 2021 14:05:51 -0400 Subject: [PATCH 025/276] Revert "Added descriptive amrex::Error messages in generate_code.py (#53)" This reverts commit 97499f21cad18e1fc6aaf12cf49d1a94457090b0. --- Scripts/symbolic_hermitians/generate_code.py | 21 +++----------------- Source/FlavoredNeutrinoContainer.cpp | 2 -- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 397f40f8..df07464d 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -401,12 +401,7 @@ def sgn(t,var): for fii in fdlist: code.append("sumP += " + fii + ";") code.append("error = sumP-1.0;") - code.append('if( std::abs(error) > 100.*parms->maxError) {') - code.append("std::ostringstream Convert;") - code.append('Convert << "Matrix trace (SumP) is not equal to 1, trace error exceeds 100*maxError: " << std::abs(error) << " > " << 100.*parms->maxError;') - code.append("std::string Trace_Error = Convert.str();") - code.append('amrex::Error(Trace_Error);') - code.append("}") + code.append("if( std::abs(error) > 100.*parms->maxError) amrex::Abort();") code.append("if( std::abs(error) > parms->maxError ) {") for fii in fdlist: code.append(fii + " -= error/"+str(args.N)+";") @@ -415,12 +410,7 @@ def sgn(t,var): # make sure diagonals are positive for fii in fdlist: - code.append('if('+fii+'<-100.*parms->maxError) {') - code.append("std::ostringstream Convert;") - code.append('Convert << "Diagonal element '+fii[14:20]+' is negative, less than -100*maxError: " << '+fii+' << " < " << -100.*parms->maxError;') - code.append("std::string Sign_Error = Convert.str();") - code.append('amrex::Error(Sign_Error);') - code.append("}") + code.append("if("+fii+"<-100.*parms->maxError) amrex::Abort();") code.append("if("+fii+"<-parms->maxError) "+fii+"=0;") code.append("") @@ -430,12 +420,7 @@ def sgn(t,var): target_length = "p.rdata(PIdx::L"+t+")" code.append("length = "+sympy.cxxcode(sympy.simplify(length))+";") code.append("error = length-"+str(target_length)+";") - code.append('if( std::abs(error) > 100.*parms->maxError) {') - code.append("std::ostringstream Convert;") - code.append('Convert << "flavor vector length differs from target length by more than 100*maxError: " << std::abs(error) << " > " << 100.*parms->maxError;') - code.append("std::string Length_Error = Convert.str();") - code.append('amrex::Error(Length_Error);') - code.append("}") + code.append("if( std::abs(error) > 100.*parms->maxError) amrex::Abort();") code.append("if( std::abs(error) > parms->maxError) {") for fii in flist: code.append(fii+" /= length/"+str(target_length)+";") diff --git a/Source/FlavoredNeutrinoContainer.cpp b/Source/FlavoredNeutrinoContainer.cpp index ee050e2d..e53c3386 100644 --- a/Source/FlavoredNeutrinoContainer.cpp +++ b/Source/FlavoredNeutrinoContainer.cpp @@ -1,7 +1,5 @@ #include "FlavoredNeutrinoContainer.H" #include "Constants.H" -#include -#include using namespace amrex; From 011bb8647751243e43e2f5bb62c6ebce4eff0e20 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 28 Sep 2021 14:17:34 -0400 Subject: [PATCH 026/276] add host tag to reduction lambda to allow code to compile on GPUs --- Source/FlavoredNeutrinoContainerInit.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 876de472..f0131d2c 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -655,7 +655,7 @@ InitParticles(const TestParams* parms) } // get the minimum neutrino energy for calculating the timestep - Real pupt_min = amrex::ReduceMin(*this, [=] AMREX_GPU_DEVICE (const FlavoredNeutrinoContainer::ParticleType& p) -> Real { return p.rdata(PIdx::pupt); }); + Real pupt_min = amrex::ReduceMin(*this, [=] AMREX_GPU_HOST_DEVICE (const FlavoredNeutrinoContainer::ParticleType& p) -> Real { return p.rdata(PIdx::pupt); }); ParallelDescriptor::ReduceRealMin(pupt_min); #include "generated_files/FlavoredNeutrinoContainerInit.cpp_Vvac_fill" } From 1747b1bd848c8609b44ec5ab3d8907650eb09c59 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Wed, 1 Jun 2022 16:07:59 -0700 Subject: [PATCH 027/276] Code comparison ic (#70) * Revert "Added descriptive amrex::Error messages in generate_code.py (#53)" This reverts commit 97499f21cad18e1fc6aaf12cf49d1a94457090b0. * add host tag to reduction lambda to allow code to compile on GPUs * added input parameters for code comparison project * fix code comparison initial conditions. Broadcast random nubers so all grids have the same set. Multiply perturbation amplitude by 0.5 to match convention with polarization vectors * fix perturbation of antineutrinos to have correct sign relative to neutrinos * added local perturbations initial conditions Co-authored-by: Sherwood Richers Co-authored-by: Sherwood Richers --- Source/FlavoredNeutrinoContainerInit.cpp | 116 ++++++++++++++++++++++- Source/Parameters.H | 35 +++++++ 2 files changed, 150 insertions(+), 1 deletion(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index f0131d2c..11f1e955 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -1,6 +1,7 @@ #include "FlavoredNeutrinoContainer.H" #include "Constants.H" #include +#include using namespace amrex; @@ -124,6 +125,13 @@ namespace } } +// angular structure as determined by the gaussian profile of Martin et al (2019). + AMREX_GPU_HOST_DEVICE void gaussian_profile(Real* result, const Real sigma, const Real mu, const Real mu0){ + Real Ainverse = sigma * std::sqrt(M_PI/2.0) * std::erf(std::sqrt(2)/sigma); + Real A = 1.0 / Ainverse; + *result = 2.0 * A * std::exp(-(mu-mu0)*(mu-mu0) / (2.0*sigma*sigma)); + } + FlavoredNeutrinoContainer:: FlavoredNeutrinoContainer(const Geometry & a_geom, const DistributionMapping & a_dmap, @@ -147,12 +155,23 @@ InitParticles(const TestParams* parms) const int nlocs_per_cell = AMREX_D_TERM( parms->nppc[0], *parms->nppc[1], *parms->nppc[2]); - + + // array of direction vectors Gpu::ManagedVector > direction_vectors = uniform_sphere_xyz(parms->nphi_equator); auto* direction_vectors_p = direction_vectors.dataPtr(); int ndirs_per_loc = direction_vectors.size(); amrex::Print() << "Using " << ndirs_per_loc << " directions based on " << parms->nphi_equator << " directions at the equator." << std::endl; + // array of random numbers, one for each grid cell + int nrandom = parms->ncell[0] * parms->ncell[1] * parms->ncell[2]; + Gpu::ManagedVector random_numbers(nrandom); + if (ParallelDescriptor::IOProcessor()) + for(int i=0; isimulation_type==6){ + AMREX_ASSERT(NUM_FLAVORS==2); + AMREX_ASSERT(parms->ncell[0] == 1); + AMREX_ASSERT(parms->ncell[1] == 1); + + // set energy to 50 MeV + p.rdata(PIdx::pupt) = 50. * 1e6*CGSUnitsConst::eV; + p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); + p.rdata(PIdx::pupy) = u[1] * p.rdata(PIdx::pupt); + p.rdata(PIdx::pupz) = u[2] * p.rdata(PIdx::pupt); + + // get the number of each flavor in this particle. + Real angular_factor; + gaussian_profile(&angular_factor, parms->st6_sigma , u[2], parms->st6_mu0 ); + Real Nnue_thisparticle = parms->st6_nnue*scale_fac * angular_factor; + gaussian_profile(&angular_factor, parms->st6_sigmabar, u[2], parms->st6_mu0bar); + Real Nnua_thisparticle = parms->st6_nnua*scale_fac * angular_factor; + +// // set total number of neutrinos the particle has as the sum of the flavors + p.rdata(PIdx::N ) = Nnue_thisparticle; + p.rdata(PIdx::Nbar) = Nnua_thisparticle; + +// // set on-diagonals to have relative proportion of each flavor + p.rdata(PIdx::f00_Re) = 1; + p.rdata(PIdx::f11_Re) = 0; + p.rdata(PIdx::f00_Rebar) = 1; + p.rdata(PIdx::f11_Rebar) = 0; + +// // random perturbations to the off-diagonals + p.rdata(PIdx::f01_Re) = 0; + p.rdata(PIdx::f01_Im) = 0; + int Nz = parms->ncell[2]; + int amax = parms->st6_amax * Nz/2; + for(int a=-amax; a<=amax; a++){ + if(a==0) continue; + Real ka = 2.*M_PI * a / parms->Lz; + Real phase = ka*z + 2.*M_PI*random_numbers_p[a+Nz/2]; + Real B = parms->st6_amplitude / std::abs(float(a)); + p.rdata(PIdx::f01_Re) += 0.5 * B * cos(phase); + p.rdata(PIdx::f01_Im) += 0.5 * B * sin(phase); + } + + // Perturb the antineutrinos in a way that preserves the symmetries of the neutrino hamiltonian + p.rdata(PIdx::f01_Rebar) = p.rdata(PIdx::f01_Re); + p.rdata(PIdx::f01_Imbar) = -p.rdata(PIdx::f01_Im); + } + + //==============================// + // 7 - Code Comparison Gaussian // + //==============================// + else if(parms->simulation_type==7){ + AMREX_ASSERT(NUM_FLAVORS==2); + AMREX_ASSERT(parms->ncell[0] == 1); + AMREX_ASSERT(parms->ncell[1] == 1); + + // set energy to 50 MeV + p.rdata(PIdx::pupt) = 50. * 1e6*CGSUnitsConst::eV; + p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); + p.rdata(PIdx::pupy) = u[1] * p.rdata(PIdx::pupt); + p.rdata(PIdx::pupz) = u[2] * p.rdata(PIdx::pupt); + + // get the number of each flavor in this particle. + Real angular_factor; + gaussian_profile(&angular_factor, parms->st7_sigma , u[2], parms->st7_mu0 ); + Real Nnue_thisparticle = parms->st7_nnue*scale_fac * angular_factor; + gaussian_profile(&angular_factor, parms->st7_sigmabar, u[2], parms->st7_mu0bar); + Real Nnua_thisparticle = parms->st7_nnua*scale_fac * angular_factor; + +// // set total number of neutrinos the particle has as the sum of the flavors + p.rdata(PIdx::N ) = Nnue_thisparticle; + p.rdata(PIdx::Nbar) = Nnua_thisparticle; + +// // set on-diagonals to have relative proportion of each flavor + p.rdata(PIdx::f00_Re) = 1; + p.rdata(PIdx::f11_Re) = 0; + p.rdata(PIdx::f00_Rebar) = 1; + p.rdata(PIdx::f11_Rebar) = 0; + +// // random perturbations to the off-diagonals + p.rdata(PIdx::f01_Re) = 0; + p.rdata(PIdx::f01_Im) = 0; + int Nz = parms->ncell[2]; + Real zprime = z - parms->Lz; + Real P1 = parms->st7_amplitude * std::exp(-zprime*zprime/(2.*parms->st7_sigma_pert*parms->st7_sigma_pert)); + p.rdata(PIdx::f01_Re) = P1 / 2.0; + p.rdata(PIdx::f01_Im) = 0; + + // Perturb the antineutrinos in a way that preserves the symmetries of the neutrino hamiltonian + p.rdata(PIdx::f01_Rebar) = p.rdata(PIdx::f01_Re); + p.rdata(PIdx::f01_Imbar) = -p.rdata(PIdx::f01_Im); + } + else{ amrex::Error("Invalid simulation type"); } diff --git a/Source/Parameters.H b/Source/Parameters.H index 170d0a8e..460d54d2 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -50,6 +50,20 @@ struct TestParams : public amrex::Gpu::Managed Real st5_avgE_MeV; Real st5_amplitude; + // simulation_type==6 + Real st6_nnue , st6_nnua; + Real st6_sigma, st6_sigmabar; + Real st6_mu0, st6_mu0bar; + Real st6_amplitude; + Real st6_amax; + + // simulation_type==7 + Real st7_nnue , st7_nnua; + Real st7_sigma, st7_sigmabar; + Real st7_mu0, st7_mu0bar; + Real st7_amplitude; + Real st7_sigma_pert; + void Initialize(){ ParmParse pp; pp.get("simulation_type", simulation_type); @@ -130,6 +144,27 @@ struct TestParams : public amrex::Gpu::Managed pp.get("st5_avgE_MeV",st5_avgE_MeV); pp.get("st5_amplitude",st5_amplitude); } + + if(simulation_type==6){ + pp.get("st6_amplitude",st6_amplitude); + pp.get("st6_amax",st6_amax); + pp.get("st6_nnue",st6_nnue); + pp.get("st6_nnua",st6_nnua); + pp.get("st6_sigma",st6_sigma); + pp.get("st6_sigmabar",st6_sigmabar); + pp.get("st6_mu0",st6_mu0); + pp.get("st6_mu0bar",st6_mu0bar); + } + if(simulation_type==7){ + pp.get("st7_amplitude",st7_amplitude); + pp.get("st7_nnue",st7_nnue); + pp.get("st7_nnua",st7_nnua); + pp.get("st7_sigma",st7_sigma); + pp.get("st7_sigmabar",st7_sigmabar); + pp.get("st7_mu0",st7_mu0); + pp.get("st7_mu0bar",st7_mu0bar); + pp.get("st7_sigma_pert",st7_sigma_pert); + } } }; From 5cfbe2ecf54191676fba222b2f0761497d8209bd Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Thu, 2 Jun 2022 05:30:34 -0700 Subject: [PATCH 028/276] Jenkins ci (#71) * first attempt at files to run Emu on Jenkins * remove python package to allow python-is-python3 to install without conflicts * add sympy python package * fix capitalization * tiny file rename * use GPU-capable dockerfile, although GPU usage in dockerfile still has to be enabled somehow * enable GPU usage * enable GPU use in jenkins? * remove debugging from compile and use proper name for compiled executable in jenkins script * update dockerfile to version that includes nvidia development tools * capitalization issue Co-authored-by: Sherwood Richers --- .travis.yml | 36 ---------- Dockerfile | 7 ++ Jenkinsfile | 65 +++++++++++++++++++ ...GNUmakefile_travis => GNUmakefile_jenkins} | 5 +- 4 files changed, 75 insertions(+), 38 deletions(-) delete mode 100644 .travis.yml create mode 100644 Dockerfile create mode 100644 Jenkinsfile rename makefiles/{GNUmakefile_travis => GNUmakefile_jenkins} (83%) diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index bd44f824..00000000 --- a/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ -dist: xenial - -language: python - -python: - - "3.7-dev" - -before_install: - - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") - - pip install sympy numpy - -addons: - apt: - packages: - - mpich - - libmpich-dev - - g++ - - gfortran - - libgmp-dev - - libmpfr-dev - - pandoc - - doxygen - - texlive - - texlive-latex-extra - - texlive-lang-cjk - - latexmk - -install: -- git submodule init; git submodule update -- cp makefiles/GNUmakefile_travis Exec/GNUmakefile; cd Exec; make - -script: -- cd Exec; mpirun -np 2 ./main3d.gnu.DEBUG.TPROF.MPI.ex ../sample_inputs/inputs_msw_test; python ../Scripts/tests/msw_test.py -- cd Exec; mpirun -np 2 ./main3d.gnu.DEBUG.TPROF.MPI.ex ../sample_inputs/inputs_bipolar_test -- cd Exec; mpirun -np 2 ./main3d.gnu.DEBUG.TPROF.MPI.ex ../sample_inputs/inputs_fast_flavor; python ../Scripts/tests/fast_flavor_test.py -- cd Exec; mpirun -np 2 ./main3d.gnu.DEBUG.TPROF.MPI.ex ../sample_inputs/inputs_fast_flavor_nonzerok; python ../Scripts/tests/fast_flavor_k_test.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..bc578e3c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM nvidia/cuda:11.4.0-devel-ubuntu20.04 +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update +RUN apt-get install -y python3 python3-pip gfortran build-essential libhdf5-openmpi-dev openmpi-bin pkg-config libopenmpi-dev openmpi-bin libblas-dev liblapack-dev libpnetcdf-dev git python-is-python3 +RUN pip3 install numpy matplotlib h5py scipy sympy +ENV USER=jenkins +ENV LOGNAME=jenkins diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..10aaa4fd --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,65 @@ +pipeline { + triggers { pollSCM('') } // Run tests whenever a new commit is detected. + agent { dockerfile {args '--gpus all'}} // Use the Dockerfile defined in the root Flash-X directory + stages { + + //=============================// + // Set up submodules and amrex // + //=============================// + stage('Prerequisites'){ steps{ + sh 'mpicc -v' + sh 'nvidia-smi' + sh 'nvcc -V' + sh 'git submodule update --init' + sh 'cp makefiles/GNUmakefile_jenkins Exec/GNUmakefile' + dir('Exec'){ + sh 'make generate; make -j12' + } + }} + + + + //=======// + // Tests // + //=======// + stage('MSW'){ steps{ + dir('Exec'){ + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_msw_test' + sh 'python ../Scripts/tests/msw_test.py' + } + }} + + stage('Bipolar'){ steps{ + dir('Exec'){ + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_bipolar_test' + } + }} + + stage('Fast Flavor'){ steps{ + dir('Exec'){ + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor' + sh 'python ../Scripts/tests/fast_flavor_test.py' + } + }} + + stage('Fast Flavor k'){ steps{ + dir('Exec'){ + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor_nonzerok' + sh 'python ../Scripts/tests/fast_flavor_k_test.py' + } + }} + + } // stages{ + + post { + always { + cleanWs( + cleanWhenNotBuilt: true, + deleteDirs: true, + disableDeferredWipeout: false, + notFailBuild: true, + patterns: [[pattern: 'amrex', type: 'EXCLUDE']] ) // allow amrex to be cached + } + } + +} // pipeline{ diff --git a/makefiles/GNUmakefile_travis b/makefiles/GNUmakefile_jenkins similarity index 83% rename from makefiles/GNUmakefile_travis rename to makefiles/GNUmakefile_jenkins index 880ccf94..62488162 100644 --- a/makefiles/GNUmakefile_travis +++ b/makefiles/GNUmakefile_jenkins @@ -9,12 +9,13 @@ SHAPE_FACTOR_ORDER = 2 COMP = gnu -DEBUG = TRUE +DEBUG = FALSE USE_MPI = TRUE USE_OMP = FALSE USE_ACC = FALSE -USE_CUDA = FALSE +USE_CUDA = TRUE +AMREX_CUDA_ARCH=60 TINY_PROFILE = TRUE USE_PARTICLES = TRUE From 587105e41c63da66c84395a2df2f0ec1a366f3a5 Mon Sep 17 00:00:00 2001 From: "Don E. Willcox" Date: Fri, 3 Jun 2022 14:57:37 -0700 Subject: [PATCH 029/276] Update to the latest AMReX development branch (#72) * Update Emu to use the latest AMReX with some modifications to the AMReX TimeIntegrator class. * Update RNG API to use the new AMReX RNG interface (needed for DPC++ compiler). * Compile with C++17 standard. * Fixed compiler warnings when compiling with DEBUG=TRUE. * Update RNG API for CPU-side RNG call. * Update AMReX submodule hash to latest AMReX development branch. * Until this line gets merged into AMReX for the Particle Saxpy function in the time integrator, this is a workaround. --- Make.Emu | 2 + Source/Evolve.H | 2 +- Source/Evolve.cpp | 2 +- Source/FlavoredNeutrinoContainer.cpp | 3 - Source/FlavoredNeutrinoContainerInit.cpp | 83 +++++++++++++----------- Source/IO.cpp | 2 - Source/main.cpp | 12 ++-- amrex | 2 +- 8 files changed, 56 insertions(+), 52 deletions(-) diff --git a/Make.Emu b/Make.Emu index 24d9d145..1b756472 100644 --- a/Make.Emu +++ b/Make.Emu @@ -6,6 +6,8 @@ TOP := $(EMU_HOME) EBASE := main +CXXSTD := c++17 + include $(AMREX_HOME)/Tools/GNUMake/Make.defs Bdirs := Source diff --git a/Source/Evolve.H b/Source/Evolve.H index ed387f1a..55f23dec 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -24,7 +24,7 @@ namespace GIdx extern amrex::Vector names; void Initialize(); -}; +} amrex::Real compute_dt(const amrex::Geometry& geom, const amrex::Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor, const Real max_adaptive_speedup); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index a1fbfe98..3a8c92c0 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -19,7 +19,7 @@ namespace GIdx } } -Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor, const Real max_adaptive_speedup) +Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const Real flavor_cfl_factor, const Real max_adaptive_speedup) { AMREX_ASSERT(cfl_factor > 0.0 || flavor_cfl_factor > 0.0); diff --git a/Source/FlavoredNeutrinoContainer.cpp b/Source/FlavoredNeutrinoContainer.cpp index e53c3386..cc15a333 100644 --- a/Source/FlavoredNeutrinoContainer.cpp +++ b/Source/FlavoredNeutrinoContainer.cpp @@ -91,9 +91,6 @@ Renormalize(const TestParams* parms) const int lev = 0; - const auto dxi = Geom(lev).InvCellSizeArray(); - const auto plo = Geom(lev).ProbLoArray(); - #ifdef _OPENMP #pragma omp parallel #endif diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 11f1e955..7ec4e10d 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -47,7 +47,7 @@ Gpu::ManagedVector > uniform_sphere_xyz(int nphi_at_equator){ Real minerbo_residual(const Real fluxfac, const Real Z){ return fluxfac - 1.0/std::tanh(Z) + 1.0 / Z; } -Real minerbo_residual_derivative(const Real fluxfac, const Real Z){ +Real minerbo_residual_derivative(const Real /* fluxfac */, const Real Z){ return 1.0/(std::sinh(Z)*std::sinh(Z)) - 1.0/(Z*Z); } Real minerbo_Z(const Real fluxfac){ @@ -97,20 +97,22 @@ namespace r[2] = (0.5+iz_part)/nz; } - AMREX_GPU_HOST_DEVICE void get_random_direction(Real* u) { +/* // Commented only to avoid compiler warning -- we currently do not use this function + AMREX_GPU_HOST_DEVICE void get_random_direction(Real* u, amrex::RandomEngine const& engine) { // Returns components of u normalized so |u| = 1 // in random directions in 3D space - Real theta = amrex::Random() * MathConst::pi; // theta from [0, pi) - Real phi = amrex::Random() * 2.0 * MathConst::pi; // phi from [0, 2*pi) + Real theta = amrex::Random(engine) * MathConst::pi; // theta from [0, pi) + Real phi = amrex::Random(engine) * 2.0 * MathConst::pi; // phi from [0, 2*pi) u[0] = std::sin(theta) * std::cos(phi); u[1] = std::sin(theta) * std::sin(phi); u[2] = std::cos(theta); } +*/ - AMREX_GPU_HOST_DEVICE void symmetric_uniform(Real* Usymmetric){ - *Usymmetric = 2. * (amrex::Random()-0.5); + AMREX_GPU_HOST_DEVICE void symmetric_uniform(Real* Usymmetric, amrex::RandomEngine const& engine){ + *Usymmetric = 2. * (amrex::Random(engine)-0.5); } // angular structure as determined by the Minerbo closure @@ -165,12 +167,15 @@ InitParticles(const TestParams* parms) // array of random numbers, one for each grid cell int nrandom = parms->ncell[0] * parms->ncell[1] * parms->ncell[2]; Gpu::ManagedVector random_numbers(nrandom); - if (ParallelDescriptor::IOProcessor()) - for(int i=0; i u = direction_vectors_p[i_direction]; - //get_random_direction(u); + //get_random_direction(u, engine); //=========================// // VACUUM OSCILLATION TEST // @@ -415,9 +420,9 @@ InitParticles(const TestParams* parms) // set particle weight such that density is // 10 dm2 c^4 / (2 sqrt(2) GF E) Real dm2 = (parms->mass2-parms->mass1)*(parms->mass2-parms->mass1); //g^2 - double omega = dm2*PhysConst::c4 / (2.*p.rdata(PIdx::pupt)); + // double omega = dm2*PhysConst::c4 / (2.*p.rdata(PIdx::pupt)); double ndens = 10. * dm2*PhysConst::c4 / (2.*sqrt(2.) * PhysConst::GF * p.rdata(PIdx::pupt)); - double mu = sqrt(2.)*PhysConst::GF * ndens; + // double mu = sqrt(2.)*PhysConst::GF * ndens; p.rdata(PIdx::N) = ndens * scale_fac; p.rdata(PIdx::Nbar) = ndens * scale_fac; } @@ -476,15 +481,15 @@ InitParticles(const TestParams* parms) // perturbation parameters Real lambda = domain_length_z/(Real)parms->st3_wavelength_fraction_of_domain; - Real k = (2.*M_PI) / lambda; + Real nu_k = (2.*M_PI) / lambda; // Set particle flavor p.rdata(PIdx::f00_Re) = 1.0; - p.rdata(PIdx::f01_Re) = parms->st3_amplitude*sin(k*p.pos(2)); + p.rdata(PIdx::f01_Re) = parms->st3_amplitude*sin(nu_k*p.pos(2)); p.rdata(PIdx::f01_Im) = 0.0; p.rdata(PIdx::f11_Re) = 0.0; p.rdata(PIdx::f00_Rebar) = 1.0; - p.rdata(PIdx::f01_Rebar) = parms->st3_amplitude*sin(k*p.pos(2)); + p.rdata(PIdx::f01_Rebar) = parms->st3_amplitude*sin(nu_k*p.pos(2)); p.rdata(PIdx::f01_Imbar) = 0.0; p.rdata(PIdx::f11_Rebar) = 0.0; @@ -514,7 +519,7 @@ InitParticles(const TestParams* parms) Real dm2 = (parms->mass2-parms->mass1)*(parms->mass2-parms->mass1); //g^2 Real omega = dm2*PhysConst::c4 / (2.* p.rdata(PIdx::pupt)); Real mu_ndens = sqrt(2.) * PhysConst::GF; // SI potential divided by the number density - Real ndens = (omega+k*PhysConst::hbarc) / (2.*mu_ndens); // want omega/2mu to be 1 + Real ndens = (omega+nu_k*PhysConst::hbarc) / (2.*mu_ndens); // want omega/2mu to be 1 p.rdata(PIdx::N) = ndens * scale_fac * (1. + u[2]); p.rdata(PIdx::Nbar) = ndens * scale_fac * (1. - u[2]); } @@ -527,10 +532,10 @@ InitParticles(const TestParams* parms) // Set particle flavor Real rand1, rand2, rand3, rand4; - symmetric_uniform(&rand1); - symmetric_uniform(&rand2); - symmetric_uniform(&rand3); - symmetric_uniform(&rand4); + symmetric_uniform(&rand1, engine); + symmetric_uniform(&rand2, engine); + symmetric_uniform(&rand3, engine); + symmetric_uniform(&rand4, engine); p.rdata(PIdx::f00_Re) = 1.0; p.rdata(PIdx::f01_Re) = parms->st4_amplitude*rand1; p.rdata(PIdx::f01_Im) = parms->st4_amplitude*rand2; @@ -540,10 +545,10 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::f01_Imbar) = parms->st4_amplitude*rand4; p.rdata(PIdx::f11_Rebar) = 0.0; #if (NUM_FLAVORS==3) - symmetric_uniform(&rand1); - symmetric_uniform(&rand2); - symmetric_uniform(&rand3); - symmetric_uniform(&rand4); + symmetric_uniform(&rand1, engine); + symmetric_uniform(&rand2, engine); + symmetric_uniform(&rand3, engine); + symmetric_uniform(&rand4, engine); p.rdata(PIdx::f22_Re) = 0.0; p.rdata(PIdx::f22_Rebar) = 0.0; p.rdata(PIdx::f02_Re) = parms->st4_amplitude*rand1; @@ -635,30 +640,30 @@ InitParticles(const TestParams* parms) // random perturbations to the off-diagonals Real rand; - symmetric_uniform(&rand); + symmetric_uniform(&rand, engine); p.rdata(PIdx::f01_Re) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); - symmetric_uniform(&rand); + symmetric_uniform(&rand, engine); p.rdata(PIdx::f01_Im) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); - symmetric_uniform(&rand); + symmetric_uniform(&rand, engine); p.rdata(PIdx::f01_Rebar) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); - symmetric_uniform(&rand); + symmetric_uniform(&rand, engine); p.rdata(PIdx::f01_Imbar) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); #if NUM_FLAVORS==3 - symmetric_uniform(&rand); + symmetric_uniform(&rand, engine); p.rdata(PIdx::f02_Re) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); - symmetric_uniform(&rand); + symmetric_uniform(&rand, engine); p.rdata(PIdx::f02_Im) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); - symmetric_uniform(&rand); + symmetric_uniform(&rand, engine); p.rdata(PIdx::f12_Re) = parms->st5_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); - symmetric_uniform(&rand); + symmetric_uniform(&rand, engine); p.rdata(PIdx::f12_Im) = parms->st5_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); - symmetric_uniform(&rand); + symmetric_uniform(&rand, engine); p.rdata(PIdx::f02_Rebar) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); - symmetric_uniform(&rand); + symmetric_uniform(&rand, engine); p.rdata(PIdx::f02_Imbar) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); - symmetric_uniform(&rand); + symmetric_uniform(&rand, engine); p.rdata(PIdx::f12_Rebar) = parms->st5_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); - symmetric_uniform(&rand); + symmetric_uniform(&rand, engine); p.rdata(PIdx::f12_Imbar) = parms->st5_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); #endif } diff --git a/Source/IO.cpp b/Source/IO.cpp index a1b09cea..a115fdfb 100644 --- a/Source/IO.cpp +++ b/Source/IO.cpp @@ -19,8 +19,6 @@ WritePlotFile (const amrex::MultiFab& state, BoxArray grids = state.boxArray(); grids.convert(IntVect()); - const DistributionMapping& dmap = state.DistributionMap(); - const std::string& plotfilename = amrex::Concatenate("plt", step); amrex::Print() << " Writing plotfile " << plotfilename << "\n"; diff --git a/Source/main.cpp b/Source/main.cpp index edd2d6c3..b17266da 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -126,10 +126,10 @@ void evolve_flavor(const TestParams* parms) amrex::Print() << "Done. " << std::endl; - TimeIntegrator integrator(neutrinos_old, neutrinos_new, initial_time, initial_step); + TimeIntegrator integrator(neutrinos_old); // Create a RHS source function we will integrate - auto source_fun = [&] (FlavoredNeutrinoContainer& neutrinos_rhs, const FlavoredNeutrinoContainer& neutrinos, Real time) { + auto source_fun = [&] (FlavoredNeutrinoContainer& neutrinos_rhs, const FlavoredNeutrinoContainer& neutrinos, Real /* time */) { /* Evaluate the neutrino distribution matrix RHS */ // Step 1: Deposit Particle Data to Mesh & fill domain boundaries/ghost cells @@ -155,8 +155,8 @@ void evolve_flavor(const TestParams* parms) auto post_timestep_fun = [&] () { /* Post-timestep function. The integrator new-time data is the latest data available. */ - // Get the latest neutrino data - auto& neutrinos = integrator.get_new_data(); + // Use the latest-time neutrino data + auto& neutrinos = neutrinos_new; // Update the new time particle locations in the domain with their // integrated coordinates. @@ -210,7 +210,7 @@ void evolve_flavor(const TestParams* parms) Real start_time = amrex::second(); - integrator.integrate(starting_dt, parms->end_time, parms->nsteps); + integrator.integrate(neutrinos_old, neutrinos_new, initial_time, starting_dt, parms->end_time, initial_step, parms->nsteps); Real stop_time = amrex::second(); Real advance_time = stop_time - start_time; @@ -230,6 +230,8 @@ int main(int argc, char* argv[]) { amrex::Initialize(argc,argv); + MFIter::allowMultipleMFIters(true); + // write build information to screen if (ParallelDescriptor::IOProcessor()) { writeBuildInfo(); diff --git a/amrex b/amrex index ee5c09d4..ff9e8324 160000 --- a/amrex +++ b/amrex @@ -1 +1 @@ -Subproject commit ee5c09d41e1f4bd9f1a3754b4c513b76322cd20e +Subproject commit ff9e832483f142a34da72f9bea5fed72e49fea33 From 12fce3cadbee3fc3cef062075162f65c81f37c49 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Thu, 16 Jun 2022 10:45:53 -0700 Subject: [PATCH 030/276] move cell size division from interpolation to deposition (#73) Co-authored-by: Sherwood Richers --- .gitignore | 3 ++- Scripts/symbolic_hermitians/generate_code.py | 15 +++++++-------- Source/Evolve.cpp | 5 +---- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 1c8c4b5f..1b24a0da 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,5 @@ Scripts/symbolic_hermitians/__pycache__ .pydevproject .settings .vscode -Source/generated_files \ No newline at end of file +Source/generated_files +*~ \ No newline at end of file diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index df07464d..b0f00a57 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -152,7 +152,7 @@ def delete_generated_files(): #=================================# tails = ["","bar"] string1 = "amrex::Gpu::Atomic::AddNoRet(&sarr(i, j, k, GIdx::" - string2 = "-start_comp), sx(i) * sy(j) * sz(k) * p.rdata(PIdx::" + string2 = "-start_comp), sx(i) * sy(j) * sz(k) * inv_cell_volume * p.rdata(PIdx::" string4 = [");", "*p.rdata(PIdx::pupx)/p.rdata(PIdx::pupt));", "*p.rdata(PIdx::pupy)/p.rdata(PIdx::pupt));", @@ -259,7 +259,6 @@ def delete_generated_files(): #============================# code = [] length = sympy.symbols("length",real=True) - cell_volume = sympy.symbols("cell_volume",real=True) rho = sympy.symbols("fab(i\,j\,k\,GIdx\:\:rho)",real=True) Ye = sympy.symbols("fab(i\,j\,k\,GIdx\:\:Ye)",real=True) mp = sympy.symbols("PhysConst\:\:Mp",real=True) @@ -269,7 +268,7 @@ def delete_generated_files(): N = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::N{}{}_{})") Nbar = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::N{}{}_{}bar)") HSI = (N-Nbar.conjugate()) - HSI.H[0,0] += rho*Ye/mp * cell_volume + HSI.H[0,0] += rho*Ye/mp V_adaptive2 = HSI.SU_vector_magnitude2() code.append("V_adaptive2 += "+sympy.cxxcode(sympy.simplify(V_adaptive2))+";") @@ -282,15 +281,15 @@ def delete_generated_files(): code.append("V_adaptive2 += "+sympy.cxxcode(sympy.simplify(V_adaptive2))+";") # put in the units - code.append("V_adaptive = sqrt(V_adaptive2)*"+sympy.cxxcode(sqrt2GF/cell_volume)+";") + code.append("V_adaptive = sqrt(V_adaptive2)*"+sympy.cxxcode(sqrt2GF)+";") # old "stupid" way of computing the timestep. # the factor of 2 accounts for potential worst-case effects of neutrinos and antineutrinos for i in range(args.N): code.append("V_stupid = max(V_stupid,"+sympy.cxxcode(N.H[i,i])+");") code.append("V_stupid = max(V_stupid,"+sympy.cxxcode(Nbar.H[i,i])+");") - code.append("V_stupid = max(V_stupid,"+sympy.cxxcode(rho*Ye/mp*cell_volume)+");") - code.append("V_stupid *= "+sympy.cxxcode(2.0*args.N*sqrt2GF/cell_volume)+";") + code.append("V_stupid = max(V_stupid,"+sympy.cxxcode(rho*Ye/mp)+");") + code.append("V_stupid *= "+sympy.cxxcode(2.0*args.N*sqrt2GF)+";") write_code(code, os.path.join(args.emu_home,"Source/generated_files","Evolve.cpp_compute_dt_fill")) #=======================================# @@ -309,7 +308,7 @@ def delete_generated_files(): Vlist = HermitianMatrix(args.N, "V{}{}_{}").header() Nlist = HermitianMatrix(args.N, "N{}{}_{}").header() Flist = [HermitianMatrix(args.N, "F"+d+"{}{}_{}").header() for d in direction] - rhoye = string_interp+"rho)*"+string_interp+"Ye)/PhysConst::Mp/inv_cell_volume" + rhoye = string_interp+"rho)*"+string_interp+"Ye)/PhysConst::Mp" code.append("double SI_partial, SI_partialbar, inside_parentheses;") code.append("") @@ -351,7 +350,7 @@ def sgn(t,var): else: line += " -= " - line += "sqrt(2.) * PhysConst::GF * inv_cell_volume * sx(i) * sy(j) * sz(k) * (inside_parentheses);" + line += "sqrt(2.) * PhysConst::GF * sx(i) * sy(j) * sz(k) * (inside_parentheses);" code.append(line) code.append("") write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_interpolate_from_mesh_fill")) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 3a8c92c0..f6f63d4e 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -23,9 +23,6 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta { AMREX_ASSERT(cfl_factor > 0.0 || flavor_cfl_factor > 0.0); - const auto dx = geom.CellSizeArray(); - const Real cell_volume = dx[0]*dx[1]*dx[2]; - // translation part of timestep limit const auto dxi = geom.CellSizeArray(); Real dt_translation = 0.0; @@ -92,6 +89,7 @@ void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, MultiFab& state { const auto plo = geom.ProbLoArray(); const auto dxi = geom.InvCellSizeArray(); + const Real inv_cell_volume = dxi[0]*dxi[1]*dxi[2]; // Create an alias of the MultiFab so ParticleToMesh only erases the quantities // that will be set by the neutrinos. @@ -129,7 +127,6 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M { const auto plo = geom.ProbLoArray(); const auto dxi = geom.InvCellSizeArray(); - const Real inv_cell_volume = dxi[0]*dxi[1]*dxi[2]; const int shape_factor_order_x = geom.Domain().length(0) > 1 ? SHAPE_FACTOR_ORDER : 0; const int shape_factor_order_y = geom.Domain().length(1) > 1 ? SHAPE_FACTOR_ORDER : 0; From f68d2ffcfcd75f19e85e2ee846904a6ffcc15b4c Mon Sep 17 00:00:00 2001 From: Sherwood Richers <5142652+srichers@users.noreply.github.com> Date: Fri, 26 Aug 2022 09:45:36 -0700 Subject: [PATCH 031/276] Hdf5 io (#76) (#77) * Hdf5 io (#76) * add in C HDF5 functions to write data to HDF5, including timestep and physical time as attributes * remove unnecessary change from last commit * changed format of steps and time info in particle hdf5 file to match that in the plotfile. Also, wrap including the hdf5 header with ifdefs to prevent the compile stage breaking when not using HDF5 * reduce complexity - do not write to particle file, and just grab metadata from the plot file. Requires that the plot file be present, but thats fine Co-authored-by: Sherwood Richers * Pointwise initial conditions (#79) * compute isotropic momentum vectors instead of direction vectors so arbitrary momenta can be read in * set momentum in one place, combine nearby if blocks * simplify code by setting everything to zero by default * moved st5-specific calculations to parameter file. May be an abuse of the use of lambdas, but it is convenient * some formatting cleanup and comments * remove support for correlated spatial perturbations to reduce initial conditions complexity * spacing changes and separating out perturbation from simulation type to reduce number of parameters and repeated code * remove perturbation logic from st3 initial conditions since perturbations are taken care of later on * restoring accidentally removed variable * forgot to add new parameters to bipolar test inputs * add parameter to distinguish between the filled sphere angular distribution and one read in from file * fix silly compile errors * allow compiling on all cores * fix angular_grid_type logic and turn wave perturbation parameter to an actual length instead of a fraction of the domain * restructured initial condition for msw test to be generated via python script instead of within Emu. Must convert the other tests as well * add bipolar test new initial conditions. Actually include the initial conditions scripts that should have been there last time * add fast flavor test * added nonzero k fast flavor test * move st4 over to separate initial conditions script * moved minerbo initial conditions to python scripts * remove simulation_type parameter Co-authored-by: Sherwood Richers Co-authored-by: Sherwood Richers --- Exec/GNUmakefile | 12 +- Jenkinsfile | 6 +- Scripts/initial_conditions/.gitignore | 2 + .../initial_condition_tools.py | 96 ++ Scripts/initial_conditions/st0_msw_test.py | 46 + .../initial_conditions/st1_bipolar_test.py | 50 + .../st2_2beam_fast_flavor.py | 52 + .../st3_2beam_fast_flavor_nonzerok.py | 54 + .../st4_linear_moment_ffi.py | 59 + Scripts/initial_conditions/st5_minerbo.py | 81 ++ Scripts/visualization/amrex_plot_tools.py | 6 +- Source/FlavoredNeutrinoContainerInit.cpp | 1001 +++++------------ Source/IO.cpp | 39 + Source/Parameters.H | 110 +- sample_inputs/inputs_1d_fiducial | 14 +- sample_inputs/inputs_bipolar_test | 6 +- sample_inputs/inputs_fast_flavor | 6 +- sample_inputs/inputs_fast_flavor_nonzerok | 8 +- sample_inputs/inputs_msw_test | 6 +- 19 files changed, 812 insertions(+), 842 deletions(-) create mode 100644 Scripts/initial_conditions/.gitignore create mode 100644 Scripts/initial_conditions/initial_condition_tools.py create mode 100644 Scripts/initial_conditions/st0_msw_test.py create mode 100644 Scripts/initial_conditions/st1_bipolar_test.py create mode 100644 Scripts/initial_conditions/st2_2beam_fast_flavor.py create mode 100644 Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py create mode 100644 Scripts/initial_conditions/st4_linear_moment_ffi.py create mode 100644 Scripts/initial_conditions/st5_minerbo.py diff --git a/Exec/GNUmakefile b/Exec/GNUmakefile index faffd2b4..3577d5e8 100644 --- a/Exec/GNUmakefile +++ b/Exec/GNUmakefile @@ -12,7 +12,10 @@ USE_MPI = TRUE USE_OMP = FALSE USE_ACC = FALSE USE_CUDA = FALSE -USE_HDF5 = FALSE +AMREX_CUDA_ARCH=60 + +USE_HDF5=FALSE +HDF5_HOME=/usr/local/hdf5-1.12.2_gnu9.4.0 TINY_PROFILE = TRUE USE_PARTICLES = TRUE @@ -22,11 +25,4 @@ PRECISION = DOUBLE Bpack := Blocs := . -ifeq ($(USE_HDF5), TRUE) -HDF5_HOME = -DEFINES += -DAMREX_USE_HDF5 -INCLUDE_LOCATIONS += $(HDF5_HOME)/include -LIBRARIES += -L$(HDF5_HOME)/lib -lhdf5 -lz -ldl -endif - include $(EMU_HOME)/Make.Emu diff --git a/Jenkinsfile b/Jenkinsfile index 10aaa4fd..4d401eb9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,7 +13,7 @@ pipeline { sh 'git submodule update --init' sh 'cp makefiles/GNUmakefile_jenkins Exec/GNUmakefile' dir('Exec'){ - sh 'make generate; make -j12' + sh 'make generate; make -j' } }} @@ -24,6 +24,7 @@ pipeline { //=======// stage('MSW'){ steps{ dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st0_msw_test.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_msw_test' sh 'python ../Scripts/tests/msw_test.py' } @@ -31,12 +32,14 @@ pipeline { stage('Bipolar'){ steps{ dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st1_bipolar_test.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_bipolar_test' } }} stage('Fast Flavor'){ steps{ dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st2_2beam_fast_flavor.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor' sh 'python ../Scripts/tests/fast_flavor_test.py' } @@ -44,6 +47,7 @@ pipeline { stage('Fast Flavor k'){ steps{ dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor_nonzerok' sh 'python ../Scripts/tests/fast_flavor_k_test.py' } diff --git a/Scripts/initial_conditions/.gitignore b/Scripts/initial_conditions/.gitignore new file mode 100644 index 00000000..8025bc04 --- /dev/null +++ b/Scripts/initial_conditions/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +*~ diff --git a/Scripts/initial_conditions/initial_condition_tools.py b/Scripts/initial_conditions/initial_condition_tools.py new file mode 100644 index 00000000..13dff1aa --- /dev/null +++ b/Scripts/initial_conditions/initial_condition_tools.py @@ -0,0 +1,96 @@ +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath+"/../visualization") + +# generate an array of theta,phi pairs that uniformily cover the surface of a sphere +# based on DOI: 10.1080/10586458.2003.10504492 section 3.3 but specifying n_j=0 instead of n +def uniform_sphere(nphi_at_equator): + assert(nphi_at_equator > 0) + + dtheta = np.pi * np.sqrt(3) / nphi_at_equator + + xyz = [] + theta = 0 + phi0 = 0 + while(theta < np.pi/2): + nphi = nphi_at_equator if theta==0 else int(round(nphi_at_equator * np.cos(theta))) + dphi = 2*np.pi/nphi + if(nphi==1): theta = np.pi/2 + + for iphi in range(nphi): + phi = phi0 = iphi*dphi + x = np.cos(theta) * np.cos(phi) + y = np.cos(theta) * np.sin(phi) + z = np.sin(theta) + xyz.append(np.array([x,y,z])) + if(theta>0): xyz.append(np.array([-x,-y,-z])) + theta += dtheta + phi0 += 0.5 * dphi # offset by half step so adjacent latitudes are not always aligned in longitude + + return np.array(xyz) + + +def write_particles(p, NF, filename): + with open(filename, "w") as f: + # write the number of flavors + f.write(str(NF)+"\n") + + for i in range(len(p)): + for j in range(len(p[i])): + f.write(str(p[i,j])+" ") + f.write("\n") + +# angular structure as determined by the Minerbo closure +# Z is a parameter determined by the flux factor +# mu is the cosine of the angle relative to the flux direction +# Coefficients set such that the expectation value is 1 +def minerbo_closure(Z, mu): + minfluxfac = 1e-3 + result = np.exp(Z*mu) + if(Z/3.0 > minfluxfac): + result *= Z/np.sinh(Z) + return result + + +# residual for the root finder +# Z needs to be bigger if residual is positive +# Minerbo (1978) (unfortunately under Elsevier paywall) +# Can also see Richers (2020) https://ui.adsabs.harvard.edu/abs/2020PhRvD.102h3017R +# Eq.41 (where a is Z), but in the non-degenerate limit +# k->0, eta->0, N->Z/(4pi sinh(Z)) (just to make it integrate to 1) +# minerbo_residual is the "f" equation between eq.42 and 43 +def minerbo_residual(fluxfac, Z): + return fluxfac - 1.0/np.tanh(Z) + 1.0 / Z + +def minerbo_residual_derivative(fluxfac, Z): + return 1.0/np.sinh(Z)**2 - 1.0/Z**2 + +def minerbo_Z(fluxfac): + # hard-code in these parameters because they are not + # really very important... + maxresidual = 1e-6 + maxcount = 20 + minfluxfac = 1e-3 + + # set the initial conditions + Z = 1.0 + + # catch the small flux factor case to prevent nans + if(fluxfac < minfluxfac): + Z = 3.*fluxfac + else: + residual = 1.0 + count = 0 + while(abs(residual)>maxresidual and countmaxresidual: + print("Failed to converge on a solution.") + assert(False) + + print("fluxfac=",fluxfac," Z=",Z) + return Z diff --git a/Scripts/initial_conditions/st0_msw_test.py b/Scripts/initial_conditions/st0_msw_test.py new file mode 100644 index 00000000..aecb56ce --- /dev/null +++ b/Scripts/initial_conditions/st0_msw_test.py @@ -0,0 +1,46 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../visualization") +from initial_condition_tools import uniform_sphere, write_particles +import amrex_plot_tools as amrex + +# generation parameters +# MUST MATCH THE INPUTS IN THE EMU INPUT FILE! +NF = 2 +nphi_equator = 1 +ndens_per_particle = 1 # cm^-3 +m2 = 0 # erg +m1 = -0.008596511*amrex.eV/amrex.clight**2 # g +theta12 = 33.82 * np.pi/180 + +# set energy so that a vacuum oscillation wavelength occurs over a distance of 1cm +dm2 = (m2-m1)**2 #g^2 +energy_erg = dm2*amrex.clight**4 * np.sin(2.*theta12) / (8.*np.pi*amrex.hbar*amrex.clight) # *1cm for units + +# get variable keys +rkey, ikey = amrex.get_particle_keys(ignore_pos=True) +nelements = len(rkey) + +# generate the grid of direction coordinates +phat = uniform_sphere(nphi_equator) +nparticles = len(phat) + +# generate the list of particle info +particles = np.zeros((nparticles,nelements)) +for ip in range(len(phat)): + p = particles[ip] + p[rkey["pupt"]] = energy_erg + p[rkey["pupx"]] = phat[ip,0] * energy_erg + p[rkey["pupy"]] = phat[ip,1] * energy_erg + p[rkey["pupz"]] = phat[ip,2] * energy_erg + p[rkey["N"] ] = ndens_per_particle + p[rkey["Nbar"]] = ndens_per_particle + p[rkey["f00_Re"]] = 1 + p[rkey["f00_Rebar"]] = 1 + + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st1_bipolar_test.py b/Scripts/initial_conditions/st1_bipolar_test.py new file mode 100644 index 00000000..8d98c742 --- /dev/null +++ b/Scripts/initial_conditions/st1_bipolar_test.py @@ -0,0 +1,50 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../visualization") +from initial_condition_tools import uniform_sphere, write_particles +import amrex_plot_tools as amrex + +# generation parameters +# MUST MATCH THE INPUTS IN THE EMU INPUT FILE! +NF = 2 +nphi_equator = 1 +m2 = 0 # erg +m1 = -0.008596511*amrex.eV/amrex.clight**2 # g + +# generate the grid of direction coordinates +phat = uniform_sphere(nphi_equator) +nparticles = len(phat) + +# set particle weight such that density is +# 10 dm2 c^4 / (2 sqrt(2) GF E) +energy_erg = 50 * 1e6*amrex.eV +dm2 = (m2-m1)**2 #g^2 +# double omega = dm2*PhysConst::c4 / (2.*p.rdata(PIdx::pupt)); +ndens = 10. * dm2*amrex.clight**4 / (2.*np.sqrt(2.) * amrex.GF * energy_erg) +# double mu = sqrt(2.)*PhysConst::GF * ndens +ndens_per_particle = ndens / nparticles # cm^-3 + +# get variable keys +rkey, ikey = amrex.get_particle_keys(ignore_pos=True) +nelements = len(rkey) + + +# generate the list of particle info +particles = np.zeros((nparticles,nelements)) +for ip in range(len(phat)): + p = particles[ip] + p[rkey["pupt"]] = energy_erg + p[rkey["pupx"]] = phat[ip,0] * energy_erg + p[rkey["pupy"]] = phat[ip,1] * energy_erg + p[rkey["pupz"]] = phat[ip,2] * energy_erg + p[rkey["N"] ] = ndens_per_particle + p[rkey["Nbar"]] = ndens_per_particle + p[rkey["f00_Re"]] = 1 + p[rkey["f00_Rebar"]] = 1 + + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st2_2beam_fast_flavor.py b/Scripts/initial_conditions/st2_2beam_fast_flavor.py new file mode 100644 index 00000000..83c74492 --- /dev/null +++ b/Scripts/initial_conditions/st2_2beam_fast_flavor.py @@ -0,0 +1,52 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../visualization") +from initial_condition_tools import uniform_sphere, write_particles +import amrex_plot_tools as amrex + +# generation parameters +# MUST MATCH THE INPUTS IN THE EMU INPUT FILE! +NF = 2 +nphi_equator = 1 +m2 = 0 # erg +m1 = -0.008596511*amrex.eV/amrex.clight**2 # g + +# generate the grid of direction coordinates +phat = uniform_sphere(nphi_equator) +nparticles = len(phat) + +# set particle weight such that density is +# 0.5 dm2 c^4 / (2 sqrt(2) GF E) +# to get maximal growth according to Chakraborty 2016 Equation 2.10 +energy_erg = 50 * 1e6*amrex.eV +dm2 = (m2-m1)**2 #g^2 +omega = dm2*amrex.clight**4 / (2.* energy_erg) +mu_ndens = np.sqrt(2.) * amrex.GF # SI potential divided by the number density +ndens = omega / (2.*mu_ndens) # want omega/2mu to be 1 +ndens_per_particle = ndens / nparticles # cm^-3 + +# get variable keys +rkey, ikey = amrex.get_particle_keys(ignore_pos=True) +nelements = len(rkey) + + +# generate the list of particle info +particles = np.zeros((nparticles,nelements)) +for ip in range(len(phat)): + u = phat[ip] + p = particles[ip] + p[rkey["pupt"]] = energy_erg + p[rkey["pupx"]] = u[0] * energy_erg + p[rkey["pupy"]] = u[1] * energy_erg + p[rkey["pupz"]] = u[2] * energy_erg + p[rkey["N"] ] = ndens_per_particle * (1. + u[2]) + p[rkey["Nbar"]] = ndens_per_particle * (1. - u[2]) + p[rkey["f00_Re"]] = 1 + p[rkey["f00_Rebar"]] = 1 + + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py b/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py new file mode 100644 index 00000000..b27b65a0 --- /dev/null +++ b/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py @@ -0,0 +1,54 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../visualization") +from initial_condition_tools import uniform_sphere, write_particles +import amrex_plot_tools as amrex + +# generation parameters +# MUST MATCH THE INPUTS IN THE EMU INPUT FILE! +NF = 2 +nphi_equator = 1 +m2 = 0 # erg +m1 = -0.008596511*amrex.eV/amrex.clight**2 # g +perturbation_wavelength_cm = 1 + +# generate the grid of direction coordinates +phat = uniform_sphere(nphi_equator) +nparticles = len(phat) + +# set particle weight such that density is +# 0.5 dm2 c^4 / (2 sqrt(2) GF E) +# to get maximal growth according to Chakraborty 2016 Equation 2.10 +energy_erg = 50 * 1e6*amrex.eV +dm2 = (m2-m1)**2 #g^2 +omega = dm2*amrex.clight**4 / (2.* energy_erg) +mu_ndens = np.sqrt(2.) * amrex.GF # SI potential divided by the number density +nu_k = (2.*np.pi) / perturbation_wavelength_cm +ndens = (omega + nu_k*amrex.hbar*amrex.clight) / (2.*mu_ndens) # want omega/2mu to be 1 +ndens_per_particle = ndens / nparticles # cm^-3 + +# get variable keys +rkey, ikey = amrex.get_particle_keys(ignore_pos=True) +nelements = len(rkey) + + +# generate the list of particle info +particles = np.zeros((nparticles,nelements)) +for ip in range(len(phat)): + u = phat[ip] + p = particles[ip] + p[rkey["pupt"]] = energy_erg + p[rkey["pupx"]] = u[0] * energy_erg + p[rkey["pupy"]] = u[1] * energy_erg + p[rkey["pupz"]] = u[2] * energy_erg + p[rkey["N"] ] = ndens_per_particle * (1. + u[2]) + p[rkey["Nbar"]] = ndens_per_particle * (1. - u[2]) + p[rkey["f00_Re"]] = 1 + p[rkey["f00_Rebar"]] = 1 + + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st4_linear_moment_ffi.py b/Scripts/initial_conditions/st4_linear_moment_ffi.py new file mode 100644 index 00000000..7364f65f --- /dev/null +++ b/Scripts/initial_conditions/st4_linear_moment_ffi.py @@ -0,0 +1,59 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../visualization") +from initial_condition_tools import uniform_sphere, write_particles +import amrex_plot_tools as amrex + +# generation parameters +# MUST MATCH THE INPUTS IN THE EMU INPUT FILE! +NF = 2 +nphi_equator = 16 +theta = 0 +thetabar = 3.14159265359 +phi = 0 +phibar=0 +ndens = 4.891290819e+32 +ndensbar = 4.891290819e+32 +fluxfac = .333333333333333 +fluxfacbar = .333333333333333 +energy_erg = 50 * 1e6*amrex.eV + +# generate the grid of direction coordinates +phat = uniform_sphere(nphi_equator) +nparticles = len(phat) + +# flux factor vectors +fhat = [np.cos(phi) *np.sin(theta ), + np.sin(phi) *np.sin(theta ), + np.cos(theta )] +fhatbar = [np.cos(phibar)*np.sin(thetabar), + np.sin(phibar)*np.sin(thetabar), + np.cos(thetabar)] + +# get variable keys +rkey, ikey = amrex.get_particle_keys(ignore_pos=True) +nelements = len(rkey) + +# generate the list of particle info +particles = np.zeros((nparticles,nelements)) +for ip in range(nparticles): + u = phat[ip] + p = particles[ip] + p[rkey["pupt"]] = energy_erg + p[rkey["pupx"]] = u[0] * energy_erg + p[rkey["pupy"]] = u[1] * energy_erg + p[rkey["pupz"]] = u[2] * energy_erg + costheta = fhat [0]*u[0] + fhat [1]*u[1] + fhat [2]*u[2] + costhetabar = fhatbar[0]*u[0] + fhatbar[1]*u[1] + fhatbar[2]*u[2] + + p[rkey["N" ]] = ndens /nparticles * (1. + 3.*fluxfac *costheta ); + p[rkey["Nbar"]] = ndensbar/nparticles * (1. + 3.*fluxfacbar*costhetabar); + p[rkey["f00_Re" ]] = 1 + p[rkey["f00_Rebar"]] = 1 + + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st5_minerbo.py b/Scripts/initial_conditions/st5_minerbo.py new file mode 100644 index 00000000..7bd5bde2 --- /dev/null +++ b/Scripts/initial_conditions/st5_minerbo.py @@ -0,0 +1,81 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../visualization") +from initial_condition_tools import uniform_sphere, write_particles, minerbo_Z, minerbo_closure +import amrex_plot_tools as amrex + +# generation parameters +# from ix=37 iy=86 iz=75 in Francois' data +NF = 2 +nphi_equator = 16 +nnue = 1.421954234999705e+33 +nnua = 1.9146237131657563e+33 +nnux = 1.9645407875568215e+33 +fnue = np.array([0.0974572, 0.04217632, -0.13433261]) +fnua = np.array([0.07237959, 0.03132354, -0.3446878]) +fnux = np.array([-0.02165833, 0.07431613, -0.53545951]) +energy_erg = 20.05473294163565 * 1e6*amrex.eV + +# flux factor +fluxfac_e = np.sqrt(np.sum(fnue**2)) +fluxfac_a = np.sqrt(np.sum(fnua**2)) +fluxfac_x = np.sqrt(np.sum(fnux**2)) + +# get the Z parameters for the Minerbo closure +Ze = minerbo_Z(fluxfac_e); +Za = minerbo_Z(fluxfac_a); +Zx = minerbo_Z(fluxfac_x); + +# generate the grid of direction coordinates +phat = uniform_sphere(nphi_equator) +nparticles = len(phat) + +# get variable keys +rkey, ikey = amrex.get_particle_keys(ignore_pos=True) +nelements = len(rkey) + +# generate the list of particle info +particles = np.zeros((nparticles,nelements)) +for ip in range(nparticles): + u = phat[ip] + p = particles[ip] + p[rkey["pupt"]] = energy_erg + p[rkey["pupx"]] = u[0] * energy_erg + p[rkey["pupy"]] = u[1] * energy_erg + p[rkey["pupz"]] = u[2] * energy_erg + + # get the cosine of the angle between the direction and each flavor's flux vector + mue = np.sum(fnue*u)/fluxfac_e if fluxfac_e>0 else 0 + mua = np.sum(fnua*u)/fluxfac_a if fluxfac_a>0 else 0 + mux = np.sum(fnux*u)/fluxfac_x if fluxfac_x>0 else 0 + + # get the number of each flavor in this particle. + # nnux contains the number density of mu+tau neutrinos+antineutrinos + # Nnux_thisparticle contains the number of EACH of mu/tau anti/neutrinos (hence the factor of 4) + Nnue_thisparticle = nnue/nparticles * minerbo_closure(Ze, mue) + Nnua_thisparticle = nnua/nparticles * minerbo_closure(Za, mua) + Nnux_thisparticle = nnux/nparticles * minerbo_closure(Zx, mux) / 4.0 + + # set total number of neutrinos the particle has as the sum of the flavors + p[rkey["N" ]] = Nnue_thisparticle + Nnux_thisparticle + p[rkey["Nbar"]] = Nnua_thisparticle + Nnux_thisparticle + if NF==3: + p[rkey["N" ]] += Nnux_thisparticle + p[rkey["Nbar"]] += Nnux_thisparticle + + # set on-diagonals to have relative proportion of each flavor + p[rkey["f00_Re"]] = Nnue_thisparticle / p[rkey["N" ]] + p[rkey["f11_Re"]] = Nnux_thisparticle / p[rkey["N" ]] + p[rkey["f00_Rebar"]] = Nnua_thisparticle / p[rkey["Nbar"]] + p[rkey["f11_Rebar"]] = Nnux_thisparticle / p[rkey["Nbar"]] + if NF==3: + p[rkey["f22_Re"]] = Nnux_thisparticle / p[rkey["N" ]] + p[rkey["f22_Rebar"]] = Nnux_thisparticle / p[rkey["Nbar"]] + + + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/visualization/amrex_plot_tools.py b/Scripts/visualization/amrex_plot_tools.py index a05ab691..fb359cf9 100644 --- a/Scripts/visualization/amrex_plot_tools.py +++ b/Scripts/visualization/amrex_plot_tools.py @@ -9,7 +9,7 @@ mp = 1.6726219e-24 # g GF = 1.1663787e-5 / (1e9*eV)**2 * (hbar*clight)**3 #erg cm^3 -def get_particle_keys(): +def get_particle_keys(ignore_pos=False): real_quantities = ["pos_x", "pos_y", "pos_z", @@ -33,6 +33,7 @@ def get_particle_keys(): "f01_Rebar", "f01_Imbar", "f11_Rebar"] + if ignore_pos: real_quantities = real_quantities[7:] rkey = {} for i, rlabel in enumerate(real_quantities): @@ -44,7 +45,7 @@ def get_particle_keys(): return rkey, ikey -def get_3flavor_particle_keys(): +def get_3flavor_particle_keys(ignore_pos=False): real_quantities = ["pos_x", "pos_y", "pos_z", @@ -78,6 +79,7 @@ def get_3flavor_particle_keys(): "f12_Rebar", "f12_Imbar", "f22_Rebar"] + if ignore_pos: real_quantities = real_quantities[7:] rkey = {} for i, rlabel in enumerate(real_quantities): diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 7ec4e10d..47089138 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -1,780 +1,339 @@ #include "FlavoredNeutrinoContainer.H" - #include "Constants.H" +#include "Constants.H" #include #include +#include using namespace amrex; -// generate an array of theta,phi pairs that uniformily cover the surface of a sphere -// based on DOI: 10.1080/10586458.2003.10504492 section 3.3 but specifying n_j=0 instead of n -Gpu::ManagedVector > uniform_sphere_xyz(int nphi_at_equator){ - AMREX_ASSERT(nphi_at_equator>0); - - Real dtheta = M_PI*std::sqrt(3)/nphi_at_equator; - - Gpu::ManagedVector > xyz; - Real theta = 0; - Real phi0 = 0; - while(theta < M_PI/2.){ - int nphi = theta==0 ? nphi_at_equator : lround(nphi_at_equator * std::cos(theta)); - Real dphi = 2.*M_PI/nphi; - if(nphi==1) theta = M_PI/2.; - - for(int iphi=0; iphi{x,y,z}); - // construct exactly opposing vectors to limit subtractive cancellation errors - // and be able to represent isotropy exactly (all odd moments == 0) - if(theta>0) xyz.push_back(GpuArray{-x,-y,-z}); - } - theta += dtheta; - phi0 = phi0 + 0.5*dphi; // offset by half step so adjacent latitudes are not always aligned in longitude - } +//=========================================// +// Particle distribution in momentum space // +//=========================================// + +Gpu::ManagedVector > read_particle_data(std::string filename){ + Gpu::ManagedVector > particle_data; + + // open the file as a stream + std::ifstream file(filename); + + // temporary string/stream + std::string line; + std::stringstream ss; + + // create zero particle + GpuArray temp_particle; + for(int i=0; i> NF_in; + AMREX_ASSERT(NF_in == NUM_FLAVORS); + + while(std::getline(file, line)){ + ss = std::stringstream(line); + + // skip over the first four attributes (x,y,z,t) + for(int i=4; i> temp_particle[i]; + particle_data.push_back(temp_particle); + } - return xyz; -} - -// residual for the root finder -// Z needs to be bigger if residual is positive -// Minerbo (1978) (unfortunately under Elsevier paywall) -// Can also see Richers (2020) https://ui.adsabs.harvard.edu/abs/2020PhRvD.102h3017R -// Eq.41 (where a is Z), but in the non-degenerate limit -// k->0, eta->0, N->Z/(4pi sinh(Z)) (just to make it integrate to 1) -// minerbo_residual is the "f" equation between eq.42 and 43 -Real minerbo_residual(const Real fluxfac, const Real Z){ - return fluxfac - 1.0/std::tanh(Z) + 1.0 / Z; -} -Real minerbo_residual_derivative(const Real /* fluxfac */, const Real Z){ - return 1.0/(std::sinh(Z)*std::sinh(Z)) - 1.0/(Z*Z); -} -Real minerbo_Z(const Real fluxfac){ - // hard-code in these parameters because they are not - // really very important... - Real maxresidual = 1e-6; - Real maxcount = 20; - Real minfluxfac = 1e-3; - - // set the initial conditions - Real Z = 1.0; - - // catch the small flux factor case to prevent nans - if(fluxfac < minfluxfac) - Z = 3.*fluxfac; - else{ - Real residual = 1.0; - int count = 0; - while(std::abs(residual)>maxresidual and countmaxresidual) - amrex::Error("Failed to converge on a solution."); - } - - amrex::Print() << "fluxfac="<nppc[0], - *parms->nppc[1], - *parms->nppc[2]); - - // array of direction vectors - Gpu::ManagedVector > direction_vectors = uniform_sphere_xyz(parms->nphi_equator); - auto* direction_vectors_p = direction_vectors.dataPtr(); - int ndirs_per_loc = direction_vectors.size(); - amrex::Print() << "Using " << ndirs_per_loc << " directions based on " << parms->nphi_equator << " directions at the equator." << std::endl; - - // array of random numbers, one for each grid cell - int nrandom = parms->ncell[0] * parms->ncell[1] * parms->ncell[2]; - Gpu::ManagedVector random_numbers(nrandom); - if (ParallelDescriptor::IOProcessor()) { - RandomEngine engine; - for (int i=0; isimulation_type==5){ - fluxfac_e = std::sqrt( - parms->st5_fxnue*parms->st5_fxnue + - parms->st5_fynue*parms->st5_fynue + - parms->st5_fznue*parms->st5_fznue ); - fluxfac_a = std::sqrt( - parms->st5_fxnua*parms->st5_fxnua + - parms->st5_fynua*parms->st5_fynua + - parms->st5_fznua*parms->st5_fznua ); - fluxfac_x = std::sqrt( - parms->st5_fxnux*parms->st5_fxnux + - parms->st5_fynux*parms->st5_fynux + - parms->st5_fznux*parms->st5_fznux ); - Ze = minerbo_Z(fluxfac_e); - Za = minerbo_Z(fluxfac_a); - Zx = minerbo_Z(fluxfac_x); - } + const int nlocs_per_cell = AMREX_D_TERM( parms->nppc[0], + *parms->nppc[1], + *parms->nppc[2]); + + // array of direction vectors + Gpu::ManagedVector > particle_data = read_particle_data(parms->particle_data_filename);; + auto* particle_data_p = particle_data.dataPtr(); + + // determine the number of directions per location + int ndirs_per_loc = particle_data.size(); + amrex::Print() << "Using " << ndirs_per_loc << " directions." << std::endl; + const Real scale_fac = dx[0]*dx[1]*dx[2]/nlocs_per_cell; + + // Loop over multifabs // #ifdef _OPENMP #pragma omp parallel #endif - for (MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) + for (MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) { - const Box& tile_box = mfi.tilebox(); + const Box& tile_box = mfi.tilebox(); - const auto lo = amrex::lbound(tile_box); - const auto hi = amrex::ubound(tile_box); + const auto lo = amrex::lbound(tile_box); + const auto hi = amrex::ubound(tile_box); - Gpu::ManagedVector counts(tile_box.numPts(), 0); - unsigned int* pcount = counts.dataPtr(); + Gpu::ManagedVector counts(tile_box.numPts(), 0); + unsigned int* pcount = counts.dataPtr(); - Gpu::ManagedVector offsets(tile_box.numPts()); - unsigned int* poffset = offsets.dataPtr(); + Gpu::ManagedVector offsets(tile_box.numPts()); + unsigned int* poffset = offsets.dataPtr(); - // Determine how many particles to add to the particle tile per cell - amrex::ParallelFor(tile_box, - [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - for (int i_part=0; i_partnppc, i_part); + get_position_unit_cell(r, parms->nppc, i_part); - Real x = plo[0] + (i + r[0])*dx[0]; - Real y = plo[1] + (j + r[1])*dx[1]; - Real z = plo[2] + (k + r[2])*dx[2]; + Real x = plo[0] + (i + r[0])*dx[0]; + Real y = plo[1] + (j + r[1])*dx[1]; + Real z = plo[2] + (k + r[2])*dx[2]; - if (x >= a_bounds.hi(0) || x < a_bounds.lo(0) || - y >= a_bounds.hi(1) || y < a_bounds.lo(1) || - z >= a_bounds.hi(2) || z < a_bounds.lo(2) ) continue; + if (x >= a_bounds.hi(0) || x < a_bounds.lo(0) || + y >= a_bounds.hi(1) || y < a_bounds.lo(1) || + z >= a_bounds.hi(2) || z < a_bounds.lo(2) ) continue; - int ix = i - lo.x; - int iy = j - lo.y; - int iz = k - lo.z; - int nx = hi.x-lo.x+1; - int ny = hi.y-lo.y+1; - int nz = hi.z-lo.z+1; - unsigned int uix = amrex::min(nx-1,amrex::max(0,ix)); - unsigned int uiy = amrex::min(ny-1,amrex::max(0,iy)); - unsigned int uiz = amrex::min(nz-1,amrex::max(0,iz)); - unsigned int cellid = (uix * ny + uiy) * nz + uiz; - pcount[cellid] += ndirs_per_loc; - } - }); - - // Determine total number of particles to add to the particle tile - Gpu::inclusive_scan(counts.begin(), counts.end(), offsets.begin()); - - int num_to_add = offsets[tile_box.numPts()-1]; - if (num_to_add == 0) continue; - - // this will be the particle ID for the first new particle in the tile - long new_pid; - ParticleType* pstruct; - #ifdef _OPENMP - #pragma omp critical - #endif - { - - auto& particles = GetParticles(lev); - auto& particle_tile = particles[std::make_pair(mfi.index(), mfi.LocalTileIndex())]; - - // Resize the particle container - auto old_size = particle_tile.GetArrayOfStructs().size(); - auto new_size = old_size + num_to_add; - particle_tile.resize(new_size); - - // get the next particle ID - new_pid = ParticleType::NextID(); - - // set the starting particle ID for the next tile of particles - ParticleType::NextID(new_pid + num_to_add); - - pstruct = particle_tile.GetArrayOfStructs()().data(); - } - - int procID = ParallelDescriptor::MyProc(); - - Real domain_length_z = Geom(lev).ProbLength(2); - - // Initialize particle data in the particle tile - amrex::ParallelForRNG(tile_box, - [=] AMREX_GPU_DEVICE (int i, int j, int k, amrex::RandomEngine const& engine) noexcept - { - int ix = i - lo.x; - int iy = j - lo.y; - int iz = k - lo.z; - int nx = hi.x-lo.x+1; - int ny = hi.y-lo.y+1; - int nz = hi.z-lo.z+1; - unsigned int uix = amrex::min(nx-1,amrex::max(0,ix)); - unsigned int uiy = amrex::min(ny-1,amrex::max(0,iy)); - unsigned int uiz = amrex::min(nz-1,amrex::max(0,iz)); - unsigned int cellid = (uix * ny + uiy) * nz + uiz; - - for (int i_loc=0; i_locnppc, i_loc); - - Real x = plo[0] + (i + r[0])*dx[0]; - Real y = plo[1] + (j + r[1])*dx[1]; - Real z = plo[2] + (k + r[2])*dx[2]; - - if (x >= a_bounds.hi(0) || x < a_bounds.lo(0) || - y >= a_bounds.hi(1) || y < a_bounds.lo(1) || - z >= a_bounds.hi(2) || z < a_bounds.lo(2) ) continue; - + int ix = i - lo.x; + int iy = j - lo.y; + int iz = k - lo.z; + int nx = hi.x-lo.x+1; + int ny = hi.y-lo.y+1; + int nz = hi.z-lo.z+1; + unsigned int uix = amrex::min(nx-1,amrex::max(0,ix)); + unsigned int uiy = amrex::min(ny-1,amrex::max(0,iy)); + unsigned int uiz = amrex::min(nz-1,amrex::max(0,iz)); + unsigned int cellid = (uix * ny + uiy) * nz + uiz; + pcount[cellid] += ndirs_per_loc; + } + }); - for(int i_direction=0; i_direction u = direction_vectors_p[i_direction]; - //get_random_direction(u, engine); - - //=========================// - // VACUUM OSCILLATION TEST // - //=========================// - if(parms->simulation_type==0){ - // set all particles to start in electron state (and anti-state) - // Set N to be small enough that self-interaction is not important - // Set all particle momenta to be such that one oscillation wavelength is 1cm - AMREX_ASSERT(NUM_FLAVORS==3 or NUM_FLAVORS==2); - - // Set particle flavor - p.rdata(PIdx::N) = 1.0; - p.rdata(PIdx::Nbar) = 1.0; - p.rdata(PIdx::f00_Re) = 1.0; - p.rdata(PIdx::f01_Re) = 0.0; - p.rdata(PIdx::f01_Im) = 0.0; - p.rdata(PIdx::f11_Re) = 0.0; - p.rdata(PIdx::f00_Rebar) = 1.0; - p.rdata(PIdx::f01_Rebar) = 0.0; - p.rdata(PIdx::f01_Imbar) = 0.0; - p.rdata(PIdx::f11_Rebar) = 0.0; - -#if (NUM_FLAVORS==3) - p.rdata(PIdx::f22_Re) = 0.0; - p.rdata(PIdx::f22_Rebar) = 0.0; - p.rdata(PIdx::f02_Re) = 0.0; - p.rdata(PIdx::f02_Im) = 0.0; - p.rdata(PIdx::f12_Re) = 0.0; - p.rdata(PIdx::f12_Im) = 0.0; - p.rdata(PIdx::f02_Rebar) = 0.0; - p.rdata(PIdx::f02_Imbar) = 0.0; - p.rdata(PIdx::f12_Rebar) = 0.0; - p.rdata(PIdx::f12_Imbar) = 0.0; -#endif + // Determine total number of particles to add to the particle tile + Gpu::inclusive_scan(counts.begin(), counts.end(), offsets.begin()); - // set momentum so that a vacuum oscillation wavelength occurs over a distance of 1cm - // Set particle velocity to c in a random direction - Real dm2 = (parms->mass2-parms->mass1)*(parms->mass2-parms->mass1); //g^2 - p.rdata(PIdx::pupt) = dm2*PhysConst::c4 * sin(2.*parms->theta12) / (8.*M_PI*PhysConst::hbarc); // *1cm for units - p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupy) = u[1] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupz) = u[2] * p.rdata(PIdx::pupt); - } - - //==========================// - // BIPOLAR OSCILLATION TEST // - //==========================// - else if(parms->simulation_type==1){ - AMREX_ASSERT(NUM_FLAVORS==3 or NUM_FLAVORS==2); - - // Set particle flavor - p.rdata(PIdx::f00_Re) = 1.0; - p.rdata(PIdx::f01_Re) = 0.0; - p.rdata(PIdx::f01_Im) = 0.0; - p.rdata(PIdx::f11_Re) = 0.0; - p.rdata(PIdx::f00_Rebar) = 1.0; - p.rdata(PIdx::f01_Rebar) = 0.0; - p.rdata(PIdx::f01_Imbar) = 0.0; - p.rdata(PIdx::f11_Rebar) = 0.0; - -#if (NUM_FLAVORS==3) - p.rdata(PIdx::f22_Re) = 0.0; - p.rdata(PIdx::f22_Rebar) = 0.0; - p.rdata(PIdx::f02_Re) = 0.0; - p.rdata(PIdx::f02_Im) = 0.0; - p.rdata(PIdx::f12_Re) = 0.0; - p.rdata(PIdx::f12_Im) = 0.0; - p.rdata(PIdx::f02_Rebar) = 0.0; - p.rdata(PIdx::f02_Imbar) = 0.0; - p.rdata(PIdx::f12_Rebar) = 0.0; - p.rdata(PIdx::f12_Imbar) = 0.0; -#endif + int num_to_add = offsets[tile_box.numPts()-1]; + if (num_to_add == 0) continue; - // set energy to 50 MeV to match Richers+(2019) - p.rdata(PIdx::pupt) = 50. * 1e6*CGSUnitsConst::eV; - p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupy) = u[1] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupz) = u[2] * p.rdata(PIdx::pupt); - - // set particle weight such that density is - // 10 dm2 c^4 / (2 sqrt(2) GF E) - Real dm2 = (parms->mass2-parms->mass1)*(parms->mass2-parms->mass1); //g^2 - // double omega = dm2*PhysConst::c4 / (2.*p.rdata(PIdx::pupt)); - double ndens = 10. * dm2*PhysConst::c4 / (2.*sqrt(2.) * PhysConst::GF * p.rdata(PIdx::pupt)); - // double mu = sqrt(2.)*PhysConst::GF * ndens; - p.rdata(PIdx::N) = ndens * scale_fac; - p.rdata(PIdx::Nbar) = ndens * scale_fac; - } - - //========================// - // 2-BEAM FAST FLAVOR TEST// - //========================// - else if(parms->simulation_type==2){ - AMREX_ASSERT(NUM_FLAVORS==3 or NUM_FLAVORS==2); - - // Set particle flavor - p.rdata(PIdx::f00_Re) = 1.0; - p.rdata(PIdx::f01_Re) = 0.0; - p.rdata(PIdx::f01_Im) = 0.0; - p.rdata(PIdx::f11_Re) = 0.0; - p.rdata(PIdx::f00_Rebar) = 1.0; - p.rdata(PIdx::f01_Rebar) = 0.0; - p.rdata(PIdx::f01_Imbar) = 0.0; - p.rdata(PIdx::f11_Rebar) = 0.0; - -#if (NUM_FLAVORS==3) - p.rdata(PIdx::f22_Re) = 0.0; - p.rdata(PIdx::f22_Rebar) = 0.0; - p.rdata(PIdx::f02_Re) = 0.0; - p.rdata(PIdx::f02_Im) = 0.0; - p.rdata(PIdx::f12_Re) = 0.0; - p.rdata(PIdx::f12_Im) = 0.0; - p.rdata(PIdx::f02_Rebar) = 0.0; - p.rdata(PIdx::f02_Imbar) = 0.0; - p.rdata(PIdx::f12_Rebar) = 0.0; - p.rdata(PIdx::f12_Imbar) = 0.0; + // this will be the particle ID for the first new particle in the tile + long new_pid; + ParticleType* pstruct; +#ifdef _OPENMP +#pragma omp critical #endif + { - // set energy to 50 MeV to match Richers+(2019) - p.rdata(PIdx::pupt) = 50. * 1e6*CGSUnitsConst::eV; - p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupy) = u[1] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupz) = u[2] * p.rdata(PIdx::pupt); - - // set particle weight such that density is - // 0.5 dm2 c^4 / (2 sqrt(2) GF E) - // to get maximal growth according to Chakraborty 2016 Equation 2.10 - Real dm2 = (parms->mass2-parms->mass1)*(parms->mass2-parms->mass1); //g^2 - Real omega = dm2*PhysConst::c4 / (2.* p.rdata(PIdx::pupt)); - Real mu_ndens = sqrt(2.) * PhysConst::GF; // SI potential divided by the number density - double ndens = omega / (2.*mu_ndens); // want omega/2mu to be 1 - p.rdata(PIdx::N) = ndens * scale_fac * (1. + u[2]); - p.rdata(PIdx::Nbar) = ndens * scale_fac * (1. - u[2]); - } - - //===============================// - // 3- k!=0 BEAM FAST FLAVOR TEST // - //===============================// - else if(parms->simulation_type==3){ - AMREX_ASSERT(NUM_FLAVORS==3 or NUM_FLAVORS==2); - - // perturbation parameters - Real lambda = domain_length_z/(Real)parms->st3_wavelength_fraction_of_domain; - Real nu_k = (2.*M_PI) / lambda; - - // Set particle flavor - p.rdata(PIdx::f00_Re) = 1.0; - p.rdata(PIdx::f01_Re) = parms->st3_amplitude*sin(nu_k*p.pos(2)); - p.rdata(PIdx::f01_Im) = 0.0; - p.rdata(PIdx::f11_Re) = 0.0; - p.rdata(PIdx::f00_Rebar) = 1.0; - p.rdata(PIdx::f01_Rebar) = parms->st3_amplitude*sin(nu_k*p.pos(2)); - p.rdata(PIdx::f01_Imbar) = 0.0; - p.rdata(PIdx::f11_Rebar) = 0.0; - -#if (NUM_FLAVORS==3) - //just perturbing the electron-muon flavor state, other terms can stay = 0.0 for simplicity - p.rdata(PIdx::f22_Re) = 0.0; - p.rdata(PIdx::f22_Rebar) = 0.0; - p.rdata(PIdx::f02_Re) = 0.0; - p.rdata(PIdx::f02_Im) = 0.0; - p.rdata(PIdx::f12_Re) = 0.0; - p.rdata(PIdx::f12_Im) = 0.0; - p.rdata(PIdx::f02_Rebar) = 0.0; - p.rdata(PIdx::f02_Imbar) = 0.0; - p.rdata(PIdx::f12_Rebar) = 0.0; - p.rdata(PIdx::f12_Imbar) = 0.0; -#endif + auto& particles = GetParticles(lev); + auto& particle_tile = particles[std::make_pair(mfi.index(), mfi.LocalTileIndex())]; - // set energy to 50 MeV to match Richers+(2019) - p.rdata(PIdx::pupt) = 50. * 1e6*CGSUnitsConst::eV; - p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupy) = u[1] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupz) = u[2] * p.rdata(PIdx::pupt); - - // set particle weight such that density is - // 0.5 dm2 c^4 / (2 sqrt(2) GF E) - // to get maximal growth according to Chakraborty 2016 Equation 2.10 - Real dm2 = (parms->mass2-parms->mass1)*(parms->mass2-parms->mass1); //g^2 - Real omega = dm2*PhysConst::c4 / (2.* p.rdata(PIdx::pupt)); - Real mu_ndens = sqrt(2.) * PhysConst::GF; // SI potential divided by the number density - Real ndens = (omega+nu_k*PhysConst::hbarc) / (2.*mu_ndens); // want omega/2mu to be 1 - p.rdata(PIdx::N) = ndens * scale_fac * (1. + u[2]); - p.rdata(PIdx::Nbar) = ndens * scale_fac * (1. - u[2]); - } - - //====================// - // 4- k!=0 RANDOMIZED // - //====================// - else if(parms->simulation_type==4){ - AMREX_ASSERT(NUM_FLAVORS==3 or NUM_FLAVORS==2); - - // Set particle flavor - Real rand1, rand2, rand3, rand4; - symmetric_uniform(&rand1, engine); - symmetric_uniform(&rand2, engine); - symmetric_uniform(&rand3, engine); - symmetric_uniform(&rand4, engine); - p.rdata(PIdx::f00_Re) = 1.0; - p.rdata(PIdx::f01_Re) = parms->st4_amplitude*rand1; - p.rdata(PIdx::f01_Im) = parms->st4_amplitude*rand2; - p.rdata(PIdx::f11_Re) = 0.0; - p.rdata(PIdx::f00_Rebar) = 1.0; - p.rdata(PIdx::f01_Rebar) = parms->st4_amplitude*rand3; - p.rdata(PIdx::f01_Imbar) = parms->st4_amplitude*rand4; - p.rdata(PIdx::f11_Rebar) = 0.0; -#if (NUM_FLAVORS==3) - symmetric_uniform(&rand1, engine); - symmetric_uniform(&rand2, engine); - symmetric_uniform(&rand3, engine); - symmetric_uniform(&rand4, engine); - p.rdata(PIdx::f22_Re) = 0.0; - p.rdata(PIdx::f22_Rebar) = 0.0; - p.rdata(PIdx::f02_Re) = parms->st4_amplitude*rand1; - p.rdata(PIdx::f02_Im) = parms->st4_amplitude*rand2; - p.rdata(PIdx::f12_Re) = 0; - p.rdata(PIdx::f12_Im) = 0; - p.rdata(PIdx::f02_Rebar) = parms->st4_amplitude*rand3; - p.rdata(PIdx::f02_Imbar) = parms->st4_amplitude*rand4; - p.rdata(PIdx::f12_Rebar) = 0; - p.rdata(PIdx::f12_Imbar) = 0; -#endif + // Resize the particle container + auto old_size = particle_tile.GetArrayOfStructs().size(); + auto new_size = old_size + num_to_add; + particle_tile.resize(new_size); - // set energy to 50 MeV to match Richers+(2019) - p.rdata(PIdx::pupt) = 50. * 1e6*CGSUnitsConst::eV; - p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupy) = u[1] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupz) = u[2] * p.rdata(PIdx::pupt); - - // set particle weight such that density is - // 0.5 dm2 c^4 / (2 sqrt(2) GF E) - // to get maximal growth according to Chakraborty 2016 Equation 2.10 - //Real dm2 = (parms->mass2-parms->mass1)*(parms->mass2-parms->mass1); //g^2 - //Real omega = dm2*PhysConst::c4 / (2.* p.rdata(PIdx::pupt)); - //Real mu_ndens = sqrt(2.) * PhysConst::GF; // SI potential divided by the number density - //Real k_expected = (2.*M_PI)/1.0;// corresponding to wavelength of 1cm - //Real ndens_fiducial = (omega+k_expected*PhysConst::hbarc) / (2.*mu_ndens); // want omega/2mu to be 1 - //amrex::Print() << "fiducial ndens would be " << ndens_fiducial << std::endl; - - Real ndens = parms->st4_ndens; - Real ndensbar = parms->st4_ndensbar; - Real fhat[3] = {cos(parms->st4_phi) *sin(parms->st4_theta ), - sin(parms->st4_phi) *sin(parms->st4_theta ), - cos(parms->st4_theta )}; - Real fhatbar[3] = {cos(parms->st4_phibar)*sin(parms->st4_thetabar), - sin(parms->st4_phibar)*sin(parms->st4_thetabar), - cos(parms->st4_thetabar)}; - Real costheta = fhat [0]*u[0] + fhat [1]*u[1] + fhat [2]*u[2]; - Real costhetabar = fhatbar[0]*u[0] + fhatbar[1]*u[1] + fhatbar[2]*u[2]; - - p.rdata(PIdx::N ) = ndens *scale_fac * (1. + 3.*parms->st4_fluxfac *costheta ); - p.rdata(PIdx::Nbar) = ndensbar*scale_fac * (1. + 3.*parms->st4_fluxfacbar*costhetabar); - } - - //====================// - // 5- Minerbo Closure // - //====================// - else if(parms->simulation_type==5){ - AMREX_ASSERT(NUM_FLAVORS==3 or NUM_FLAVORS==2); - - // set energy to 50 MeV - p.rdata(PIdx::pupt) = parms->st5_avgE_MeV * 1e6*CGSUnitsConst::eV; - p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupy) = u[1] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupz) = u[2] * p.rdata(PIdx::pupt); - - // get the cosine of the angle between the direction and each flavor's flux vector - Real mue = fluxfac_e>0 ? (parms->st5_fxnue*u[0] + parms->st5_fynue*u[1] + parms->st5_fznue*u[2])/fluxfac_e : 0; - Real mua = fluxfac_a>0 ? (parms->st5_fxnua*u[0] + parms->st5_fynua*u[1] + parms->st5_fznua*u[2])/fluxfac_a : 0; - Real mux = fluxfac_x>0 ? (parms->st5_fxnux*u[0] + parms->st5_fynux*u[1] + parms->st5_fznux*u[2])/fluxfac_x : 0; - - // get the number of each flavor in this particle. - // parms->st5_nnux contains the number density of mu+tau neutrinos+antineutrinos - // Nnux_thisparticle contains the number of EACH of mu/tau anti/neutrinos (hence the factor of 4) - Real angular_factor; - minerbo_closure(&angular_factor, Ze, mue); - Real Nnue_thisparticle = parms->st5_nnue*scale_fac * angular_factor; - minerbo_closure(&angular_factor, Za, mua); - Real Nnua_thisparticle = parms->st5_nnua*scale_fac * angular_factor; - minerbo_closure(&angular_factor, Zx, mux); - Real Nnux_thisparticle = parms->st5_nnux*scale_fac * angular_factor / 4.0; - - // set total number of neutrinos the particle has as the sum of the flavors - p.rdata(PIdx::N ) = Nnue_thisparticle + Nnux_thisparticle; - p.rdata(PIdx::Nbar) = Nnua_thisparticle + Nnux_thisparticle; -#if NUM_FLAVORS==3 - p.rdata(PIdx::N ) += Nnux_thisparticle; - p.rdata(PIdx::Nbar) += Nnux_thisparticle; -#endif + // get the next particle ID + new_pid = ParticleType::NextID(); + + // set the starting particle ID for the next tile of particles + ParticleType::NextID(new_pid + num_to_add); - // set on-diagonals to have relative proportion of each flavor - p.rdata(PIdx::f00_Re) = Nnue_thisparticle / p.rdata(PIdx::N ); - p.rdata(PIdx::f11_Re) = Nnux_thisparticle / p.rdata(PIdx::N ); - p.rdata(PIdx::f00_Rebar) = Nnua_thisparticle / p.rdata(PIdx::Nbar); - p.rdata(PIdx::f11_Rebar) = Nnux_thisparticle / p.rdata(PIdx::Nbar); + pstruct = particle_tile.GetArrayOfStructs()().data(); + } + + int procID = ParallelDescriptor::MyProc(); + + //===============================================// + // Initialize particle data in the particle tile // + //===============================================// + amrex::ParallelForRNG(tile_box, + [=] AMREX_GPU_DEVICE (int i, int j, int k, amrex::RandomEngine const& engine) noexcept + { + int ix = i - lo.x; + int iy = j - lo.y; + int iz = k - lo.z; + int nx = hi.x-lo.x+1; + int ny = hi.y-lo.y+1; + int nz = hi.z-lo.z+1; + unsigned int uix = amrex::min(nx-1,amrex::max(0,ix)); + unsigned int uiy = amrex::min(ny-1,amrex::max(0,iy)); + unsigned int uiz = amrex::min(nz-1,amrex::max(0,iz)); + unsigned int cellid = (uix * ny + uiy) * nz + uiz; + + for (int i_loc=0; i_locnppc, i_loc); + + Real x = plo[0] + (i + r[0])*dx[0]; + Real y = plo[1] + (j + r[1])*dx[1]; + Real z = plo[2] + (k + r[2])*dx[2]; + + if (x >= a_bounds.hi(0) || x < a_bounds.lo(0) || + y >= a_bounds.hi(1) || y < a_bounds.lo(1) || + z >= a_bounds.hi(2) || z < a_bounds.lo(2) ) continue; + + for(int i_direction=0; i_direction= 0); + AMREX_ASSERT(p.rdata(PIdx::Nbar ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::L ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::Lbar ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::f00_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::f11_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::f00_Rebar) >= 0); + AMREX_ASSERT(p.rdata(PIdx::f11_Rebar) >= 0); + Real trace = p.rdata(PIdx::f00_Re ) + p.rdata(PIdx::f11_Re ); + Real tracebar = p.rdata(PIdx::f00_Rebar) + p.rdata(PIdx::f11_Rebar); #if NUM_FLAVORS==3 - p.rdata(PIdx::f22_Re) = Nnux_thisparticle / p.rdata(PIdx::N ); - p.rdata(PIdx::f22_Rebar) = Nnux_thisparticle / p.rdata(PIdx::Nbar); + AMREX_ASSERT(p.rdata(PIdx::f22_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::f22_Rebar) >= 0); + trace += p.rdata(PIdx::f22_Re ); + tracebar += p.rdata(PIdx::f22_Rebar); #endif - - // random perturbations to the off-diagonals - Real rand; - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Re) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Im) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Rebar) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Imbar) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + AMREX_ASSERT(std::abs(trace -1)<1e-6); + AMREX_ASSERT(std::abs(tracebar-1)<1e-6); + + // Set particle position + p.pos(0) = x; + p.pos(1) = y; + p.pos(2) = z; + + // Set particle integrated position + p.rdata(PIdx::x) = x; + p.rdata(PIdx::y) = y; + p.rdata(PIdx::z) = z; + p.rdata(PIdx::time) = 0; + + // scale particle numbers based on number of points per cell and the cell volume + p.rdata(PIdx::N ) *= scale_fac; + p.rdata(PIdx::Nbar) *= scale_fac; + + //=====================// + // Apply Perturbations // + //=====================// + if(parms->perturbation_type == 0){ + // random perturbations to the off-diagonals + Real rand; + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f01_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); #if NUM_FLAVORS==3 - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Re) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Im) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Re) = parms->st5_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Im) = parms->st5_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Rebar) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Imbar) = parms->st5_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Rebar) = parms->st5_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Imbar) = parms->st5_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); #endif - } - - //============================// - // 6 - Code Comparison Random // - //============================// - else if(parms->simulation_type==6){ - AMREX_ASSERT(NUM_FLAVORS==2); - AMREX_ASSERT(parms->ncell[0] == 1); - AMREX_ASSERT(parms->ncell[1] == 1); - - // set energy to 50 MeV - p.rdata(PIdx::pupt) = 50. * 1e6*CGSUnitsConst::eV; - p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupy) = u[1] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupz) = u[2] * p.rdata(PIdx::pupt); - - // get the number of each flavor in this particle. - Real angular_factor; - gaussian_profile(&angular_factor, parms->st6_sigma , u[2], parms->st6_mu0 ); - Real Nnue_thisparticle = parms->st6_nnue*scale_fac * angular_factor; - gaussian_profile(&angular_factor, parms->st6_sigmabar, u[2], parms->st6_mu0bar); - Real Nnua_thisparticle = parms->st6_nnua*scale_fac * angular_factor; - -// // set total number of neutrinos the particle has as the sum of the flavors - p.rdata(PIdx::N ) = Nnue_thisparticle; - p.rdata(PIdx::Nbar) = Nnua_thisparticle; - -// // set on-diagonals to have relative proportion of each flavor - p.rdata(PIdx::f00_Re) = 1; - p.rdata(PIdx::f11_Re) = 0; - p.rdata(PIdx::f00_Rebar) = 1; - p.rdata(PIdx::f11_Rebar) = 0; - -// // random perturbations to the off-diagonals - p.rdata(PIdx::f01_Re) = 0; - p.rdata(PIdx::f01_Im) = 0; - int Nz = parms->ncell[2]; - int amax = parms->st6_amax * Nz/2; - for(int a=-amax; a<=amax; a++){ - if(a==0) continue; - Real ka = 2.*M_PI * a / parms->Lz; - Real phase = ka*z + 2.*M_PI*random_numbers_p[a+Nz/2]; - Real B = parms->st6_amplitude / std::abs(float(a)); - p.rdata(PIdx::f01_Re) += 0.5 * B * cos(phase); - p.rdata(PIdx::f01_Im) += 0.5 * B * sin(phase); - } - - // Perturb the antineutrinos in a way that preserves the symmetries of the neutrino hamiltonian - p.rdata(PIdx::f01_Rebar) = p.rdata(PIdx::f01_Re); - p.rdata(PIdx::f01_Imbar) = -p.rdata(PIdx::f01_Im); - } - - //==============================// - // 7 - Code Comparison Gaussian // - //==============================// - else if(parms->simulation_type==7){ - AMREX_ASSERT(NUM_FLAVORS==2); - AMREX_ASSERT(parms->ncell[0] == 1); - AMREX_ASSERT(parms->ncell[1] == 1); - - // set energy to 50 MeV - p.rdata(PIdx::pupt) = 50. * 1e6*CGSUnitsConst::eV; - p.rdata(PIdx::pupx) = u[0] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupy) = u[1] * p.rdata(PIdx::pupt); - p.rdata(PIdx::pupz) = u[2] * p.rdata(PIdx::pupt); - - // get the number of each flavor in this particle. - Real angular_factor; - gaussian_profile(&angular_factor, parms->st7_sigma , u[2], parms->st7_mu0 ); - Real Nnue_thisparticle = parms->st7_nnue*scale_fac * angular_factor; - gaussian_profile(&angular_factor, parms->st7_sigmabar, u[2], parms->st7_mu0bar); - Real Nnua_thisparticle = parms->st7_nnua*scale_fac * angular_factor; - -// // set total number of neutrinos the particle has as the sum of the flavors - p.rdata(PIdx::N ) = Nnue_thisparticle; - p.rdata(PIdx::Nbar) = Nnua_thisparticle; - -// // set on-diagonals to have relative proportion of each flavor - p.rdata(PIdx::f00_Re) = 1; - p.rdata(PIdx::f11_Re) = 0; - p.rdata(PIdx::f00_Rebar) = 1; - p.rdata(PIdx::f11_Rebar) = 0; - -// // random perturbations to the off-diagonals - p.rdata(PIdx::f01_Re) = 0; - p.rdata(PIdx::f01_Im) = 0; - int Nz = parms->ncell[2]; - Real zprime = z - parms->Lz; - Real P1 = parms->st7_amplitude * std::exp(-zprime*zprime/(2.*parms->st7_sigma_pert*parms->st7_sigma_pert)); - p.rdata(PIdx::f01_Re) = P1 / 2.0; - p.rdata(PIdx::f01_Im) = 0; - - // Perturb the antineutrinos in a way that preserves the symmetries of the neutrino hamiltonian - p.rdata(PIdx::f01_Rebar) = p.rdata(PIdx::f01_Re); - p.rdata(PIdx::f01_Imbar) = -p.rdata(PIdx::f01_Im); - } - - else{ - amrex::Error("Invalid simulation type"); - } - - #include "generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length" - } - } - }); - } - - // get the minimum neutrino energy for calculating the timestep - Real pupt_min = amrex::ReduceMin(*this, [=] AMREX_GPU_HOST_DEVICE (const FlavoredNeutrinoContainer::ParticleType& p) -> Real { return p.rdata(PIdx::pupt); }); - ParallelDescriptor::ReduceRealMin(pupt_min); - #include "generated_files/FlavoredNeutrinoContainerInit.cpp_Vvac_fill" -} + } + if(parms->perturbation_type == 1){ + // Perturb real part of e-mu component only sinusoidally in z + Real nu_k = (2.*M_PI) / parms->perturbation_wavelength_cm; + p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); + p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + } + + +#include "generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length" + + } // loop over direction + } // loop over location + }); // loop over grid cells + } // loop over multifabs + + // get the minimum neutrino energy for calculating the timestep + Real pupt_min = amrex::ReduceMin(*this, [=] AMREX_GPU_HOST_DEVICE (const FlavoredNeutrinoContainer::ParticleType& p) -> Real { return p.rdata(PIdx::pupt); }); + ParallelDescriptor::ReduceRealMin(pupt_min); +#include "generated_files/FlavoredNeutrinoContainerInit.cpp_Vvac_fill" +} // InitParticles() diff --git a/Source/IO.cpp b/Source/IO.cpp index a115fdfb..8d5bd473 100644 --- a/Source/IO.cpp +++ b/Source/IO.cpp @@ -2,6 +2,10 @@ #include #include +#ifdef AMREX_USE_HDF5 +#include "hdf5.h" +#endif + #include "FlavoredNeutrinoContainer.H" #include "IO.H" #include "Evolve.H" @@ -23,12 +27,21 @@ WritePlotFile (const amrex::MultiFab& state, amrex::Print() << " Writing plotfile " << plotfilename << "\n"; + // write the grid file +#ifdef AMREX_USE_HDF5 + amrex::WriteSingleLevelPlotfileHDF5(plotfilename, state, GIdx::names, geom, time, step); +#else amrex::WriteSingleLevelPlotfile(plotfilename, state, GIdx::names, geom, time, step); +#endif if (write_plot_particles == 1) { auto neutrino_varnames = neutrinos.get_attribute_names(); +#ifdef AMREX_USE_HDF5 + neutrinos.CheckpointHDF5(plotfilename, "neutrinos", true, neutrino_varnames); +#else neutrinos.Checkpoint(plotfilename, "neutrinos", true, neutrino_varnames); +#endif } // write job information @@ -42,6 +55,31 @@ RecoverParticles (const std::string& dir, { BL_PROFILE("RecoverParticles()"); +#ifdef AMREX_USE_HDF5 + // open the plotfile to get the metadata + std::string plotfilename = dir; + plotfilename.append(".h5"); + hid_t file = H5Fopen(plotfilename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); + hid_t group = H5Gopen(file, "level_0", H5P_DEFAULT); + + // read the attributes to get step number + hid_t attr = H5Aopen(group, "steps", H5P_DEFAULT); + H5Aread(attr, H5T_NATIVE_INT, &step); + H5Aclose(attr); + + // read the attributes to get the current time + attr = H5Aopen(group, "time", H5P_DEFAULT); + H5Aread(attr, H5T_NATIVE_DOUBLE, &time); + H5Aclose(attr); + + // close all the pointers + H5Fclose(file); + H5Gclose(group); + + // initialize our particle container from the plotfile + std::string checkpointfilename("neutrinos/neutrinos"); + neutrinos.RestartHDF5(dir, checkpointfilename); +#else // load the metadata from this plotfile PlotFileData plotfile(dir); @@ -55,6 +93,7 @@ RecoverParticles (const std::string& dir, // initialize our particle container from the plotfile std::string file("neutrinos"); neutrinos.Restart(dir, file); +#endif // print the step/time for the restart amrex::Print() << "Restarting after time step: " << step-1 << " t = " << time << " s. ct = " << PhysConst::c * time << " cm" << std::endl; diff --git a/Source/Parameters.H b/Source/Parameters.H index 460d54d2..4ee8121a 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -8,11 +8,14 @@ using namespace amrex; + +//=========================================// +// Structure of global parameter variables // +//=========================================// struct TestParams : public amrex::Gpu::Managed { IntVect ncell; // num cells in domain IntVect nppc; // number of particles per cell in each dim - int nphi_equator; // number of directions in x-y plane. Real Lx, Ly, Lz; int max_grid_size; int nsteps; @@ -20,59 +23,33 @@ struct TestParams : public amrex::Gpu::Managed int write_plot_every; int write_plot_particles_every; Real rho_in, Ye_in, T_in; // g/ccm, 1, MeV - int simulation_type; Real cfl_factor, flavor_cfl_factor; Real max_adaptive_speedup; bool do_restart; std::string restart_dir; Real maxError; + // angular grid + std::string particle_data_filename; + // neutrino physics parameters. See first column of table 14.7 in http://pdg.lbl.gov/2019/reviews/rpp2019-rev-neutrino-mixing.pdf Real mass1, mass2, mass3; // neutrino masses in grams Real theta12, theta13, theta23; // neutrino mixing angles in radians Real alpha1, alpha2; // Majorana phases, radians Real deltaCP; // CP violating phases in radians - // simulation_type==3 - int st3_wavelength_fraction_of_domain; - Real st3_amplitude; - - // simulation_type==4 - Real st4_ndens , st4_theta , st4_phi , st4_fluxfac ; - Real st4_ndensbar, st4_thetabar, st4_phibar, st4_fluxfacbar; - Real st4_amplitude; - - // simulation_type==5 - Real st5_nnue , st5_nnua , st5_nnux ; - Real st5_fxnue, st5_fxnua, st5_fxnux; - Real st5_fynue, st5_fynua, st5_fynux; - Real st5_fznue, st5_fznua, st5_fznux; - Real st5_avgE_MeV; - Real st5_amplitude; - - // simulation_type==6 - Real st6_nnue , st6_nnua; - Real st6_sigma, st6_sigmabar; - Real st6_mu0, st6_mu0bar; - Real st6_amplitude; - Real st6_amax; - - // simulation_type==7 - Real st7_nnue , st7_nnua; - Real st7_sigma, st7_sigmabar; - Real st7_mu0, st7_mu0bar; - Real st7_amplitude; - Real st7_sigma_pert; - + // perturbation parameters + int perturbation_type; + Real perturbation_wavelength_cm; + Real perturbation_amplitude; + void Initialize(){ ParmParse pp; - pp.get("simulation_type", simulation_type); pp.get("ncell", ncell); pp.get("Lx", Lx); pp.get("Ly", Ly); pp.get("Lz", Lz); pp.get("nppc", nppc); - pp.get("nphi_equator", nphi_equator); pp.get("max_grid_size", max_grid_size); pp.get("nsteps", nsteps); pp.get("end_time", end_time); @@ -88,6 +65,15 @@ struct TestParams : public amrex::Gpu::Managed pp.get("restart_dir", restart_dir); pp.get("maxError", maxError); + // angular grid + pp.get("particle_data_filename",particle_data_filename); + + // perturbation parameters + pp.get("perturbation_amplitude", perturbation_amplitude); + pp.get("perturbation_type", perturbation_type); + if(perturbation_type == 1) + pp.get("perturbation_wavelength_cm", perturbation_wavelength_cm); + // neutrino physics parameters for 2-flavor pp.get("mass1_eV", mass1); pp.get("mass2_eV", mass2); @@ -111,60 +97,6 @@ struct TestParams : public amrex::Gpu::Managed deltaCP *= M_PI/180.; } - if(simulation_type==3){ - pp.get("st3_amplitude", st3_amplitude); - pp.get("st3_wavelength_fraction_of_domain", st3_wavelength_fraction_of_domain); - } - - if(simulation_type==4){ - pp.get("st4_theta" , st4_theta ); - pp.get("st4_thetabar", st4_thetabar); - pp.get("st4_phi" , st4_phi); - pp.get("st4_phibar", st4_phibar); - pp.get("st4_ndens" , st4_ndens); - pp.get("st4_ndensbar", st4_ndensbar); - pp.get("st4_fluxfac" , st4_fluxfac); - pp.get("st4_fluxfacbar", st4_fluxfacbar); - pp.get("st4_amplitude", st4_amplitude); - } - - if(simulation_type==5){ - pp.get("st5_nnue",st5_nnue); - pp.get("st5_nnua",st5_nnua); - pp.get("st5_nnux",st5_nnux); - pp.get("st5_fxnue",st5_fxnue); - pp.get("st5_fxnua",st5_fxnua); - pp.get("st5_fxnux",st5_fxnux); - pp.get("st5_fynua",st5_fynua); - pp.get("st5_fynux",st5_fynux); - pp.get("st5_fynue",st5_fynue); - pp.get("st5_fznue",st5_fznue); - pp.get("st5_fznua",st5_fznua); - pp.get("st5_fznux",st5_fznux); - pp.get("st5_avgE_MeV",st5_avgE_MeV); - pp.get("st5_amplitude",st5_amplitude); - } - - if(simulation_type==6){ - pp.get("st6_amplitude",st6_amplitude); - pp.get("st6_amax",st6_amax); - pp.get("st6_nnue",st6_nnue); - pp.get("st6_nnua",st6_nnua); - pp.get("st6_sigma",st6_sigma); - pp.get("st6_sigmabar",st6_sigmabar); - pp.get("st6_mu0",st6_mu0); - pp.get("st6_mu0bar",st6_mu0bar); - } - if(simulation_type==7){ - pp.get("st7_amplitude",st7_amplitude); - pp.get("st7_nnue",st7_nnue); - pp.get("st7_nnua",st7_nnua); - pp.get("st7_sigma",st7_sigma); - pp.get("st7_sigmabar",st7_sigmabar); - pp.get("st7_mu0",st7_mu0); - pp.get("st7_mu0bar",st7_mu0bar); - pp.get("st7_sigma_pert",st7_sigma_pert); - } } }; diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 72ee3bfb..833421e8 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -1,13 +1,5 @@ -simulation_type = 4 -st4_theta = 0 -st4_thetabar = 3.14159265359 -st4_phi = 0 -st4_phibar=0 -st4_ndens = 4.891290819e+32 -st4_ndensbar = 4.891290819e+32 -st4_fluxfac = .333333333333333 -st4_fluxfacbar = .333333333333333 -st4_amplitude = 1e-6 +perturbation_type = 0 +perturbation_amplitude = 1e-6 cfl_factor = 0.5 flavor_cfl_factor = 0.5 @@ -25,7 +17,7 @@ Lz = 64.0 # Number of particles per cell nppc = (1, 1, 1) -nphi_equator = 16 +particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain max_grid_size = 16 diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 305ddf47..e4d58c56 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -1,9 +1,11 @@ -simulation_type = 1 cfl_factor = 0.5 max_adaptive_speedup = 0 flavor_cfl_factor = .5 maxError = 1e-6 +perturbation_type = 0 +perturbation_amplitude = 0 + integration.type = 1 integration.rk.type = 4 @@ -15,7 +17,7 @@ Lz = 1e7 # Number of particles per cell nppc = (1, 1, 1) -nphi_equator = 1 +particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain max_grid_size = 64 diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index adc998b1..2406fbeb 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -1,9 +1,11 @@ -simulation_type = 2 cfl_factor = 0.5 flavor_cfl_factor = .5 max_adaptive_speedup = 0 maxError = 1e-6 +perturbation_type = 0 +perturbation_amplitude = 0 + integration.type = 1 integration.rk.type = 4 @@ -15,7 +17,7 @@ Lz = 1e7 # Number of particles per cell nppc = (1, 1, 1) -nphi_equator = 1 +particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain max_grid_size = 64 diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 32ccb2ae..1996e71a 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -1,6 +1,6 @@ -simulation_type = 3 -st3_amplitude = 1e-6 -st3_wavelength_fraction_of_domain = 1 +perturbation_type = 1 +perturbation_amplitude = 1e-6 +perturbation_wavelength_cm = 1 cfl_factor = 0.5 flavor_cfl_factor = 0.5 @@ -18,7 +18,7 @@ Lz = 1.0 # Number of particles per cell nppc = (1, 1, 1) -nphi_equator = 1 +particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain max_grid_size = 1000 diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index e674a58c..cdd14ac7 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -1,9 +1,11 @@ -simulation_type = 0 cfl_factor = 0.5 flavor_cfl_factor = 0.1 max_adaptive_speedup = 0 maxError = 1e-6 +perturbation_type = 0 +perturbation_amplitude = 0 + integration.type = 1 integration.rk.type = 4 @@ -15,7 +17,7 @@ Lz = 1.0 # Number of particles per cell nppc = (1, 1, 1) -nphi_equator = 1 +particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain max_grid_size = 64 From 0a204e3251a40549b7926914a0730e2d2dfc94c8 Mon Sep 17 00:00:00 2001 From: Sherwood Richers <5142652+srichers@users.noreply.github.com> Date: Thu, 2 Feb 2023 10:41:33 -0500 Subject: [PATCH 032/276] Sherwood development (#81) * Hdf5 io (#76) * add in C HDF5 functions to write data to HDF5, including timestep and physical time as attributes * remove unnecessary change from last commit * changed format of steps and time info in particle hdf5 file to match that in the plotfile. Also, wrap including the hdf5 header with ifdefs to prevent the compile stage breaking when not using HDF5 * reduce complexity - do not write to particle file, and just grab metadata from the plot file. Requires that the plot file be present, but thats fine Co-authored-by: Sherwood Richers * Pointwise initial conditions (#79) * compute isotropic momentum vectors instead of direction vectors so arbitrary momenta can be read in * set momentum in one place, combine nearby if blocks * simplify code by setting everything to zero by default * moved st5-specific calculations to parameter file. May be an abuse of the use of lambdas, but it is convenient * some formatting cleanup and comments * remove support for correlated spatial perturbations to reduce initial conditions complexity * spacing changes and separating out perturbation from simulation type to reduce number of parameters and repeated code * remove perturbation logic from st3 initial conditions since perturbations are taken care of later on * restoring accidentally removed variable * forgot to add new parameters to bipolar test inputs * add parameter to distinguish between the filled sphere angular distribution and one read in from file * fix silly compile errors * allow compiling on all cores * fix angular_grid_type logic and turn wave perturbation parameter to an actual length instead of a fraction of the domain * restructured initial condition for msw test to be generated via python script instead of within Emu. Must convert the other tests as well * add bipolar test new initial conditions. Actually include the initial conditions scripts that should have been there last time * add fast flavor test * added nonzero k fast flavor test * move st4 over to separate initial conditions script * moved minerbo initial conditions to python scripts * remove simulation_type parameter Co-authored-by: Sherwood Richers * add uniform grid function to initial condition tools * fix bug in grid initial condition script based on definition of polar angle being relative to pole or equator * added levermore closure to initial conditions * move regularly-used scripts to main repository * add pointwise initial conditions complete with linear interpolation to refine the grid * add function to print an input file from a dictionary * gitignore * significant modifications to reduce_data.py in order to make it callable as a function * modified combine_files.py so it can be called as a function * modified convert_to_hdf5 script so it can be called as a python function * function to create a new simulation * added function to create a simulation restart * added the ignore_pos optional argument for the rkey dictionary. I must have missed this while I was merging the two different versions together * added function to further reduce code replication in initial conditions routines * added linear interpolant and made particle generator more robust against divide by zeros * make linear initial condition use the same interpolating framework to reduce code replication * remove unnecessary example scripts * merged particle key functions into a single function for less code replication * convert input dictionary to reasonable types * automatically set restart directory * added initial condition functions that allow one to randomly generate moments and Poincare transform those moments such that the distribution has net zero flux and net eln flux along the z axis * added new NF argument to the variable key function in initial condition scripts * fixed python library paths in test scripts * fixed indexing error in new amrex plot tools function * update cuda version for new hardware * fix bug in combine files scripts that resulted in the k array being duplicated * fix argument propagation to initial condition sette * add all flavors to reduced data * update babysitting scripts to work with current code output --------- Co-authored-by: Sherwood Richers Co-authored-by: Sherwood Richers Co-authored-by: Sherwood Richers --- Scripts/babysitting/angular_power_spectrum.py | 72 +++ Scripts/babysitting/avgfee.py | 79 +++ Scripts/babysitting/power_spectrum.py | 72 +++ Scripts/data_reduction/.gitignore | 2 + .../amrex_plot_tools.py | 156 ++--- Scripts/data_reduction/combine_files.py | 46 ++ Scripts/data_reduction/convertToHDF5.py | 88 +++ Scripts/data_reduction/emu_yt_module.py | 382 +++++++++++ Scripts/data_reduction/glass_viz_extract.py | 28 + Scripts/data_reduction/reduce_data.py | 607 ++++++++++++++++++ Scripts/data_reduction/reduce_data_fft.py | 147 +++++ Scripts/data_reduction/submit_reduce_data.sh | 44 ++ .../initial_condition_tools.py | 274 +++++++- Scripts/initial_conditions/st0_msw_test.py | 2 +- .../initial_conditions/st1_bipolar_test.py | 2 +- .../st2_2beam_fast_flavor.py | 2 +- .../st3_2beam_fast_flavor_nonzerok.py | 2 +- .../st4_linear_moment_ffi.py | 54 +- Scripts/initial_conditions/st5_minerbo.py | 68 +- .../initial_conditions/st6_emu_pointwise.py | 192 ++++++ Scripts/runtools/.gitignore | 2 + Scripts/runtools/runtools.py | 159 +++++ Scripts/tests/fast_flavor_k_test.py | 5 +- Scripts/tests/fast_flavor_test.py | 5 +- Scripts/tests/msw_test.py | 5 +- makefiles/GNUmakefile_jenkins | 2 +- 26 files changed, 2319 insertions(+), 178 deletions(-) create mode 100644 Scripts/babysitting/angular_power_spectrum.py create mode 100755 Scripts/babysitting/avgfee.py create mode 100644 Scripts/babysitting/power_spectrum.py create mode 100644 Scripts/data_reduction/.gitignore rename Scripts/{visualization => data_reduction}/amrex_plot_tools.py (59%) create mode 100644 Scripts/data_reduction/combine_files.py create mode 100755 Scripts/data_reduction/convertToHDF5.py create mode 100755 Scripts/data_reduction/emu_yt_module.py create mode 100644 Scripts/data_reduction/glass_viz_extract.py create mode 100644 Scripts/data_reduction/reduce_data.py create mode 100755 Scripts/data_reduction/reduce_data_fft.py create mode 100644 Scripts/data_reduction/submit_reduce_data.sh create mode 100644 Scripts/initial_conditions/st6_emu_pointwise.py create mode 100644 Scripts/runtools/.gitignore create mode 100644 Scripts/runtools/runtools.py diff --git a/Scripts/babysitting/angular_power_spectrum.py b/Scripts/babysitting/angular_power_spectrum.py new file mode 100644 index 00000000..7e9f1f12 --- /dev/null +++ b/Scripts/babysitting/angular_power_spectrum.py @@ -0,0 +1,72 @@ +import numpy as np +import matplotlib.pyplot as plt +import glob +import h5py +import matplotlib as mpl +from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator,LogLocator) + +cmap=mpl.cm.jet + +################ +# plot options # +################ +mpl.rcParams['font.size'] = 22 +mpl.rcParams['font.family'] = 'serif' +#mpl.rc('text', usetex=True) +mpl.rcParams['xtick.major.size'] = 7 +mpl.rcParams['xtick.major.width'] = 2 +mpl.rcParams['xtick.major.pad'] = 8 +mpl.rcParams['xtick.minor.size'] = 4 +mpl.rcParams['xtick.minor.width'] = 2 +mpl.rcParams['ytick.major.size'] = 7 +mpl.rcParams['ytick.major.width'] = 2 +mpl.rcParams['ytick.minor.size'] = 4 +mpl.rcParams['ytick.minor.width'] = 2 +mpl.rcParams['axes.linewidth'] = 2 + +component_list = ["00","01","02","11","12","22"] + +f = h5py.File("reduced_data_angular_power_spectrum.h5","r") +t = np.array(f["t"]) +spectra = np.array(f["angular_spectrum"]) +f.close() + +tnorm = t[-1] + +def makeplot(icomponent,t, spectra): + plt.close('all') + fig, ax = plt.subplots(1,1, figsize=(8,6)) + + # get appropriate data + this_spectra = spectra[:,:,0,icomponent] + nl = np.shape(this_spectra)[1] + l = np.array(range(nl)) + for it in range(len(t)): + print(it,t) + total_power = np.sum(this_spectra[it]) + ax.semilogy(l, this_spectra[it,:], color=cmap(t[it]/tnorm)) + ax.grid() + + # colorbar + cax = fig.add_axes([0.125, .89, .775, 0.02]) + cax.tick_params(axis='both', which='both', direction='in', labeltop='on') + norm = mpl.colors.Normalize(vmin=0, vmax=tnorm) + cbar = plt.colorbar(mpl.cm.ScalarMappable(norm=norm, cmap=cmap),cax=cax,orientation='horizontal') + cbar.set_label(r"$t\,(10^{-9}\,\mathrm{s})$",labelpad=10) + cax.minorticks_on() + cax.xaxis.set_ticks_position('top') + cax.xaxis.set_label_position('top') + cax.xaxis.set_minor_locator(MultipleLocator(0.1)) + + # axis labels + ax.set_xlabel(r"$l$") + ax.set_ylabel(r"$|f_l|^2$") + ax.set_ylim(1e50, 1e61) + + plt.savefig("angular_power"+component_list[icomponent]+".pdf", bbox_inches='tight') + +nl = np.shape(spectra)[1] +for icomponent in range(len(component_list)): + print(component_list[icomponent]) + makeplot(icomponent, t, spectra) + diff --git a/Scripts/babysitting/avgfee.py b/Scripts/babysitting/avgfee.py new file mode 100755 index 00000000..4b17d038 --- /dev/null +++ b/Scripts/babysitting/avgfee.py @@ -0,0 +1,79 @@ +# Run from /ocean/projects/phy200048p/shared to generate plot showing time evolution of at different dimensionalities + +import os +os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' +import numpy as np +import matplotlib.pyplot as plt +import glob +import h5py +import matplotlib as mpl +from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator,LogLocator) + + +base=["N","Fx","Fy","Fz"] +diag_flavor=["00","11"]#,"22"] +offdiag_flavor=["01"]#,"02","12"] +re=["Re","Im"] +# real/imag +R=0 +I=1 + + +###################### +# read averaged data # +###################### +def plotdata(filename,a,b): + avgData = h5py.File(filename,"r") + t=np.array(avgData["t(s)"])*1e9 + N=np.array(avgData["N_avg_mag(1|ccm)"])[:,a,b] + avgData.close() + return t, N + +################ +# plot options # +################ +mpl.rcParams['font.size'] = 22 +mpl.rcParams['font.family'] = 'serif' +#mpl.rc('text', usetex=True) +mpl.rcParams['xtick.major.size'] = 7 +mpl.rcParams['xtick.major.width'] = 2 +mpl.rcParams['xtick.major.pad'] = 8 +mpl.rcParams['xtick.minor.size'] = 4 +mpl.rcParams['xtick.minor.width'] = 2 +mpl.rcParams['ytick.major.size'] = 7 +mpl.rcParams['ytick.major.width'] = 2 +mpl.rcParams['ytick.minor.size'] = 4 +mpl.rcParams['ytick.minor.width'] = 2 +mpl.rcParams['axes.linewidth'] = 2 + + +fig, ax = plt.subplots(1,1, figsize=(6,5)) + +############## +# formatting # +############## +ax.axhline(1./3., color="green") +ax.set_ylabel(r"$\langle N\rangle$ (cm$^{-3}$)") +ax.set_xlabel(r"$t\,(10^{-9}\,\mathrm{s})$") +ax.tick_params(axis='both', which='both', direction='in', right=True,top=True) +ax.xaxis.set_minor_locator(AutoMinorLocator()) +ax.yaxis.set_minor_locator(AutoMinorLocator()) +ax.minorticks_on() +ax.grid(which='both') + +############# +# plot data # +############# +filename = "plt_reduced_data.h5" +t,N = plotdata(filename,0,0) +ax.plot(t, N) + +############ +# save pdf # +############ +plt.savefig("avgfee.pdf", bbox_inches="tight") + +# same for f_e\mu +t,N = plotdata(filename,0,1) +ax.semilogy(t, N) +plt.savefig("avgfemu.pdf", bbox_inches="tight") diff --git a/Scripts/babysitting/power_spectrum.py b/Scripts/babysitting/power_spectrum.py new file mode 100644 index 00000000..88783b97 --- /dev/null +++ b/Scripts/babysitting/power_spectrum.py @@ -0,0 +1,72 @@ +import os +os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' +import numpy as np +import matplotlib.pyplot as plt +import glob +import h5py +import matplotlib as mpl +from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator,LogLocator) + +variables=["N","Fx","Fy","Fz"] +flavors=["00","11","22","01","02","12"] +cmap=mpl.cm.jet + +################ +# plot options # +################ +mpl.rcParams['font.size'] = 22 +mpl.rcParams['font.family'] = 'serif' +#mpl.rc('text', usetex=True) +mpl.rcParams['xtick.major.size'] = 7 +mpl.rcParams['xtick.major.width'] = 2 +mpl.rcParams['xtick.major.pad'] = 8 +mpl.rcParams['xtick.minor.size'] = 4 +mpl.rcParams['xtick.minor.width'] = 2 +mpl.rcParams['ytick.major.size'] = 7 +mpl.rcParams['ytick.major.width'] = 2 +mpl.rcParams['ytick.minor.size'] = 4 +mpl.rcParams['ytick.minor.width'] = 2 +mpl.rcParams['axes.linewidth'] = 2 + + +def makeplot(v,f,data): + + # get appropriate data + t=np.array(data["t(s)"]) + k=np.array(data["k(1|cm)"]) + fft = data[v+f+"_FFT(cm^-2)"] + total_power = np.sum(fft) + + plt.close('all') + fig, ax = plt.subplots(1,1, figsize=(8,6)) + for it in range(len(t)): + ax.semilogy(k, fft[it,:-1]/total_power, color=cmap(t[it]/t[-1])) + + # colorbar + cax = fig.add_axes([0.125, .89, .775, 0.02]) + cax.tick_params(axis='both', which='both', direction='in', labeltop='on') + norm = mpl.colors.Normalize(vmin=0, vmax=t[-1]*1e9) + cbar = plt.colorbar(mpl.cm.ScalarMappable(norm=norm, cmap=cmap),cax=cax,orientation='horizontal') + cbar.set_label(r"$t\,(10^{-9}\,\mathrm{s})$",labelpad=10) + cax.minorticks_on() + cax.xaxis.set_ticks_position('top') + cax.xaxis.set_label_position('top') + cax.xaxis.set_minor_locator(MultipleLocator(0.1)) + + # axis labels + ax.set_xlabel(r"$k\,(\mathrm{cm}^{-1})$") + ax.set_ylabel(r"$|\widetilde{f}|^2\,(\mathrm{cm}^{-2})$") + ax.minorticks_on() + ax.grid(which='both') + + plt.savefig(v+f+"_FFT_power.png", bbox_inches='tight') + +data = h5py.File("plt_reduced_data_fft_power.h5","r") + +for v in variables: + for f in flavors: + if v+f+"_FFT(cm^-2)" in data: + print(v+f) + makeplot(v,f, data) + +data.close() diff --git a/Scripts/data_reduction/.gitignore b/Scripts/data_reduction/.gitignore new file mode 100644 index 00000000..c24e9012 --- /dev/null +++ b/Scripts/data_reduction/.gitignore @@ -0,0 +1,2 @@ +*~ +__pycache__ \ No newline at end of file diff --git a/Scripts/visualization/amrex_plot_tools.py b/Scripts/data_reduction/amrex_plot_tools.py similarity index 59% rename from Scripts/visualization/amrex_plot_tools.py rename to Scripts/data_reduction/amrex_plot_tools.py index fb359cf9..94c8d094 100644 --- a/Scripts/visualization/amrex_plot_tools.py +++ b/Scripts/data_reduction/amrex_plot_tools.py @@ -9,78 +9,73 @@ mp = 1.6726219e-24 # g GF = 1.1663787e-5 / (1e9*eV)**2 * (hbar*clight)**3 #erg cm^3 -def get_particle_keys(ignore_pos=False): - real_quantities = ["pos_x", - "pos_y", - "pos_z", - "time", - "x", - "y", - "z", - "pupx", - "pupy", - "pupz", - "pupt", - "N", - "L", - "f00_Re", - "f01_Re", - "f01_Im", - "f11_Re", - "Nbar", - "Lbar", - "f00_Rebar", - "f01_Rebar", - "f01_Imbar", - "f11_Rebar"] +# NF is the number of flavors +# ignore_pos causes the first three elements to be ignored +# xp_only returns the position and momentum keys, but not the f keys +def get_particle_keys(NF, ignore_pos=False, xp_only=False): + assert(NF==2 or NF==3) + if(NF==2): + real_quantities = ["pos_x", + "pos_y", + "pos_z", + "time", + "x", + "y", + "z", + "pupx", + "pupy", + "pupz", + "pupt", + "N", + "L", + "f00_Re", + "f01_Re", + "f01_Im", + "f11_Re", + "Nbar", + "Lbar", + "f00_Rebar", + "f01_Rebar", + "f01_Imbar", + "f11_Rebar"] + if(NF==3): + real_quantities = ["pos_x", + "pos_y", + "pos_z", + "time", + "x", + "y", + "z", + "pupx", + "pupy", + "pupz", + "pupt", + "N", + "L", + "f00_Re", + "f01_Re", + "f01_Im", + "f02_Re", + "f02_Im", + "f11_Re", + "f12_Re", + "f12_Im", + "f22_Re", + "Nbar", + "Lbar", + "f00_Rebar", + "f01_Rebar", + "f01_Imbar", + "f02_Rebar", + "f02_Imbar", + "f11_Rebar", + "f12_Rebar", + "f12_Imbar", + "f22_Rebar"] + + if xp_only: real_quantities = real_quantities[:11] if ignore_pos: real_quantities = real_quantities[7:] - - rkey = {} - for i, rlabel in enumerate(real_quantities): - rkey[rlabel] = i - - ikey = { - # no ints are stored - } - - return rkey, ikey - -def get_3flavor_particle_keys(ignore_pos=False): - real_quantities = ["pos_x", - "pos_y", - "pos_z", - "time", - "x", - "y", - "z", - "pupx", - "pupy", - "pupz", - "pupt", - "N", - "L", - "f00_Re", - "f01_Re", - "f01_Im", - "f02_Re", - "f02_Im", - "f11_Re", - "f12_Re", - "f12_Im", - "f22_Re", - "Nbar", - "Lbar", - "f00_Rebar", - "f01_Rebar", - "f01_Imbar", - "f02_Rebar", - "f02_Imbar", - "f11_Rebar", - "f12_Rebar", - "f12_Imbar", - "f22_Rebar"] - if ignore_pos: real_quantities = real_quantities[7:] - + rkey = {} for i, rlabel in enumerate(real_quantities): rkey[rlabel] = i @@ -158,7 +153,7 @@ def __init__(self, header_filename): self.grids[level_num].append(tuple(entry)) -def read_particle_data(fn, ptype="particle0"): +def read_particle_data(fn, ptype="particle0", level_gridID=None): ''' This function returns the particle data stored in a particular @@ -183,11 +178,22 @@ def read_particle_data(fn, ptype="particle0"): elif header.real_type == np.float32: fdtype = "(%d,)f4" % header.num_real - idata = np.empty((header.num_particles, header.num_int )) - rdata = np.empty((header.num_particles, header.num_real)) + if level_gridID==None: + level_gridlist = enumerate(header.grids) + idata = np.empty((header.num_particles, header.num_int )) + rdata = np.empty((header.num_particles, header.num_real)) + else: + level = level_gridID[0] + gridID = level_gridID[1] + level_grid = header.grids[level][gridID] + which, count, where = level_grid + level_gridlist = enumerate([[level_grid,],]) + idata = np.empty((count, header.num_int )) + rdata = np.empty((count, header.num_real)) + ip = 0 - for lvl, level_grids in enumerate(header.grids): + for lvl, level_grids in level_gridlist: for (which, count, where) in level_grids: if count == 0: continue fn = base_fn + "/Level_%d/DATA_%05d" % (lvl, which) diff --git a/Scripts/data_reduction/combine_files.py b/Scripts/data_reduction/combine_files.py new file mode 100644 index 00000000..12de9c8c --- /dev/null +++ b/Scripts/data_reduction/combine_files.py @@ -0,0 +1,46 @@ +import glob +import h5py +import numpy as np +import sys + +def combine_files(directory, filename_base, filename_tail): + file_list = sorted(glob.glob(directory+"/"+filename_base+"*"+filename_tail)) + + # get the number of datasets in the file + f = h5py.File(file_list[0],"r") + keylist = [key for key in f.keys()] + ndatasets = len(keylist) + f.close() + + # collect the data in appended arrays + print() + datasets = [[] for i in range(ndatasets)] + for filename in file_list: + print("getting data from",filename) + f = h5py.File(filename,"r") + for i, key in enumerate(keylist): + datasets[i].append(np.array(f[key])) + f.close() + + # concatenate the arrays together + # output to file + print() + output_filename = directory+"/"+filename_base+filename_tail + print("Outputting datasets to "+output_filename) + f = h5py.File(filename_base+filename_tail,"w") + for i, key in enumerate(keylist): + if key=="k(1|cm)" or key=="phat": + datasets[i] = datasets[i][0] + else: + datasets[i] = np.concatenate(datasets[i], axis=0) + f[key] = datasets[i] + print(key, "\t",np.shape(datasets[i]) ) + f.close() + +if __name__ == "__main__": + if(len(sys.argv) != 3): + print() + print("Usage: [combine_files.py filename_base filename_tail], where filename is contained in each of the run* subdirectories") + print() + exit() + combine_files(".", sys.argv[1], sys.argv[2]) diff --git a/Scripts/data_reduction/convertToHDF5.py b/Scripts/data_reduction/convertToHDF5.py new file mode 100755 index 00000000..08e70909 --- /dev/null +++ b/Scripts/data_reduction/convertToHDF5.py @@ -0,0 +1,88 @@ +# used to make plots but now just generates a hdf5 file with domain-averaged data. +# Run in the directory of the simulation the data should be generated for. +# Still has functionality for per-snapshot plots, but the line is commented out. +# This version averages the magnitudes of off-diagonal components rather than the real/imaginary parts +# also normalizes fluxes by sumtrace of N rather than F. +# This data is used for the growth plot. +# Note - also tried a version using maxima rather than averages, and it did not make the growth plot look any better. + +import os +os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' +import sys +sys.path.append(os.path.dirname(os.path.abspath(__file__))) +import numpy as np +import matplotlib.pyplot as plt +import yt +import glob +import multiprocessing as mp +import h5py +import amrex_plot_tools as amrex +import emu_yt_module as emu +from multiprocessing import Pool +import scipy.special +import shutil + +# NOTE - assumes particle output is done at a multiple of fluid output + +######## +# misc # +######## +fluid_vars = ["N","Fx","Fy","Fz"] +nunubar = ["","bar"] + +def convert_to_HDF5(sim_directory, DELETE_ALL_BUT_LAST_RESTART=False): + ######################### + # loop over directories # + ######################### + fluid_directories = sorted(glob.glob(sim_directory+"/plt?????")) + nfluid = len(fluid_directories) + + print(fluid_directories) + + for d in fluid_directories: + print(d) + eds = emu.EmuDataset(d) + t = eds.ds.current_time + ad = eds.ds.all_data() + datatype = ad['boxlib',"N00_Re"].d.dtype + + if d==fluid_directories[0]: + NF = eds.get_num_flavors() + allData = h5py.File(sim_directory+"/allData.h5","w") + allData["dz(cm)"] = eds.dz + allData.create_dataset("t(s)", data=np.zeros(0), maxshape=(None,), dtype=datatype) + allData.create_dataset("it", data=np.zeros(0), maxshape=(None,), dtype=int) + + # create fluid data sets + maxshape = (None, eds.Nz) + chunkshape = (1, eds.Nz) + zeros = np.zeros((0,eds.Nz)) + varlist = [] + for v in fluid_vars: + for f1 in range(NF): + for f2 in range(f1,NF): + for b in nunubar: + varlist.append(v+str(f1)+str(f2)+"_Re"+b+"(1|ccm)") + if f2!=f1: + varlist.append(v+str(f1)+str(f2)+"_Im"+b+"(1|ccm)") + for v in varlist: + allData.create_dataset(v, data=zeros, maxshape=maxshape, chunks=chunkshape, dtype=datatype) + + # resize the datasets + allData["t(s)"].resize((len(allData["t(s)"]) + 1, )) + allData["t(s)"][-1] = eds.ds.current_time + allData["it"].resize((len(allData["it"]) + 1, )) + allData["it"][-1] = int(d[-5:]) + for v in varlist: + allData[v].resize(np.shape(allData[v])[0] + 1, axis=0) + allData[v][-1,:] = eds.cg[v[:-7]].d / ad['index',"cell_volume"][0] + + if DELETE_ALL_BUT_LAST_RESTART: + particle_directories = [d[:-10] for d in sorted(glob.glob(sim_directory+"/plt*/neutrinos"))] + last_particle_directory = particle_directories[-1] + for d in fluid_directories: + if d != last_particle_directory: + shutil.rmtree(d) + +if __name__ == "__main__": + convert_to_HDF5(".") diff --git a/Scripts/data_reduction/emu_yt_module.py b/Scripts/data_reduction/emu_yt_module.py new file mode 100755 index 00000000..79c620cf --- /dev/null +++ b/Scripts/data_reduction/emu_yt_module.py @@ -0,0 +1,382 @@ +import yt +import numpy as np +import math +import scipy.fft as fft +from yt import derived_field +import yt.units.dimensions as dimensions + +class Dim3(object): + def __init__(self, x=0, y=0, z=0): + if type(x) == np.ndarray: + self.x = x[0] + self.y = x[1] + self.z = x[2] + else: + self.x = x + self.y = y + self.z = z + + @staticmethod + def dimensions(lo, hi): + # return the number of elements (lo, hi) + # contain along each dimension + return [hi.x - lo.x + 1, + hi.y - lo.y + 1, + hi.z - lo.z + 1] + +class FourierData(object): + def __init__(self, time, kx, ky, kz, FT_magnitude, FT_phase): + # Dataset time + self.time = time + + # Wavenumbers + self.kx = kx + self.ky = ky + self.kz = kz + + # Fourier transform magnitude & phase + self.magnitude = FT_magnitude + self.phase = FT_phase + +class EmuDataset(object): + def __init__(self, plotfile=None): + # initialize from a supplied plotfile, otherwise just make an empty dataset object + # and initialize it later with init_from_data() + if plotfile: + self.setup_dataset(yt.load(plotfile)) + else: + self.ds = None + + def init_from_data(self, data, left_edge=None, right_edge=None, sim_time=0.0, dimensions=None, + length_unit=(1.0, "cm"), periodicity=(True, True, True), nprocs=1): + + assert(left_edge is not None and right_edge is not None) + + # initialize the dataset using a dictionary of numpy arrays in data, keyed by field name + domain_bounds = np.array([[left_edge[0], right_edge[0]], + [left_edge[1], right_edge[1]], + [left_edge[2], right_edge[2]]]) + + self.setup_dataset(yt.load_uniform_grid(data, dimensions, length_unit=length_unit, sim_time=sim_time, + bbox=domain_bounds, periodicity=periodicity, nprocs=nprocs)) + + def setup_dataset(self, yt_dataset): + self.ds = yt_dataset + self.construct_covering_grid() + #self.add_emu_fields() + + def construct_covering_grid(self): + self.cg = self.ds.covering_grid(level=0, left_edge=self.ds.domain_left_edge, + dims=self.ds.domain_dimensions) + + # number of cells in each dimension + self.Nx = self.ds.domain_dimensions[0] + self.Ny = self.ds.domain_dimensions[1] + self.Nz = self.ds.domain_dimensions[2] + + # find dx, dy, dz in each of X,Y,Z + # this is the spacing between cell centers in the domain + # it is the same as the spacing between cell edges + self.dx = (self.ds.domain_right_edge[0] - self.ds.domain_left_edge[0])/self.ds.domain_dimensions[0] + self.dy = (self.ds.domain_right_edge[1] - self.ds.domain_left_edge[1])/self.ds.domain_dimensions[1] + self.dz = (self.ds.domain_right_edge[2] - self.ds.domain_left_edge[2])/self.ds.domain_dimensions[2] + + if self.Nx > 1: + # low, high edge locations in x domain + xlo = self.ds.domain_left_edge[0] + xhi = self.ds.domain_right_edge[0] + + # the offset between the edges xlo, xhi and the interior cell centers + x_cc_offset = 0.5 * self.dx + + self.X, DX = np.linspace(xlo + x_cc_offset, # first cell centered location in the interior of x domain + xhi - x_cc_offset, # last cell centered location in the interior of x domain + num=self.Nx, # Nx evenly spaced samples + endpoint=True, # include interval endpoint for the last cell-centered location in the domain + retstep=True) # return the stepsize between cell centers to check consistency with dx + + # the spacing we calculated should be the same as what linspace finds between cell centers + # using our edge-to-cell-center offset and the number of samples + #print("dx, DX = ", dx, DX) + assert math.isclose(self.dx,DX) + + if self.Ny > 1: + # low, high edge locations in y domain + ylo = self.ds.domain_left_edge[1] + yhi = self.ds.domain_right_edge[1] + + # the offset between the edges ylo, yhi and the interior cell centers + y_cc_offset = 0.5 * self.dy + + self.Y, DY = np.linspace(ylo + y_cc_offset, # first cell centered location in the interior of y domain + yhi - y_cc_offset, # last cell centered location in the interior of y domain + num=self.Ny, # Ny evenly spaced samples + endpoint=True, # include interval endpoint for the last cell-centered location in the domain + retstep=True) # return the stepsize between cell centers to check consistency with dy + + # the spacing we calculated should be the same as what linspace finds between cell centers + # using our edge-to-cell-center offset and the number of samples + #print("dy, DY = ", dy, DY) + assert math.isclose(self.dy,DY) + + + if self.Nz > 1: + # low, high edge locations in z domain + zlo = self.ds.domain_left_edge[2] + zhi = self.ds.domain_right_edge[2] + + # the offset between the edges zlo, zhi and the interior cell centers + z_cc_offset = 0.5 * self.dz + + self.Z, DZ = np.linspace(zlo + z_cc_offset, # first cell centered location in the interior of z domain + zhi - z_cc_offset, # last cell centered location in the interior of z domain + num=self.Nz, # Nz evenly spaced samples + endpoint=True, # include interval endpoint for the last cell-centered location in the domain + retstep=True) # return the stepsize between cell centers to check consistency with dz + + # the spacing we calculated should be the same as what linspace finds between cell centers + # using our edge-to-cell-center offset and the number of samples + #print("dz, DZ = ", dz, DZ) + assert math.isclose(self.dz,DZ) + + def get_num_flavors(self): + just_the_fields = [f for ftype, f in self.ds.field_list] + if "N33_Re" in just_the_fields: + raise NotImplementedError("Analysis script currently only supports 2 and 3 flavor simulations") + elif "N22_Re" in just_the_fields: + return 3 + else: + return 2 + + def get_num_dimensions(self): + dim = self.ds.domain_dimensions + return np.sum(dim > 1) + + #def add_emu_fields(self): + # # first, define the trace + # def _make_trace(ds): + # if self.get_num_flavors() == 3: + # def _trace(field, data): + # return data["N00_Re"] + data["N11_Re"] + data["N22_Re"] + # return _trace + # else: + # def _trace(field, data): + # return data["N00_Re"] + data["N11_Re"] + # return _trace + # + # _trace = _make_trace(self.ds) + # + # self.ds.add_field(("gas", "trace"), function=_trace, sampling_type="local", units="auto", dimensions=dimensions.dimensionless) + # + # # now, define normalized fields + # for f in self.ds.field_list: + # if "_Re" in f[1] or "_Im" in f[1]: + # fname = f[1] + # fname_norm = "{}_norm_tr".format(fname) + # + # def _make_derived_field(f): + # def _derived_field(field, data): + # return data[f]/data[("gas", "trace")] + # return _derived_field + # + # _norm_derived_f = _make_derived_field(f) + # self.ds.add_field(("gas", fname_norm), function=_norm_derived_f, sampling_type="local", units="auto", dimensions=dimensions.dimensionless) + + def fourier(self, field_Re, field_Im=None, nproc=None): + if field_Im: + FT = np.squeeze(self.cg[field_Re][:,:,:].d + 1j * self.cg[field_Im][:,:,:].d) + else: + FT = np.squeeze(self.cg[field_Re][:,:,:].d) + + # use fftn to do an N-dimensional FFT on an N-dimensional numpy array + FT = fft.fftn(FT,workers=nproc) + + # we're shifting the sampling frequencies next, so we have to shift the FFT values + FT = fft.fftshift(FT) + + # get the absolute value of the fft + FT_mag = np.abs(FT) + + # get the phase of the fft + FT_phi = np.angle(FT) + + if self.Nx > 1: + # find the sampling frequencies in X & shift them + kx = fft.fftfreq(self.Nx, self.dx) + kx = fft.fftshift(kx) + else: + kx = None + + if self.Ny > 1: + # find the sampling frequencies in Y & shift them + ky = fft.fftfreq(self.Ny, self.dy) + ky = fft.fftshift(ky) + else: + ky = None + + if self.Nz > 1: + # find the sampling frequencies in Z & shift them + kz = fft.fftfreq(self.Nz, self.dz) + kz = fft.fftshift(kz) + else: + kz = None + + return FourierData(self.ds.current_time, kx, ky, kz, FT_mag, FT_phi) + + def get_rectangle(self, left_edge, right_edge): + # returns an EmuDataset containing only the rectangular region + # defined by [left_edge, right_edge] in the current dataset + + data, data_dimensions = self.get_data(bounds=(left_edge, right_edge)) + + # return a new EmuDataset object + new_dataset = EmuDataset() + new_dataset.init_from_data(data, left_edge=left_edge, right_edge=right_edge, sim_time=self.ds.current_time.in_units("s"), + dimensions=data_dimensions, length_unit=self.ds.length_unit, + periodicity=(False, False, False), nprocs=1) + return new_dataset + + def get_data(self, bounds=None): + # gets a dictionary of numpy arrays with the raw data in this dataset, keyed by field + # if bounds = None, returns the entire domain, otherwise interpret bounds = (left_edge, right_edge) + # where left_edge and right_edge are each numpy arrays with the physical positions of the selection edges + # and return the subdomain inside those bounds at the same resolution as the original dataset + cg = self.ds.covering_grid(left_edge=self.ds.domain_left_edge, dims=self.ds.domain_dimensions, level=0) + + ddim = self.ds.domain_dimensions + + # lo, hi Dim3's are defined in the AMReX style where they are inclusive indices + # defined for Fortran-style array slicing (not Python) + lo = Dim3(0,0,0) + hi = Dim3(ddim - 1) + + if bounds: + left_edge, right_edge = bounds + dleft = self.ds.domain_left_edge + dright = self.ds.domain_right_edge + delta = [self.dx, self.dy, self.dz] + + # initialize lo to the first indices in the domain + lo = np.array([0,0,0]) + # initialize hi to the last indices in the domain + hi = ddim - 1 + + for i in range(3): + if ddim[i] > 1: + # set lo[i] to the first cell-centered index to the right of left_edge + lo[i] = round((left_edge[i].d - dleft[i].d) / delta[i].d) + # set hi[i] to the last cell-centered index to the left of right_edge + hi[i] = hi[i] - round((dright[i].d - right_edge[i].d) / delta[i].d) + + lo = Dim3(lo) + hi = Dim3(hi) + + data = {} + data_dimensions = Dim3.dimensions(lo, hi) + + # Note: we have to throw away the field type, e.g. 'boxlib' because YT's uniform data loader + # gives its fields a 'stream' type. If we were to keep the field type here, we would be unable + # to call to_3D() on a dataset returned by to_2D() since the field types would not match. + for ftype, f in self.ds.field_list: + data[f] = (cg[f][lo.x:hi.x+1, lo.y:hi.y+1, lo.z:hi.z+1].d, "") + + return data, data_dimensions + + def to_2D(self, extend_dims=None): + # transform this 1D dataset into a 2D EmuDataset object + + # first, assert this dataset is 1D along z + assert(self.ds.domain_dimensions[0] == 1 and self.ds.domain_dimensions[1] == 1 and self.ds.domain_dimensions[2] > 1) + + # get the 1D data + data, _ = self.get_data() + + data_2D = {} + + # by default extend y to match z unless extend_dims is passed + length_z = self.ds.domain_dimensions[2] + length_y = length_z + if extend_dims: + length_y = extend_dims + + for f in data.keys(): + df, df_units = data[f] + data_2D[f] = (np.tile(df, (1, length_y, 1)), df_units) + + left_edge = self.ds.domain_left_edge + left_edge[1] = left_edge[2] + right_edge = self.ds.domain_right_edge + right_edge[1] = right_edge[2] + dimensions = self.ds.domain_dimensions + dimensions[1] = length_y + + # return a new EmuDataset object + new_dataset = EmuDataset() + new_dataset.init_from_data(data_2D, left_edge=left_edge, right_edge=right_edge, sim_time=self.ds.current_time.in_units("s"), + dimensions=dimensions, length_unit=self.ds.length_unit, + periodicity=self.ds.periodicity, nprocs=1) + return new_dataset + + def to_3D(self, extend_dims=None): + # transform this 1D or 2D dataset into a 3D EmuDataset object + + # check if this dataset is 1D or 2D and extended along z + assert(self.ds.domain_dimensions[0] == 1 and self.ds.domain_dimensions[2] > 1) + + # get the current data + data, _ = self.get_data() + + data_3D = {} + left_edge = None + right_edge = None + dimensions = None + + if self.ds.domain_dimensions[1] == 1: + # we are 1D, extended along z + # by default extend x, y to match z unless extend_dims is passed + length_z = self.ds.domain_dimensions[2] + length_y = length_z + length_x = length_z + if extend_dims: + length_x, length_y = extend_dims + + for f in data.keys(): + df, df_units = data[f] + data_3D[f] = (np.tile(df, (length_x, length_y, 1)), df_units) + + left_edge = self.ds.domain_left_edge + left_edge[0] = left_edge[2] + left_edge[1] = left_edge[2] + right_edge = self.ds.domain_right_edge + right_edge[0] = right_edge[2] + right_edge[1] = right_edge[2] + dimensions = self.ds.domain_dimensions + dimensions[0] = length_x + dimensions[1] = length_y + + else: + # we are 2D, extended along y and z + # by default extend x to match y unless extend_dims is passed + length_y = self.ds.domain_dimensions[1] + length_x = length_y + if extend_dims: + length_x = extend_dims + + for f in data.keys(): + df, df_units = data[f] + data_3D[f] = (np.tile(df, (length_x, 1, 1)), df_units) + + left_edge = self.ds.domain_left_edge + left_edge[0] = left_edge[1] + right_edge = self.ds.domain_right_edge + right_edge[0] = right_edge[1] + dimensions = self.ds.domain_dimensions + dimensions[0] = length_x + + # return a new EmuDataset object + new_dataset = EmuDataset() + new_dataset.init_from_data(data_3D, left_edge=left_edge, right_edge=right_edge, sim_time=self.ds.current_time.in_units("s"), + dimensions=dimensions, length_unit=self.ds.length_unit, + periodicity=self.ds.periodicity, nprocs=1) + return new_dataset + diff --git a/Scripts/data_reduction/glass_viz_extract.py b/Scripts/data_reduction/glass_viz_extract.py new file mode 100644 index 00000000..7e3bef94 --- /dev/null +++ b/Scripts/data_reduction/glass_viz_extract.py @@ -0,0 +1,28 @@ +# script to extract data cube to give to Bathsheba Grossman (crystalproteins.com) + +import h5py +import yt +import os +import sys +sys.path.append(os.path.dirname(os.path.abspath(__file__))) +import emu_yt_module as emu +import numpy as np + +dirname = "plt02200" +component = "N01" +suffix = "" + +eds = emu.EmuDataset(dirname) +t = eds.ds.current_time + +NR = eds.cg[component+"_Re"+suffix] +NI = eds.cg[component+"_Im"+suffix] + +phi = np.arctan2(NI,NR) +print(np.min(phi), np.max(phi)) +print(np.shape(phi)) + +f = h5py.File(dirname+"_glass.h5","w") +f.create_dataset("phi",data=phi, dtype='float32') +#f["phi"] = phi +f.close() diff --git a/Scripts/data_reduction/reduce_data.py b/Scripts/data_reduction/reduce_data.py new file mode 100644 index 00000000..665f3866 --- /dev/null +++ b/Scripts/data_reduction/reduce_data.py @@ -0,0 +1,607 @@ +# used to make plots but now just generates a hdf5 file with domain-averaged data. +# Run in the directory of the simulation the data should be generated for. +# Still has functionality for per-snapshot plots, but the line is commented out. +# This version averages the magnitudes of off-diagonal components rather than the real/imaginary parts +# also normalizes fluxes by sumtrace of N rather than F. +# This data is used for the growth plot. +# Note - also tried a version using maxima rather than averages, and it did not make the growth plot look any better. + +import os +os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' +import sys +sys.path.append(os.path.dirname(os.path.abspath(__file__))) +import numpy as np +import glob +import multiprocessing as mp +import h5py +import amrex_plot_tools as amrex +import emu_yt_module as emu +from multiprocessing import Pool +import scipy.special + +def dataset_name(moment, nu_nubar, i, j, ReIm): + data_format = format_dict["data_format"] + # make sure the inputs make sense + assert(i>=0) + assert(j>=0) + assert(i<=j) + if(i==j): + assert(ReIm=="Re") + assert(ReIm=="Re" or ReIm=="Im") + assert(moment=="N" or moment=="Fx" or moment=="Fy" or moment=="Fz") + assert(nu_nubar=="" or nu_nubar=="bar") + + # Emu + if(data_format=="Emu"): + return moment+str(i)+str(j)+"_"+ReIm+nu_nubar + + # FLASH + if(data_format=="FLASH"): + # make sure inputs only assume two flavors + assert(i<=1) + assert(j<=1) + + if moment=="N": momentFlash = "e" + if moment=="Fx": momentFlash = "f" + if moment=="Fy": momentFlash = "g" + if moment=="Fz": momentFlash = "h" + + if i==0 and j==0: + if nu_nubar=="": componentFlash = "e" + else: componentFlash = "a" + if j==1 and j==1: + if nu_nubar=="": componentFlash = "m" + else: componentFlash = "n" + if i==0 and j==1: + if ReIm=="Re": + if nu_nubar=="": componentFlash = "r" + else: componentFlash = "s" + else: + if nu_nubar=="": componentFlash = "i" + else: componentFlash = "j" + + return momentFlash+componentFlash+energyGroup + +# N is the corresponding flavor's number density (already converted to the proper units) +def convert_F_to_inv_ccm(N, data_format): + if(data_format=="Emu"): return 1.0 + if(data_format=="FLASH"): return N + + +##################### +# FFT preliminaries # +##################### +def get_kmid(fft): + if fft.kx is not None: + kmid = fft.kx[np.where(fft.kx>=0)] + if fft.ky is not None: + kmid = fft.ky[np.where(fft.ky>=0)] + if fft.kz is not None: + kmid = fft.kz[np.where(fft.kz>=0)] + return kmid + +def fft_coefficients(fft): + # add another point to the end of the k grid for interpolation + # MAKES POWER SPECTRUM HAVE SIZE ONE LARGER THAN KTEMPLATE + kmid = get_kmid(fft) + dk = kmid[1]-kmid[0] + kmid = np.append(kmid, kmid[-1]+dk) + + # compute the magnitude of the wavevector for every point + kmag = 0 + if fft.kx is not None: + kmag = kmag + fft.kx[:,np.newaxis,np.newaxis]**2 + if fft.ky is not None: + kmag = kmag + fft.ky[np.newaxis,:,np.newaxis]**2 + if fft.kz is not None: + kmag = kmag + fft.kz[np.newaxis,np.newaxis,:]**2 + kmag = np.sqrt(np.squeeze(kmag)) + kmag[np.where(kmag>=kmid[-1])] = kmid[-1] + + + # compute left index for interpolation + ileft = (kmag/dk).astype(int) + iright = ileft+1 + iright[np.where(iright>=len(kmid)-1)] = len(kmid)-1 + + # compute the fraction of the power that goes toward the left and right k point + cleft = (kmid[iright]-kmag)/dk + cright = 1.0-cleft + + return cleft, cright, ileft, iright, kmid + +def fft_power(fft, cleft, cright, ileft, iright, kmid): + + # compute power contributions to left and right indices + power = fft.magnitude**2 + powerLeft = power*cleft + powerRight = power*cright + + # accumulate onto spectrum + spectrum = np.array( [ + np.sum( powerLeft*(ileft ==i) + powerRight*(iright==i) ) + for i in range(len(kmid))] ) + + return spectrum + +######################### +# average preliminaries # +######################### +def get_matrix(ad,moment,nu_nubar): + NF = format_dict["NF"] + yt_descriptor = format_dict["yt_descriptor"] + f00 = ad[yt_descriptor, dataset_name(moment, nu_nubar, 0, 0, "Re")] + f01 = ad[yt_descriptor, dataset_name(moment, nu_nubar, 0, 1, "Re")] + f01I = ad[yt_descriptor, dataset_name(moment, nu_nubar, 0, 1, "Im")] + f11 = ad[yt_descriptor, dataset_name(moment, nu_nubar, 1, 1, "Re")] + if(NF>=3): + f02 = ad[yt_descriptor,dataset_name(moment, nu_nubar, 0, 2, "Re")] + f02I = ad[yt_descriptor,dataset_name(moment, nu_nubar, 0, 2, "Im")] + f12 = ad[yt_descriptor,dataset_name(moment, nu_nubar, 1, 2, "Re")] + f12I = ad[yt_descriptor,dataset_name(moment, nu_nubar, 1, 2, "Im")] + f22 = ad[yt_descriptor,dataset_name(moment, nu_nubar, 2, 2, "Re")] + zero = np.zeros(np.shape(f00)) + if(NF==2): + fR = [[f00 , f01 ], [ f01 ,f11 ]] + fI = [[zero, f01I], [-f01I,zero]] + if(NF==3): + fR = [[f00 , f01 , f02 ], [ f01 ,f11 ,f12 ], [ f02 , f12 ,f22 ]] + fI = [[zero, f01I, f02I], [-f01I,zero,f12I], [-f02I,-f12I,zero]] + return fR, fI + +def averaged_N(N, NI): + NF = format_dict["NF"] + R=0 + I=1 + + # do the averaging + # f1, f2, R/I + Nout = np.zeros((NF,NF)) + for i in range(NF): + for j in range(NF): + Nout[i][j] = float(np.average(np.sqrt(N[i][j]**2 + NI[i][j]**2))) + return np.array(Nout) + +def averaged_F(F, FI): + NF = format_dict["NF"] + R=0 + I=1 + + # do the averaging + # direction, f1, f2, R/I + Fout = np.zeros((3,NF,NF)) + for i in range(3): + for j in range(NF): + for k in range(NF): + Fout[i][j][k] = float(np.average(np.sqrt( F[i][j][k]**2 + FI[i][j][k]**2))) + + return Fout + +def offdiagMag(f): + NF = format_dict["NF"] + R = 0 + I = 1 + result = 0 + for f0 in range(NF): + for f1 in range(f0,NF): + result += f[:,f0,f1,R]**2 + f[:,f0,f1,I]**2 + return np.sqrt(result) + + + +class GridData(object): + def __init__(self, ad): + x = ad['index','x'].d + y = ad['index','y'].d + z = ad['index','z'].d + dx = ad['index','dx'].d + dy = ad['index','dy'].d + dz = ad['index','dz'].d + self.ad = ad + self.dx = dx[0] + self.dy = dy[0] + self.dz = dz[0] + self.xmin = np.min(x-dx/2.) + self.ymin = np.min(y-dy/2.) + self.zmin = np.min(z-dz/2.) + self.xmax = np.max(x+dx/2.) + self.ymax = np.max(y+dy/2.) + self.zmax = np.max(z+dz/2.) + self.nx = int((self.xmax - self.xmin) / self.dx + 0.5) + self.ny = int((self.ymax - self.ymin) / self.dy + 0.5) + self.nz = int((self.zmax - self.zmin) / self.dz + 0.5) + + + # particle cell id ON THE CURRENT GRID + # the x, y, and z values are assumed to be relative to the + # lower boundary of the grid + def get_particle_cell_ids(self,rdata): + # get coordinates + x = rdata[:,rkey["x"]] + y = rdata[:,rkey["y"]] + z = rdata[:,rkey["z"]] + ix = (x/self.dx).astype(int) + iy = (y/self.dy).astype(int) + iz = (z/self.dz).astype(int) + + # HACK - get this grid's bounds using particle locations + ix -= np.min(ix) + iy -= np.min(iy) + iz -= np.min(iz) + nx = np.max(ix)+1 + ny = np.max(iy)+1 + nz = np.max(iz)+1 + idlist = (iz + nz*iy + nz*ny*ix).astype(int) + return idlist + +# get the number of particles per cell +def get_nppc(d): + eds = emu.EmuDataset(d) + t = eds.ds.current_time + ad = eds.ds.all_data() + grid_data = GridData(ad) + level = 0 + gridID = 0 + idata, rdata = amrex.read_particle_data(d, ptype="neutrinos", level_gridID=(level,gridID)) + idlist = grid_data.get_particle_cell_ids(rdata) + ncells = np.max(idlist)+1 + nppc = len(idlist) // ncells + return nppc + + +# input list of particle data separated into grid cells +# output the same array, but sorted by zenith angle, then azimuthal angle +# also output the grid of directions in each cell (assumed to be the same) +def sort_rdata_chunk(p): + # sort first in theta + sorted_indices = p[:,rkey["pupz"]].argsort() + p = p[sorted_indices,:] + + # loop over unique values of theta + costheta = p[:,rkey["pupz"]] / p[:,rkey["pupt"]] + for unique_costheta in np.unique(costheta): + # get the array of particles with the same costheta + costheta_locs = np.where(costheta == unique_costheta)[0] + p_theta = p[costheta_locs,:] + + # sort these particles by the azimuthal angle + phi = np.arctan2(p_theta[:,rkey["pupy"]] , p_theta[:,rkey["pupx"]] ) + sorted_indices = phi.argsort() + p_theta = p_theta[sorted_indices,:] + + # put the sorted data back into p + p[costheta_locs,:] = p_theta + + # return the sorted array + return p + + +# class containing the set of coefficients to be multiplied by particle quantities +# compute once and use many times +class DiscreteSphericalHarmonic: + global Ylm_star_shared + + @staticmethod + def Ylm_indices(l): + start = l**2 + stop = (l+1)**2 + return start,stop + + # phat is the array of neutrino direction unit vectors in a single grid cell [particle, xyz] + # nl is the number of spherical harmonics to evaluate + def __init__(self, nl, nppc): + self.nl = nl + self.nppc = nppc + + # create shared object + # double the size for real+imaginary parts + global Ylm_star_shared + Ylm_star_shared = mp.RawArray('d', int(2 * (self.nl+1)**2 * self.nppc)) + + def precompute(self, phat): + # create local arrays that use this memory for easy modification + Ylm_star = np.frombuffer(Ylm_star_shared, dtype='complex').reshape( ( (self.nl+1)**2, self.nppc) ) + + # get direction coordinates + theta = np.arccos(phat[:,2]) + phi = np.arctan2(phat[:,1],phat[:,0]) + + # evaluate spherical harmonic amplitudes + for l in range(self.nl): + start,stop = self.Ylm_indices(l) + nm = stop-start + mlist = np.array(range(nm))-l + Ylm_star_thisl = [np.conj(scipy.special.sph_harm(m, l, phi, theta)) for m in mlist] + Ylm_star[start:stop] = np.array( Ylm_star_thisl ) + + def get_shared_Ylm_star(self,l): + assert(l0 + if do_average and not already_done: + thisN, thisNI = get_matrix(ad,"N","" ) + N = averaged_N(thisN,thisNI) * convert_N_to_inv_ccm + + thisFx, thisFxI = get_matrix(ad,"Fx","") + thisFy, thisFyI = get_matrix(ad,"Fy","") + thisFz, thisFzI = get_matrix(ad,"Fz","") + Ftmp = np.array([thisFx , thisFy , thisFz ]) + FtmpI = np.array([thisFxI, thisFyI, thisFzI]) + F = averaged_F(Ftmp, FtmpI) * convert_F_to_inv_ccm(N,data_format) + + thisN, thisNI = get_matrix(ad,"N","bar") + Nbar = averaged_N(thisN,thisNI) * convert_N_to_inv_ccm + + thisFx, thisFxI = get_matrix(ad,"Fx","bar") + thisFy, thisFyI = get_matrix(ad,"Fy","bar") + thisFz, thisFzI = get_matrix(ad,"Fz","bar") + Ftmp = np.array([thisFx , thisFy , thisFz ]) + FtmpI = np.array([thisFxI, thisFyI, thisFzI]) + Fbar = averaged_F(Ftmp, FtmpI) * convert_F_to_inv_ccm(Nbar,data_format) + + print("# rank",mpi_rank,"writing",outputfilename) + avgData = h5py.File(outputfilename,"w") + avgData["N_avg_mag(1|ccm)"] = [N,] + avgData["Nbar_avg_mag(1|ccm)"] = [Nbar,] + avgData["F_avg_mag(1|ccm)"] = [F,] + avgData["Fbar_avg_mag(1|ccm)"] = [Fbar,] + avgData["t(s)"] = [t,] + avgData.close() + + ############ + # FFT work # + ############ + outputfilename = d+"_reduced_data_fft_power.h5" + already_done = len(glob.glob(outputfilename))>0 + if do_fft and not already_done and len(eds.cg["N00_Re"][:,:,:].d.flatten())>1: + + print("# rank",mpi_rank,"writing",outputfilename) + fout = h5py.File(outputfilename,"w") + fout["t(s)"] = [np.array(t),] + + fft = eds.fourier(dataset_name("N", "", 0, 0, "Re"),nproc=nproc) + fout["k(1|cm)"] = get_kmid(fft) + cleft, cright, ileft, iright, kmid = fft_coefficients(fft) + N00_FFT = fft_power(fft, cleft, cright, ileft, iright, kmid) + fft = eds.fourier(dataset_name("N", "", 1, 1, "Re"),nproc=nproc) + N11_FFT = fft_power(fft, cleft, cright, ileft, iright, kmid) + fft = eds.fourier(dataset_name("N", "", 0, 1, "Re"), + dataset_name("N", "", 0, 1, "Im"),nproc=nproc) + N01_FFT = fft_power(fft, cleft, cright, ileft, iright, kmid) + fout["N00_FFT(cm^-2)"] = [np.array(N00_FFT),] + fout["N11_FFT(cm^-2)"] = [np.array(N11_FFT),] + fout["N01_FFT(cm^-2)"] = [np.array(N01_FFT),] + if format_dict["NF"]>2: + fft = eds.fourier(dataset_name("N", "", 2, 2, "Re"),nproc=nproc) + N22_FFT = fft_power(fft, cleft, cright, ileft, iright, kmid) + fft = eds.fourier(dataset_name("N", "", 0, 2, "Re"), + dataset_name("N", "", 0, 2, "Im"),nproc=nproc) + N02_FFT = fft_power(fft, cleft, cright, ileft, iright, kmid) + fft = eds.fourier(dataset_name("N", "", 1, 2, "Re"), + dataset_name("N", "", 1, 2, "Im"),nproc=nproc) + N12_FFT = fft_power(fft, cleft, cright, ileft, iright, kmid) + fout["N22_FFT(cm^-2)"] = [np.array(N22_FFT),] + fout["N02_FFT(cm^-2)"] = [np.array(N02_FFT),] + fout["N12_FFT(cm^-2)"] = [np.array(N12_FFT),] + + fout.close() + + if do_angular: + # separate loop for angular spectra so there is no aliasing and better load balancing + directories = sorted(glob.glob("plt*/neutrinos")) + directories = [directories[i].split('/')[0] for i in range(len(directories))] # remove "neutrinos" + + # get number of particles to be able to construct + # must build Ylm before pool or the internal shared memory will not be declared in pool subprocesses + nppc = get_nppc(directories[-1]) + Ylm = DiscreteSphericalHarmonic(nl,nppc) + + pool = Pool(nproc) + for d in directories: + if mpi_rank==0: + print("# working on", d) + eds = emu.EmuDataset(d) + t = eds.ds.current_time + ad = eds.ds.all_data() + + ################ + # angular work # + ################ + outputfilename = d+"_reduced_data_angular_power_spectrum.h5" + already_done = len(glob.glob(outputfilename))>0 + if not already_done: + + if mpi_rank==0: + print("Computing up to l =",nl-1) + + header = amrex.AMReXParticleHeader(d+"/neutrinos/Header") + grid_data = GridData(ad) + nlevels = len(header.grids) + assert nlevels==1 + level = 0 + ngrids = len(header.grids[level]) + + # average the angular power spectrum over many cells + # loop over all cells within each grid + if format_dict["NF"]==2: + ncomps = 3 + if format_dict["NF"]==3: + ncomps = 6 + spectrum = np.zeros((nl,2,ncomps)) + Nrho_avg = np.zeros((2,ncomps,nppc))*1j + total_ncells = 0 + for gridID in range(mpi_rank,ngrids,mpi_size): + print(" rank",mpi_rank,"grid",gridID+1,"/",ngrids) + + # read particle data on a single grid + # [particleID, quantity] + idata, rdata = amrex.read_particle_data(d, ptype="neutrinos", level_gridID=(level,gridID)) + + # get list of cell ids + idlist = grid_data.get_particle_cell_ids(rdata) + + # sort rdata based on id list + # still [particleID, quantity] + sorted_indices = idlist.argsort() + rdata = rdata[sorted_indices] + idlist = idlist[sorted_indices] + + # split up the data into cell chunks + # [cellID][particleID, quantity] + ncells = np.max(idlist)+1 + rdata = [ rdata[icell*nppc:(icell+1)*nppc,:] for icell in range(ncells)] # + chunksize = ncells//nproc + if ncells % nproc != 0: + chunksize += 1 + + # sort particles in each chunk + # still [cellID][particleID, quantity] + rdata = pool.map(sort_rdata_chunk, rdata, chunksize=chunksize) + + # initialize the shared data class. + #Only need to compute for one grid cell, since the angular grid is the same in every cell. + if gridID==mpi_rank: + phat = rdata[0][:,rkey["pupx"]:rkey["pupz"]+1] / rdata[0][:,rkey["pupt"]][:,np.newaxis] + Ylm.precompute(phat) + + # accumulate the spatial average of the angular distribution + Nrho = pool.map(get_Nrho,rdata, chunksize=chunksize) + Nrho_avg += sum(Nrho) + + # accumulate a spectrum from each cell + spectrum_each_cell = pool.map(Ylm.spherical_harmonic_power_spectrum, Nrho, chunksize=chunksize) + spectrum += sum(spectrum_each_cell) + + # count the total number of cells + total_ncells += ncells + + if do_MPI: + comm.Barrier() + spectrum = comm.reduce(spectrum , op=MPI.SUM, root=0) + Nrho_avg = comm.reduce(Nrho_avg , op=MPI.SUM, root=0) + total_ncells = comm.reduce(total_ncells, op=MPI.SUM, root=0) + + # write averaged data + if mpi_rank==0: + spectrum /= total_ncells + Nrho_avg /= total_ncells + + print("# writing",outputfilename) + avgData = h5py.File(outputfilename,"w") + avgData["angular_spectrum"] = [spectrum,] + avgData["Nrho(1|ccm)"] = [Nrho_avg,] + avgData["phat"] = phat + avgData["t(s)"] = [t,] + avgData.close() + + +if __name__ == "__main__": + reduce_data() diff --git a/Scripts/data_reduction/reduce_data_fft.py b/Scripts/data_reduction/reduce_data_fft.py new file mode 100755 index 00000000..9a5527b8 --- /dev/null +++ b/Scripts/data_reduction/reduce_data_fft.py @@ -0,0 +1,147 @@ +import numpy as np +import emu_yt_module as emu +import h5py +import glob +import scipy +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument("-o", "--output", type=str, default="reduced_data_fft.h5", help="Name of the output file (default: reduced_data_fft.h5)") +args = parser.parse_args() + +directories = sorted(glob.glob("plt*")) + +t = [] + +N00_FFT = [] +N01_FFT = [] +N02_FFT = [] +N11_FFT = [] +N12_FFT = [] +N22_FFT = [] + +Fx00_FFT = [] +Fx01_FFT = [] +Fx02_FFT = [] +Fx11_FFT = [] +Fx12_FFT = [] +Fx22_FFT = [] + +N00_FFT_phase = [] +N01_FFT_phase = [] +N02_FFT_phase = [] +N11_FFT_phase = [] +N12_FFT_phase = [] +N22_FFT_phase = [] + +Fx00_FFT_phase = [] +Fx01_FFT_phase = [] +Fx02_FFT_phase = [] +Fx11_FFT_phase = [] +Fx12_FFT_phase = [] +Fx22_FFT_phase = [] + +################################ +# read data and calculate FFTs # +################################ +for d in directories: + print(d) + eds = emu.EmuDataset(d) + NF = eds.get_num_flavors() + t.append(eds.ds.current_time) + + fft = eds.fourier("N00_Re") + N00_FFT.append(fft.magnitude) + N00_FFT_phase.append(fft.phase) + + fft = eds.fourier("N11_Re") + N11_FFT.append(fft.magnitude) + N11_FFT_phase.append(fft.phase) + + fft = eds.fourier("N01_Re","N01_Im") + N01_FFT.append(fft.magnitude) + N01_FFT_phase.append(fft.phase) + + fft = eds.fourier("Fx00_Re") + Fx00_FFT.append(fft.magnitude) + Fx00_FFT_phase.append(fft.phase) + + fft = eds.fourier("Fx11_Re") + Fx11_FFT.append(fft.magnitude) + Fx11_FFT_phase.append(fft.phase) + + fft = eds.fourier("Fx01_Re","Fx01_Im") + Fx01_FFT.append(fft.magnitude) + Fx01_FFT_phase.append(fft.phase) + + if NF>2: + fft = eds.fourier("N22_Re") + N22_FFT.append(fft.magnitude) + N22_FFT_phase.append(fft.phase) + + fft = eds.fourier("N02_Re","N02_Im") + N02_FFT.append(fft.magnitude) + N02_FFT_phase.append(fft.phase) + + fft = eds.fourier("N12_Re","N12_Im") + N12_FFT.append(fft.magnitude) + N12_FFT_phase.append(fft.phase) + + fft = eds.fourier("Fx22_Re") + Fx22_FFT.append(fft.magnitude) + Fx22_FFT_phase.append(fft.phase) + + fft = eds.fourier("Fx02_Re","Fx02_Im") + Fx02_FFT.append(fft.magnitude) + Fx02_FFT_phase.append(fft.phase) + + fft = eds.fourier("Fx12_Re","Fx12_Im") + Fx12_FFT.append(fft.magnitude) + Fx12_FFT_phase.append(fft.phase) + + +################## +# write the file # +################## +f = h5py.File(args.output,"w") + +f["t"] = np.array(t) + +if fft.kx is not None: + f["kx"] = np.array(fft.kx) +if fft.ky is not None: + f["ky"] = np.array(fft.ky) +if fft.kz is not None: + f["kz"] = np.array(fft.kz) + +f["N00_FFT"] = np.array(N00_FFT) +f["N11_FFT"] = np.array(N11_FFT) +f["N01_FFT"] = np.array(N01_FFT) +f["Fx00_FFT"] = np.array(Fx00_FFT) +f["Fx11_FFT"] = np.array(Fx11_FFT) +f["Fx01_FFT"] = np.array(Fx01_FFT) + +f["N00_FFT_phase"] = np.array(N00_FFT_phase) +f["N11_FFT_phase"] = np.array(N11_FFT_phase) +f["N01_FFT_phase"] = np.array(N01_FFT_phase) +f["Fx00_FFT_phase"] = np.array(Fx00_FFT_phase) +f["Fx11_FFT_phase"] = np.array(Fx11_FFT_phase) +f["Fx01_FFT_phase"] = np.array(Fx01_FFT_phase) + +if NF>2: + f["N22_FFT"] = np.array(N22_FFT) + f["N02_FFT"] = np.array(N02_FFT) + f["N12_FFT"] = np.array(N12_FFT) + f["Fx22_FFT"] = np.array(Fx22_FFT) + f["Fx02_FFT"] = np.array(Fx02_FFT) + f["Fx12_FFT"] = np.array(Fx12_FFT) + + f["N22_FFT_phase"] = np.array(N22_FFT_phase) + f["N02_FFT_phase"] = np.array(N02_FFT_phase) + f["N12_FFT_phase"] = np.array(N12_FFT_phase) + f["Fx22_FFT_phase"] = np.array(Fx22_FFT_phase) + f["Fx02_FFT_phase"] = np.array(Fx02_FFT_phase) + f["Fx12_FFT_phase"] = np.array(Fx12_FFT_phase) + + +f.close() diff --git a/Scripts/data_reduction/submit_reduce_data.sh b/Scripts/data_reduction/submit_reduce_data.sh new file mode 100644 index 00000000..d42b7b5d --- /dev/null +++ b/Scripts/data_reduction/submit_reduce_data.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Number of nodes: +#SBATCH --nodes=8 +# +################# +# Haswell nodes +################# +# +# Requests Cori Haswell nodes: +#SBATCH --constraint=haswell +# +# Haswell: Assign 1 MPI task to each socket +#SBATCH --tasks-per-node=16 +# +# Haswell: each socket has 32 CPUs (with hyperthreading) +#SBATCH --cpus-per-task=4 +# +################# +# Queue & Job +################# +# +# Which queue to run in: debug, regular, premium, etc. ... +#SBATCH --qos=regular +# +# Run for this much walltime: hh:mm:ss +#SBATCH --time=24:00:00 +# +# Use this job name: +#SBATCH -J emu_reduce +# +# Send notification emails here: +#SBATCH --mail-user=srichers@berkeley.edu +#SBATCH --mail-type=ALL +# +# Which allocation to use: +#SBATCH -A m3761 + +# On the compute node, change to the directory we submitted from +cd $SLURM_SUBMIT_DIR + +module load python3 + +srun -n 128 -c 4 python3 ~/emu_scripts/data_reduction/reduce_data.py diff --git a/Scripts/initial_conditions/initial_condition_tools.py b/Scripts/initial_conditions/initial_condition_tools.py index 13dff1aa..16c0f625 100644 --- a/Scripts/initial_conditions/initial_condition_tools.py +++ b/Scripts/initial_conditions/initial_condition_tools.py @@ -2,7 +2,8 @@ import sys import os importpath = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(importpath+"/../visualization") +sys.path.append(importpath+"/../data_reduction") +import amrex_plot_tools as amrex # generate an array of theta,phi pairs that uniformily cover the surface of a sphere # based on DOI: 10.1080/10586458.2003.10504492 section 3.3 but specifying n_j=0 instead of n @@ -32,6 +33,29 @@ def uniform_sphere(nphi_at_equator): return np.array(xyz) +# generate an array of theta,phi pairs that cover a sphere evenly in phi and cos(theta) +def grid_sphere(nphi): + assert(nphi > 0) + nmu = nphi // 2 + + mu_grid = np.linspace(-1,1,num=nmu+1) + costheta = np.array([ (mu_grid[i]+mu_grid[i+1])/2. for i in range(nmu)]) + sintheta = np.sqrt(1. - costheta**2) + + phi_mid = np.linspace(0, 2.*np.pi, num=nphi, endpoint=False) + cosphi = np.cos(phi_mid) + sinphi = np.sin(phi_mid) + + xyz = [] + for imu in range(nmu): + for iphi in range(nphi): + x = sintheta[imu] * cosphi[iphi] + y = sintheta[imu] * sinphi[iphi] + z = costheta[imu] + xyz.append(np.array([x,y,z])) + return np.array(xyz) + + def write_particles(p, NF, filename): with open(filename, "w") as f: # write the number of flavors @@ -94,3 +118,251 @@ def minerbo_Z(fluxfac): print("fluxfac=",fluxfac," Z=",Z) return Z + +# angular structure as determined by the Levermore closure +# from assuming that radiation is isotropic in some frame +# v is the velocity of this frame +# sign of v chosen so distribution is large when mu==1 +# Coefficients set such that the expectation value is 1 +def levermore_closure(v, mu): + gamma2 = 1/(1-v**2) + result = 1/(2*gamma2*(1-v*mu)**2) + return result + + +# residual for root finder to get v from the fluxfac +def levermore_residual(fluxfac, v): + return fluxfac - ( v-(1-v**2)*np.arctanh(v) ) / v**2 +def levermore_residual_derivative(fluxfac, v): + return 2*(v-np.arctanh(v))/v**3 + +def levermore_v(fluxfac): + # hard-code in these parameters because they are not + # really very important... + maxresidual = 1e-6 + maxcount = 20 + minfluxfac = 1e-3 + + # initial condition + v = 0.5 + + # catch the small flux factor case to prevent nans + if(fluxfac < minfluxfac): + v = 3*f/2 + else: + residual = 1.0 + count = 0 + while(abs(residual)>maxresidual and countmaxresidual: + print("Failed to converge on a solution.") + assert(False) + + print("fluxfac=",fluxfac," v=",v) + return v + +# interpolate the levermore closure +# mu is the cosine of the angle of the direction relative to the flux direction +# the expectation value of the result is 1 +def levermore_interpolate(fluxfac, mu): + assert(fluxfac >= 0 and fluxfac <= 1) + + v = levermore_v(fluxfac) + return levermore_closure(v, mu) + +# interpolate the Minerbo closure +# mu is the cosine of the angle of the direction relative to the flux direction +# the expectation value of the result is 1 +def minerbo_interpolate(fluxfac, mu): + assert(fluxfac >= 0 and fluxfac <= 1) + + Z = minerbo_Z(fluxfac) + return minerbo_closure(Z, mu) + +# interpolate linearly +# mu is the cosine of the angle of the direction relative to the flux direction +# the expectation value of the result is 1 +def linear_interpolate(fluxfac, mu): + assert(fluxfac>=0 and fluxfac<=1/3) + return 1 + mu + +# generate a list of particle data +# NF is the number of flavors +# nphi_equator is the number of directions in the xhat-yhat plane +# nnu is an array of neutrino number density of lenth NF and units 1/ccm [nu/nubar, iflavor] +# fnu is an array of neutrino flux density of shape [nu/nubar, iflavor, xyz] and units of 1/ccm +# direction_generator is either uniform_sphere or grid_sphere +# interpolate_function is either levermore_interpolate or minerbo_interpolate or linear_interpolate +# for each nu/nubar and flavor, the interpolant adds up to 1 and approximates the flux +def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_generator, interpolate_function): + # number of neutrino flavors + NF = nnu.shape[1] + + # flux magnitude and flux factor [nu/nubar, flavor] + fluxmag = np.sqrt(np.sum(fnu**2, axis=2)) + fluxfac = fluxmag / nnu + fluxfac[np.where(nnu==0)] = 0 + + # direction unit vector of fluxes [nu/nubar, flavor, xyz] + fhat = fnu / fluxmag[:,:,np.newaxis] + fhat[np.where(fhat!=fhat)] = 0 + + # generate list of momenta and direction cosines + phat = direction_generator(nphi_equator) # [iparticle, xyz] + mu = np.sum(phat[:,np.newaxis,np.newaxis,:] * fhat[np.newaxis,:,:,:],axis=3) # [iparticle, nu/nubar, flavor] + + # generate interpolant # [particle, nu/nubar, flavor] + nparticles = mu.shape[0] + interpolant = np.array( [ [ interpolate_function(fluxfac[nu_nubar,flavor], mu[:,nu_nubar,flavor]) for nu_nubar in range(2)] for flavor in range(NF)] ) # [flavor, nu/nubar, particle] + interpolant = np.swapaxes(interpolant, 0, 2) + + # make sure the interpolant adds up to 1 + norm = np.sum(interpolant, axis=(0)) # [nu/nubar, flavor] + interpolant /= norm[np.newaxis,:,:] + + # determine the number of each flavor for each particle [particle, nu/nubar, flavor] + n_particle = interpolant * nnu[np.newaxis,:,:] + + # get variable keys + rkey, ikey = amrex.get_particle_keys(NF, ignore_pos=True) + nelements = len(rkey) + + # generate the list of particle info + particles = np.zeros((nparticles,nelements)) + + # save particle momenta + particles[:,rkey["pupt"] ] = energy_erg + particles[:,rkey["pupx"]:rkey["pupz"]+1] = energy_erg * phat + + # save the total number density of neutrinos for each particle + n_flavorsummed = np.sum(n_particle, axis=2) # [particle, nu/nubar] + for nu_nubar, suffix in zip(range(2), ["","bar"]): + nvarname = "N"+suffix + particles[:,rkey[nvarname]] = n_flavorsummed[:,nu_nubar] + + for flavor in range(NF): + fvarname = "f"+str(flavor)+str(flavor)+"_Re"+suffix + particles[:,rkey[fvarname]] = n_particle[:,nu_nubar, flavor] / n_flavorsummed[:,nu_nubar] + particles[:,rkey[fvarname]][np.where(n_flavorsummed[:,nu_nubar]==0)] = 0 + + # double check that the number densities are correct + particle_n = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]]) + particle_fmag = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]] * mu[:,nu_nubar, flavor]) + print("nu/nubar,flavor =", nu_nubar, flavor) + print("output/input ndens =",particle_n, nnu[nu_nubar,flavor]) + print("output/input fluxfac =",particle_fmag / particle_n, fluxfac[nu_nubar,flavor]) + print() + + return particles + +# create a random distribution defined by number and flux densities +# NF is the number of flavors +# n is normalized such that the sum of all ns is 1 +def random_moments(NF): + bad_distro = True + count = 0 + + # create random number densities, normalize to 1 + n = np.random.rand(2,NF) + n /= np.sum(n) + + # randomly sample flux directions + mu = np.random.rand(2,NF)*2. - 1 + phi = np.random.rand(2,NF)*2.*np.pi + mag = np.random.rand(2,NF) * n + + f = np.zeros((2,NF,3)) + sintheta = np.sqrt(1-mu**2) + f[:,:,0] = mag * sintheta * np.cos(phi) + f[:,:,1] = mag * sintheta * np.sin(phi) + f[:,:,2] = mag * mu + + return n, f + +# v has 3 components +# v is dimensionless velocity +def lorentz_transform(v): + v2 = np.sum(v**2) + gamma = 1/np.sqrt(1-v2) + + L = np.identity(4) + L[3,3] = gamma + L[0:3,3] = -gamma * v + L[3,0:3] = -gamma * v + L[0:3, 0:3] += (gamma-1) * np.outer(v,v) / v2 + + return L + +# returns a rotation matrix that rotates along axis u by angle costheta +def axis_angle_rotation_matrix(u, costheta): + costheta_2 = np.sqrt((1+costheta)/2.) + sintheta_2 = np.sqrt((1-costheta)/2.) + + # get rotation quaternion + q = np.array([costheta_2, + sintheta_2 * u[0], + sintheta_2 * u[1], + sintheta_2 * u[2] + ]) + + # construct rotation matrix ala wikipedia + R = np.zeros((4,4)) + for i in range(3): + # R[0,0] = q0^2 + q1^2 - q2^2 - q3^2 + # = costheta^2 + 2q1^2 - sintheta^2 (assuming u is a unit vector) + # Let the loop over j take into accout the 2q1^2 + R[i,i] = 2.*costheta_2**2 - 1. + + for j in range(3): + R[i,j] += 2.*q[i+1]*q[j+1] # indexing = q is size 4 + R[0,1] -= 2.*q[0]*q[2+1] + R[1,0] += 2.*q[0]*q[2+1] + R[0,2] += 2.*q[0]*q[1+1] + R[2,0] -= 2.*q[0]*q[1+1] + R[1,2] -= 2.*q[0]*q[0+1] + R[2,1] += 2.*q[0]*q[0+1] + + R[3,3] = 1 + + return R + +# return a matrix that transforms a set of four-fluxes +# such that the net flux is zero +# and the ELN points along the z axis +# n = [nu/nubar, flavor] +# f = [nu/nubar, flavor, xyz] +def poincare_transform_0flux_elnZ(n,f): + # number of flavors + NF = n.shape[1] + + # construct the four-flux of all flavors + f4 = np.zeros((2,NF,4)) + f4[:,:,0:3] = f + f4[:,:, 3] = n + f4 = np.moveaxis(f4, 2, 0) # [xyzt, nu/nubar, NF] + + # net flux factor + netf4 = np.sum(f4, axis=(1,2)) + average_velocity = netf4[0:3] / netf4[3] + L = lorentz_transform(average_velocity) + f4 = np.tensordot(L,f4, axes=1) + + # get the angle of rotation to the z axis + f4_flavorsum = np.sum(f4, axis=2) # [xyzt, nu/nubar] + f4_eln = f4_flavorsum[:,0] - f4_flavorsum[:,1] # [xyz] + fn_eln_mag = np.sqrt(np.sum(f4_eln[:3]**2)) + costheta = f4_eln[2] / fn_eln_mag + + # get axis of rotation (the axis normal to the e and a fluxes) + u = np.cross(f4_eln[:3], np.array([0,0,1]), axisa=0, axisc=0) + u /= np.sqrt(np.sum(u**2)) + + R = axis_angle_rotation_matrix(u, costheta) + f4 = np.tensordot(R,f4, axes=1) + + f4 = np.moveaxis(f4, 0, 2) # [nu/nubar, flavor, xyzt] + + return f4[:,:,3], f4[:,:,0:3] diff --git a/Scripts/initial_conditions/st0_msw_test.py b/Scripts/initial_conditions/st0_msw_test.py index aecb56ce..2efafa1d 100644 --- a/Scripts/initial_conditions/st0_msw_test.py +++ b/Scripts/initial_conditions/st0_msw_test.py @@ -22,7 +22,7 @@ energy_erg = dm2*amrex.clight**4 * np.sin(2.*theta12) / (8.*np.pi*amrex.hbar*amrex.clight) # *1cm for units # get variable keys -rkey, ikey = amrex.get_particle_keys(ignore_pos=True) +rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) nelements = len(rkey) # generate the grid of direction coordinates diff --git a/Scripts/initial_conditions/st1_bipolar_test.py b/Scripts/initial_conditions/st1_bipolar_test.py index 8d98c742..f915303e 100644 --- a/Scripts/initial_conditions/st1_bipolar_test.py +++ b/Scripts/initial_conditions/st1_bipolar_test.py @@ -29,7 +29,7 @@ ndens_per_particle = ndens / nparticles # cm^-3 # get variable keys -rkey, ikey = amrex.get_particle_keys(ignore_pos=True) +rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) nelements = len(rkey) diff --git a/Scripts/initial_conditions/st2_2beam_fast_flavor.py b/Scripts/initial_conditions/st2_2beam_fast_flavor.py index 83c74492..740b5de1 100644 --- a/Scripts/initial_conditions/st2_2beam_fast_flavor.py +++ b/Scripts/initial_conditions/st2_2beam_fast_flavor.py @@ -30,7 +30,7 @@ ndens_per_particle = ndens / nparticles # cm^-3 # get variable keys -rkey, ikey = amrex.get_particle_keys(ignore_pos=True) +rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) nelements = len(rkey) diff --git a/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py b/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py index b27b65a0..bbcdfab4 100644 --- a/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py +++ b/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py @@ -32,7 +32,7 @@ ndens_per_particle = ndens / nparticles # cm^-3 # get variable keys -rkey, ikey = amrex.get_particle_keys(ignore_pos=True) +rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) nelements = len(rkey) diff --git a/Scripts/initial_conditions/st4_linear_moment_ffi.py b/Scripts/initial_conditions/st4_linear_moment_ffi.py index 7364f65f..661c5cfe 100644 --- a/Scripts/initial_conditions/st4_linear_moment_ffi.py +++ b/Scripts/initial_conditions/st4_linear_moment_ffi.py @@ -4,8 +4,8 @@ import os importpath = os.path.dirname(os.path.realpath(__file__)) sys.path.append(importpath) -sys.path.append(importpath+"/../visualization") -from initial_condition_tools import uniform_sphere, write_particles +sys.path.append(importpath+"/../data_analysis") +from initial_condition_tools import uniform_sphere, write_particles, moment_interpolate_particles, linear_interpolate import amrex_plot_tools as amrex # generation parameters @@ -22,38 +22,24 @@ fluxfacbar = .333333333333333 energy_erg = 50 * 1e6*amrex.eV -# generate the grid of direction coordinates -phat = uniform_sphere(nphi_equator) -nparticles = len(phat) - # flux factor vectors -fhat = [np.cos(phi) *np.sin(theta ), - np.sin(phi) *np.sin(theta ), - np.cos(theta )] -fhatbar = [np.cos(phibar)*np.sin(thetabar), - np.sin(phibar)*np.sin(thetabar), - np.cos(thetabar)] - -# get variable keys -rkey, ikey = amrex.get_particle_keys(ignore_pos=True) -nelements = len(rkey) - -# generate the list of particle info -particles = np.zeros((nparticles,nelements)) -for ip in range(nparticles): - u = phat[ip] - p = particles[ip] - p[rkey["pupt"]] = energy_erg - p[rkey["pupx"]] = u[0] * energy_erg - p[rkey["pupy"]] = u[1] * energy_erg - p[rkey["pupz"]] = u[2] * energy_erg - costheta = fhat [0]*u[0] + fhat [1]*u[1] + fhat [2]*u[2] - costhetabar = fhatbar[0]*u[0] + fhatbar[1]*u[1] + fhatbar[2]*u[2] - - p[rkey["N" ]] = ndens /nparticles * (1. + 3.*fluxfac *costheta ); - p[rkey["Nbar"]] = ndensbar/nparticles * (1. + 3.*fluxfacbar*costhetabar); - p[rkey["f00_Re" ]] = 1 - p[rkey["f00_Rebar"]] = 1 - +fhat = np.array([np.cos(phi) *np.sin(theta ), + np.sin(phi) *np.sin(theta ), + np.cos(theta )]) +fhatbar = np.array([np.cos(phibar)*np.sin(thetabar), + np.sin(phibar)*np.sin(thetabar), + np.cos(thetabar)]) + +nnu = np.zeros((2,NF)) +nnu[0,0] = ndens +nnu[1,0] = ndensbar +nnu[:,1:] = 0 + +fnu = np.zeros((2,NF,3)) +fnu[0,0,:] = ndens * fluxfac * fhat +fnu[1,0,:] = ndensbar * fluxfacbar * fhatbar +fnu[:,1:,:] = 0 + +particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, linear_interpolate) # [particle, variable] write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st5_minerbo.py b/Scripts/initial_conditions/st5_minerbo.py index 7bd5bde2..292b5e65 100644 --- a/Scripts/initial_conditions/st5_minerbo.py +++ b/Scripts/initial_conditions/st5_minerbo.py @@ -4,8 +4,8 @@ import os importpath = os.path.dirname(os.path.realpath(__file__)) sys.path.append(importpath) -sys.path.append(importpath+"/../visualization") -from initial_condition_tools import uniform_sphere, write_particles, minerbo_Z, minerbo_closure +sys.path.append(importpath+"/../data_reduction") +from initial_condition_tools import uniform_sphere, moment_interpolate_particles, minerbo_interpolate, write_particles import amrex_plot_tools as amrex # generation parameters @@ -20,62 +20,16 @@ fnux = np.array([-0.02165833, 0.07431613, -0.53545951]) energy_erg = 20.05473294163565 * 1e6*amrex.eV -# flux factor -fluxfac_e = np.sqrt(np.sum(fnue**2)) -fluxfac_a = np.sqrt(np.sum(fnua**2)) -fluxfac_x = np.sqrt(np.sum(fnux**2)) - -# get the Z parameters for the Minerbo closure -Ze = minerbo_Z(fluxfac_e); -Za = minerbo_Z(fluxfac_a); -Zx = minerbo_Z(fluxfac_x); - -# generate the grid of direction coordinates -phat = uniform_sphere(nphi_equator) -nparticles = len(phat) - -# get variable keys -rkey, ikey = amrex.get_particle_keys(ignore_pos=True) -nelements = len(rkey) - -# generate the list of particle info -particles = np.zeros((nparticles,nelements)) -for ip in range(nparticles): - u = phat[ip] - p = particles[ip] - p[rkey["pupt"]] = energy_erg - p[rkey["pupx"]] = u[0] * energy_erg - p[rkey["pupy"]] = u[1] * energy_erg - p[rkey["pupz"]] = u[2] * energy_erg - - # get the cosine of the angle between the direction and each flavor's flux vector - mue = np.sum(fnue*u)/fluxfac_e if fluxfac_e>0 else 0 - mua = np.sum(fnua*u)/fluxfac_a if fluxfac_a>0 else 0 - mux = np.sum(fnux*u)/fluxfac_x if fluxfac_x>0 else 0 - - # get the number of each flavor in this particle. - # nnux contains the number density of mu+tau neutrinos+antineutrinos - # Nnux_thisparticle contains the number of EACH of mu/tau anti/neutrinos (hence the factor of 4) - Nnue_thisparticle = nnue/nparticles * minerbo_closure(Ze, mue) - Nnua_thisparticle = nnua/nparticles * minerbo_closure(Za, mua) - Nnux_thisparticle = nnux/nparticles * minerbo_closure(Zx, mux) / 4.0 - - # set total number of neutrinos the particle has as the sum of the flavors - p[rkey["N" ]] = Nnue_thisparticle + Nnux_thisparticle - p[rkey["Nbar"]] = Nnua_thisparticle + Nnux_thisparticle - if NF==3: - p[rkey["N" ]] += Nnux_thisparticle - p[rkey["Nbar"]] += Nnux_thisparticle - - # set on-diagonals to have relative proportion of each flavor - p[rkey["f00_Re"]] = Nnue_thisparticle / p[rkey["N" ]] - p[rkey["f11_Re"]] = Nnux_thisparticle / p[rkey["N" ]] - p[rkey["f00_Rebar"]] = Nnua_thisparticle / p[rkey["Nbar"]] - p[rkey["f11_Rebar"]] = Nnux_thisparticle / p[rkey["Nbar"]] - if NF==3: - p[rkey["f22_Re"]] = Nnux_thisparticle / p[rkey["N" ]] - p[rkey["f22_Rebar"]] = Nnux_thisparticle / p[rkey["Nbar"]] +nnu = np.zeros((2,NF)) +nnu[0,0] = nnue +nnu[1,0] = nnua +nnu[:,1:] = nnux / 4. +fnu = np.zeros((2,NF,3)) +fnu[0,0,:] = nnue * fnue +fnu[1,0,:] = nnua * fnua +fnu[:,1:,:] = nnu[:,1:,np.newaxis] * fnux[np.newaxis,np.newaxis,:] +particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, minerbo_interpolate) # [particle, variable] write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st6_emu_pointwise.py b/Scripts/initial_conditions/st6_emu_pointwise.py new file mode 100644 index 00000000..2b9c9178 --- /dev/null +++ b/Scripts/initial_conditions/st6_emu_pointwise.py @@ -0,0 +1,192 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../visualization") +from initial_condition_tools import write_particles +import amrex_plot_tools as amrex + +# generation parameters +emu_filename = "/mnt/scratch/crossing_comparison/sedonu_data/NsNs_LS220-q1-D24_Ev_Lev0_AA-Merger_M1MC/run_8x16/fluid_00001.h5" +iEmu = 126 +jEmu = 103 +kEmu = 12 +NF = 2 +refine_factor = 3 + +# open the emu file +infile = h5py.File(emu_filename,"r") + +# get the volume of the chosen grid cell +xgrid = np.array(infile["axes/x0(cm)[edge]"]) +ygrid = np.array(infile["axes/x1(cm)[edge]"]) +zgrid = np.array(infile["axes/x2(cm)[edge]"]) +dx = xgrid[iEmu+1]-xgrid[iEmu] +dy = ygrid[jEmu+1]-ygrid[jEmu] +dz = zgrid[kEmu+1]-zgrid[kEmu] +dV = dx * dy * dz + +# get the angular grid +costheta_grid = np.array(infile["axes/distribution_costheta_grid(lab)[edge]"]) +costheta_mid = np.array(infile["axes/distribution_costheta_grid(lab)[mid]"]) +phi_grid = np.array(infile["axes/distribution_phi_grid(radians,lab)[edge]"]) +phi_mid = np.array(infile["axes/distribution_phi_grid(radians,lab)[mid]"]) +energy_grid = np.array(infile["axes/frequency(Hz)[edge]"]) * amrex.hbar*2.*np.pi +energy_mid = np.array(infile["axes/frequency(Hz)[mid]" ]) * amrex.hbar*2.*np.pi +ncostheta = len(costheta_mid) +nphi = len(phi_mid) + +# get the full distribution [nu/antinu, energy, mu, phi] +fee = np.array(infile["distribution0(erg|ccm,tet)"][iEmu][jEmu][kEmu]) +feebar = np.array(infile["distribution1(erg|ccm,tet)"][iEmu][jEmu][kEmu]) +f = np.array([fee,feebar]) +infile.close() + +# integrate over energy to get the energy density, number density, and average energy +energy_density = np.sum(f, axis=1) +number_density = np.sum(f / energy_mid[np.newaxis,:,np.newaxis,np.newaxis], axis=1) +energy_erg = np.sum(energy_density) / np.sum(number_density) + + + +# refine the distribution +def refine_grid(grid): + grid_refined = [] + for i in range(len(grid)-1): + delta = (grid[i+1]-grid[i]) / refine_factor + for j in range(refine_factor): + grid_refined.append(grid[i] + delta*j) + grid_refined.append(grid[-1]) + grid_refined = np.array(grid_refined) + mid_refined = np.array([(grid_refined[i] + grid_refined[i+1])/2. for i in range(len(grid_refined)-1)]) + return mid_refined, grid_refined + +phi_mid_refined, phi_grid_refined = refine_grid(phi_grid) +costheta_mid_refined, costheta_grid_refined = refine_grid(costheta_grid) +theta_mid_refined = np.arccos(costheta_mid_refined) +sintheta_mid_refined = np.sin(theta_mid_refined) + +ncostheta_refined = len(costheta_mid_refined) +nphi_refined = len(phi_mid_refined) + +# interpolate the distribution +def interpolate_2d_polar_1point(costheta_mid, phi_mid, f, costheta, phi): + m,n = f.shape + pole0 = np.average(f[ 0,:]) + pole1 = np.average(f[-1,:]) + + if phi <= phi_mid[0] or phi>=phi_mid[-1]: + jL = -1 + jR = 0 + else: + jL = np.where(phi_midphi)[0][ 0] + phiL = phi_mid[jL] + phiR = phi_mid[jR] + + if costheta >= costheta_mid[-1]: + iL = -1 + costhetaL = costheta_mid[iL] + costhetaR = 1 + fLL = f[iL,jL] + fLR = f[iL,jR] + fRL = pole1 + fRR = pole1 + elif costheta <= costheta_mid[0]: + iR = 0 + costhetaL = -1 + costhetaR = costheta_mid[iR] + fLL = pole0 + fLR = pole0 + fRL = f[iR,jL] + fRR = f[iR,jR] + else: + iL = np.where(costheta_midcostheta)[0][ 0] + costhetaL = costheta_mid[iL] + costhetaR = costheta_mid[iR] + fLL = f[iL,jL] + fLR = f[iL,jR] + fRL = f[iR,jL] + fRR = f[iR,jR] + + # calculate the coefficients + dcosthetaL = costheta - costhetaL + dcosthetaR = costhetaR - costheta + dphiL = phi - phiL + dphiR = phiR - phi + dV = (costhetaR-costhetaL) * (phiR-phiL) + cLL = dcosthetaR * dphiR / dV + cLR = dcosthetaR * dphiL / dV + cRL = dcosthetaL * dphiR / dV + cRR = dcosthetaL * dphiL / dV + + # evaluate the result + f_interpolated = (fLL*cLL + fLR*cLR + fRL*cRL + fRR*cRR) / refine_factor**2 + + return f_interpolated + +def interpolate_2d_polar(costheta_mid, phi_mid, f, costheta_mid_refined, phi_mid_refined): + result = np.array([[ interpolate_2d_polar_1point(costheta_mid, phi_mid, f, costheta_mid_refined[i], phi_mid_refined[j]) + for j in range(nphi_refined)] + for i in range(ncostheta_refined)]) + return result + +# interpolate onto the new mesh +number_density_refined = np.array([ + interpolate_2d_polar(costheta_mid, phi_mid, number_density[i], costheta_mid_refined, phi_mid_refined) + for i in range(2)]) +energy_density_refined = np.array([ + interpolate_2d_polar(costheta_mid, phi_mid, energy_density[i], costheta_mid_refined, phi_mid_refined) + for i in range(2)]) +shape = number_density_refined.shape +nparticles = shape[1]*shape[2] + +# print useful quantities +N = np.sum(number_density_refined, axis=(1,2)) +Fx = np.sum(number_density_refined * sintheta_mid_refined[np.newaxis,:,np.newaxis]*np.cos(phi_mid_refined[np.newaxis,np.newaxis,:]), axis=(1,2)) / N +Fy = np.sum(number_density_refined * sintheta_mid_refined[np.newaxis,:,np.newaxis]*np.sin(phi_mid_refined[np.newaxis,np.newaxis,:]), axis=(1,2)) / N +Fz = np.sum(number_density_refined * costheta_mid_refined[np.newaxis,:,np.newaxis], axis=(1,2)) / N +print("Writing",ncostheta_refined*nphi_refined,"particles") +print("ncostheta =",ncostheta_refined) +print("nphi =",nphi_refined) +print("[x,y,z](km) = [",(xgrid[iEmu]+dx/2)/1e5,(ygrid[jEmu]+dy/2)/1e5,(zgrid[kEmu]+dz/2)/1e5,"]") +print("Nee =",N[0],"cm^-3") +print("Neebar =",N[1],"cm^-3") +print("fluxfac_ee = [",Fx[0],Fy[0],Fz[0],"]") +print("fluxfac_eebar = [",Fx[1],Fy[1],Fz[1],"]") +print("|fluxfac_ee| =",np.sqrt(Fx[0]**2+Fy[0]**2+Fz[0]**2)) +print("|fluxfac_eebar| =",np.sqrt(Fx[1]**2+Fy[1]**2+Fz[1]**2)) + +# get variable keys +rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) +nelements = len(rkey) + +# generate list of particles +particles = np.zeros((nparticles, nelements)) +for i in range(ncostheta_refined): + ctheta = costheta_mid_refined[i] + theta = np.arccos(ctheta) + stheta = np.sin(theta) + for j in range(nphi_refined): + cphi = np.cos(phi_mid_refined[i]) + sphi = np.sin(phi_mid_refined[i]) + + index = i*ncostheta_refined + j + p = particles[index] + + u = np.array([stheta*cphi, stheta*sphi, ctheta]) + p[rkey["pupt"]] = energy_erg + p[rkey["pupx"]] = u[0] * energy_erg + p[rkey["pupy"]] = u[1] * energy_erg + p[rkey["pupz"]] = u[2] * energy_erg + + p[rkey["N" ]] = number_density_refined[0,i,j] + p[rkey["Nbar"]] = number_density_refined[1,i,j] + p[rkey["f00_Re" ]] = 1 + p[rkey["f00_Rebar"]] = 1 + + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/runtools/.gitignore b/Scripts/runtools/.gitignore new file mode 100644 index 00000000..c24e9012 --- /dev/null +++ b/Scripts/runtools/.gitignore @@ -0,0 +1,2 @@ +*~ +__pycache__ \ No newline at end of file diff --git a/Scripts/runtools/runtools.py b/Scripts/runtools/runtools.py new file mode 100644 index 00000000..c1ba6ab5 --- /dev/null +++ b/Scripts/runtools/runtools.py @@ -0,0 +1,159 @@ +import os +import shutil +import glob + +n_digits_rundir = 3 +rundir_base = "RUN" + +# write an input file in a particular directory with a set of inputs +def write_inputfile(directory, inputs_dict, filename="inputs"): + path = directory+"/"+filename + f = open(path,"w") + for key in inputs_dict: + f.write(key+" = ") + value = inputs_dict[key] + wrapper = "\"" if type(value)==str else "" + f.write(wrapper) + if type(value)==list: + first = True + for elem in value: + if first: f.write("(") + else: f.write(",") + first = False + f.write(str(elem)) + f.write(")") + else: + f.write(str(inputs_dict[key])) + f.write(wrapper) + f.write("\n") + f.close() + +def clean_string(s): + for character in ["\"","\'"," ","\n"]: + s = s.replace(character,"") + return s + +# given a run directory and an optional inputs filename, +# return a dictionary with key-value pairs for the parameters +# keys and values are both returned as strings +def read_inputfile(directory, filename="inputs"): + path = directory + "/"+filename + f = open(path,"r") + inputs_dict = {} + for line in f.readlines(): + # remove comments + line = line.split("#")[0] + + # skip empty lines + if "=" not in line: + continue + + # split line into key/value + key,value = line.split("=") + + # get the format + is_string = any((c in value) for c in ['\"','\'']) + is_list = "(" in value + is_float = (not is_string) and (("." in value) or ("e" in value) ) + is_int = (not is_float) and (not is_string) + + if(is_int): cast_func = int + if(is_float): cast_func = float + if(is_string): cast_func = clean_string + + if is_list: + value = value.replace("(","") + value = value.replace(")","") + value = value.split(",") + value = [cast_func(v) for v in value] + else: + value = cast_func(value) + + key = key.strip() + + # append to the dictionary + inputs_dict[key] = value + + f.close() + return inputs_dict + +# set up a new simulation in the specified location with an inputs dictionary and executable +# Refuse to overwrite unless overwrite==True +# note that the simulation still lacks a particle file! Must be created separately. +def create_new_simulation(directory, inputs_dict, executable_path, overwrite=False): + + # check whether the directory already exiss + if os.path.exists(directory): + if not overwrite: + raise(directory+" already exists. Cannot create ") + else: + os.system('rm -rf '+directory) + print(" Creating",directory) + + # create the new directories + os.mkdir(directory) + subdir = directory+"/"+rundir_base+str(1).zfill(n_digits_rundir) + os.mkdir(subdir) + + # copy in the executable + os.system("cp "+executable_path+" "+subdir) + + # write the inputs file + write_inputfile(subdir, inputs_dict) + + return subdir + +def out_step_list(directory,subdir=None): + search_string = directory+"/plt?????" + if subdir==None: + dirlist = glob.glob(search_string) + steplist = sorted([int(d.split("/")[-1][3:]) for d in dirlist]) + else: + search_string = search_string+"/"+subdir + dirlist = glob.glob(search_string) + steplist = sorted([int(d.split("/")[-2][3:]) for d in dirlist]) + return steplist + +# create a new run subdirectory based on the previous one +# replace the parameters and executable as necessary +def create_restart_simulation(directory, replace_inputs_dict=None, executable_path=None): + # determine new run dir + rundirlist = sorted(glob.glob(directory+"/"+rundir_base+"*")) + assert(len(rundirlist)>0) + lastrun = int(rundirlist[-1].split("/")[-1].replace(rundir_base,"").strip("0")) + nextrundir = directory+"/"+rundir_base+str(lastrun+1).zfill(n_digits_rundir) + os.mkdir(nextrundir) + + # print some diagnostics + print(" Restarting",directory,"at",nextrundir) + last_step_list = out_step_list(rundirlist[-1]) + print(" "+rundirlist[-1]+" outputs grid data from",str(last_step_list[0]),"to",str(last_step_list[-1])) + particle_step_list = out_step_list(rundirlist[-1],"neutrinos") + print(" "+rundirlist[-1]+" outputs particle data from",str(particle_step_list[0]),"to",str(particle_step_list[-1])) + assert(len(particle_step_list)>0) + + # get inputs dict from previous simulation + inputs_dict = read_inputfile(rundirlist[-1]) + + # activate restarting + inputs_dict["do_restart"] = 1 + + # set restart directory + inputs_dict["restart_dir"] = "../"+rundirlist[-1].split("/")[-1]+"/plt"+str(particle_step_list[-1]).zfill(5) + + # use replace_inputs_dict last to override anything else + if replace_inputs_dict != None: + print("replacing dict") + for key in replace_inputs_dict: + print(key in inputs_dict) + inputs_dict[key] = replace_inputs_dict[key] + + # write the inputs file + write_inputfile(nextrundir,inputs_dict) + + # copy in executable + if executable_path==None: + executable_path = rundirlist[-1]+"/*.ex" + os.system("cp "+executable_path+" "+nextrundir) + + return nextrundir diff --git a/Scripts/tests/fast_flavor_k_test.py b/Scripts/tests/fast_flavor_k_test.py index 31562870..5703e49c 100644 --- a/Scripts/tests/fast_flavor_k_test.py +++ b/Scripts/tests/fast_flavor_k_test.py @@ -4,7 +4,7 @@ import EmuReader import sys import os -importpath = os.path.dirname(os.path.realpath(__file__))+"/../visualization/" +importpath = os.path.dirname(os.path.realpath(__file__))+"/../data_reduction/" sys.path.append(importpath) import amrex_plot_tools as amrex @@ -17,6 +17,7 @@ tolerance = 1e-2 i0 = 100 i1 = 160 +NF = 2 # get domain size file = open("../sample_inputs/inputs_fast_flavor_nonzerok","r") @@ -26,7 +27,7 @@ if __name__ == "__main__": - rkey, ikey = amrex.get_particle_keys() + rkey, ikey = amrex.get_particle_keys(NF) t = [] fexR = [] diff --git a/Scripts/tests/fast_flavor_test.py b/Scripts/tests/fast_flavor_test.py index 86c6e8d5..ff9d7647 100644 --- a/Scripts/tests/fast_flavor_test.py +++ b/Scripts/tests/fast_flavor_test.py @@ -4,7 +4,7 @@ import EmuReader import sys import os -importpath = os.path.dirname(os.path.realpath(__file__))+"/../visualization/" +importpath = os.path.dirname(os.path.realpath(__file__))+"/../data_reduction/" sys.path.append(importpath) import amrex_plot_tools as amrex @@ -19,10 +19,11 @@ tolerance = 2e-2 i0 = 50 i1 = 70 +NF=2 if __name__ == "__main__": - rkey, ikey = amrex.get_particle_keys() + rkey, ikey = amrex.get_particle_keys(NF) t = [] fexR = [] diff --git a/Scripts/tests/msw_test.py b/Scripts/tests/msw_test.py index ce90c898..0c4f24ac 100644 --- a/Scripts/tests/msw_test.py +++ b/Scripts/tests/msw_test.py @@ -4,7 +4,7 @@ import EmuReader import sys import os -importpath = os.path.dirname(os.path.realpath(__file__))+"/../visualization/" +importpath = os.path.dirname(os.path.realpath(__file__))+"/../data_reduction/" sys.path.append(importpath) import amrex_plot_tools as amrex @@ -20,6 +20,7 @@ dm21c4 = 7.39e-5 * eV**2 # erg^2 mp = 1.6726219e-24 # g GF = 1.1663787e-5 / (1e9*eV)**2 * (hbar*clight)**3 #erg cm^3 +NF = 2 tolerance = 7e-2 @@ -31,7 +32,7 @@ if __name__ == "__main__": - rkey, ikey = amrex.get_particle_keys() + rkey, ikey = amrex.get_particle_keys(NF) t = [] fee = [] diff --git a/makefiles/GNUmakefile_jenkins b/makefiles/GNUmakefile_jenkins index 62488162..da1d700d 100644 --- a/makefiles/GNUmakefile_jenkins +++ b/makefiles/GNUmakefile_jenkins @@ -15,7 +15,7 @@ USE_MPI = TRUE USE_OMP = FALSE USE_ACC = FALSE USE_CUDA = TRUE -AMREX_CUDA_ARCH=60 +AMREX_CUDA_ARCH=75 TINY_PROFILE = TRUE USE_PARTICLES = TRUE From 85ff792c7a38256041ae23feca5b96a92d69ba9f Mon Sep 17 00:00:00 2001 From: Sherwood Richers <5142652+srichers@users.noreply.github.com> Date: Sun, 2 Apr 2023 18:20:07 -0400 Subject: [PATCH 033/276] Realtime reduced data (#82) * output time and step number to a file rather than screen. Will continue with more meaningful outputs * move all data reduction including file IO to a separate data reduction class * added missing files for the DataReducer class * bring amrex up to 23.02 * added parallel reduction operation over grid. ParReduce is a bit more straightforward than doing the array reduction followed by a parallel reduction * modify reduction operator to deal with an ArithmeticArray, which is an array that includes basic arithmetic operations * use code generator to make data reduction work for any number of flavors * add Tr(rho) to the scalar output. Also only output to the reduction file if the processor id is 0 * generate particle reduction code to work with any number of particles * added HighFive submodule * first bit of support for hdf5 realtime io * added hdf5 support for all reduced data * added support for outputting the net energy of the system. Required adding one more scalar variable to each particle. This is a good metric to track numerical errors - the number should remain constant * remove commented reduction examples * total energy should account for the relative number of real neutrinos present in a particular neutrino particle. * only output particle info at first timestep if it is output at any time * add fluxes to realtime data output * comment out print lines so initial conditions functions can be used more cleanly in scripts * Reduced max grid size on FFI test to force MPI parallelism * Run fiducial simulation in test suite. Will use it to test data reduction scripts as well. * Get rid of mpi errors in stdout * Reduce the number of steps the fiducial test takes to reduce testing time * install yt to be able to test data reduction scripts * Add more data reduction scripts to fiducial test * First attempt at including HFD5 tests in jenkins * Remove extraneous plot files to avoid accumuation of 'old' files during testing * move amrex to submodules directory * Clean up makefiles to make them a little bit more user friendly and transparent * fix glob search string so reduction script doesnt match to reduced data files * fix python script filename * further reduce number of steps for test speed * fix some paths to get hdf5 working in tests (hopefully) * Protect reduced data file initialization to only occur for IO processor. Mixed in with some formatting changes from VS Code. * cache submodules for faster testing * ignore hdf5 files * Create a three-flavor version of the initial condition script. * Add understandable error message that works even when the code is not compiled in debug mode to save future headache * Add babysitting plot scripts to plot reduced data output. Add to Jenkins to make sure they work and the reduced data look good. * add gnuplot library * Rename one of the makefiles to be a clear default * draft update of the readme * line spacing * A few more tweaks going through the readme and testing * updated change log for next version --------- Co-authored-by: Sherwood Richers --- .gitmodules | 5 +- CHANGES.md | 13 ++ Dockerfile | 4 +- Exec/.gitignore | 3 + Exec/GNUmakefile | 28 --- Jenkinsfile | 41 +++- Make.Emu | 9 +- README.md | 34 ++- Scripts/babysitting/avgfee.py | 3 +- Scripts/babysitting/avgfee_HDF5.py | 76 ++++++ Scripts/babysitting/avgfee_gnuplot.plt | 7 + Scripts/data_reduction/reduce_data_fft.py | 2 +- .../initial_condition_tools.py | 12 +- .../st4_linear_moment_ffi_3F.py | 45 ++++ Scripts/symbolic_hermitians/generate_code.py | 52 +++++ Source/ArithmeticArray.H | 93 ++++++++ Source/DataReducer.H | 37 +++ Source/DataReducer.cpp | 219 ++++++++++++++++++ Source/FlavoredNeutrinoContainerInit.cpp | 1 + Source/IO.H | 9 + Source/Make.package | 3 + Source/main.cpp | 7 +- amrex | 1 - makefiles/GNUmakefile_cassowary | 19 ++ makefiles/GNUmakefile_default | 16 ++ makefiles/GNUmakefile_ganon | 33 --- makefiles/GNUmakefile_jenkins | 18 +- makefiles/GNUmakefile_jenkins_HDF5 | 19 ++ sample_inputs/inputs_1d_fiducial | 2 +- sample_inputs/inputs_fast_flavor_nonzerok | 2 +- submodules/HighFive | 1 + submodules/amrex | 1 + 32 files changed, 707 insertions(+), 108 deletions(-) delete mode 100644 Exec/GNUmakefile create mode 100755 Scripts/babysitting/avgfee_HDF5.py create mode 100644 Scripts/babysitting/avgfee_gnuplot.plt create mode 100644 Scripts/initial_conditions/st4_linear_moment_ffi_3F.py create mode 100644 Source/ArithmeticArray.H create mode 100644 Source/DataReducer.H create mode 100644 Source/DataReducer.cpp delete mode 160000 amrex create mode 100644 makefiles/GNUmakefile_cassowary create mode 100644 makefiles/GNUmakefile_default delete mode 100644 makefiles/GNUmakefile_ganon create mode 100644 makefiles/GNUmakefile_jenkins_HDF5 create mode 160000 submodules/HighFive create mode 160000 submodules/amrex diff --git a/.gitmodules b/.gitmodules index c5213734..f2322e41 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "amrex"] - path = amrex + path = submodules/amrex url = https://github.com/dwillcox/amrex.git +[submodule "submodules/HighFive"] + path = submodules/HighFive + url = https://github.com/BlueBrain/HighFive.git diff --git a/CHANGES.md b/CHANGES.md index 6017bdfb..bf3777e0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,16 @@ +# 1.4 + + * Restructured initial conditions to have python scripts generate particle distributions instead of doing so inside the C++ code. This improves the code's flexibility. + + * Add option to output data in HDF5 format. Post-processing scripts only work with the original binary format, since yt only reads the binary format. + + * Add realtime output of scalar quantities to make basic analysis many times faster than with the post-processing scripts. + + * Include all of the basic post-processing scripts with Emu itself to avoid keeping multiple incompatible copies of them. + +# 1.3 + + * Incorporated various feature additions used for _Code Comparison for Fast Flavor Instability Simulations_ (https://doi.org/10.1103/PhysRevD.106.043011) # 1.2 diff --git a/Dockerfile b/Dockerfile index bc578e3c..1c2d49fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM nvidia/cuda:11.4.0-devel-ubuntu20.04 ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update -RUN apt-get install -y python3 python3-pip gfortran build-essential libhdf5-openmpi-dev openmpi-bin pkg-config libopenmpi-dev openmpi-bin libblas-dev liblapack-dev libpnetcdf-dev git python-is-python3 -RUN pip3 install numpy matplotlib h5py scipy sympy +RUN apt-get install -y python3 python3-pip gfortran build-essential libhdf5-openmpi-dev openmpi-bin pkg-config libopenmpi-dev openmpi-bin libblas-dev liblapack-dev libpnetcdf-dev git python-is-python3 gnuplot +RUN pip3 install numpy matplotlib h5py scipy sympy yt ENV USER=jenkins ENV LOGNAME=jenkins diff --git a/Exec/.gitignore b/Exec/.gitignore index e02b640b..7797b25a 100644 --- a/Exec/.gitignore +++ b/Exec/.gitignore @@ -5,3 +5,6 @@ plt* tmp_build_dir Backtrace.* *.png +*.dat +*.h5 +*.pdf diff --git a/Exec/GNUmakefile b/Exec/GNUmakefile deleted file mode 100644 index 3577d5e8..00000000 --- a/Exec/GNUmakefile +++ /dev/null @@ -1,28 +0,0 @@ -EMU_HOME ?= ../ -AMREX_HOME ?= ../amrex - -SHAPE_FACTOR_ORDER ?= 2 -NUM_FLAVORS ?= 3 - -COMP = gnu - -DEBUG = FALSE - -USE_MPI = TRUE -USE_OMP = FALSE -USE_ACC = FALSE -USE_CUDA = FALSE -AMREX_CUDA_ARCH=60 - -USE_HDF5=FALSE -HDF5_HOME=/usr/local/hdf5-1.12.2_gnu9.4.0 - -TINY_PROFILE = TRUE -USE_PARTICLES = TRUE - -PRECISION = DOUBLE - -Bpack := -Blocs := . - -include $(EMU_HOME)/Make.Emu diff --git a/Jenkinsfile b/Jenkinsfile index 4d401eb9..84c48283 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,11 @@ pipeline { triggers { pollSCM('') } // Run tests whenever a new commit is detected. agent { dockerfile {args '--gpus all'}} // Use the Dockerfile defined in the root Flash-X directory + environment { + // Get rid of Read -1, expected , errno =1 error + // See https://github.com/open-mpi/ompi/issues/4948 + OMPI_MCA_btl_vader_single_copy_mechanism = 'none' + } stages { //=============================// @@ -27,13 +32,15 @@ pipeline { sh 'python ../Scripts/initial_conditions/st0_msw_test.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_msw_test' sh 'python ../Scripts/tests/msw_test.py' + sh 'rm -rf plt*' } }} stage('Bipolar'){ steps{ dir('Exec'){ - sh 'python ../Scripts/initial_conditions/st1_bipolar_test.py' + sh 'python ../Scripts/initial_conditions/st1_bipolar_test.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_bipolar_test' + sh 'rm -rf plt*' } }} @@ -42,6 +49,7 @@ pipeline { sh 'python ../Scripts/initial_conditions/st2_2beam_fast_flavor.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor' sh 'python ../Scripts/tests/fast_flavor_test.py' + sh 'rm -rf plt*' } }} @@ -50,9 +58,38 @@ pipeline { sh 'python ../Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor_nonzerok' sh 'python ../Scripts/tests/fast_flavor_k_test.py' + sh 'rm -rf plt*' } }} + stage('Fiducial 2F GPU Binary'){ steps{ + dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_1d_fiducial' + sh 'python ../Scripts/data_reduction/reduce_data.py' + sh 'python ../Scripts/data_reduction/reduce_data_fft.py' + sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data.h5' + sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data_fft_power.h5' + sh 'python ../Scripts/babysitting/avgfee.py' + sh 'python ../Scripts/babysitting/power_spectrum.py' + sh 'python ../Scripts/data_reduction/convertToHDF5.py' + sh 'gnuplot ../Scripts/babysitting/avgfee_gnuplot.plt' + archiveArtifacts artifacts: '*.pdf' + sh 'rm -rf plt*' + } + }} + + stage('Fiducial 3F CPU HDF5'){ steps{ + dir('Exec'){ + sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5 GNUmakefile' + sh 'make realclean; make generate; make -j' + sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi_3F.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.ex ../sample_inputs/inputs_1d_fiducial' + sh 'python3 ../Scripts/babysitting/avgfee_HDF5.py' + sh 'rm -rf plt*' + } + }} + } // stages{ post { @@ -62,7 +99,7 @@ pipeline { deleteDirs: true, disableDeferredWipeout: false, notFailBuild: true, - patterns: [[pattern: 'amrex', type: 'EXCLUDE']] ) // allow amrex to be cached + patterns: [[pattern: 'submodules', type: 'EXCLUDE']] ) // allow submodules to be cached } } diff --git a/Make.Emu b/Make.Emu index 1b756472..d1081c91 100644 --- a/Make.Emu +++ b/Make.Emu @@ -1,6 +1,11 @@ -NUM_FLAVORS ?= 2 -SHAPE_FACTOR_ORDER ?= 2 +# things that used to be defined in the makefile in Exec DIM = 3 +TINY_PROFILE = TRUE +USE_PARTICLES = TRUE +PRECISION = DOUBLE +Bpack := +Blocs := . + TOP := $(EMU_HOME) diff --git a/README.md b/README.md index d32d487c..bcabfc9c 100644 --- a/README.md +++ b/README.md @@ -29,30 +29,40 @@ If you would like to run Emu, first clone Emu with the AMReX submodule: ``` git clone --recurse-submodules https://github.com/AMReX-Astro/Emu.git +git submodule update ``` -Then change directories to `Emu/Exec`. - -Before each compilation, you must symbolically generate Emu source code for -the number of neutrino flavors you wish to use. Do this like: +Then change directories to `Emu/Exec`. Before each compilation, you must symbolically generate Emu source code for +the number of neutrino flavors you wish to use and specify a few other compile-time settings in a file called `GNUmakefile`. +Copy in a default makefile. In this file you can specify the number of neutrino flavors, whether to compile for GPUs, etc. We have set the defaults to 2 neutrino flavors, order 2 PIC shape factors, and compiling for a single CPU. ``` -make generate NUM_FLAVORS=2 +cp ../makefiles/GNUmakefile_default GNUmakefile ``` -Then compile Emu with `make`, e.g.: - +Compiling occurs in two stages. We first have to generate code according to the number of neutrino flavors. +``` +make generate +``` +Then we have to compile Emu. ``` -make NUM_FLAVORS=2 +make -j ``` -Emu parameters are set in an input file, and we provide a series of sample -input files for various simulation setups in `Emu/sample_inputs`. +The initial particle distribution is set by an ASCII particle data file. You can generate the data file with our initial condition scripts. For instance, if we want to simulate a two-beam fast flavor instability, generate the initial conditions using +``` +python3 ../Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py +``` +You should now see a new file called `particle_input.dat`. -You can run the MSW setup in Emu by doing: +The parameters for the simulation are set in input files. These include information about things like the size of the domain, the number of grid cells, and fundamental neutrino properties. Run the fast flavor test simulation using the particle distribution generated previously using one of the test input files stored in `sample_inputs` +``` +./main3d.gnu.TPROF.ex ../sample_inputs/inputs_fast_flavor_nonzerok +``` +We have a number of data reduction, analysis, and visualization scripts in the `Scripts` directory. Generate a PDF file titled `avgfee.pdf` showing the time evolution of the average number density of electron neutrinos using ``` -./main3d.gnu.TPROF.MPI.ex inputs_msw_test +gnuplot ../Scripts/babysitting/avgfee_gnuplot.plt ``` # Open Source Development diff --git a/Scripts/babysitting/avgfee.py b/Scripts/babysitting/avgfee.py index 4b17d038..58f6c09e 100755 --- a/Scripts/babysitting/avgfee.py +++ b/Scripts/babysitting/avgfee.py @@ -1,4 +1,5 @@ -# Run from /ocean/projects/phy200048p/shared to generate plot showing time evolution of at different dimensionalities +# plots and as a function of time +# without reference to the reduced data file outputs import os os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' diff --git a/Scripts/babysitting/avgfee_HDF5.py b/Scripts/babysitting/avgfee_HDF5.py new file mode 100755 index 00000000..57c2f8eb --- /dev/null +++ b/Scripts/babysitting/avgfee_HDF5.py @@ -0,0 +1,76 @@ +# plots and as a function of time +# assuming the code was compiled with HDF5 and wrote the file reduced0D.h5 + +import os +os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' +import numpy as np +import matplotlib.pyplot as plt +import glob +import h5py +import matplotlib as mpl +from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator,LogLocator) + + +base=["N","Fx","Fy","Fz"] +diag_flavor=["00","11"]#,"22"] +offdiag_flavor=["01"]#,"02","12"] +re=["Re","Im"] +# real/imag +R=0 +I=1 + + +###################### +# read averaged data # +###################### +avgData = h5py.File("reduced0D.h5","r") +t=np.array(avgData["time(s)"])*1e9 +N00=np.array(avgData["N00(1|ccm)"]) +Noffdiag = np.array(avgData["N_offdiag_mag(1|ccm)"]) +avgData.close() + +################ +# plot options # +################ +mpl.rcParams['font.size'] = 22 +mpl.rcParams['font.family'] = 'serif' +#mpl.rc('text', usetex=True) +mpl.rcParams['xtick.major.size'] = 7 +mpl.rcParams['xtick.major.width'] = 2 +mpl.rcParams['xtick.major.pad'] = 8 +mpl.rcParams['xtick.minor.size'] = 4 +mpl.rcParams['xtick.minor.width'] = 2 +mpl.rcParams['ytick.major.size'] = 7 +mpl.rcParams['ytick.major.width'] = 2 +mpl.rcParams['ytick.minor.size'] = 4 +mpl.rcParams['ytick.minor.width'] = 2 +mpl.rcParams['axes.linewidth'] = 2 + + +fig, ax = plt.subplots(1,1, figsize=(6,5)) + +############## +# formatting # +############## +ax.axhline(1./3., color="green") +ax.set_ylabel(r"$\langle N\rangle$ (cm$^{-3}$)") +ax.set_xlabel(r"$t\,(10^{-9}\,\mathrm{s})$") +ax.tick_params(axis='both', which='both', direction='in', right=True,top=True) +ax.xaxis.set_minor_locator(AutoMinorLocator()) +ax.yaxis.set_minor_locator(AutoMinorLocator()) +ax.minorticks_on() +ax.grid(which='both') + +############# +# plot data # +############# +ax.plot(t, N00) + +############ +# save pdf # +############ +plt.savefig("avgfee.pdf", bbox_inches="tight") + +# same for f_e\mu +ax.semilogy(t, Noffdiag) +plt.savefig("avgfemu.pdf", bbox_inches="tight") diff --git a/Scripts/babysitting/avgfee_gnuplot.plt b/Scripts/babysitting/avgfee_gnuplot.plt new file mode 100644 index 00000000..234139eb --- /dev/null +++ b/Scripts/babysitting/avgfee_gnuplot.plt @@ -0,0 +1,7 @@ +set term pdf +set output 'avgfee.pdf' +set key autotitle columnhead +set xlabel 'time (s)' +set ylabel 'N00 (cm^-3)' +set yrange [0:*] +plot 'reduced0D.dat' u 2:5 w l \ No newline at end of file diff --git a/Scripts/data_reduction/reduce_data_fft.py b/Scripts/data_reduction/reduce_data_fft.py index 9a5527b8..7abbe3fe 100755 --- a/Scripts/data_reduction/reduce_data_fft.py +++ b/Scripts/data_reduction/reduce_data_fft.py @@ -9,7 +9,7 @@ parser.add_argument("-o", "--output", type=str, default="reduced_data_fft.h5", help="Name of the output file (default: reduced_data_fft.h5)") args = parser.parse_args() -directories = sorted(glob.glob("plt*")) +directories = sorted(glob.glob("plt?????")) t = [] diff --git a/Scripts/initial_conditions/initial_condition_tools.py b/Scripts/initial_conditions/initial_condition_tools.py index 16c0f625..efa8ef6c 100644 --- a/Scripts/initial_conditions/initial_condition_tools.py +++ b/Scripts/initial_conditions/initial_condition_tools.py @@ -116,7 +116,7 @@ def minerbo_Z(fluxfac): print("Failed to converge on a solution.") assert(False) - print("fluxfac=",fluxfac," Z=",Z) + #print("fluxfac=",fluxfac," Z=",Z) return Z # angular structure as determined by the Levermore closure @@ -161,7 +161,7 @@ def levermore_v(fluxfac): print("Failed to converge on a solution.") assert(False) - print("fluxfac=",fluxfac," v=",v) + #print("fluxfac=",fluxfac," v=",v) return v # interpolate the levermore closure @@ -251,10 +251,10 @@ def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_g # double check that the number densities are correct particle_n = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]]) particle_fmag = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]] * mu[:,nu_nubar, flavor]) - print("nu/nubar,flavor =", nu_nubar, flavor) - print("output/input ndens =",particle_n, nnu[nu_nubar,flavor]) - print("output/input fluxfac =",particle_fmag / particle_n, fluxfac[nu_nubar,flavor]) - print() + #print("nu/nubar,flavor =", nu_nubar, flavor) + #print("output/input ndens =",particle_n, nnu[nu_nubar,flavor]) + #print("output/input fluxfac =",particle_fmag / particle_n, fluxfac[nu_nubar,flavor]) + #print() return particles diff --git a/Scripts/initial_conditions/st4_linear_moment_ffi_3F.py b/Scripts/initial_conditions/st4_linear_moment_ffi_3F.py new file mode 100644 index 00000000..93bdc96c --- /dev/null +++ b/Scripts/initial_conditions/st4_linear_moment_ffi_3F.py @@ -0,0 +1,45 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../data_analysis") +from initial_condition_tools import uniform_sphere, write_particles, moment_interpolate_particles, linear_interpolate +import amrex_plot_tools as amrex + +# generation parameters +# MUST MATCH THE INPUTS IN THE EMU INPUT FILE! +NF = 3 +nphi_equator = 16 +theta = 0 +thetabar = 3.14159265359 +phi = 0 +phibar=0 +ndens = 4.891290819e+32 +ndensbar = 4.891290819e+32 +fluxfac = .333333333333333 +fluxfacbar = .333333333333333 +energy_erg = 50 * 1e6*amrex.eV + +# flux factor vectors +fhat = np.array([np.cos(phi) *np.sin(theta ), + np.sin(phi) *np.sin(theta ), + np.cos(theta )]) +fhatbar = np.array([np.cos(phibar)*np.sin(thetabar), + np.sin(phibar)*np.sin(thetabar), + np.cos(thetabar)]) + +nnu = np.zeros((2,NF)) +nnu[0,0] = ndens +nnu[1,0] = ndensbar +nnu[:,1:] = 0 + +fnu = np.zeros((2,NF,3)) +fnu[0,0,:] = ndens * fluxfac * fhat +fnu[1,0,:] = ndensbar * fluxfacbar * fhatbar +fnu[:,1:,:] = 0 + +particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, linear_interpolate) # [particle, variable] + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index b0f00a57..98d536a6 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -99,6 +99,7 @@ def delete_generated_files(): for v in vars: A = HermitianMatrix(args.N, v+"{}{}_{}"+t) code += A.header() + code += ["TrHf"] code = [code[i]+"," for i in range(len(code))] write_code(code, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.H_fill")) @@ -115,6 +116,7 @@ def delete_generated_files(): for v in vars: A = HermitianMatrix(args.N, v+"{}{}_{}"+t) code += A.header() + code += ["TrHf"] code_string = 'attribute_names = {"time", "x", "y", "z", "pupx", "pupy", "pupz", "pupt", ' code = ['"{}"'.format(c) for c in code] code_string = code_string + ", ".join(code) + "};" @@ -168,6 +170,51 @@ def delete_generated_files(): code.append(string1+deplist[icomp]+string2+flist[icomp]+string3+string4[ivar]) write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_deposit_to_mesh_fill")) + #================================# + # DataReducer.cpp_fill_particles # + #================================# + tails = ["","bar"] + code = [] + for t in tails: + # diagonal averages + N = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") + Nlist = N.header_diagonals(); + for i in range(len(Nlist)): + code.append("Trf += "+Nlist[i]+";") + + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "DataReducer.cpp_fill_particles")) + + #======================# + # DataReducer.cpp_fill # + #======================# + tails = ["","bar"] + code = [] + for t in tails: + # diagonal averages + N = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::N{}{}_{}"+t+")") + Nlist = N.header_diagonals(); + Fx = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::Fx{}{}_{}"+t+")") + Fxlist = Fx.header_diagonals(); + Fy = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::Fy{}{}_{}"+t+")") + Fylist = Fy.header_diagonals(); + Fz = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::Fz{}{}_{}"+t+")") + Fzlist = Fz.header_diagonals(); + for i in range(len(Nlist)): + code.append("Ndiag"+t+"["+str(i)+"] = "+Nlist[i]+";") + code.append("Fxdiag"+t+"["+str(i)+"] = "+Fxlist[i]+";") + code.append("Fydiag"+t+"["+str(i)+"] = "+Fylist[i]+";") + code.append("Fzdiag"+t+"["+str(i)+"] = "+Fzlist[i]+";") + + # off-diagonal magnitude + mag2 = 0 + for i in range(N.size): + for j in range(i+1,N.size): + re,im = N.H[i,j].as_real_imag() + mag2 += re**2 + im**2 + code.append("N_offdiag_mag2 += "+sympy.cxxcode(sympy.simplify(mag2))+";") + + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "DataReducer.cpp_fill")) + #==================# # Evolve.H_M2_fill # #==================# @@ -384,6 +431,11 @@ def sgn(t,var): # Write out dFdt->F code.append(dFdt.code()) + + # store Tr(H*F) for estimating numerical errors + TrHf = (H*F).trace(); + code.append(["p.rdata(PIdx::TrHf) += p.rdata(PIdx::N"+t+") * ("+sympy.cxxcode(sympy.simplify(TrHf))+");"]) + code = [line for sublist in code for line in sublist] write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill")) diff --git a/Source/ArithmeticArray.H b/Source/ArithmeticArray.H new file mode 100644 index 00000000..36d32403 --- /dev/null +++ b/Source/ArithmeticArray.H @@ -0,0 +1,93 @@ +template +class ArithmeticArray{ +public: + std::array data; + + AMREX_GPU_HOST_DEVICE + ArithmeticArray(){} + + AMREX_GPU_HOST_DEVICE + ArithmeticArray(T in){ + for(size_t i=0; i + AMREX_GPU_HOST_DEVICE + ArithmeticArray(ArithmeticArray in){ + for(size_t i=0; i + AMREX_GPU_HOST_DEVICE + ArithmeticArray& operator=(const ArithmeticArray& input){ + for(size_t i=0; i& operator=(const double input){ + for(size_t i=0; i operator*(const double scale) const{ + ArithmeticArray result; + for(size_t i=0; i operator/(const double scale) const{ + double inv_scale = 1./scale; + return operator*(inv_scale); + } + template + AMREX_GPU_HOST_DEVICE + const ArithmeticArray operator+(const ArithmeticArray& input) const{ + ArithmeticArray result; + for(size_t i=0; i + AMREX_GPU_HOST_DEVICE + const ArithmeticArray operator-(const ArithmeticArray& input) const{ + ArithmeticArray result; + for(size_t i=0; i operator-() const{ + ArithmeticArray result; + for(size_t i=0; i + AMREX_GPU_HOST_DEVICE + void operator+=(const ArithmeticArray& input){ + for(size_t i=0; i + AMREX_GPU_HOST_DEVICE + bool operator==(const ArithmeticArray& input){ + bool isequal = true; + for(size_t i=0; i +#include +#include +#include +#include +#include +#ifdef AMREX_USE_HDF5 +#include <../submodules/HighFive/include/highfive/H5File.hpp> +#include <../submodules/HighFive/include/highfive/H5DataSpace.hpp> +#include <../submodules/HighFive/include/highfive/H5DataSet.hpp> +#endif + +class DataReducer{ +private: + +#ifdef AMREX_USE_HDF5 + const char* filename0D = "reduced0D.h5"; + const HighFive::DataSpace dataspace = HighFive::DataSpace({0}, {HighFive::DataSpace::UNLIMITED}); +#else + const char* filename0D = "reduced0D.dat"; +#endif + +public: + void + InitializeFiles(); + + void + WriteReducedData0D(const amrex::Geometry& geom, + const MultiFab& state, + const FlavoredNeutrinoContainer& neutrinos, + amrex::Real time, int step); +}; + +#endif diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp new file mode 100644 index 00000000..cb283d11 --- /dev/null +++ b/Source/DataReducer.cpp @@ -0,0 +1,219 @@ +#include "Evolve.H" +#include "Constants.H" +#include "DataReducer.H" +#include "ArithmeticArray.H" +#include +#include +#ifdef AMREX_USE_HDF5 +#include <../submodules/HighFive/include/highfive/H5File.hpp> +#include <../submodules/HighFive/include/highfive/H5DataSpace.hpp> +#include <../submodules/HighFive/include/highfive/H5DataSet.hpp> +#endif + +#ifdef AMREX_USE_HDF5 +// append a single scalar to the specified file and dataset +template +void append_0D(HighFive::File& file0D, const std::string& datasetname, const T value){ + HighFive::DataSet dataset_step = file0D.getDataSet(datasetname); + std::vector dims = dataset_step.getDimensions(); + dims[0] ++; + dataset_step.resize(dims); + dataset_step.select({dims[0]-1},{1}).write((T[1]){value}); +} +#endif + +void DataReducer::InitializeFiles() +{ + if (ParallelDescriptor::IOProcessor()) + { + +#ifdef AMREX_USE_HDF5 + + using namespace HighFive; + File file0D(filename0D, File::Truncate | File::Create); + + DataSetCreateProps props; + props.add(Chunking(std::vector{1})); + file0D.createDataSet("step", dataspace, create_datatype(), props); + file0D.createDataSet("time(s)", dataspace, create_datatype(), props); + file0D.createDataSet("Ntot(1|ccm)", dataspace, create_datatype(), props); + file0D.createDataSet("Ndiff(1|ccm)", dataspace, create_datatype(), props); + for (int i = 0; i < NUM_FLAVORS; i++) + { + file0D.createDataSet(std::string("N") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("N") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fx") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fy") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fz") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fx") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fy") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fz") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + } + file0D.createDataSet("N_offdiag_mag(1|ccm)", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrf", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrHf", dataspace, create_datatype(), props); + +#else + + std::ofstream outfile; + outfile.open(filename0D, std::ofstream::out); + int j = 0; + j++; + outfile << j << ":step\t"; + j++; + outfile << j << ":time(s)\t"; + j++; + outfile << j << ":Ntot(1|ccm)\t"; + j++; + outfile << j << ":Ndiff(1|ccm)\t"; + for (int i = 0; i < NUM_FLAVORS; i++) + { + j++; + outfile << j << ":N" << i << i << "(1|ccm)\t"; + j++; + outfile << j << ":N" << i << i << "bar(1|ccm)\t"; + j++; + outfile << j << ":Fx" << i << i << "(1|ccm)\t"; + j++; + outfile << j << ":Fy" << i << i << "(1|ccm)\t"; + j++; + outfile << j << ":Fz" << i << i << "(1|ccm)\t"; + j++; + outfile << j << ":Fx" << i << i << "bar(1|ccm)\t"; + j++; + outfile << j << ":Fy" << i << i << "bar(1|ccm)\t"; + j++; + outfile << j << ":Fz" << i << i << "bar(1|ccm)\t"; + } + j++; + outfile << j << ":N_offdiag_mag(1|ccm)\t"; + j++; + outfile << j << ":sumTrf\t"; + j++; + outfile << j << ":sumTrHf\t"; + outfile << std::endl; + outfile.close(); + +#endif + } +} + +void +DataReducer::WriteReducedData0D(const amrex::Geometry& geom, + const MultiFab& state, + const FlavoredNeutrinoContainer& neutrinos, + const amrex::Real time, const int step) +{ + // get index volume of the domain + int ncells = geom.Domain().volume(); + + //==================================// + // Do reductions over the particles // + //==================================// + using PType = typename FlavoredNeutrinoContainer::ParticleType; + amrex::ReduceOps reduce_ops; + auto particleResult = amrex::ParticleReduce< ReduceData >(neutrinos, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { + Real TrHf = p.rdata(PIdx::TrHf); + Real Trf = 0; +#include "generated_files/DataReducer.cpp_fill_particles" + return GpuTuple{Trf,TrHf}; + }, reduce_ops); + Real Trf = amrex::get<0>(particleResult); + Real TrHf = amrex::get<1>(particleResult); + ParallelDescriptor::ReduceRealSum(Trf); + ParallelDescriptor::ReduceRealSum(TrHf); + + //=============================// + // Do reductions over the grid // + //=============================// + // first, get a reference to the data arrays + auto const& ma = state.const_arrays(); + IntVect nghost(AMREX_D_DECL(0, 0, 0)); + + // use the ParReduce function to define reduction operator + GpuTuple< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > result = + ParReduce(TypeList{}, + TypeList, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray >{}, + state, nghost, + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> + GpuTuple, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > { + Array4 const& a = ma[box_no]; + + // Doing the actual work + ArithmeticArray Ndiag, Ndiagbar; + ArithmeticArray Fxdiag, Fxdiagbar; + ArithmeticArray Fydiag, Fydiagbar; + ArithmeticArray Fzdiag, Fzdiagbar; + Real N_offdiag_mag2 = 0; + #include "generated_files/DataReducer.cpp_fill" + return {Ndiag, Ndiagbar, N_offdiag_mag2, Fxdiag, Fydiag, Fzdiag, Fxdiagbar,Fydiagbar,Fzdiagbar}; + + }); + + // retrieve the reduced data values + ArithmeticArray N = amrex::get<0>(result) / ncells; + ArithmeticArray Nbar = amrex::get<1>(result) / ncells; + Real N_offdiag_mag = sqrt(amrex::get<2>(result)) / ncells; + ArithmeticArray Fx = amrex::get<3>(result) / ncells; + ArithmeticArray Fy = amrex::get<4>(result) / ncells; + ArithmeticArray Fz = amrex::get<5>(result) / ncells; + ArithmeticArray Fxbar = amrex::get<6>(result) / ncells; + ArithmeticArray Fybar = amrex::get<7>(result) / ncells; + ArithmeticArray Fzbar = amrex::get<8>(result) / ncells; + + // calculate net number of neutrinos and antineutrinos + Real Ntot=0, Ndiff=0; + for(int i=0; i > read_particle_data(std::strin ss = std::stringstream(line); int NF_in; ss >> NF_in; + if(NF_in != NUM_FLAVORS) amrex::Print() << "Error: number of flavors in particle data file does not match the number of flavors Emu was compiled for." << std::endl; AMREX_ASSERT(NF_in == NUM_FLAVORS); while(std::getline(file, line)){ diff --git a/Source/IO.H b/Source/IO.H index e851f0a2..c159b065 100644 --- a/Source/IO.H +++ b/Source/IO.H @@ -6,6 +6,15 @@ #include #include +void +InitializeReducedData0D(); + +void +WriteReducedData0D(const amrex::MultiFab& state, + const FlavoredNeutrinoContainer& neutrinos, + const amrex::Geometry& geom, amrex::Real time, + int step); + void WritePlotFile (const amrex::MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, diff --git a/Source/Make.package b/Source/Make.package index 191e7a7f..b038507d 100644 --- a/Source/Make.package +++ b/Source/Make.package @@ -1,4 +1,5 @@ CEXE_sources += main.cpp +CEXE_sources += DataReducer.cpp CEXE_sources += IO.cpp CEXE_sources += FlavoredNeutrinoContainerInit.cpp CEXE_sources += Evolve.cpp @@ -11,3 +12,5 @@ CEXE_headers += Particles.H CEXE_headers += IO.H CEXE_headers += Parameters.H CEXE_headers += ParticleInterpolator.H +CEXE_headers += DataReducer.H +CEXE_headers += ArithmeticArray.H \ No newline at end of file diff --git a/Source/main.cpp b/Source/main.cpp index b17266da..0b6eac4b 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -31,6 +31,7 @@ #include "Evolve.H" #include "Constants.H" #include "IO.H" +#include "DataReducer.H" using namespace amrex; @@ -118,10 +119,12 @@ void evolve_flavor(const TestParams* parms) deposit_to_mesh(neutrinos_old, state, geom); // Write plotfile after initialization + DataReducer rd; if (not parms->do_restart) { // If we have just initialized, then always save the particle data for reference - const int write_particles_after_init = 1; + const int write_particles_after_init = (parms->write_plot_particles_every>0); WritePlotFile(state, neutrinos_old, geom, initial_time, initial_step, write_particles_after_init); + rd.InitializeFiles(); } amrex::Print() << "Done. " << std::endl; @@ -176,7 +179,7 @@ void evolve_flavor(const TestParams* parms) const int step = integrator.get_step_number(); const Real time = integrator.get_time(); - amrex::Print() << "Completed time step: " << step << " t = " << time << " s. ct = " << PhysConst::c * time << " cm" << std::endl; + rd.WriteReducedData0D(geom, state, neutrinos, time, step+1); run_fom += neutrinos.TotalNumberOfParticles(); diff --git a/amrex b/amrex deleted file mode 160000 index ff9e8324..00000000 --- a/amrex +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ff9e832483f142a34da72f9bea5fed72e49fea33 diff --git a/makefiles/GNUmakefile_cassowary b/makefiles/GNUmakefile_cassowary new file mode 100644 index 00000000..4ed76b6f --- /dev/null +++ b/makefiles/GNUmakefile_cassowary @@ -0,0 +1,19 @@ +NUM_FLAVORS = 2 +SHAPE_FACTOR_ORDER = 2 + +COMP = gnu + +DEBUG = FALSE + +USE_MPI = TRUE +USE_OMP = FALSE +USE_ACC = FALSE +USE_CUDA = FALSE +AMREX_CUDA_ARCH=60 + +USE_HDF5=FALSE +HDF5_HOME=/usr/lib/x86_64-linux-gnu/hdf5/serial + +EMU_HOME = .. +AMREX_HOME = ../submodules/amrex +include ../Make.Emu diff --git a/makefiles/GNUmakefile_default b/makefiles/GNUmakefile_default new file mode 100644 index 00000000..74d83b04 --- /dev/null +++ b/makefiles/GNUmakefile_default @@ -0,0 +1,16 @@ +NUM_FLAVORS = 2 +SHAPE_FACTOR_ORDER = 2 + +COMP = gnu + +DEBUG = FALSE + +USE_MPI = FALSE +USE_OMP = FALSE +USE_ACC = FALSE +USE_CUDA = FALSE +USE_HDF5 = FALSE + +EMU_HOME = .. +AMREX_HOME = ../submodules/amrex +include ../Make.Emu diff --git a/makefiles/GNUmakefile_ganon b/makefiles/GNUmakefile_ganon deleted file mode 100644 index e6c6fa45..00000000 --- a/makefiles/GNUmakefile_ganon +++ /dev/null @@ -1,33 +0,0 @@ -EMU_HOME ?= ../ -AMREX_HOME ?= ../amrex - -DIM = 3 - -NUM_FLAVORS = 2 - -COMP = gnu - -DEBUG = TRUE - -USE_MPI = TRUE -USE_OMP = FALSE -USE_ACC = FALSE -USE_CUDA = FALSE -USE_HDF5 = FALSE - -TINY_PROFILE = TRUE -USE_PARTICLES = TRUE - -PRECISION = DOUBLE - -Bpack := -Blocs := . - -ifeq ($(USE_HDF5), TRUE) -HDF5_HOME = /usr/local/hdf5-1.12.0_gnu7.5.0 -DEFINES += -DAMREX_USE_HDF5 -INCLUDE_LOCATIONS += $(HDF5_HOME)/include -LIBRARIES += -L$(HDF5_HOME)/lib -lhdf5 -lz -ldl -endif - -include ../Make.Emu diff --git a/makefiles/GNUmakefile_jenkins b/makefiles/GNUmakefile_jenkins index da1d700d..431901c8 100644 --- a/makefiles/GNUmakefile_jenkins +++ b/makefiles/GNUmakefile_jenkins @@ -1,15 +1,9 @@ -EMU_HOME ?= ../ -AMREX_HOME ?= ../amrex - -DIM = 3 - NUM_FLAVORS = 2 - SHAPE_FACTOR_ORDER = 2 COMP = gnu -DEBUG = FALSE +DEBUG = FALSE USE_MPI = TRUE USE_OMP = FALSE @@ -17,12 +11,6 @@ USE_ACC = FALSE USE_CUDA = TRUE AMREX_CUDA_ARCH=75 -TINY_PROFILE = TRUE -USE_PARTICLES = TRUE - -PRECISION = DOUBLE - -Bpack := -Blocs := . - +EMU_HOME = .. +AMREX_HOME = ../submodules/amrex include ../Make.Emu diff --git a/makefiles/GNUmakefile_jenkins_HDF5 b/makefiles/GNUmakefile_jenkins_HDF5 new file mode 100644 index 00000000..1a5b3993 --- /dev/null +++ b/makefiles/GNUmakefile_jenkins_HDF5 @@ -0,0 +1,19 @@ +NUM_FLAVORS = 3 +SHAPE_FACTOR_ORDER = 2 + +COMP = gnu + +DEBUG = FALSE + +USE_MPI = TRUE +USE_OMP = FALSE +USE_ACC = FALSE +USE_CUDA = FALSE +AMREX_CUDA_ARCH=75 + +USE_HDF5=TRUE +HDF5_HOME=/usr/lib/x86_64-linux-gnu/hdf5/openmpi + +EMU_HOME = .. +AMREX_HOME = ../submodules/amrex +include ../Make.Emu diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 833421e8..c3e89d37 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -23,7 +23,7 @@ particle_data_filename = "particle_input.dat" max_grid_size = 16 # Number of steps to run -nsteps = 5000 +nsteps = 1000 # Simulation end time end_time = 5.0e-9 diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 1996e71a..dc858a8e 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -21,7 +21,7 @@ nppc = (1, 1, 1) particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain -max_grid_size = 1000 +max_grid_size = 10 # Number of steps to run nsteps = 5000 diff --git a/submodules/HighFive b/submodules/HighFive new file mode 160000 index 00000000..4d29deeb --- /dev/null +++ b/submodules/HighFive @@ -0,0 +1 @@ +Subproject commit 4d29deebf939ba47f8f28f0594b813fd9289c0fd diff --git a/submodules/amrex b/submodules/amrex new file mode 160000 index 00000000..eefe2463 --- /dev/null +++ b/submodules/amrex @@ -0,0 +1 @@ +Subproject commit eefe2463884081a27025973d4a99dc909657b4f0 From 602c433d51013f198dd8aff2ba27d42babb4d910 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 19:01:25 -0400 Subject: [PATCH 034/276] remove gitpod to decrease maintenance need --- .gitpod.dockerfile | 4 -- .gitpod.yml | 98 ---------------------------------------------- README.md | 19 --------- 3 files changed, 121 deletions(-) delete mode 100644 .gitpod.dockerfile delete mode 100644 .gitpod.yml diff --git a/.gitpod.dockerfile b/.gitpod.dockerfile deleted file mode 100644 index 4ae39609..00000000 --- a/.gitpod.dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM gitpod/workspace-full - -RUN sudo apt-get update && sudo apt-get install -y gfortran libopenmpi3 libopenmpi-dev && sudo rm -rf /var/lib/apt/lists/* - diff --git a/.gitpod.yml b/.gitpod.yml deleted file mode 100644 index 05d8da7b..00000000 --- a/.gitpod.yml +++ /dev/null @@ -1,98 +0,0 @@ -image: - file: .gitpod.dockerfile - -tasks: - - init: |- - sudo apt -y install gfortran libopenmpi3 libopenmpi-dev - mkdir /workspace/python - python -m venv /workspace/python/emu --system-site-packages - source /workspace/python/emu/bin/activate - pip install sympy numpy matplotlib - - pushd Scripts/visualization - python setup.py install - popd - - cd Exec - - make generate NUM_FLAVORS=2 - make -j NUM_FLAVORS=2 - mkdir 2-Flavors - mv *.ex 2-Flavors/. - pushd 2-Flavors - - mkdir bipolar - pushd bipolar - cp ../*.ex . - cp ../../../sample_inputs/inputs_bipolar_test . - cp ../../../Scripts/visualization/plot_first_particle.py . - popd - - mkdir msw - pushd msw - cp ../*.ex . - cp ../../../sample_inputs/inputs_msw_test . - cp ../../../Scripts/visualization/plot_first_particle.py . - popd - - mkdir FFI_zero_wavenumber - pushd FFI_zero_wavenumber - cp ../*.ex . - cp ../../../sample_inputs/inputs_fast_flavor . - popd - - mkdir FFI - pushd FFI - cp ../*.ex . - cp ../../../sample_inputs/inputs_fast_flavor_nonzerok . - popd - - popd - - make realclean - make generate NUM_FLAVORS=3 - make -j NUM_FLAVORS=3 - mkdir 3-Flavors - mv *.ex 3-Flavors/. - pushd 3-Flavors - - mkdir FFI - pushd FFI - cp ../*.ex . - cp ../../../sample_inputs/inputs_1d_fiducial . - popd - - popd - - make realclean - - mkdir Examples - mv 2-Flavors Examples/. - mv 3-Flavors Examples/. - mv Examples ../. - - clear - - - command: |- - source /workspace/python/emu/bin/activate - clear - -github: - prebuilds: - # enable for the master/default branch (defaults to true) - master: true - # enable for all branches in this repo (defaults to false) - branches: false - # enable for pull requests coming from this repo (defaults to true) - pullRequests: false - # enable for pull requests coming from forks (defaults to false) - pullRequestsFromForks: false - # add a check to pull requests (defaults to true) - addCheck: true - # add a "Review in Gitpod" button as a comment to pull requests (defaults to false) - addComment: true - # add a "Review in Gitpod" button to the pull request's description (defaults to false) - addBadge: true - # add a label once the prebuild is ready to pull requests (defaults to false) - addLabel: false - diff --git a/README.md b/README.md index 9aee5049..afa5121b 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ [![DOI](https://zenodo.org/badge/228717670.svg)](https://zenodo.org/badge/latestdoi/228717670) [![AMReX](https://amrex-codes.github.io/badges/powered%20by-AMReX-red.svg)](https://amrex-codes.github.io) -[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/AMReX-Astro/Emu) - ![Emu](https://github.com/AMReX-Astro/Emu/blob/development/Docs/Emu_logo_transparent.png) # Emu @@ -25,23 +23,6 @@ Emu is implemented in C++ and is based on the AMReX library for high-performance, block-structured adaptive mesh refinement. Emu is parallelized with MPI + OpenMP for CPUs and MPI + CUDA for GPUs. -# Try Emu in Your Browser! - -To quickly try Emu out using your browser, you can -[open an interactive Emu workspace in Gitpod!](https://gitpod.io/#https://github.com/AMReX-Astro/Emu) - -Emu's prebuilt Gitpod workspace tracks the current release branch, and you can find pre-compiled examples in the `Examples` directory. - -For example, to run and visualize the MSW setup: - -``` -cd Examples/2-Flavors/msw -./main3d.gnu.TPROF.MPI.ex inputs_msw_test -python plot_first_particle.py -``` - -And then open the plot through the file browser on the left of the screen. - # Getting Started From Scratch If you would like to run Emu on your own machine, first clone Emu with the AMReX submodule: From 1fef7a6b97366c0ab44a9453c89c77ff3901730e Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 4 Apr 2023 09:58:54 -0400 Subject: [PATCH 035/276] track official amrex repository, since that is possible now --- .gitmodules | 2 +- submodules/amrex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index f2322e41..5d5d624a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "amrex"] path = submodules/amrex - url = https://github.com/dwillcox/amrex.git + url = https://github.com/AMReX-Codes/amrex.git [submodule "submodules/HighFive"] path = submodules/HighFive url = https://github.com/BlueBrain/HighFive.git diff --git a/submodules/amrex b/submodules/amrex index eefe2463..bacaa10a 160000 --- a/submodules/amrex +++ b/submodules/amrex @@ -1 +1 @@ -Subproject commit eefe2463884081a27025973d4a99dc909657b4f0 +Subproject commit bacaa10a76366ed40b90b422cf5c7da43565281a From f5af309f5af8f21bd3f742146592621d8e737a96 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 4 Apr 2023 10:18:28 -0400 Subject: [PATCH 036/276] Fix bug in initial conditions that causes density matrix to have trace 0 if particle weight is zero, causing crash in Emu --- Scripts/initial_conditions/initial_condition_tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Scripts/initial_conditions/initial_condition_tools.py b/Scripts/initial_conditions/initial_condition_tools.py index efa8ef6c..7808018a 100644 --- a/Scripts/initial_conditions/initial_condition_tools.py +++ b/Scripts/initial_conditions/initial_condition_tools.py @@ -207,8 +207,9 @@ def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_g fluxfac[np.where(nnu==0)] = 0 # direction unit vector of fluxes [nu/nubar, flavor, xyz] + # point in arbitrary direction if fluxmag is zero fhat = fnu / fluxmag[:,:,np.newaxis] - fhat[np.where(fhat!=fhat)] = 0 + fhat[np.where(fhat!=fhat)] = 1./np.sqrt(3) # generate list of momenta and direction cosines phat = direction_generator(nphi_equator) # [iparticle, xyz] @@ -246,7 +247,7 @@ def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_g for flavor in range(NF): fvarname = "f"+str(flavor)+str(flavor)+"_Re"+suffix particles[:,rkey[fvarname]] = n_particle[:,nu_nubar, flavor] / n_flavorsummed[:,nu_nubar] - particles[:,rkey[fvarname]][np.where(n_flavorsummed[:,nu_nubar]==0)] = 0 + particles[:,rkey[fvarname]][np.where(n_flavorsummed[:,nu_nubar]==0)] = 1./NF # ensure that trace stays equal to 1 # double check that the number densities are correct particle_n = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]]) From 26a21c00380b594ef52a662cb10fdfd4a49231ec Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 4 Apr 2023 14:35:53 -0400 Subject: [PATCH 037/276] some tweaks to plotting scripts to make the compatible with current output formats --- Scripts/babysitting/power_spectrum.py | 2 +- Scripts/visualization/plot_grid.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Scripts/babysitting/power_spectrum.py b/Scripts/babysitting/power_spectrum.py index 88783b97..156932a4 100644 --- a/Scripts/babysitting/power_spectrum.py +++ b/Scripts/babysitting/power_spectrum.py @@ -33,7 +33,7 @@ def makeplot(v,f,data): # get appropriate data t=np.array(data["t(s)"]) - k=np.array(data["k(1|cm)"]) + k=np.array(data["k(1|cm, FFT format not including 2 pi)"]) fft = data[v+f+"_FFT(cm^-2)"] total_power = np.sum(fft) diff --git a/Scripts/visualization/plot_grid.py b/Scripts/visualization/plot_grid.py index 70d23e01..e0f1fd6b 100644 --- a/Scripts/visualization/plot_grid.py +++ b/Scripts/visualization/plot_grid.py @@ -10,6 +10,7 @@ def make_plot(d): plt.clf() #plt.ylim(-1.5e40,1.5e40) ds = yt.load(d) + t = ds.current_time ad = ds.all_data() Re = ad['boxlib',base+"_Re"] Im = ad['boxlib',base+"_Im"] @@ -22,6 +23,7 @@ def make_plot(d): plt.plot(Re,color="blue",linestyle="--") plt.plot(Im,color="orange",linestyle="--") # plt.plot(mag) + plt.text(0,0,"t="+str(t)+" s") plt.savefig(base+"_"+d+".png") directories = sorted(glob.glob("plt*")) @@ -31,4 +33,4 @@ def make_plot(d): for d in directories: make_plot(d) -# ffmpeg -i N01_%05d.png -pix_fmt yuv420p movie.mp4 +# ffmpeg -i %*.png -pix_fmt yuv420p movie.mp4 From d49ff057f7b240a0bd9ce215620bc5d259cbb110 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 4 Apr 2023 20:48:30 -0400 Subject: [PATCH 038/276] clarify condition for outputting files and prevent divide by zero --- Source/main.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/main.cpp b/Source/main.cpp index 0b6eac4b..c19a282f 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -184,9 +184,9 @@ void evolve_flavor(const TestParams* parms) run_fom += neutrinos.TotalNumberOfParticles(); // Write the Mesh Data to Plotfile if required - if ((step+1) % parms->write_plot_every == 0 || - (parms->write_plot_particles_every > 0 && - (step+1) % parms->write_plot_particles_every == 0)) { + bool write_plotfile = parms->write_plot_every > 0 && (step+1) % parms->write_plot_every == 0; + bool write_plot_particles = parms->write_plot_particles_every > 0 && (step+1) % parms->write_plot_particles_every == 0; + if (write_plotfile || write_plot_particles) { // Only include the Particle Data if write_plot_particles_every is satisfied int write_plot_particles = parms->write_plot_particles_every > 0 && (step+1) % parms->write_plot_particles_every == 0; From 9d395fee59330b51d44d7bc0cafb32ff5db64557 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 4 Apr 2023 20:49:50 -0400 Subject: [PATCH 039/276] added ability to create set of particles - one particle for each species in the direction of the specified flux --- .../initial_condition_tools.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/Scripts/initial_conditions/initial_condition_tools.py b/Scripts/initial_conditions/initial_condition_tools.py index 7808018a..a3639f9f 100644 --- a/Scripts/initial_conditions/initial_condition_tools.py +++ b/Scripts/initial_conditions/initial_condition_tools.py @@ -259,6 +259,50 @@ def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_g return particles +# generate a list of particle data, assuming each flavor's flux represents a single beam +# fnu is an array of neutrino flux density of shape [nu/nubar, iflavor, xyz] and units of 1/ccm +def beam_particles(fnu, energy_erg): + # number of neutrino flavors + NF = fnu.shape[1] + + # flux magnitude and flux factor [nu/nubar, flavor] + fluxmag = np.sqrt(np.sum(fnu**2, axis=2)) + + # direction unit vector of fluxes [nu/nubar, flavor, xyz] + # point in arbitrary direction if fluxmag is zero + fhat = fnu / fluxmag[:,:,np.newaxis] + fhat[np.where(fhat!=fhat)] = 1./np.sqrt(3) + + # generate interpolant # [particle, nu/nubar, flavor] + nparticles = 2*NF + + # get variable keys + rkey, ikey = amrex.get_particle_keys(NF, ignore_pos=True) + nelements = len(rkey) + + # generate the list of particle info + particles = np.zeros((nparticles,nelements)) + + # save the total number density of neutrinos for each particle + for flavor in range(NF): + fvarname = "f"+str(flavor)+str(flavor)+"_Re" + for nu_nubar, suffix in zip(range(2), ["","bar"]): + ind = nu_nubar*NF + flavor + + # set only the one flavor-flavor diagonal to 1 + # Also set an element for the anti-particle to be one so that density matrix still has unit trace, even though it has zero weight + particles[ind,rkey[fvarname ]] = 1 + particles[ind,rkey[fvarname+"bar"]] = 1 + + # set the number density to be equal to the magnitude of the flux + particles[ind,rkey["N"+suffix]] = fluxmag[nu_nubar,flavor] + + # set the direction to be equal to the direction of the flux + particles[ind,rkey["pupt"]] = energy_erg + particles[ind,rkey["pupx"]:rkey["pupz"]+1] = energy_erg * fhat[nu_nubar,flavor,:] + + return particles + # create a random distribution defined by number and flux densities # NF is the number of flavors # n is normalized such that the sum of all ns is 1 From f34efee714b69183d7b50b8e59ec35b8f2d43f64 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Wed, 5 Jul 2023 12:49:03 -0400 Subject: [PATCH 040/276] update version of cuda base image, since docker seems unable to find the old one --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1c2d49fd..4369f495 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:11.4.0-devel-ubuntu20.04 +FROM nvidia/cuda:12.2.0-devel-ubuntu20.04 ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update RUN apt-get install -y python3 python3-pip gfortran build-essential libhdf5-openmpi-dev openmpi-bin pkg-config libopenmpi-dev openmpi-bin libblas-dev liblapack-dev libpnetcdf-dev git python-is-python3 gnuplot From 58db302364d465e9634ea0b59a2a8c4bcc47151d Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Thu, 27 Jul 2023 01:47:49 -0400 Subject: [PATCH 041/276] move install to same line as update or update has no effect --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4369f495..55270821 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,6 @@ FROM nvidia/cuda:12.2.0-devel-ubuntu20.04 ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update -RUN apt-get install -y python3 python3-pip gfortran build-essential libhdf5-openmpi-dev openmpi-bin pkg-config libopenmpi-dev openmpi-bin libblas-dev liblapack-dev libpnetcdf-dev git python-is-python3 gnuplot +RUN apt-get update && apt-get install -y python3 python3-pip gfortran build-essential libhdf5-openmpi-dev openmpi-bin pkg-config libopenmpi-dev openmpi-bin libblas-dev liblapack-dev libpnetcdf-dev git python-is-python3 gnuplot RUN pip3 install numpy matplotlib h5py scipy sympy yt ENV USER=jenkins ENV LOGNAME=jenkins From 6a4e2828276a37049e1c42c1e4bd30a9be7b121a Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Thu, 27 Jul 2023 02:38:26 -0400 Subject: [PATCH 042/276] going back to cuda 11.4, but a version that is actually availabel on dockerhub --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 55270821..5447ec41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:12.2.0-devel-ubuntu20.04 +FROM nvidia/cuda:11.4.3-devel-ubuntu20.04 ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y python3 python3-pip gfortran build-essential libhdf5-openmpi-dev openmpi-bin pkg-config libopenmpi-dev openmpi-bin libblas-dev liblapack-dev libpnetcdf-dev git python-is-python3 gnuplot RUN pip3 install numpy matplotlib h5py scipy sympy yt From 940d56a6ff0d967f858bdce7f16664bc5d86fe71 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Thu, 27 Jul 2023 03:26:51 -0400 Subject: [PATCH 043/276] some more plotting tweaks. Fixed error with dataset name - I don't know why it was changed in the first place... --- Scripts/babysitting/power_spectrum.py | 2 +- Scripts/visualization/plot_all_particles.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Scripts/babysitting/power_spectrum.py b/Scripts/babysitting/power_spectrum.py index 156932a4..37b6b079 100644 --- a/Scripts/babysitting/power_spectrum.py +++ b/Scripts/babysitting/power_spectrum.py @@ -33,7 +33,7 @@ def makeplot(v,f,data): # get appropriate data t=np.array(data["t(s)"]) - k=np.array(data["k(1|cm, FFT format not including 2 pi)"]) + k=np.array(data["k(1|cm)"]) #FFT format not including 2 pi fft = data[v+f+"_FFT(cm^-2)"] total_power = np.sum(fft) diff --git a/Scripts/visualization/plot_all_particles.py b/Scripts/visualization/plot_all_particles.py index b2f6f97b..67f45b20 100644 --- a/Scripts/visualization/plot_all_particles.py +++ b/Scripts/visualization/plot_all_particles.py @@ -1,12 +1,17 @@ import numpy as np import argparse import glob +import os +import sys +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../data_reduction") import amrex_plot_tools as amrex if __name__ == "__main__": import pylab as plt - rkey, ikey = amrex.get_particle_keys() + rkey, ikey = amrex.get_particle_keys(2) t = [] fee = [] From 2b1c6a804494791e7d48739669663e9eb71c304c Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Thu, 14 Sep 2023 14:17:19 -0400 Subject: [PATCH 044/276] reduce scalar output over MPI --- Source/DataReducer.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index cb283d11..a869235d 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -134,11 +134,12 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, // use the ParReduce function to define reduction operator GpuTuple< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > result = ParReduce(TypeList{}, - TypeList, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray >{}, + TypeList< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray >{}, state, nghost, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> - GpuTuple, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > { + GpuTuple< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > { Array4 const& a = ma[box_no]; + std::cout << "box_no = " << box_no << std::endl; // Doing the actual work ArithmeticArray Ndiag, Ndiagbar; @@ -162,6 +163,19 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, ArithmeticArray Fybar = amrex::get<7>(result) / ncells; ArithmeticArray Fzbar = amrex::get<8>(result) / ncells; + // further reduce over mpi ranks + for(int i=0; i Date: Thu, 14 Sep 2023 14:18:41 -0400 Subject: [PATCH 045/276] reduce scalar output over MPI (#86) Co-authored-by: Sherwood Richers --- Source/DataReducer.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index cb283d11..a869235d 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -134,11 +134,12 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, // use the ParReduce function to define reduction operator GpuTuple< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > result = ParReduce(TypeList{}, - TypeList, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray >{}, + TypeList< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray >{}, state, nghost, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> - GpuTuple, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > { + GpuTuple< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > { Array4 const& a = ma[box_no]; + std::cout << "box_no = " << box_no << std::endl; // Doing the actual work ArithmeticArray Ndiag, Ndiagbar; @@ -162,6 +163,19 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, ArithmeticArray Fybar = amrex::get<7>(result) / ncells; ArithmeticArray Fzbar = amrex::get<8>(result) / ncells; + // further reduce over mpi ranks + for(int i=0; i Date: Thu, 14 Sep 2023 14:28:05 -0400 Subject: [PATCH 046/276] remove print line --- Source/DataReducer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index a869235d..24e53747 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -139,7 +139,6 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> GpuTuple< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > { Array4 const& a = ma[box_no]; - std::cout << "box_no = " << box_no << std::endl; // Doing the actual work ArithmeticArray Ndiag, Ndiagbar; From c06bd06b8bf4c79de573dbac9877f72b233cccd9 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Mon, 18 Sep 2023 20:52:12 -0400 Subject: [PATCH 047/276] reduce offdiag mag properly by only taking sqrt after mpi reduction --- Source/DataReducer.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index 24e53747..0dca6612 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -152,9 +152,9 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, }); // retrieve the reduced data values - ArithmeticArray N = amrex::get<0>(result) / ncells; - ArithmeticArray Nbar = amrex::get<1>(result) / ncells; - Real N_offdiag_mag = sqrt(amrex::get<2>(result)) / ncells; + ArithmeticArray N = amrex::get<0>(result) / ncells; + ArithmeticArray Nbar = amrex::get<1>(result) / ncells; + Real N_offdiag_mag2 = amrex::get<2>(result) / ncells; ArithmeticArray Fx = amrex::get<3>(result) / ncells; ArithmeticArray Fy = amrex::get<4>(result) / ncells; ArithmeticArray Fz = amrex::get<5>(result) / ncells; @@ -173,7 +173,10 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, ParallelDescriptor::ReduceRealSum(Fybar[i], ParallelDescriptor::IOProcessorNumber()); ParallelDescriptor::ReduceRealSum(Fzbar[i], ParallelDescriptor::IOProcessorNumber()); } - ParallelDescriptor::ReduceRealSum(N_offdiag_mag, ParallelDescriptor::IOProcessorNumber()); + ParallelDescriptor::ReduceRealSum(N_offdiag_mag2, ParallelDescriptor::IOProcessorNumber()); + + // take square root of N_offdiag_mag2 + Real N_offdiag_mag = std::sqrt(N_offdiag_mag2); // calculate net number of neutrinos and antineutrinos Real Ntot=0, Ndiff=0; From 39ed26d62bf8094c0c8a0ee84e11baa567f94845 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Mon, 18 Sep 2023 21:00:41 -0400 Subject: [PATCH 048/276] remove print line...again? --- Source/DataReducer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index f06ea6f9..0dca6612 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -139,7 +139,6 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> GpuTuple< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > { Array4 const& a = ma[box_no]; - std::cout << "box_no = " << box_no << std::endl; // Doing the actual work ArithmeticArray Ndiag, Ndiagbar; From 0fc2f43df263b93230ef71af33dea14e47c4d784 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Wed, 20 Sep 2023 18:02:30 -0400 Subject: [PATCH 049/276] make perturbations only on the diagonal elements, let vacuum potential do off-diagonal perturbations. This is a more physically palettable way of doing perturbations --- Scripts/symbolic_hermitians/generate_code.py | 11 +++++++ Source/FlavoredNeutrinoContainerInit.cpp | 34 ++------------------ 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 98d536a6..46152719 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -491,3 +491,14 @@ def sgn(t,var): f = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") code.append("p.rdata(PIdx::L"+t+") = "+sympy.cxxcode(sympy.simplify(f.SU_vector_magnitude()))+";" ) write_code(code, os.path.join(args.emu_home, "Source/generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length")) + + #=====================================================# + # FlavoredNeutrinoContainerInit.cpp_perturb_diagonals # + #=====================================================# + code = [] + for i in range(args.N): + for t in tails: + quantity = "p.rdata(PIdx::f"+str(i)+str(i)+"_Re"+t+")" + code.append("symmetric_uniform(&rand, engine);") + code.append(quantity + "+= parms->perturbation_amplitude*rand * "+quantity+";") + write_code(code, os.path.join(args.emu_home, "Source/generated_files/FlavoredNeutrinoContainerInit.cpp_perturb_diagonals")) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index e47d042a..e8253847 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -291,43 +291,15 @@ InitParticles(const TestParams* parms) if(parms->perturbation_type == 0){ // random perturbations to the off-diagonals Real rand; - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); -#if NUM_FLAVORS==3 - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); -#endif - } + #include "generated_files/FlavoredNeutrinoContainerInit.cpp_perturb_diagonals" + } if(parms->perturbation_type == 1){ // Perturb real part of e-mu component only sinusoidally in z Real nu_k = (2.*M_PI) / parms->perturbation_wavelength_cm; p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + #include "generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length" } - - -#include "generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length" - } // loop over direction } // loop over location }); // loop over grid cells From 9dc27731dec166619a4b0f1f95cd906430505e60 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Wed, 20 Sep 2023 18:02:42 -0400 Subject: [PATCH 050/276] update amrex --- submodules/amrex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/amrex b/submodules/amrex index bacaa10a..b2052f2d 160000 --- a/submodules/amrex +++ b/submodules/amrex @@ -1 +1 @@ -Subproject commit bacaa10a76366ed40b90b422cf5c7da43565281a +Subproject commit b2052f2d2e5ce44317450ba13de705a3e01ef0ea From 7a2ee967a24b84cc8204539dc6e0346dd17c776b Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Mon, 25 Sep 2023 17:04:51 -0400 Subject: [PATCH 051/276] Revert "make perturbations only on the diagonal elements, let vacuum potential do off-diagonal perturbations. This is a more physically palettable way of doing perturbations" This reverts commit 0fc2f43df263b93230ef71af33dea14e47c4d784. --- Scripts/symbolic_hermitians/generate_code.py | 11 ------- Source/FlavoredNeutrinoContainerInit.cpp | 34 ++++++++++++++++++-- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 46152719..98d536a6 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -491,14 +491,3 @@ def sgn(t,var): f = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") code.append("p.rdata(PIdx::L"+t+") = "+sympy.cxxcode(sympy.simplify(f.SU_vector_magnitude()))+";" ) write_code(code, os.path.join(args.emu_home, "Source/generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length")) - - #=====================================================# - # FlavoredNeutrinoContainerInit.cpp_perturb_diagonals # - #=====================================================# - code = [] - for i in range(args.N): - for t in tails: - quantity = "p.rdata(PIdx::f"+str(i)+str(i)+"_Re"+t+")" - code.append("symmetric_uniform(&rand, engine);") - code.append(quantity + "+= parms->perturbation_amplitude*rand * "+quantity+";") - write_code(code, os.path.join(args.emu_home, "Source/generated_files/FlavoredNeutrinoContainerInit.cpp_perturb_diagonals")) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index e8253847..e47d042a 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -291,15 +291,43 @@ InitParticles(const TestParams* parms) if(parms->perturbation_type == 0){ // random perturbations to the off-diagonals Real rand; - #include "generated_files/FlavoredNeutrinoContainerInit.cpp_perturb_diagonals" - } + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f01_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); +#if NUM_FLAVORS==3 + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); +#endif + } if(parms->perturbation_type == 1){ // Perturb real part of e-mu component only sinusoidally in z Real nu_k = (2.*M_PI) / parms->perturbation_wavelength_cm; p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); - #include "generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length" } + + +#include "generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length" + } // loop over direction } // loop over location }); // loop over grid cells From 6e66398ddf9d6114051ff771bc5d75a855e28f32 Mon Sep 17 00:00:00 2001 From: Javier Gomez <145796242+javierigm@users.noreply.github.com> Date: Thu, 19 Oct 2023 10:37:53 -0400 Subject: [PATCH 052/276] Specify the modules used in Emu in README (#87) * Specify the modules used in Emu in README * Incorporating feedback from review * Correct a small typo * Testing subtitles in README * Dividing instructions for Linux/WSL and macOS * Fixing subtitles * Fixing subtitles * Changing subtitle format * To show where to start reading after following the Linux/macOS instructios * Incorporating suggestions --- README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index afa5121b..4580aa59 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,27 @@ high-performance, block-structured adaptive mesh refinement. Emu is parallelized with MPI + OpenMP for CPUs and MPI + CUDA for GPUs. # Getting Started From Scratch +If you would like to run Emu on your own machine, there are some packages +you will need to install depending on your operating system. -If you would like to run Emu on your own machine, first clone Emu with the AMReX submodule: +## Linux or WSL + +If you are running Emu on Linux or a WSL, you will need to install the +following packages: + +``` +apt-get install g++ libopenmpi-dev python-3 gfortran gnuplot-x11 +pip install sympy h5py +``` + +## macOS + +If you are running Emu on macOS... + +## Using Emu + + +After installing those modules, clone Emu with the AMReX submodule: ``` git clone --recurse-submodules https://github.com/AMReX-Astro/Emu.git From 0f85532adacd3db661ccd4d8d792012a9022ae05 Mon Sep 17 00:00:00 2001 From: mikaelalatkinson <89419094+mikaelalatkinson@users.noreply.github.com> Date: Thu, 7 Dec 2023 10:26:05 -0500 Subject: [PATCH 053/276] Readmechanges (#89) * Specify the modules used in Emu in README * Incorporating feedback from review * Correct a small typo * Testing subtitles in README * Dividing instructions for Linux/WSL and macOS * Fixing subtitles * Fixing subtitles * Changing subtitle format * To show where to start reading after following the Linux/macOS instructios * Incorporating suggestions * macOS instructions * mac makefile --------- Co-authored-by: Javier Gomez Co-authored-by: Sherwood Richers <5142652+srichers@users.noreply.github.com> --- README.md | 20 ++++++++++++++++++-- makefiles/GNUmakefile_macOS | 22 ++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 makefiles/GNUmakefile_macOS diff --git a/README.md b/README.md index 4580aa59..ecf2545e 100644 --- a/README.md +++ b/README.md @@ -37,9 +37,14 @@ apt-get install g++ libopenmpi-dev python-3 gfortran gnuplot-x11 pip install sympy h5py ``` -## macOS +## MacOS + +If you are running Emu on macOS you will need gcc and a mpi wrapper: + +``` +brew install mpich --cc=gcc-13 +``` -If you are running Emu on macOS... ## Using Emu @@ -55,10 +60,21 @@ Then change directories to `Emu/Exec`. Before each compilation, you must symboli the number of neutrino flavors you wish to use and specify a few other compile-time settings in a file called `GNUmakefile`. Copy in a default makefile. In this file you can specify the number of neutrino flavors, whether to compile for GPUs, etc. We have set the defaults to 2 neutrino flavors, order 2 PIC shape factors, and compiling for a single CPU. + +For Linux or WSL: + ``` cp ../makefiles/GNUmakefile_default GNUmakefile ``` +For MacOS: + +``` +cp ../makefiles/GNUmakefile_macOS GNUmakefile +``` + + + Compiling occurs in two stages. We first have to generate code according to the number of neutrino flavors. ``` make generate diff --git a/makefiles/GNUmakefile_macOS b/makefiles/GNUmakefile_macOS new file mode 100644 index 00000000..a87167e3 --- /dev/null +++ b/makefiles/GNUmakefile_macOS @@ -0,0 +1,22 @@ +NUM_FLAVORS = 2 +SHAPE_FACTOR_ORDER = 2 + +COMP = gnu + +DEBUG = FALSE + +USE_MPI = FALSE +USE_OMP = FALSE +USE_ACC = FALSE +USE_CUDA = FALSE +USE_HDF5 = FALSE + +EMU_HOME = .. +AMREX_HOME = ../submodules/amrex +include ../Make.Emu + +CXX = mpicxx +CC = mpicc +FC = gfortran-13 +F90 = gfortran-13 +INCLUDE_LOCATIONS += /usr/local/include From 279ed28ce8eab2848cc75f9a70768f7c0d508dde Mon Sep 17 00:00:00 2001 From: Sherwood Richers <5142652+srichers@users.noreply.github.com> Date: Thu, 7 Dec 2023 11:21:25 -0500 Subject: [PATCH 054/276] Pressure tensor output (#88) * add option to code generator. Not yet used * add pressure tensor calculation and output. Not verified to be correct yet. Also changed the form of the compile variable determining how many moments to output to make it usable both by the C++ code and the python code * remove unused libraries * fix typo in dataset name * quick script to create a text file with the largest wavenumber and amplitude * script to draw eln from initial conditions --------- Co-authored-by: Sherwood Richers --- Make.Emu | 4 +- Scripts/data_reduction/kmax_t.py | 62 ++++++++ Scripts/initial_conditions/draw_eln.py | 99 +++++++++++++ Scripts/symbolic_hermitians/generate_code.py | 43 +++++- Source/DataReducer.cpp | 148 +++++++++++++++---- makefiles/GNUmakefile_bridges | 4 +- makefiles/GNUmakefile_cassowary | 1 + makefiles/GNUmakefile_default | 1 + makefiles/GNUmakefile_jenkins | 1 + makefiles/GNUmakefile_jenkins_HDF5 | 1 + 10 files changed, 332 insertions(+), 32 deletions(-) create mode 100755 Scripts/data_reduction/kmax_t.py create mode 100644 Scripts/initial_conditions/draw_eln.py diff --git a/Make.Emu b/Make.Emu index d1081c91..056f7940 100644 --- a/Make.Emu +++ b/Make.Emu @@ -29,13 +29,13 @@ Ppack += $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.pack include $(Ppack) -DEFINES += -DNUM_FLAVORS=$(NUM_FLAVORS) -DSHAPE_FACTOR_ORDER=$(SHAPE_FACTOR_ORDER) +DEFINES += -DNUM_FLAVORS=$(NUM_FLAVORS) -DSHAPE_FACTOR_ORDER=$(SHAPE_FACTOR_ORDER) -DNUM_MOMENTS=$(NUM_MOMENTS) all: generate $(objEXETempDir)/AMReX_buildInfo.o $(executable) @echo SUCCESS generate: - python3 $(EMU_HOME)/Scripts/symbolic_hermitians/generate_code.py $(NUM_FLAVORS) --emu_home $(EMU_HOME) + python3 $(EMU_HOME)/Scripts/symbolic_hermitians/generate_code.py $(NUM_FLAVORS) --emu_home $(EMU_HOME) --num_moments $(NUM_MOMENTS) #------------------------------------------------------------------------------ # build info (from Castro/Exec/Make.auto_source) diff --git a/Scripts/data_reduction/kmax_t.py b/Scripts/data_reduction/kmax_t.py new file mode 100755 index 00000000..14288400 --- /dev/null +++ b/Scripts/data_reduction/kmax_t.py @@ -0,0 +1,62 @@ +# Create a file called kmax_t.dat that contains the amplitude, phase, and wavenumber of the biggest Fourier mode on the domain at every point in time. + +import numpy as np +import emu_yt_module as emu +import glob +import scipy +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument("-o", "--output", type=str, default="kmax_t", help="Name of the output file") +parser.add_argument("-d", "--dataset", type=str, default="N01", help="Name of the dataset in which to find kmax. Omit _Re and _Im, as they will be added as needed.") +args = parser.parse_args() + +directories = sorted(glob.glob("plt?????")) + +# Need to know if the chosen dataset is flavor diagonal +# If so, we have to assume all imaginary values are zero +isDiagonal = (args.dataset[-1]==args.dataset[-2]) + +################################ +# read data and calculate FFTs # +################################ +f = open(args.output+"_"+args.dataset+".dat","w") +f.write("1:t(s)\t2:kx(1|cm,with2pi)\t3:ky(1|cm,with2pi)\t4:kz(1|cm,with2pi)\t5:magnitude(1|ccm)\t6:phase\n") +for d in directories: + print(d) + eds = emu.EmuDataset(d) + NF = eds.get_num_flavors() + + if isDiagonal: + fft = eds.fourier(args.dataset+"_Re") + else: + fft = eds.fourier(args.dataset+"_Re",args.dataset+"_Im") + + f.write(str(float(fft.time))+"\t") + + maxloc = np.unravel_index(np.argmax(fft.magnitude), fft.magnitude.shape) + + index = 0 + if fft.kx is not None: + f.write(str(fft.kx[maxloc[index]]*2.*np.pi)+"\t") + index += 1 + else: + f.write(str(0)+"\t") + + if fft.ky is not None: + f.write(str(fft.ky[maxloc[index]]*2.*np.pi)+"\t") + index += 1 + else: + f.write(str(0)+"\t") + + if fft.kz is not None: + f.write(str(fft.kz[maxloc[index]]*2.*np.pi)+"\t") + index += 1 + else: + f.write(str(0)+"\t") + + f.write(str(fft.magnitude[maxloc])+"\t") + f.write(str(fft.phase[maxloc])+"\n") + print(np.max(fft.magnitude), fft.magnitude[maxloc]) + +f.close() diff --git a/Scripts/initial_conditions/draw_eln.py b/Scripts/initial_conditions/draw_eln.py new file mode 100644 index 00000000..e153cab4 --- /dev/null +++ b/Scripts/initial_conditions/draw_eln.py @@ -0,0 +1,99 @@ +import matplotlib.pyplot as plt +import matplotlib as mpl +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../data_reduction") +from initial_condition_tools import uniform_sphere, moment_interpolate_particles, minerbo_interpolate, write_particles +import amrex_plot_tools as amrex + +NF = 2 +nphi_equator = 1024 +nnue = 1.421954234999705e+33 +nnua = 1.9146237131657563e+33 +nnux = 1.9645407875568215e+33 +fnue = np.array([0.0974572, 0.04217632, -0.13433261]) +fnua = np.array([0.07237959, 0.03132354, -0.3446878]) +fnux = np.array([-0.02165833, 0.07431613, -0.53545951]) +energy_erg = 20.05473294163565 * 1e6*amrex.eV + +nnu = np.zeros((2,NF)) +nnu[0,0] = nnue +nnu[1,0] = nnua +nnu[:,1:] = nnux / 4. + +fnu = np.zeros((2,NF,3)) +fnu[0,0,:] = nnue * fnue +fnu[1,0,:] = nnua * fnua +fnu[:,1:,:] = nnu[:,1:,np.newaxis] * fnux[np.newaxis,np.newaxis,:] + +particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, minerbo_interpolate) # [particle, variable] + +rkey, ikey = amrex.get_particle_keys(NF, ignore_pos=True) + +pxhat = particles[:,rkey["pupx"]] / particles[:,rkey["pupt"]] +pyhat = particles[:,rkey["pupy"]] / particles[:,rkey["pupt"]] +pzhat = particles[:,rkey["pupz"]] / particles[:,rkey["pupt"]] + +peq = np.sqrt(pxhat**2+pyhat**2) +pmag = np.sqrt(pxhat**2+pyhat**2+pzhat**2) + +mu = pzhat / pmag +phi = np.arctan2(pxhat,pyhat) + +N = particles[:,rkey["N"]] +Nbar = particles[:,rkey["Nbar"]] +f00 = particles[:,rkey["f00_Re"]] +f00bar = particles[:,rkey["f00_Rebar"]] + +eln = N*f00 - Nbar*f00bar + +#==============# +# plot options # +#==============# +mpl.rcParams['font.size'] = 22 +mpl.rcParams['font.family'] = 'serif' +mpl.rc('text', usetex=True) +mpl.rcParams['xtick.major.size'] = 7 +mpl.rcParams['xtick.major.width'] = 2 +mpl.rcParams['xtick.major.pad'] = 8 +mpl.rcParams['xtick.minor.size'] = 4 +mpl.rcParams['xtick.minor.width'] = 2 +mpl.rcParams['ytick.major.size'] = 7 +mpl.rcParams['ytick.major.width'] = 2 +mpl.rcParams['ytick.minor.size'] = 4 +mpl.rcParams['ytick.minor.width'] = 2 +mpl.rcParams['axes.linewidth'] = 2 + +#==========# +# subplots # +#==========# +fig,axes=plt.subplots(1,1, figsize=(8,6)) +plt.subplots_adjust(wspace=0, hspace=0) + +elnmax = np.max(np.abs(eln)) + +sc0 = axes.scatter(phi, mu, c=eln, cmap=mpl.cm.seismic, s=3, vmin=-elnmax, vmax=elnmax) + +particle_direction = np.array([3.03e-6, 2.718e-5, -1.687e-5]) +particle_mag = np.sqrt(np.sum(particle_direction**2)) +particle_mu = particle_direction[2] / particle_mag +particle_phi = np.arctan2(particle_direction[0], particle_direction[1]) +print(particle_mag) +axes.scatter(particle_phi, particle_mu, c='green', s=3) + +particle_direction = np.array([-1.321e-5, -6.981e-6, 2.840e-5]) +particle_mag = np.sqrt(np.sum(particle_direction**2)) +particle_mu = particle_direction[2] / particle_mag +particle_phi = np.arctan2(particle_direction[0], particle_direction[1]) +print(particle_mag) +axes.scatter(particle_phi, particle_mu, c='black', s=3) +#plt.colorbar() + +axes.set_xlabel(r"$\phi$") +axes.set_ylabel(r"$\mu$") + +plt.savefig("eln.pdf",bbox_inches="tight") diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 98d536a6..ebb8d6db 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -4,10 +4,8 @@ import argparse import os import sympy -from sympy.codegen.ast import Assignment -from HermitianUtils import HermitianMatrix,SU_vector_ideal_magnitude +from HermitianUtils import HermitianMatrix import shutil -import math parser = argparse.ArgumentParser(description="Generates code for calculating C = i * [A,B] for symbolic NxN Hermitian matrices A, B, C, using real-valued Real and Imaginary components.") parser.add_argument("N", type=int, help="Size of NxN Hermitian matrices.") @@ -15,9 +13,15 @@ parser.add_argument("-eh", "--emu_home", type=str, default=".", help="Path to Emu home directory.") parser.add_argument("-c", "--clean", action="store_true", help="Clean up any previously generated files.") parser.add_argument("-rn", "--rhs_normalize", action="store_true", help="Normalize F when applying the RHS update F += dt * dFdt (limits to 2nd order in time).") +parser.add_argument("-nm", "--num_moments", type=int, default=2, help="Number of moments to compute.") args = parser.parse_args() +# make sure arguments make sense +assert(args.N>0) +assert(args.num_moments>=2) # just N and F +assert(args.num_moments<=3) # also include P. Higher moments not implemented + def write_code(code, output_file, template=None): ## If a template file is supplied, this will insert the generated code ## where the "<>code<>" string is found. @@ -127,6 +131,8 @@ def delete_generated_files(): # Evolve.H_fill # #===============# vars = ["N","Fx","Fy","Fz"] + if args.num_moments>=3: + vars.extend(["Pxx","Pxy","Pxz","Pyy","Pyz","Pzz"]) tails = ["","bar"] code = [] for v in vars: @@ -140,6 +146,8 @@ def delete_generated_files(): # Evolve.cpp_grid_names_fill # #============================# vars = ["N","Fx","Fy","Fz"] + if args.num_moments>=3: + vars.extend(["Pxx","Pxy","Pxz","Pyy","Pyz","Pzz"]) tails = ["","bar"] code = [] for v in vars: @@ -160,6 +168,14 @@ def delete_generated_files(): "*p.rdata(PIdx::pupy)/p.rdata(PIdx::pupt));", "*p.rdata(PIdx::pupz)/p.rdata(PIdx::pupt));"] deposit_vars = ["N","Fx","Fy","Fz"] + if args.num_moments >= 3: + deposit_vars.extend(["Pxx","Pxy","Pxz","Pyy","Pyz","Pzz"]) + string4.extend(["*p.rdata(PIdx::pupx)*p.rdata(PIdx::pupx)/p.rdata(PIdx::pupt)/p.rdata(PIdx::pupt));", + "*p.rdata(PIdx::pupx)*p.rdata(PIdx::pupy)/p.rdata(PIdx::pupt)/p.rdata(PIdx::pupt));", + "*p.rdata(PIdx::pupx)*p.rdata(PIdx::pupz)/p.rdata(PIdx::pupt)/p.rdata(PIdx::pupt));", + "*p.rdata(PIdx::pupy)*p.rdata(PIdx::pupy)/p.rdata(PIdx::pupt)/p.rdata(PIdx::pupt));", + "*p.rdata(PIdx::pupy)*p.rdata(PIdx::pupz)/p.rdata(PIdx::pupt)/p.rdata(PIdx::pupt));", + "*p.rdata(PIdx::pupz)*p.rdata(PIdx::pupz)/p.rdata(PIdx::pupt)/p.rdata(PIdx::pupt));"]) code = [] for t in tails: string3 = ")*p.rdata(PIdx::N"+t+")" @@ -205,6 +221,27 @@ def delete_generated_files(): code.append("Fydiag"+t+"["+str(i)+"] = "+Fylist[i]+";") code.append("Fzdiag"+t+"["+str(i)+"] = "+Fzlist[i]+";") + if args.num_moments>=3: + Pxx = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::Pxx{}{}_{}"+t+")") + Pxxlist = Pxx.header_diagonals(); + Pxy = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::Pxy{}{}_{}"+t+")") + Pxylist = Pxy.header_diagonals(); + Pxz = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::Pxz{}{}_{}"+t+")") + Pxzlist = Pxz.header_diagonals(); + Pyy = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::Pyy{}{}_{}"+t+")") + Pyylist = Pyy.header_diagonals(); + Pyz = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::Pyz{}{}_{}"+t+")") + Pyzlist = Pyz.header_diagonals(); + Pzz = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::Pzz{}{}_{}"+t+")") + Pzzlist = Pzz.header_diagonals(); + for i in range(len(Nlist)): + code.append("Pxxdiag"+t+"["+str(i)+"] = "+Pxxlist[i]+";") + code.append("Pxydiag"+t+"["+str(i)+"] = "+Pxylist[i]+";") + code.append("Pxzdiag"+t+"["+str(i)+"] = "+Pxzlist[i]+";") + code.append("Pyydiag"+t+"["+str(i)+"] = "+Pyylist[i]+";") + code.append("Pyzdiag"+t+"["+str(i)+"] = "+Pyzlist[i]+";") + code.append("Pzzdiag"+t+"["+str(i)+"] = "+Pzzlist[i]+";") + # off-diagonal magnitude mag2 = 0 for i in range(N.size): diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index 0dca6612..d339beb6 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -48,6 +48,20 @@ void DataReducer::InitializeFiles() file0D.createDataSet(std::string("Fx") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); file0D.createDataSet(std::string("Fy") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); file0D.createDataSet(std::string("Fz") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + #if NUM_MOMENTS == 3 + file0D.createDataSet(std::string("Pxx") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Pxy") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Pxz") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Pyy") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Pyz") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Pzz") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Pxx") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Pxy") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Pxz") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Pyy") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Pyz") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Pzz") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + #endif } file0D.createDataSet("N_offdiag_mag(1|ccm)", dataspace, create_datatype(), props); file0D.createDataSet("sumTrf", dataspace, create_datatype(), props); @@ -58,32 +72,34 @@ void DataReducer::InitializeFiles() std::ofstream outfile; outfile.open(filename0D, std::ofstream::out); int j = 0; - j++; - outfile << j << ":step\t"; - j++; - outfile << j << ":time(s)\t"; - j++; - outfile << j << ":Ntot(1|ccm)\t"; - j++; - outfile << j << ":Ndiff(1|ccm)\t"; + j++; outfile << j << ":step\t"; + j++; outfile << j << ":time(s)\t"; + j++; outfile << j << ":Ntot(1|ccm)\t"; + j++; outfile << j << ":Ndiff(1|ccm)\t"; for (int i = 0; i < NUM_FLAVORS; i++) { - j++; - outfile << j << ":N" << i << i << "(1|ccm)\t"; - j++; - outfile << j << ":N" << i << i << "bar(1|ccm)\t"; - j++; - outfile << j << ":Fx" << i << i << "(1|ccm)\t"; - j++; - outfile << j << ":Fy" << i << i << "(1|ccm)\t"; - j++; - outfile << j << ":Fz" << i << i << "(1|ccm)\t"; - j++; - outfile << j << ":Fx" << i << i << "bar(1|ccm)\t"; - j++; - outfile << j << ":Fy" << i << i << "bar(1|ccm)\t"; - j++; - outfile << j << ":Fz" << i << i << "bar(1|ccm)\t"; + j++; outfile << j << ":N" << i << i << "(1|ccm)\t"; + j++; outfile << j << ":N" << i << i << "bar(1|ccm)\t"; + j++; outfile << j << ":Fx" << i << i << "(1|ccm)\t"; + j++; outfile << j << ":Fy" << i << i << "(1|ccm)\t"; + j++; outfile << j << ":Fz" << i << i << "(1|ccm)\t"; + j++; outfile << j << ":Fx" << i << i << "bar(1|ccm)\t"; + j++; outfile << j << ":Fy" << i << i << "bar(1|ccm)\t"; + j++; outfile << j << ":Fz" << i << i << "bar(1|ccm)\t"; + #if NUM_MOMENTS == 3 + j++; outfile << j << ":Pxx" << i << i << "(1|ccm)\t"; + j++; outfile << j << ":Pxy" << i << i << "(1|ccm)\t"; + j++; outfile << j << ":Pxz" << i << i << "(1|ccm)\t"; + j++; outfile << j << ":Pyy" << i << i << "(1|ccm)\t"; + j++; outfile << j << ":Pyz" << i << i << "(1|ccm)\t"; + j++; outfile << j << ":Pzz" << i << i << "(1|ccm)\t"; + j++; outfile << j << ":Pxx" << i << i << "bar(1|ccm)\t"; + j++; outfile << j << ":Pxy" << i << i << "bar(1|ccm)\t"; + j++; outfile << j << ":Pxz" << i << i << "bar(1|ccm)\t"; + j++; outfile << j << ":Pyy" << i << i << "bar(1|ccm)\t"; + j++; outfile << j << ":Pyz" << i << i << "bar(1|ccm)\t"; + j++; outfile << j << ":Pzz" << i << i << "bar(1|ccm)\t"; + #endif } j++; outfile << j << ":N_offdiag_mag(1|ccm)\t"; @@ -132,12 +148,23 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, IntVect nghost(AMREX_D_DECL(0, 0, 0)); // use the ParReduce function to define reduction operator + #if NUM_MOMENTS == 2 GpuTuple< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > result = ParReduce(TypeList{}, TypeList< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray >{}, state, nghost, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> GpuTuple< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > { + #elif NUM_MOMENTS == 3 + GpuTuple< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > result = + ParReduce(TypeList{}, + TypeList< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray >{}, + state, nghost, + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> + GpuTuple< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > { + #else + #error "NUM_MOMENTS must be 2 or 3" + #endif Array4 const& a = ma[box_no]; // Doing the actual work @@ -145,10 +172,25 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, ArithmeticArray Fxdiag, Fxdiagbar; ArithmeticArray Fydiag, Fydiagbar; ArithmeticArray Fzdiag, Fzdiagbar; + #if NUM_MOMENTS >= 2 + ArithmeticArray Pxxdiag, Pxxdiagbar; + ArithmeticArray Pxydiag, Pxydiagbar; + ArithmeticArray Pxzdiag, Pxzdiagbar; + ArithmeticArray Pyydiag, Pyydiagbar; + ArithmeticArray Pyzdiag, Pyzdiagbar; + ArithmeticArray Pzzdiag, Pzzdiagbar; + #endif + Real N_offdiag_mag2 = 0; + #include "generated_files/DataReducer.cpp_fill" + #if NUM_MOMENTS == 2 return {Ndiag, Ndiagbar, N_offdiag_mag2, Fxdiag, Fydiag, Fzdiag, Fxdiagbar,Fydiagbar,Fzdiagbar}; - + #elif NUM_MOMENTS == 3 + return {Ndiag, Ndiagbar, N_offdiag_mag2, Fxdiag, Fydiag, Fzdiag, Fxdiagbar,Fydiagbar,Fzdiagbar, Pxxdiag, Pxydiag, Pxzdiag, Pyydiag, Pyzdiag, Pzzdiag, Pxxdiagbar, Pxydiagbar, Pxzdiagbar, Pyydiagbar, Pyzdiagbar, Pzzdiagbar}; + #else + #error "NUM_MOMENTS must be 2 or 3" + #endif }); // retrieve the reduced data values @@ -161,6 +203,20 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, ArithmeticArray Fxbar = amrex::get<6>(result) / ncells; ArithmeticArray Fybar = amrex::get<7>(result) / ncells; ArithmeticArray Fzbar = amrex::get<8>(result) / ncells; + #if NUM_MOMENTS == 3 + ArithmeticArray Pxx = amrex::get< 9>(result) / ncells; + ArithmeticArray Pxy = amrex::get<10>(result) / ncells; + ArithmeticArray Pxz = amrex::get<11>(result) / ncells; + ArithmeticArray Pyy = amrex::get<12>(result) / ncells; + ArithmeticArray Pyz = amrex::get<13>(result) / ncells; + ArithmeticArray Pzz = amrex::get<14>(result) / ncells; + ArithmeticArray Pxxbar= amrex::get<15>(result) / ncells; + ArithmeticArray Pxybar= amrex::get<16>(result) / ncells; + ArithmeticArray Pxzbar= amrex::get<17>(result) / ncells; + ArithmeticArray Pyybar= amrex::get<18>(result) / ncells; + ArithmeticArray Pyzbar= amrex::get<19>(result) / ncells; + ArithmeticArray Pzzbar= amrex::get<20>(result) / ncells; + #endif // further reduce over mpi ranks for(int i=0; i Date: Wed, 17 Jan 2024 20:40:55 -0500 Subject: [PATCH 055/276] Particles store the phase space volume they occupy --- Scripts/symbolic_hermitians/generate_code.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index ebb8d6db..f17319d3 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -104,23 +104,14 @@ def delete_generated_files(): A = HermitianMatrix(args.N, v+"{}{}_{}"+t) code += A.header() code += ["TrHf"] + code += ["Vphase"] - code = [code[i]+"," for i in range(len(code))] - write_code(code, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.H_fill")) + code_lines = [code[i]+"," for i in range(len(code))] + write_code(code_lines, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.H_fill")) #========================================================# # FlavoredNeutrinoContainerInit.H_particle_varnames_fill # #========================================================# - vars = ["f"] - tails = ["","bar"] - code = [] - for t in tails: - code += ["N"+t] - code += ["L"+t] - for v in vars: - A = HermitianMatrix(args.N, v+"{}{}_{}"+t) - code += A.header() - code += ["TrHf"] code_string = 'attribute_names = {"time", "x", "y", "z", "pupx", "pupy", "pupz", "pupt", ' code = ['"{}"'.format(c) for c in code] code_string = code_string + ", ".join(code) + "};" From f777df1cbf9d6252fbab360caccb93ac9d76a18d Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Wed, 17 Jan 2024 20:50:30 -0500 Subject: [PATCH 056/276] set evolution equations for N and L to be zero in code generation script in anticipation of changing them based on opacities. Set Vphase to not evolve in time. --- Scripts/symbolic_hermitians/generate_code.py | 8 ++++++++ Source/Evolve.cpp | 5 +---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index f17319d3..5e6e7439 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -460,6 +460,14 @@ def sgn(t,var): # Write out dFdt->F code.append(dFdt.code()) + # evolution equations for N and Nbar, stored as dNdt-->N + line = "p.rdata(PIdx::N"+t+") = 0;" + code.append([line]) + + # evolution equations for L and Lbar, stored as dLdt-->L + line = "p.rdata(PIdx::L"+t+") = 0;" + code.append([line]) + # store Tr(H*F) for estimating numerical errors TrHf = (H*F).trace(); code.append(["p.rdata(PIdx::TrHf) += p.rdata(PIdx::N"+t+") * ("+sympy.cxxcode(sympy.simplify(TrHf))+");"]) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index f6f63d4e..e3943cd5 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -163,10 +163,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M p.rdata(PIdx::pupy) = 0; p.rdata(PIdx::pupz) = 0; p.rdata(PIdx::pupt) = 0; - p.rdata(PIdx::N) = 0; - p.rdata(PIdx::Nbar) = 0; - p.rdata(PIdx::L) = 0; - p.rdata(PIdx::Lbar) = 0; + p.rdata(PIdx::Vphase) = 0; #include "generated_files/Evolve.cpp_dfdt_fill" }); From f042f180cc4da2b1a845a5c7dd4d0d3c29e8e407 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Wed, 17 Jan 2024 20:59:39 -0500 Subject: [PATCH 057/276] whitespace --- Source/Parameters.H | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index 4ee8121a..bcc9b99c 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -74,28 +74,27 @@ struct TestParams : public amrex::Gpu::Managed if(perturbation_type == 1) pp.get("perturbation_wavelength_cm", perturbation_wavelength_cm); - // neutrino physics parameters for 2-flavor - pp.get("mass1_eV", mass1); - pp.get("mass2_eV", mass2); - pp.get("theta12_degrees", theta12); - pp.get("alpha1_degrees", alpha1); - mass1 *= CGSUnitsConst::eV/PhysConst::c2; - mass2 *= CGSUnitsConst::eV/PhysConst::c2; - theta12 *= M_PI/180.; - alpha1 *= M_PI/180.; - - if(NUM_FLAVORS>=2){ - pp.get("mass3_eV", mass3); - pp.get("theta13_degrees", theta13); - pp.get("theta23_degrees", theta23); - pp.get("alpha2_degrees", alpha2); - pp.get("deltaCP_degrees", deltaCP); - mass3 *= CGSUnitsConst::eV/PhysConst::c2; - theta13 *= M_PI/180.; - theta23 *= M_PI/180.; - alpha2 *= M_PI/180.; - deltaCP *= M_PI/180.; - } + // neutrino physics parameters for 2-flavor + pp.get("mass1_eV", mass1); + pp.get("mass2_eV", mass2); + pp.get("theta12_degrees", theta12); + pp.get("alpha1_degrees", alpha1); + mass1 *= CGSUnitsConst::eV/PhysConst::c2; + mass2 *= CGSUnitsConst::eV/PhysConst::c2; + theta12 *= M_PI/180.; + alpha1 *= M_PI/180.; + if(NUM_FLAVORS>=2){ + pp.get("mass3_eV", mass3); + pp.get("theta13_degrees", theta13); + pp.get("theta23_degrees", theta23); + pp.get("alpha2_degrees", alpha2); + pp.get("deltaCP_degrees", deltaCP); + mass3 *= CGSUnitsConst::eV/PhysConst::c2; + theta13 *= M_PI/180.; + theta23 *= M_PI/180.; + alpha2 *= M_PI/180.; + deltaCP *= M_PI/180.; + } } }; From 9251ec11324025c478378971f3285081000228dc Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Wed, 17 Jan 2024 22:00:25 -0500 Subject: [PATCH 058/276] infrastructure to set IMFPs and chemical potentials in the parameter file --- Source/Constants.H | 1 + Source/Evolve.cpp | 41 ++++++++++++++++++++++ Source/Parameters.H | 42 +++++++++++++++++++++-- Source/main.cpp | 2 +- sample_inputs/inputs_1d_fiducial | 4 +++ sample_inputs/inputs_bipolar_test | 5 +++ sample_inputs/inputs_fast_flavor | 5 +++ sample_inputs/inputs_fast_flavor_nonzerok | 4 +++ sample_inputs/inputs_msw_test | 5 +++ 9 files changed, 106 insertions(+), 3 deletions(-) diff --git a/Source/Constants.H b/Source/Constants.H index 1e9eeab6..21883969 100644 --- a/Source/Constants.H +++ b/Source/Constants.H @@ -18,6 +18,7 @@ namespace PhysConst static constexpr amrex::Real GF = 1.1663787e-5/*GeV^-2*//(1e9*1e9*CGSUnitsConst::eV*CGSUnitsConst::eV) * hbarc*hbarc*hbarc; //erg cm^3 static constexpr amrex::Real Mp = 1.6726219e-24; // g static constexpr amrex::Real sin2thetaW = 0.23122; + static constexpr amrex::Real kB = 1.380658e-16; // erg/K } namespace MathConst diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index e3943cd5..09d72780 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -154,6 +154,47 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } } + // determine the IMFPs and equilibrium distribution value + // create 2 x NF matrix to store absorption IMFPs + // and 2 x NF matrix to store scattering IMFPs + // and 2 x NF matrix to store equilibrium distribution values + Real IMFP_abs[2][NUM_FLAVORS]; + Real IMFP_scat[2][NUM_FLAVORS]; + Real f_eq[2][NUM_FLAVORS]; // equilibrium distribution function (dimensionless) + Real munu[2][NUM_FLAVORS]; // equilibrium chemical potential (erg) + + // fill the IMFP values + if(parms->IMFP_method==0){ + // fill with all zeros + for (int i=0; i<2; ++i) { + for (int j=0; jIMFP_method==1){ + // use the IMFPs from the input file + for(int i=0; i<2; i++){ + for(int j=0; jIMFP_abs[i][j]; + IMFP_scat[i][j] = parms->IMFP_scat[i][j]; + munu[i][j] = parms->munu[i][j]; + } + } + } + else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0 or 1"); + + // calculate the equilibrium distribution. Really munu and temperature should be interpolated from the grid. + for(int i=0; i<2; i++){ + for(int j=0; jkT_in; + f_eq[i][j] = 1. / (1. + exp(exponent)); + } + } + // set the dfdt values into p.rdata p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; diff --git a/Source/Parameters.H b/Source/Parameters.H index bcc9b99c..2ee70a4f 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -22,7 +22,7 @@ struct TestParams : public amrex::Gpu::Managed Real end_time; int write_plot_every; int write_plot_particles_every; - Real rho_in, Ye_in, T_in; // g/ccm, 1, MeV + Real rho_in, Ye_in, kT_in; // g/ccm, 1, erg Real cfl_factor, flavor_cfl_factor; Real max_adaptive_speedup; bool do_restart; @@ -42,6 +42,12 @@ struct TestParams : public amrex::Gpu::Managed int perturbation_type; Real perturbation_wavelength_cm; Real perturbation_amplitude; + + // absorption opacities and equilibrium neutrino chemical potentials + int IMFP_method; + Real IMFP_abs[2][NUM_FLAVORS]; + Real IMFP_scat[2][NUM_FLAVORS]; + Real munu[2][NUM_FLAVORS]; // equilibrium electron neutrino chemical potential in MeV void Initialize(){ ParmParse pp; @@ -55,7 +61,7 @@ struct TestParams : public amrex::Gpu::Managed pp.get("end_time", end_time); pp.get("rho_g_ccm", rho_in); pp.get("Ye", Ye_in); - pp.get("T_MeV", T_in); + pp.get("T_MeV", kT_in); pp.get("cfl_factor", cfl_factor); pp.get("flavor_cfl_factor", flavor_cfl_factor); pp.get("max_adaptive_speedup", max_adaptive_speedup); @@ -65,6 +71,9 @@ struct TestParams : public amrex::Gpu::Managed pp.get("restart_dir", restart_dir); pp.get("maxError", maxError); + // convert to cgs + kT_in *= 1e6 * CGSUnitsConst::eV; // erg + // angular grid pp.get("particle_data_filename",particle_data_filename); @@ -96,6 +105,35 @@ struct TestParams : public amrex::Gpu::Managed deltaCP *= M_PI/180.; } + // absorption opacities and equilibrium neutrino chemical potentials + pp.get("IMFP_method", IMFP_method); + if(IMFP_method==0){ + // do nothing - all values will be set to zero + } + else if(IMFP_method==1){ + for(int i=0;irho_in,GIdx::rho,1); // g/ccm state.setVal(parms->Ye_in,GIdx::Ye,1); - state.setVal(parms->T_in,GIdx::T,1); // MeV + state.setVal(parms->kT_in,GIdx::T,1); // erg state.FillBoundary(geom.periodicity()); // initialize the grid variable names diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index c3e89d37..ab3bd0f9 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -78,3 +78,7 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 +################# +# opacity stuff # +################# +IMFP_method = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index e4d58c56..2ca553b5 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -78,3 +78,8 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index 2406fbeb..d5bbf174 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -78,3 +78,8 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index dc858a8e..38a4da5e 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -79,3 +79,7 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 +################# +# opacity stuff # +################# +IMFP_method = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index cdd14ac7..33f09834 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -78,3 +78,8 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 0 \ No newline at end of file From d5a1f63d3314c09b385784d5c91468e747da9793 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 26 Jan 2024 09:47:22 -0500 Subject: [PATCH 059/276] redefining f --> Nrho (before was f ---> Nrho) --- Scripts/symbolic_hermitians/generate_code.py | 4 ++-- Source/FlavoredNeutrinoContainer.cpp | 2 +- Source/FlavoredNeutrinoContainerInit.cpp | 18 ++++++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 5e6e7439..3b1acf7a 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -169,7 +169,7 @@ def delete_generated_files(): "*p.rdata(PIdx::pupz)*p.rdata(PIdx::pupz)/p.rdata(PIdx::pupt)/p.rdata(PIdx::pupt));"]) code = [] for t in tails: - string3 = ")*p.rdata(PIdx::N"+t+")" + string3 = ")" flist = HermitianMatrix(args.N, "f{}{}_{}"+t).header() for ivar in range(len(deposit_vars)): deplist = HermitianMatrix(args.N, deposit_vars[ivar]+"{}{}_{}"+t).header() @@ -470,7 +470,7 @@ def sgn(t,var): # store Tr(H*F) for estimating numerical errors TrHf = (H*F).trace(); - code.append(["p.rdata(PIdx::TrHf) += p.rdata(PIdx::N"+t+") * ("+sympy.cxxcode(sympy.simplify(TrHf))+");"]) + code.append(["p.rdata(PIdx::TrHf) += ("+sympy.cxxcode(sympy.simplify(TrHf))+");"]) code = [line for sublist in code for line in sublist] write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill")) diff --git a/Source/FlavoredNeutrinoContainer.cpp b/Source/FlavoredNeutrinoContainer.cpp index cc15a333..a264fd49 100644 --- a/Source/FlavoredNeutrinoContainer.cpp +++ b/Source/FlavoredNeutrinoContainer.cpp @@ -102,7 +102,7 @@ Renormalize(const TestParams* parms) amrex::ParallelFor (np, [=] AMREX_GPU_DEVICE (int i) { ParticleType& p = pstruct[i]; Real sumP, length, error; - #include "generated_files/FlavoredNeutrinoContainer.cpp_Renormalize_fill" + // #include "generated_files/FlavoredNeutrinoContainer.cpp_Renormalize_fill" }); } } diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index e47d042a..bc1de720 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -285,6 +285,24 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::N ) *= scale_fac; p.rdata(PIdx::Nbar) *= scale_fac; + // set f ---> N*rho (before f ---> rho) + p.rdata(PIdx::f00_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::f00_Re); + p.rdata(PIdx::f01_Im) = p.rdata(PIdx::N )* p.rdata(PIdx::f01_Im); + p.rdata(PIdx::f01_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::f01_Re); + p.rdata(PIdx::f11_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::f11_Re); + p.rdata(PIdx::f00_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f00_Rebar); + p.rdata(PIdx::f01_Imbar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f01_Imbar); + p.rdata(PIdx::f01_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f01_Rebar); + p.rdata(PIdx::f11_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f11_Rebar); +#if NUM_FLAVORS==3 + p.rdata(PIdx::f02_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::f02_Re); + p.rdata(PIdx::f02_Im) = p.rdata(PIdx::N )* p.rdata(PIdx::f02_Im); + p.rdata(PIdx::f22_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::f22_Re); + p.rdata(PIdx::f02_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f02_Rebar); + p.rdata(PIdx::f02_Imbar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f02_Imbar); + p.rdata(PIdx::f22_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f22_Rebar); +#endif + //=====================// // Apply Perturbations // //=====================// From 8b2ef955c2ebcb5c44b6bb8c699ac0852e12fc7f Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Thu, 1 Feb 2024 18:46:45 -0500 Subject: [PATCH 060/276] do not divide by cell volume, since this is already done in the code itself. --- Scripts/data_reduction/convertToHDF5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/data_reduction/convertToHDF5.py b/Scripts/data_reduction/convertToHDF5.py index 08e70909..6489aaa8 100755 --- a/Scripts/data_reduction/convertToHDF5.py +++ b/Scripts/data_reduction/convertToHDF5.py @@ -75,7 +75,7 @@ def convert_to_HDF5(sim_directory, DELETE_ALL_BUT_LAST_RESTART=False): allData["it"][-1] = int(d[-5:]) for v in varlist: allData[v].resize(np.shape(allData[v])[0] + 1, axis=0) - allData[v][-1,:] = eds.cg[v[:-7]].d / ad['index',"cell_volume"][0] + allData[v][-1,:] = eds.cg[v[:-7]].d if DELETE_ALL_BUT_LAST_RESTART: particle_directories = [d[:-10] for d in sorted(glob.glob(sim_directory+"/plt*/neutrinos"))] From ce3e01f9b945dcfe42f87e721dddf39459b13779 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 2 Feb 2024 08:26:08 -0500 Subject: [PATCH 061/276] computing volume of phase space --- Source/FlavoredNeutrinoContainerInit.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index bc1de720..26a75d34 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -285,6 +285,10 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::N ) *= scale_fac; p.rdata(PIdx::Nbar) *= scale_fac; + if(parms->IMFP_method == 1){ + p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/(3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); + } + // set f ---> N*rho (before f ---> rho) p.rdata(PIdx::f00_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::f00_Re); p.rdata(PIdx::f01_Im) = p.rdata(PIdx::N )* p.rdata(PIdx::f01_Im); From f1adb1b76c916acb3600d07ea2c3771eb9d71248 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 2 Feb 2024 08:31:07 -0500 Subject: [PATCH 062/276] deleting lines that convert twice mu to erg --- Source/Parameters.H | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index 2ee70a4f..42d0a925 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -126,11 +126,6 @@ struct TestParams : public amrex::Gpu::Managed munu[0][i] *= 1e6*CGSUnitsConst::eV; munu[1][i] = -munu[0][i]; } - - // convert munu to erg - for(int j=0;j<2;j++) - for(int i=0;i Date: Fri, 2 Feb 2024 08:36:12 -0500 Subject: [PATCH 063/276] adding delta E as an input parameter to compute phase space volume --- Source/Parameters.H | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Source/Parameters.H b/Source/Parameters.H index 42d0a925..838fffd2 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -48,6 +48,7 @@ struct TestParams : public amrex::Gpu::Managed Real IMFP_abs[2][NUM_FLAVORS]; Real IMFP_scat[2][NUM_FLAVORS]; Real munu[2][NUM_FLAVORS]; // equilibrium electron neutrino chemical potential in MeV + Real delta_E; void Initialize(){ ParmParse pp; @@ -126,6 +127,8 @@ struct TestParams : public amrex::Gpu::Managed munu[0][i] *= 1e6*CGSUnitsConst::eV; munu[1][i] = -munu[0][i]; } + pp.get("delta_E", delta_E); + delta_E*= 1e6*CGSUnitsConst::eV; // erg } else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 1"); From 3baca4924b6244753f5a7e9f87999fd4173c6ace Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 2 Feb 2024 08:40:54 -0500 Subject: [PATCH 064/276] adding new input parameters to input files to run collitions --- sample_inputs/inputs_1d_fiducial | 22 +++++++++++++++++++++- sample_inputs/inputs_bipolar_test | 22 +++++++++++++++++++++- sample_inputs/inputs_fast_flavor | 22 +++++++++++++++++++++- sample_inputs/inputs_fast_flavor_nonzerok | 22 +++++++++++++++++++++- sample_inputs/inputs_msw_test | 22 +++++++++++++++++++++- 5 files changed, 105 insertions(+), 5 deletions(-) diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index ab3bd0f9..c80dfb26 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -81,4 +81,24 @@ deltaCP_degrees = 222 ################# # opacity stuff # ################# -IMFP_method = 0 \ No newline at end of file +IMFP_method = 0 + +IMFP_abs0_cm = 0 +IMFP_abs1_cm = 0 +IMFP_abs2_cm = 0 +IMFP_abs0bar_cm = 0 +IMFP_abs1bar_cm = 0 +IMFP_abs2bar_cm = 0 + +munu0_MeV=0 +munu1_MeV=0 +munu2_MeV=0 + +IMFP_scat0_cm = 0 +IMFP_scat1_cm = 0 +IMFP_scat2_cm = 0 +IMFP_scat0bar_cm = 0 +IMFP_scat1bar_cm = 0 +IMFP_scat2bar_cm = 0 + +delta_E = 0 # Mev \ No newline at end of file diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 2ca553b5..398f5f92 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -82,4 +82,24 @@ deltaCP_degrees = 222 ################# # opacity stuff # ################# -IMFP_method = 0 \ No newline at end of file +IMFP_method = 0 + +IMFP_abs0_cm = 0 +IMFP_abs1_cm = 0 +IMFP_abs2_cm = 0 +IMFP_abs0bar_cm = 0 +IMFP_abs1bar_cm = 0 +IMFP_abs2bar_cm = 0 + +munu0_MeV=0 +munu1_MeV=0 +munu2_MeV=0 + +IMFP_scat0_cm = 0 +IMFP_scat1_cm = 0 +IMFP_scat2_cm = 0 +IMFP_scat0bar_cm = 0 +IMFP_scat1bar_cm = 0 +IMFP_scat2bar_cm = 0 + +delta_E = 0 # Mev \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index d5bbf174..4be5bc45 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -82,4 +82,24 @@ deltaCP_degrees = 222 ################# # opacity stuff # ################# -IMFP_method = 0 \ No newline at end of file +IMFP_method = 0 + +IMFP_abs0_cm = 0 +IMFP_abs1_cm = 0 +IMFP_abs2_cm = 0 +IMFP_abs0bar_cm = 0 +IMFP_abs1bar_cm = 0 +IMFP_abs2bar_cm = 0 + +munu0_MeV=0 +munu1_MeV=0 +munu2_MeV=0 + +IMFP_scat0_cm = 0 +IMFP_scat1_cm = 0 +IMFP_scat2_cm = 0 +IMFP_scat0bar_cm = 0 +IMFP_scat1bar_cm = 0 +IMFP_scat2bar_cm = 0 + +delta_E = 0 # Mev \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 38a4da5e..8d9698c8 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -82,4 +82,24 @@ deltaCP_degrees = 222 ################# # opacity stuff # ################# -IMFP_method = 0 \ No newline at end of file +IMFP_method = 0 + +IMFP_abs0_cm = 0 +IMFP_abs1_cm = 0 +IMFP_abs2_cm = 0 +IMFP_abs0bar_cm = 0 +IMFP_abs1bar_cm = 0 +IMFP_abs2bar_cm = 0 + +munu0_MeV=0 +munu1_MeV=0 +munu2_MeV=0 + +IMFP_scat0_cm = 0 +IMFP_scat1_cm = 0 +IMFP_scat2_cm = 0 +IMFP_scat0bar_cm = 0 +IMFP_scat1bar_cm = 0 +IMFP_scat2bar_cm = 0 + +delta_E = 0 # Mev \ No newline at end of file diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index 33f09834..a088d93b 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -82,4 +82,24 @@ deltaCP_degrees = 222 ################# # opacity stuff # ################# -IMFP_method = 0 \ No newline at end of file +IMFP_method = 0 + +IMFP_abs0_cm = 0 +IMFP_abs1_cm = 0 +IMFP_abs2_cm = 0 +IMFP_abs0bar_cm = 0 +IMFP_abs1bar_cm = 0 +IMFP_abs2bar_cm = 0 + +munu0_MeV=0 +munu1_MeV=0 +munu2_MeV=0 + +IMFP_scat0_cm = 0 +IMFP_scat1_cm = 0 +IMFP_scat2_cm = 0 +IMFP_scat0bar_cm = 0 +IMFP_scat1bar_cm = 0 +IMFP_scat2bar_cm = 0 + +delta_E = 0 # Mev \ No newline at end of file From 764e827b7b03f5ea3c0e5985668de82975896b31 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 2 Feb 2024 08:45:10 -0500 Subject: [PATCH 065/276] adding collition term in QKE --- Scripts/symbolic_hermitians/generate_code.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 3b1acf7a..8f8506ec 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -452,6 +452,22 @@ def sgn(t,var): Gdeclare = ["amrex::Real {}".format(line) for line in G.code()] code.append(Gdeclare) + #collision term (emmission and apsortion) + s=0 + if(t == ""): s=0 + if(t == "bar"): s=1 + + code.append(["dfdt00_Re"+t+" += IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] * p.rdata(PIdx::Vphase) * pow( 2 * MathConst::pi / PhysConst::hbar , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][0] ) / 2 ) * p.rdata(PIdx::f00_Re"+t+");"]) + code.append(["dfdt11_Re"+t+" += IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] * p.rdata(PIdx::Vphase) * pow( 2 * MathConst::pi / PhysConst::hbar , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] + IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] ) / 2 + ( IMFP_abs["+str(s)+"][1] + IMFP_abs["+str(s)+"][1] ) / 2 ) * p.rdata(PIdx::f11_Re"+t+");"]) + code.append(["dfdt01_Re"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] ) / 2 ) * p.rdata(PIdx::f01_Re"+t+");"]) + code.append(["dfdt01_Im"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] ) / 2 ) * p.rdata(PIdx::f01_Im"+t+");"]) + if(args.N == 3): + code.append(["dfdt22_Re"+t+" += IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] * p.rdata(PIdx::Vphase) * pow( 2 * MathConst::pi / PhysConst::hbar , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][2] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f22_Re"+t+");"]) + code.append(["dfdt02_Re"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f02_Re"+t+");"]) + code.append(["dfdt02_Im"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f02_Im"+t+");"]) + code.append(["dfdt12_Re"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f02_Re"+t+");"]) + code.append(["dfdt12_Im"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f02_Im"+t+");"]) + # Store dFdt back into the particle data for F dFdt = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") Gempty = HermitianMatrix(args.N, "dfdt{}{}_{}"+t) From 3f9ef87ab1119685f53983dab03f104279a66635 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 2 Feb 2024 08:50:27 -0500 Subject: [PATCH 066/276] moving line that compute derivates, now it set the phase volume to zero meaning its time derivative its zero after it is used in collision term --- Source/Evolve.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 09d72780..955bd887 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -195,6 +195,8 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } } + #include "generated_files/Evolve.cpp_dfdt_fill" + // set the dfdt values into p.rdata p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; @@ -205,7 +207,5 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M p.rdata(PIdx::pupz) = 0; p.rdata(PIdx::pupt) = 0; p.rdata(PIdx::Vphase) = 0; - - #include "generated_files/Evolve.cpp_dfdt_fill" }); } From f0936684bb289a6b2aa657f7a65c4ad91a6e2291 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 2 Feb 2024 15:59:13 -0500 Subject: [PATCH 067/276] considering the particle that carry antineutrinos in the fast flavor test --- Scripts/tests/fast_flavor_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Scripts/tests/fast_flavor_test.py b/Scripts/tests/fast_flavor_test.py index ff9d7647..557ab791 100644 --- a/Scripts/tests/fast_flavor_test.py +++ b/Scripts/tests/fast_flavor_test.py @@ -41,6 +41,7 @@ t.append(p[rkey["time"]]) fexR.append(p[rkey["f01_Re"]]) fexI.append(p[rkey["f01_Im"]]) + p = rdata[1] fexRbar.append(p[rkey["f01_Rebar"]]) fexIbar.append(p[rkey["f01_Imbar"]]) pupt.append(p[rkey["pupt"]]) From 275f5a08c4bba76e36f18e83d831ba5214b24766 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 13 Feb 2024 23:23:45 -0500 Subject: [PATCH 068/276] Fixing an error in the collision term --- Scripts/symbolic_hermitians/generate_code.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 8f8506ec..c4dacb93 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -457,12 +457,14 @@ def sgn(t,var): if(t == ""): s=0 if(t == "bar"): s=1 - code.append(["dfdt00_Re"+t+" += IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] * p.rdata(PIdx::Vphase) * pow( 2 * MathConst::pi / PhysConst::hbar , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][0] ) / 2 ) * p.rdata(PIdx::f00_Re"+t+");"]) - code.append(["dfdt11_Re"+t+" += IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] * p.rdata(PIdx::Vphase) * pow( 2 * MathConst::pi / PhysConst::hbar , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] + IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] ) / 2 + ( IMFP_abs["+str(s)+"][1] + IMFP_abs["+str(s)+"][1] ) / 2 ) * p.rdata(PIdx::f11_Re"+t+");"]) + code.append(["dfdt00_Re"+t+" += IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] * p.rdata(PIdx::Vphase) * pow( 1 / ( 2 * MathConst::pi * PhysConst::hbar ) , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][0] ) / 2 ) * p.rdata(PIdx::f00_Re"+t+");"]) + code.append(["dfdt11_Re"+t+" += IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] * p.rdata(PIdx::Vphase) * pow( 1 / ( 2 * MathConst::pi * PhysConst::hbar ) , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] + IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] ) / 2 + ( IMFP_abs["+str(s)+"][1] + IMFP_abs["+str(s)+"][1] ) / 2 ) * p.rdata(PIdx::f11_Re"+t+");"]) code.append(["dfdt01_Re"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] ) / 2 ) * p.rdata(PIdx::f01_Re"+t+");"]) code.append(["dfdt01_Im"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] ) / 2 ) * p.rdata(PIdx::f01_Im"+t+");"]) + if(args.N == 3): - code.append(["dfdt22_Re"+t+" += IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] * p.rdata(PIdx::Vphase) * pow( 2 * MathConst::pi / PhysConst::hbar , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][2] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f22_Re"+t+");"]) + + code.append(["dfdt22_Re"+t+" += IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] * p.rdata(PIdx::Vphase) * pow( 1 / ( 2 * MathConst::pi * PhysConst::hbar ) , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][2] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f22_Re"+t+");"]) code.append(["dfdt02_Re"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f02_Re"+t+");"]) code.append(["dfdt02_Im"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f02_Im"+t+");"]) code.append(["dfdt12_Re"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f02_Re"+t+");"]) From 2614233a0c2f15088c296236fdf25e58c6fa8a11 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 28 Feb 2024 09:13:29 -0500 Subject: [PATCH 069/276] including adaptative time step for absortion and emission --- Source/Evolve.H | 2 +- Source/Evolve.cpp | 30 +++++++++++++++++++++-- Source/Parameters.H | 3 ++- Source/main.cpp | 4 +-- sample_inputs/inputs_1d_fiducial | 1 + sample_inputs/inputs_bipolar_test | 1 + sample_inputs/inputs_fast_flavor | 1 + sample_inputs/inputs_fast_flavor_nonzerok | 1 + sample_inputs/inputs_msw_test | 1 + 9 files changed, 38 insertions(+), 6 deletions(-) diff --git a/Source/Evolve.H b/Source/Evolve.H index 55f23dec..893b0049 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -26,7 +26,7 @@ namespace GIdx void Initialize(); } -amrex::Real compute_dt(const amrex::Geometry& geom, const amrex::Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor, const Real max_adaptive_speedup); +amrex::Real compute_dt(const amrex::Geometry& geom, const amrex::Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor, const Real max_adaptive_speedup, const TestParams* parms); void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 955bd887..f817c83a 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -19,7 +19,7 @@ namespace GIdx } } -Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const Real flavor_cfl_factor, const Real max_adaptive_speedup) +Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const Real flavor_cfl_factor, const Real max_adaptive_speedup, const TestParams* parms) { AMREX_ASSERT(cfl_factor > 0.0 || flavor_cfl_factor > 0.0); @@ -64,11 +64,37 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta // define the dt associated with each method Real dt_flavor_adaptive = PhysConst::hbar/Vmax_adaptive*flavor_cfl_factor; Real dt_flavor_stupid = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; + Real dt_flavor_absortion = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; // it will change only if opacity_method is 1 + + // get the inverse mean free path for absortion + Real IMFP_abs[2][NUM_FLAVORS]; + + if(parms->IMFP_method==1){ + // use the IMFPs from the input file + for(int i=0; i<2; i++){ + for(int j=0; jIMFP_abs[i][j]; + } + } + + // this will store the max absortion IMFP + double max_IMFP_abs = std::numeric_limits::lowest(); // Initialize max to lowest possible value + + // Iterate through the array and update max_IMFP_abs if a larger value is found + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < NUM_FLAVORS; ++j) { + if (IMFP_abs[i][j] > max_IMFP_abs) { + max_IMFP_abs = IMFP_abs[i][j]; + } + } + } + dt_flavor_absortion = (1/(PhysConst::c*max_IMFP_abs)) *parms->collision_cfl_factor; + } // pick the appropriate timestep dt_flavor = dt_flavor_stupid; if(max_adaptive_speedup>1) - dt_flavor = min(dt_flavor_stupid*max_adaptive_speedup, dt_flavor_adaptive); + dt_flavor = min(dt_flavor_stupid*max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absortion); } Real dt = 0.0; diff --git a/Source/Parameters.H b/Source/Parameters.H index 838fffd2..c4dc3509 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -23,7 +23,7 @@ struct TestParams : public amrex::Gpu::Managed int write_plot_every; int write_plot_particles_every; Real rho_in, Ye_in, kT_in; // g/ccm, 1, erg - Real cfl_factor, flavor_cfl_factor; + Real cfl_factor, flavor_cfl_factor,collision_cfl_factor; Real max_adaptive_speedup; bool do_restart; std::string restart_dir; @@ -65,6 +65,7 @@ struct TestParams : public amrex::Gpu::Managed pp.get("T_MeV", kT_in); pp.get("cfl_factor", cfl_factor); pp.get("flavor_cfl_factor", flavor_cfl_factor); + pp.get("collision_cfl_factor", collision_cfl_factor); pp.get("max_adaptive_speedup", max_adaptive_speedup); pp.get("write_plot_every", write_plot_every); pp.get("write_plot_particles_every", write_plot_particles_every); diff --git a/Source/main.cpp b/Source/main.cpp index e68449e4..b5ff87ad 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -197,7 +197,7 @@ void evolve_flavor(const TestParams* parms) // Note: this won't be the same as the new-time grid data // because the last deposit_to_mesh call was at either the old time (forward Euler) // or the final RK stage, if using Runge-Kutta. - const Real dt = compute_dt(geom,parms->cfl_factor,state,neutrinos,parms->flavor_cfl_factor,parms->max_adaptive_speedup); + const Real dt = compute_dt(geom,parms->cfl_factor,state,neutrinos,parms->flavor_cfl_factor,parms->max_adaptive_speedup, parms); integrator.set_timestep(dt); }; @@ -206,7 +206,7 @@ void evolve_flavor(const TestParams* parms) integrator.set_post_timestep(post_timestep_fun); // Get a starting timestep - const Real starting_dt = compute_dt(geom,parms->cfl_factor,state,neutrinos_old,parms->flavor_cfl_factor, parms->max_adaptive_speedup); + const Real starting_dt = compute_dt(geom,parms->cfl_factor, state,neutrinos_old,parms->flavor_cfl_factor, parms->max_adaptive_speedup, parms); // Do all the science! amrex::Print() << "Starting timestepping loop... " << std::endl; diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index c80dfb26..3ec8a462 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -1,6 +1,7 @@ perturbation_type = 0 perturbation_amplitude = 1e-6 +collision_cfl_factor = 0.5 cfl_factor = 0.5 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 398f5f92..2cd1f6e7 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -1,3 +1,4 @@ +collision_cfl_factor = 0.5 cfl_factor = 0.5 max_adaptive_speedup = 0 flavor_cfl_factor = .5 diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index 4be5bc45..e8036706 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -1,3 +1,4 @@ +collision_cfl_factor = 0.5 cfl_factor = 0.5 flavor_cfl_factor = .5 max_adaptive_speedup = 0 diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 8d9698c8..298d3725 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -2,6 +2,7 @@ perturbation_type = 1 perturbation_amplitude = 1e-6 perturbation_wavelength_cm = 1 +collision_cfl_factor = 0.5 cfl_factor = 0.5 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index a088d93b..80a2d6c6 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -1,3 +1,4 @@ +collision_cfl_factor = 0.5 cfl_factor = 0.5 flavor_cfl_factor = 0.1 max_adaptive_speedup = 0 From 8f7b9281e923b8c1f5d03c5ad94d9c8991f85ef0 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 7 Mar 2024 13:31:54 -0500 Subject: [PATCH 070/276] creating vacuum paticles initial condition script --- .../initial_conditions/st7_empty_particles.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 Scripts/initial_conditions/st7_empty_particles.py diff --git a/Scripts/initial_conditions/st7_empty_particles.py b/Scripts/initial_conditions/st7_empty_particles.py new file mode 100644 index 00000000..df370ba6 --- /dev/null +++ b/Scripts/initial_conditions/st7_empty_particles.py @@ -0,0 +1,45 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../data_analysis") +from initial_condition_tools import uniform_sphere, write_particles, moment_interpolate_particles, linear_interpolate +import amrex_plot_tools as amrex + +# generation parameters +# MUST MATCH THE INPUTS IN THE EMU INPUT FILE! +NF = 3 +nphi_equator = 16 +theta = 0 +thetabar = 3.14159265359 +phi = 0 +phibar=0 +ndens = 0 +ndensbar = 0 +fluxfac = .333333333333333 +fluxfacbar = .333333333333333 +energy_erg = 50 * 1e6*amrex.eV + +# flux factor vectors +fhat = np.array([np.cos(phi) *np.sin(theta ), + np.sin(phi) *np.sin(theta ), + np.cos(theta )]) +fhatbar = np.array([np.cos(phibar)*np.sin(thetabar), + np.sin(phibar)*np.sin(thetabar), + np.cos(thetabar)]) + +nnu = np.zeros((2,NF)) +nnu[0,0] = ndens +nnu[1,0] = ndensbar +nnu[:,1:] = 0 + +fnu = np.zeros((2,NF,3)) +fnu[0,0,:] = ndens * fluxfac * fhat +fnu[1,0,:] = ndensbar * fluxfacbar * fhatbar +fnu[:,1:,:] = 0 + +particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, linear_interpolate) # [particle, variable] + +write_particles(np.array(particles), NF, "particle_input.dat") From fd6964aa6f3a84b277e5ec21a671d6edf09976e6 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 7 Mar 2024 16:08:03 -0500 Subject: [PATCH 071/276] adding attenuation parameter to the time derivative of N_Ab due to the hamiltonians --- Scripts/symbolic_hermitians/generate_code.py | 14 ++++++++++++++ Source/Evolve.cpp | 1 + Source/Parameters.H | 8 +++++++- sample_inputs/inputs_1d_fiducial | 3 +++ sample_inputs/inputs_bipolar_test | 3 +++ sample_inputs/inputs_fast_flavor | 3 +++ sample_inputs/inputs_fast_flavor_nonzerok | 3 +++ sample_inputs/inputs_msw_test | 3 +++ 8 files changed, 37 insertions(+), 1 deletion(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index c4dacb93..3bf26349 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -452,6 +452,20 @@ def sgn(t,var): Gdeclare = ["amrex::Real {}".format(line) for line in G.code()] code.append(Gdeclare) + # time derivative due to hamiltonians attenuation parameter + code.append(["dfdt00_Re"+t+" *= att_ham;"]) + code.append(["dfdt11_Re"+t+" *= att_ham;"]) + code.append(["dfdt01_Re"+t+" *= att_ham;"]) + code.append(["dfdt01_Im"+t+" *= att_ham;"]) + + if(args.N == 3): + + code.append(["dfdt22_Re"+t+" *= att_ham;"]) + code.append(["dfdt02_Re"+t+" *= att_ham;"]) + code.append(["dfdt02_Im"+t+" *= att_ham;"]) + code.append(["dfdt12_Re"+t+" *= att_ham;"]) + code.append(["dfdt12_Im"+t+" *= att_ham;"]) + #collision term (emmission and apsortion) s=0 if(t == ""): s=0 diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index f817c83a..60596120 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -188,6 +188,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M Real IMFP_scat[2][NUM_FLAVORS]; Real f_eq[2][NUM_FLAVORS]; // equilibrium distribution function (dimensionless) Real munu[2][NUM_FLAVORS]; // equilibrium chemical potential (erg) + Real att_ham = parms->attenuation_hamiltonians; // fill the IMFP values if(parms->IMFP_method==0){ diff --git a/Source/Parameters.H b/Source/Parameters.H index c4dc3509..771fccb3 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -49,7 +49,10 @@ struct TestParams : public amrex::Gpu::Managed Real IMFP_scat[2][NUM_FLAVORS]; Real munu[2][NUM_FLAVORS]; // equilibrium electron neutrino chemical potential in MeV Real delta_E; - + + // attenuation to hamiltonians + Real attenuation_hamiltonians; + void Initialize(){ ParmParse pp; pp.get("ncell", ncell); @@ -107,6 +110,9 @@ struct TestParams : public amrex::Gpu::Managed deltaCP *= M_PI/180.; } + // attenuation to hamiltonians + pp.get("attenuation_hamiltonians", attenuation_hamiltonians); + // absorption opacities and equilibrium neutrino chemical potentials pp.get("IMFP_method", IMFP_method); if(IMFP_method==0){ diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 3ec8a462..033c5e2f 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -1,6 +1,9 @@ perturbation_type = 0 perturbation_amplitude = 1e-6 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + collision_cfl_factor = 0.5 cfl_factor = 0.5 flavor_cfl_factor = 0.5 diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 2cd1f6e7..105b3db7 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -4,6 +4,9 @@ max_adaptive_speedup = 0 flavor_cfl_factor = .5 maxError = 1e-6 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + perturbation_type = 0 perturbation_amplitude = 0 diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index e8036706..9f52f0a6 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -4,6 +4,9 @@ flavor_cfl_factor = .5 max_adaptive_speedup = 0 maxError = 1e-6 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + perturbation_type = 0 perturbation_amplitude = 0 diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 298d3725..070a0599 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -2,6 +2,9 @@ perturbation_type = 1 perturbation_amplitude = 1e-6 perturbation_wavelength_cm = 1 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + collision_cfl_factor = 0.5 cfl_factor = 0.5 flavor_cfl_factor = 0.5 diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index 80a2d6c6..eab8e0bc 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -4,6 +4,9 @@ flavor_cfl_factor = 0.1 max_adaptive_speedup = 0 maxError = 1e-6 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + perturbation_type = 0 perturbation_amplitude = 0 From 00f830706d5fb39cd12e466054257eb059345167 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 7 Mar 2024 21:24:24 -0500 Subject: [PATCH 072/276] solving mistake to compute time step due to collisions --- Source/Evolve.cpp | 39 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 60596120..faec7f03 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -62,39 +62,32 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta ParallelDescriptor::ReduceRealMax(Vmax_stupid ); // define the dt associated with each method - Real dt_flavor_adaptive = PhysConst::hbar/Vmax_adaptive*flavor_cfl_factor; - Real dt_flavor_stupid = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; - Real dt_flavor_absortion = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; // it will change only if opacity_method is 1 + Real dt_flavor_adaptive = PhysConst::hbar/Vmax_adaptive*flavor_cfl_factor; + Real dt_flavor_stupid = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; + Real dt_flavor_absorption = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; // it will change only if opacity_method is 1 - // get the inverse mean free path for absortion - Real IMFP_abs[2][NUM_FLAVORS]; - - if(parms->IMFP_method==1){ - // use the IMFPs from the input file - for(int i=0; i<2; i++){ - for(int j=0; jIMFP_abs[i][j]; - } - } - - // this will store the max absortion IMFP + if (parms->IMFP_method == 1) { + // Use the IMFPs from the input file and find the maximum absorption IMFP double max_IMFP_abs = std::numeric_limits::lowest(); // Initialize max to lowest possible value - - // Iterate through the array and update max_IMFP_abs if a larger value is found for (int i = 0; i < 2; ++i) { for (int j = 0; j < NUM_FLAVORS; ++j) { - if (IMFP_abs[i][j] > max_IMFP_abs) { - max_IMFP_abs = IMFP_abs[i][j]; - } + max_IMFP_abs = std::max(max_IMFP_abs, parms->IMFP_abs[i][j]); } } - dt_flavor_absortion = (1/(PhysConst::c*max_IMFP_abs)) *parms->collision_cfl_factor; + + // Calculate dt_flavor_absorption + dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor / 500; + + if (parms->attenuation_hamiltonians==0){ + dt_flavor_adaptive = dt_flavor_absorption; + dt_flavor_stupid = dt_flavor_absorption; + } } // pick the appropriate timestep - dt_flavor = dt_flavor_stupid; + dt_flavor = dt_flavor_stupid; if(max_adaptive_speedup>1) - dt_flavor = min(dt_flavor_stupid*max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absortion); + dt_flavor = min(dt_flavor_stupid*max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); } Real dt = 0.0; From 19916f7dd9764a0ec588bde250da54c7c71f3e8e Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 7 Mar 2024 21:24:46 -0500 Subject: [PATCH 073/276] adding collision test input file --- sample_inputs/inputs_coll_equi_test | 108 ++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 sample_inputs/inputs_coll_equi_test diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test new file mode 100644 index 00000000..3609b0fe --- /dev/null +++ b/sample_inputs/inputs_coll_equi_test @@ -0,0 +1,108 @@ +perturbation_type = 0 +perturbation_amplitude = 1e-6 + +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 0 + +collision_cfl_factor = 2 +cfl_factor = 1e10 +flavor_cfl_factor = 0.5 +max_adaptive_speedup = 0 +maxError = 1e-6 + +integration.type = 1 +integration.rk.type = 4 + +# Domain size in 3D index space +ncell = (2, 2, 2) +Lx = 20 +Ly = 20 +Lz = 20 + +# Number of particles per cell +nppc = (1, 1, 1) +particle_data_filename = "particle_input.dat" + +# Maximum size of each grid in the domain +max_grid_size = 16 + +# Number of steps to run +nsteps = 5000 + +# Simulation end time +end_time = 5.0e9 + +# Make FPE signal errors so we get a Backtrace +amrex.fpe_trap_invalid=1 + +# give background fluid conditions +rho_g_ccm = 0 +T_MeV = 10 +Ye = 1 + +# Write plotfiles +write_plot_every = 25 + +# Write particle data in plotfiles +write_plot_particles_every = 25 + +# checkpointing +do_restart = 0 +restart_dir = "" + +############################### +# NEUTRINO PHYSICS PARAMETERS # +############################### +# see first column of table 14.7 in http://pdg.lbl.gov/2019/reviews/rpp2019-rev-neutrino-mixing.pdf + +# mass state 1 mass in eV [NO/IO:-sqrt(7.39e-5)] +mass1_eV = 0 #-0.008596511 + +# mass state 2 mass in eV (define at 0 arbitrarily because oscillations only sensitive to deltaM^2) +mass2_eV = 0 + +# mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] +mass3_eV = 0.049487372 + +# 1-2 mixing angle in degrees [NO/IO:33.82] +theta12_degrees = 1e-6 + +# 2-3 mixing angle in degrees [NO:8.61 IO:8.65] +theta23_degrees = 8.61 + +# 1-3 mixing angle in degrees [NO:48.3 IO:48.6] +theta13_degrees = 48.3 + +# Majorana angle 1 in degrees +alpha1_degrees = 0 + +# Majorana angle 2 in degrees +alpha2_degrees = 0 + +# CP-violating phase in degrees [NO:222 IO:285] +deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 1 + +IMFP_abs0_cm = 5e-4 +IMFP_abs1_cm = 5e-4 +IMFP_abs2_cm = 5e-4 +IMFP_abs0bar_cm = 5e-4 +IMFP_abs1bar_cm = 5e-4 +IMFP_abs2bar_cm = 5e-4 + +munu0_MeV=0 +munu1_MeV=0 +munu2_MeV=0 + +IMFP_scat0_cm = 0 +IMFP_scat1_cm = 0 +IMFP_scat2_cm = 0 +IMFP_scat0bar_cm = 0 +IMFP_scat1bar_cm = 0 +IMFP_scat2bar_cm = 0 + +delta_E = 0.8394810651882109 # Mev From 8105f03c5bfdf72b9345ca06f9caac5b4db91526 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 7 Mar 2024 22:44:09 -0500 Subject: [PATCH 074/276] testing if the particles converge to the expected equilibrium distribution --- Jenkinsfile | 11 ++++++ Scripts/tests/coll_equi_test.py | 60 +++++++++++++++++++++++++++++ sample_inputs/inputs_coll_equi_test | 8 ++-- 3 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 Scripts/tests/coll_equi_test.py diff --git a/Jenkinsfile b/Jenkinsfile index 84c48283..a871e56f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -90,6 +90,17 @@ pipeline { } }} + stage('Collisions to equilibrium'){ steps{ + dir('Exec'){ + sh 'cp makefiles/GNUmakefile_jenkins Exec/GNUmakefile' + sh 'make realclean; make generate NUM_FLAVORS=3; make NUM_FLAVORS=3' + sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' + sh 'python ../Scripts/tests/coll_equi_test.py' + sh 'rm -rf plt*' + } + }} + } // stages{ post { diff --git a/Scripts/tests/coll_equi_test.py b/Scripts/tests/coll_equi_test.py new file mode 100644 index 00000000..75a80f7a --- /dev/null +++ b/Scripts/tests/coll_equi_test.py @@ -0,0 +1,60 @@ +import numpy as np +import argparse +import glob +import EmuReader +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__))+"/../data_reduction/" +sys.path.append(importpath) +import amrex_plot_tools as amrex + +parser = argparse.ArgumentParser() +parser.add_argument("-na", "--no_assert", action="store_true", help="If --no_assert is supplied, do not raise assertion errors if the test error > tolerance.") +args = parser.parse_args() + +# physical constants +theta12 = 33.82*np.pi/180. # radians +dm21c4 = 7.39e-5 * amrex.eV**2 # erg^2 + +tolerance = 2e-2 +NF=3 + +if __name__ == "__main__": + + rkey, ikey = amrex.get_particle_keys(NF) + + N_ee = [] + N_uu = [] + N_tt = [] + N_eebar = [] + N_uubar = [] + N_ttbar = [] + + idata, rdata = EmuReader.read_particle_data('plt01000', ptype="neutrinos") + + for i in range(len(rdata)): + p = rdata[i] + N_ee.append(p[rkey["f00_Re"]]) + N_uu.append(p[rkey["f11_Re"]]) + N_tt.append(p[rkey["f22_Re"]]) + N_eebar.append(p[rkey["f00_Rebar"]]) + N_uubar.append(p[rkey["f11_Rebar"]]) + N_ttbar.append(p[rkey["f22_Rebar"]]) + + print(f'average N_ee {np.average(N_ee)}') + print(f'average N_uu {np.average(N_uu)}') + print(f'average N_tt {np.average(N_tt)}') + print(f'average N_eebar {np.average(N_eebar)}') + print(f'average N_uubar {np.average(N_uubar)}') + print(f'average N_ttbar {np.average(N_ttbar)}') + + def myassert(condition): + if not args.no_assert: + assert(condition) + + myassert( np.all(np.isclose(N_ee, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_uu, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_tt, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_eebar, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_uubar, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_ttbar, np.array(1e33), atol=1e33/100)) ) \ No newline at end of file diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index 3609b0fe..f504bf0e 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -4,7 +4,7 @@ perturbation_amplitude = 1e-6 # attenuation parameters to time derivative of N due to hamiltonians attenuation_hamiltonians = 0 -collision_cfl_factor = 2 +collision_cfl_factor = 20 cfl_factor = 1e10 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 @@ -27,7 +27,7 @@ particle_data_filename = "particle_input.dat" max_grid_size = 16 # Number of steps to run -nsteps = 5000 +nsteps = 1000 # Simulation end time end_time = 5.0e9 @@ -41,10 +41,10 @@ T_MeV = 10 Ye = 1 # Write plotfiles -write_plot_every = 25 +write_plot_every = 1000 # Write particle data in plotfiles -write_plot_particles_every = 25 +write_plot_particles_every = 1000 # checkpointing do_restart = 0 From 2507f0f69cba8e676d6d8cdca77079c8694eca0c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 8 Mar 2024 08:32:55 -0500 Subject: [PATCH 075/276] solving issue to copy GNUmakefile_jenkins to Exec directory --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a871e56f..f6a504d1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -92,7 +92,7 @@ pipeline { stage('Collisions to equilibrium'){ steps{ dir('Exec'){ - sh 'cp makefiles/GNUmakefile_jenkins Exec/GNUmakefile' + sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' sh 'make realclean; make generate NUM_FLAVORS=3; make NUM_FLAVORS=3' sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' From 0c1708deab644550ea5a11880075be7befd6386f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 15 Mar 2024 11:18:38 -0400 Subject: [PATCH 076/276] doing fast compilation in collision test --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f6a504d1..d24bb3d5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -93,7 +93,7 @@ pipeline { stage('Collisions to equilibrium'){ steps{ dir('Exec'){ sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' - sh 'make realclean; make generate NUM_FLAVORS=3; make NUM_FLAVORS=3' + sh 'make realclean; make generate NUM_FLAVORS=3; make -j' sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' sh 'python ../Scripts/tests/coll_equi_test.py' From 95fdc84396ee3c3c41d90500d0a008e6c5e28f1f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 15 Mar 2024 11:23:57 -0400 Subject: [PATCH 077/276] clarifying better the collision time steps, now delta t collision is collision_CFL / ( c * max_IMFP ) --- Source/Evolve.cpp | 2 +- sample_inputs/inputs_1d_fiducial | 2 +- sample_inputs/inputs_bipolar_test | 2 +- sample_inputs/inputs_coll_equi_test | 2 +- sample_inputs/inputs_fast_flavor | 2 +- sample_inputs/inputs_fast_flavor_nonzerok | 2 +- sample_inputs/inputs_msw_test | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index faec7f03..dd6d5b8b 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -76,7 +76,7 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta } // Calculate dt_flavor_absorption - dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor / 500; + dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor; if (parms->attenuation_hamiltonians==0){ dt_flavor_adaptive = dt_flavor_absorption; diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 033c5e2f..b590b37a 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -4,7 +4,7 @@ perturbation_amplitude = 1e-6 # attenuation parameters to time derivative of N due to hamiltonians attenuation_hamiltonians = 1 -collision_cfl_factor = 0.5 +collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 105b3db7..7e428465 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -1,4 +1,4 @@ -collision_cfl_factor = 0.5 +collision_cfl_factor = 1e-3 cfl_factor = 0.5 max_adaptive_speedup = 0 flavor_cfl_factor = .5 diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index f504bf0e..acbf6502 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -4,7 +4,7 @@ perturbation_amplitude = 1e-6 # attenuation parameters to time derivative of N due to hamiltonians attenuation_hamiltonians = 0 -collision_cfl_factor = 20 +collision_cfl_factor = 0.04 cfl_factor = 1e10 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index 9f52f0a6..3174a67b 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -1,4 +1,4 @@ -collision_cfl_factor = 0.5 +collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = .5 max_adaptive_speedup = 0 diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 070a0599..120d0b32 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -5,7 +5,7 @@ perturbation_wavelength_cm = 1 # attenuation parameters to time derivative of N due to hamiltonians attenuation_hamiltonians = 1 -collision_cfl_factor = 0.5 +collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index eab8e0bc..baebba23 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -1,4 +1,4 @@ -collision_cfl_factor = 0.5 +collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = 0.1 max_adaptive_speedup = 0 From 1b8a938aa68fa1c8eab9fdb484e8992563e2bdc3 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 15 Mar 2024 11:27:20 -0400 Subject: [PATCH 078/276] deletting renomarlization function. It will not be used anymore --- Source/FlavoredNeutrinoContainer.H | 2 -- Source/FlavoredNeutrinoContainer.cpp | 25 +------------------------ Source/main.cpp | 3 --- 3 files changed, 1 insertion(+), 29 deletions(-) diff --git a/Source/FlavoredNeutrinoContainer.H b/Source/FlavoredNeutrinoContainer.H index cb6bcd68..42e13c2a 100644 --- a/Source/FlavoredNeutrinoContainer.H +++ b/Source/FlavoredNeutrinoContainer.H @@ -89,8 +89,6 @@ public: Redistribute(lev_min, lev_max, nGrow, local); } - void Renormalize(const TestParams* parms); - amrex::Vector get_attribute_names() const { return attribute_names; diff --git a/Source/FlavoredNeutrinoContainer.cpp b/Source/FlavoredNeutrinoContainer.cpp index a264fd49..6abd1907 100644 --- a/Source/FlavoredNeutrinoContainer.cpp +++ b/Source/FlavoredNeutrinoContainer.cpp @@ -82,27 +82,4 @@ UpdateLocationFrom(FlavoredNeutrinoContainer& Ploc) } }); } -} - -void FlavoredNeutrinoContainer:: -Renormalize(const TestParams* parms) -{ - BL_PROFILE("FlavoredNeutrinoContainer::Renormalize"); - - const int lev = 0; - -#ifdef _OPENMP -#pragma omp parallel -#endif - for (FNParIter pti(*this, lev); pti.isValid(); ++pti) - { - const int np = pti.numParticles(); - ParticleType * pstruct = &(pti.GetArrayOfStructs()[0]); - - amrex::ParallelFor (np, [=] AMREX_GPU_DEVICE (int i) { - ParticleType& p = pstruct[i]; - Real sumP, length, error; - // #include "generated_files/FlavoredNeutrinoContainer.cpp_Renormalize_fill" - }); - } -} +} \ No newline at end of file diff --git a/Source/main.cpp b/Source/main.cpp index b5ff87ad..609d47be 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -172,9 +172,6 @@ void evolve_flavor(const TestParams* parms) // since Redistribute() applies periodic boundary conditions. neutrinos.SyncLocation(Sync::PositionToCoordinate); - // Renormalize the neutrino state - neutrinos.Renormalize(parms); - // Get which step the integrator is on const int step = integrator.get_step_number(); const Real time = integrator.get_time(); From a6d325529d707fab1067318cb077bdbc576aaf63 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 15 Mar 2024 12:48:25 -0400 Subject: [PATCH 079/276] solving mistake in compilation comand to compile the code for 3 flavors --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d24bb3d5..5a2d6243 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -93,7 +93,7 @@ pipeline { stage('Collisions to equilibrium'){ steps{ dir('Exec'){ sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' - sh 'make realclean; make generate NUM_FLAVORS=3; make -j' + sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3' sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' sh 'python ../Scripts/tests/coll_equi_test.py' From 5b12e3908b02a78224bf70f6dc7051c9652a353a Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 16 Mar 2024 19:00:00 -0400 Subject: [PATCH 080/276] Setting default collision time step to infinite. It will change if the code do collisions. --- Source/Evolve.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index dd6d5b8b..886ba036 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -64,7 +64,7 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta // define the dt associated with each method Real dt_flavor_adaptive = PhysConst::hbar/Vmax_adaptive*flavor_cfl_factor; Real dt_flavor_stupid = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; - Real dt_flavor_absorption = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; // it will change only if opacity_method is 1 + Real dt_flavor_absorption = std::numeric_limits::infinity(); // it will change only if opacity_method is 1 if (parms->IMFP_method == 1) { // Use the IMFPs from the input file and find the maximum absorption IMFP From b24f1f534168d4c00479f8a8fbecf98deeea3055 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 16 Mar 2024 20:29:41 -0400 Subject: [PATCH 081/276] setting right indentation --- Source/Evolve.cpp | 74 +++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 886ba036..8601937e 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -39,10 +39,9 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta ReduceOps reduce_op; ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; - for (MFIter mfi(state); mfi.isValid(); ++mfi) - { + for (MFIter mfi(state); mfi.isValid(); ++mfi) { const Box& bx = mfi.fabbox(); - auto const& fab = state.array(mfi); + auto const& fab = state.array(mfi); reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { @@ -50,44 +49,45 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta #include "generated_files/Evolve.cpp_compute_dt_fill" return {V_adaptive, V_stupid}; }); - } - - // extract the reduced values from the combined reduced data structure - auto rv = reduce_data.value(); - Real Vmax_adaptive = amrex::get<0>(rv) + FlavoredNeutrinoContainer::Vvac_max; - Real Vmax_stupid = amrex::get<1>(rv) + FlavoredNeutrinoContainer::Vvac_max; - - // reduce across MPI ranks - ParallelDescriptor::ReduceRealMax(Vmax_adaptive); - ParallelDescriptor::ReduceRealMax(Vmax_stupid ); - - // define the dt associated with each method - Real dt_flavor_adaptive = PhysConst::hbar/Vmax_adaptive*flavor_cfl_factor; - Real dt_flavor_stupid = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; - Real dt_flavor_absorption = std::numeric_limits::infinity(); // it will change only if opacity_method is 1 - - if (parms->IMFP_method == 1) { - // Use the IMFPs from the input file and find the maximum absorption IMFP - double max_IMFP_abs = std::numeric_limits::lowest(); // Initialize max to lowest possible value - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < NUM_FLAVORS; ++j) { - max_IMFP_abs = std::max(max_IMFP_abs, parms->IMFP_abs[i][j]); + } + + // extract the reduced values from the combined reduced data structure + auto rv = reduce_data.value(); + Real Vmax_adaptive = amrex::get<0>(rv) + FlavoredNeutrinoContainer::Vvac_max; + Real Vmax_stupid = amrex::get<1>(rv) + FlavoredNeutrinoContainer::Vvac_max; + + // reduce across MPI ranks + ParallelDescriptor::ReduceRealMax(Vmax_adaptive); + ParallelDescriptor::ReduceRealMax(Vmax_stupid ); + + // define the dt associated with each method + Real dt_flavor_adaptive = PhysConst::hbar/Vmax_adaptive*flavor_cfl_factor; + Real dt_flavor_stupid = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; + Real dt_flavor_absorption = std::numeric_limits::infinity(); // it will change only if opacity_method is 1 + + if (parms->IMFP_method == 1) { + // Use the IMFPs from the input file and find the maximum absorption IMFP + double max_IMFP_abs = std::numeric_limits::lowest(); // Initialize max to lowest possible value + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < NUM_FLAVORS; ++j) { + max_IMFP_abs = std::max(max_IMFP_abs, parms->IMFP_abs[i][j]); + } } - } - - // Calculate dt_flavor_absorption - dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor; + + // Calculate dt_flavor_absorption + dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor; - if (parms->attenuation_hamiltonians==0){ - dt_flavor_adaptive = dt_flavor_absorption; - dt_flavor_stupid = dt_flavor_absorption; + if (parms->attenuation_hamiltonians==0){ + dt_flavor_adaptive = dt_flavor_absorption; + dt_flavor_stupid = dt_flavor_absorption; + } } - } - // pick the appropriate timestep - dt_flavor = dt_flavor_stupid; - if(max_adaptive_speedup>1) - dt_flavor = min(dt_flavor_stupid*max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); + // pick the appropriate timestep + dt_flavor = dt_flavor_stupid; + if(max_adaptive_speedup>1) { + dt_flavor = min(dt_flavor_stupid*max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); + } } Real dt = 0.0; From f8338bde39f657c565e1e5a35e3952e1931e61b7 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 16 Mar 2024 20:41:04 -0400 Subject: [PATCH 082/276] including attenuation parameter to compute the time steps due to potentials --- Source/Evolve.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 8601937e..f16ce705 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -61,9 +61,14 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta ParallelDescriptor::ReduceRealMax(Vmax_stupid ); // define the dt associated with each method - Real dt_flavor_adaptive = PhysConst::hbar/Vmax_adaptive*flavor_cfl_factor; - Real dt_flavor_stupid = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; - Real dt_flavor_absorption = std::numeric_limits::infinity(); // it will change only if opacity_method is 1 + Real dt_flavor_adaptive = std::numeric_limits::max(); + Real dt_flavor_stupid = std::numeric_limits::max(); + Real dt_flavor_absorption = std::numeric_limits::max(); // Initialize with infinity + + if (parms->attenuation_hamiltonians != 0) { + dt_flavor_adaptive = PhysConst::hbar / Vmax_adaptive * flavor_cfl_factor / parms->attenuation_hamiltonians; + dt_flavor_stupid = PhysConst::hbar / Vmax_stupid * flavor_cfl_factor / parms->attenuation_hamiltonians; + } if (parms->IMFP_method == 1) { // Use the IMFPs from the input file and find the maximum absorption IMFP @@ -73,14 +78,8 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta max_IMFP_abs = std::max(max_IMFP_abs, parms->IMFP_abs[i][j]); } } - // Calculate dt_flavor_absorption dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor; - - if (parms->attenuation_hamiltonians==0){ - dt_flavor_adaptive = dt_flavor_absorption; - dt_flavor_stupid = dt_flavor_absorption; - } } // pick the appropriate timestep From 1d925c36e087ccb255ffa49d3107b8eb87cbe97e Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 16 Mar 2024 20:43:34 -0400 Subject: [PATCH 083/276] computing minimum between time steps due to potentials and collision even if max_adaptive_speedup>1 is not satisfy --- Source/Evolve.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index f16ce705..b4ea99ad 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -83,7 +83,7 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta } // pick the appropriate timestep - dt_flavor = dt_flavor_stupid; + dt_flavor = min(dt_flavor_stupid, dt_flavor_adaptive, dt_flavor_absorption); if(max_adaptive_speedup>1) { dt_flavor = min(dt_flavor_stupid*max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); } From 618e685cb0070ecdf55d52da3931d3e25c26b7d7 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 16 Mar 2024 20:49:49 -0400 Subject: [PATCH 084/276] solve mistake in chemical potential units in coment --- Source/Parameters.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index 771fccb3..f9916cd2 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -47,7 +47,7 @@ struct TestParams : public amrex::Gpu::Managed int IMFP_method; Real IMFP_abs[2][NUM_FLAVORS]; Real IMFP_scat[2][NUM_FLAVORS]; - Real munu[2][NUM_FLAVORS]; // equilibrium electron neutrino chemical potential in MeV + Real munu[2][NUM_FLAVORS]; // equilibrium electron neutrino chemical potential in erg (CGS unist) Real delta_E; // attenuation to hamiltonians From a804c0a229ffe913592769f48626b0e79554705c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 16 Mar 2024 20:51:03 -0400 Subject: [PATCH 085/276] add coment on the energy units --- Source/Parameters.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index f9916cd2..3cc31a45 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -48,7 +48,7 @@ struct TestParams : public amrex::Gpu::Managed Real IMFP_abs[2][NUM_FLAVORS]; Real IMFP_scat[2][NUM_FLAVORS]; Real munu[2][NUM_FLAVORS]; // equilibrium electron neutrino chemical potential in erg (CGS unist) - Real delta_E; + Real delta_E; // erg (CGS unist) // attenuation to hamiltonians Real attenuation_hamiltonians; From be9a3189f50d4eb4fe7d6dc9963731611a7cc58f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 16 Mar 2024 21:11:38 -0400 Subject: [PATCH 086/276] delete the individual parms arguments --- Source/Evolve.H | 2 +- Source/Evolve.cpp | 18 +++++++++--------- Source/main.cpp | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Source/Evolve.H b/Source/Evolve.H index 893b0049..395634e7 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -26,7 +26,7 @@ namespace GIdx void Initialize(); } -amrex::Real compute_dt(const amrex::Geometry& geom, const amrex::Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor, const Real max_adaptive_speedup, const TestParams* parms); +amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index b4ea99ad..2b6dcb73 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -19,19 +19,19 @@ namespace GIdx } } -Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const Real flavor_cfl_factor, const Real max_adaptive_speedup, const TestParams* parms) +Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const TestParams* parms) { - AMREX_ASSERT(cfl_factor > 0.0 || flavor_cfl_factor > 0.0); + AMREX_ASSERT(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0); // translation part of timestep limit const auto dxi = geom.CellSizeArray(); Real dt_translation = 0.0; - if (cfl_factor > 0.0) { - dt_translation = std::min(std::min(dxi[0],dxi[1]), dxi[2]) / PhysConst::c * cfl_factor; + if (parms->cfl_factor > 0.0) { + dt_translation = std::min(std::min(dxi[0],dxi[1]), dxi[2]) / PhysConst::c * parms->cfl_factor; } Real dt_flavor = 0.0; - if (flavor_cfl_factor > 0.0) { + if (parms->flavor_cfl_factor > 0.0 && parms->collision_cfl_factor > 0.0) { // define the reduction operator to get the max contribution to // the potential from matter and neutrinos // compute "effective" potential (ergs) that produces characteristic timescale @@ -66,8 +66,8 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta Real dt_flavor_absorption = std::numeric_limits::max(); // Initialize with infinity if (parms->attenuation_hamiltonians != 0) { - dt_flavor_adaptive = PhysConst::hbar / Vmax_adaptive * flavor_cfl_factor / parms->attenuation_hamiltonians; - dt_flavor_stupid = PhysConst::hbar / Vmax_stupid * flavor_cfl_factor / parms->attenuation_hamiltonians; + dt_flavor_adaptive = PhysConst::hbar / Vmax_adaptive * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; + dt_flavor_stupid = PhysConst::hbar / Vmax_stupid * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; } if (parms->IMFP_method == 1) { @@ -84,8 +84,8 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta // pick the appropriate timestep dt_flavor = min(dt_flavor_stupid, dt_flavor_adaptive, dt_flavor_absorption); - if(max_adaptive_speedup>1) { - dt_flavor = min(dt_flavor_stupid*max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); + if(parms->max_adaptive_speedup>1) { + dt_flavor = min(dt_flavor_stupid*parms->max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); } } diff --git a/Source/main.cpp b/Source/main.cpp index 609d47be..3973d4b5 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -194,7 +194,7 @@ void evolve_flavor(const TestParams* parms) // Note: this won't be the same as the new-time grid data // because the last deposit_to_mesh call was at either the old time (forward Euler) // or the final RK stage, if using Runge-Kutta. - const Real dt = compute_dt(geom,parms->cfl_factor,state,neutrinos,parms->flavor_cfl_factor,parms->max_adaptive_speedup, parms); + const Real dt = compute_dt(geom, state, neutrinos, parms); integrator.set_timestep(dt); }; @@ -203,7 +203,7 @@ void evolve_flavor(const TestParams* parms) integrator.set_post_timestep(post_timestep_fun); // Get a starting timestep - const Real starting_dt = compute_dt(geom,parms->cfl_factor, state,neutrinos_old,parms->flavor_cfl_factor, parms->max_adaptive_speedup, parms); + const Real starting_dt = compute_dt(geom, state, neutrinos_old, parms); // Do all the science! amrex::Print() << "Starting timestepping loop... " << std::endl; From c8474a6fcb04c71a6008e3249e55b559228ce566 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sun, 24 Mar 2024 15:55:51 -0400 Subject: [PATCH 087/276] creating collisional flavor instability test --- .../initial_conditions/st8_coll_inst_test.py | 37 ++++++ .../inputs_collisional_instability_test | 108 ++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 Scripts/initial_conditions/st8_coll_inst_test.py create mode 100644 sample_inputs/inputs_collisional_instability_test diff --git a/Scripts/initial_conditions/st8_coll_inst_test.py b/Scripts/initial_conditions/st8_coll_inst_test.py new file mode 100644 index 00000000..86a97086 --- /dev/null +++ b/Scripts/initial_conditions/st8_coll_inst_test.py @@ -0,0 +1,37 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../data_reduction") +from initial_condition_tools import uniform_sphere, moment_interpolate_particles, minerbo_interpolate, write_particles +import amrex_plot_tools as amrex + +# generation parameters +# from ix=37 iy=86 iz=75 in Francois' data +NF = 2 +nphi_equator = 16 +nnue = 6.789813908916637e+33 +nnua = 7.733526735756642e+33 +nnux = 4.445468787928724e+33 +fnue = np.array([-7.54557877e+30, -5.43212748e+30, -7.69897358e+31]) / np.array(6.789813908916637e+33) +fnua = np.array([-7.54557877e+30, -5.43212748e+30, -2.94417496e+32]) / np.array(7.733526735756642e+33) +fnux = np.array([-9.35603382e+30, -2.95170204e+31, -2.04503596e+32]) / np.array(4.445468787928724e+33) +energy_erg = np.average([37.39393896350027, 38.51187069862436, 42.44995883633579, 42.44995883633579]) * 1e6*amrex.eV + +nnu = np.zeros((2,NF)) +nnu[0,0] = nnue +nnu[1,0] = nnua +nnu[:,1:] = nnux +print(f'nnu {nnu}') + +fnu = np.zeros((2,NF,3)) +fnu[0,0,:] = nnue * fnue +fnu[1,0,:] = nnua * fnua +fnu[:,1:,:] = nnu[:,1:,np.newaxis] * fnux[np.newaxis,np.newaxis,:] +print(f'fnu {fnu}') + +particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, minerbo_interpolate) # [particle, variable] + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test new file mode 100644 index 00000000..2f55554d --- /dev/null +++ b/sample_inputs/inputs_collisional_instability_test @@ -0,0 +1,108 @@ +perturbation_type = 0 +perturbation_amplitude = 1e-6 + +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + +collision_cfl_factor = 1e-3 +cfl_factor = 0.5 +flavor_cfl_factor = 0.5 +max_adaptive_speedup = 0 +maxError = 1e-6 + +integration.type = 1 +integration.rk.type = 4 + +# Domain size in 3D index space +ncell = (1, 1, 256) +Lx = 1.0 +Ly = 1.0 +Lz = 64.0 + +# Number of particles per cell +nppc = (1, 1, 1) +particle_data_filename = "particle_input.dat" + +# Maximum size of each grid in the domain +max_grid_size = 16 + +# Number of steps to run +nsteps = 1000 + +# Simulation end time +end_time = 5.0e-9 + +# Make FPE signal errors so we get a Backtrace +amrex.fpe_trap_invalid=1 + +# give background fluid conditions +rho_g_ccm = 2104856022650.468 +T_MeV = 92.30803950999913 +Ye = 0.09924428291081952 + +# Write plotfiles +write_plot_every = 25 + +# Write particle data in plotfiles +write_plot_particles_every = 25 + +# checkpointing +do_restart = 0 +restart_dir = "" + +############################### +# NEUTRINO PHYSICS PARAMETERS # +############################### +# see first column of table 14.7 in http://pdg.lbl.gov/2019/reviews/rpp2019-rev-neutrino-mixing.pdf + +# mass state 1 mass in eV [NO/IO:-sqrt(7.39e-5)] +mass1_eV = 0 #-0.008596511 + +# mass state 2 mass in eV (define at 0 arbitrarily because oscillations only sensitive to deltaM^2) +mass2_eV = 0 + +# mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] +mass3_eV = 0.049487372 + +# 1-2 mixing angle in degrees [NO/IO:33.82] +theta12_degrees = 1e-6 + +# 2-3 mixing angle in degrees [NO:8.61 IO:8.65] +theta23_degrees = 8.61 + +# 1-3 mixing angle in degrees [NO:48.3 IO:48.6] +theta13_degrees = 48.3 + +# Majorana angle 1 in degrees +alpha1_degrees = 0 + +# Majorana angle 2 in degrees +alpha2_degrees = 0 + +# CP-violating phase in degrees [NO:222 IO:285] +deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 1 + +IMFP_abs0_cm = 1.403900319813648e-04 +IMFP_abs1_cm = 4.175459950002267e-08 +IMFP_abs2_cm = 0 +IMFP_abs0bar_cm = 7.950517300555263e-06 +IMFP_abs1bar_cm = 4.175459950002267e-08 +IMFP_abs2bar_cm = 0 + +munu0_MeV=-4.171266344694622 +munu1_MeV=0 +munu2_MeV=0 + +IMFP_scat0_cm = 0 +IMFP_scat1_cm = 0 +IMFP_scat2_cm = 0 +IMFP_scat0bar_cm = 0 +IMFP_scat1bar_cm = 0 +IMFP_scat2bar_cm = 0 + +delta_E = 10.444945789118993 # Mev \ No newline at end of file From 5c8dc11ab3f4343981f4e65bc542224e8f48d65c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 26 Mar 2024 23:11:41 -0400 Subject: [PATCH 088/276] delete print line --- Scripts/initial_conditions/st8_coll_inst_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Scripts/initial_conditions/st8_coll_inst_test.py b/Scripts/initial_conditions/st8_coll_inst_test.py index 86a97086..a5e58276 100644 --- a/Scripts/initial_conditions/st8_coll_inst_test.py +++ b/Scripts/initial_conditions/st8_coll_inst_test.py @@ -24,13 +24,11 @@ nnu[0,0] = nnue nnu[1,0] = nnua nnu[:,1:] = nnux -print(f'nnu {nnu}') fnu = np.zeros((2,NF,3)) fnu[0,0,:] = nnue * fnue fnu[1,0,:] = nnua * fnua fnu[:,1:,:] = nnu[:,1:,np.newaxis] * fnux[np.newaxis,np.newaxis,:] -print(f'fnu {fnu}') particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, minerbo_interpolate) # [particle, variable] From 6c875ea17dda578bdfc278df591654151d673b03 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 26 Mar 2024 23:15:35 -0400 Subject: [PATCH 089/276] now reads plt* files for time steps of more than 5 digits --- Scripts/data_reduction/reduce_data_fft.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Scripts/data_reduction/reduce_data_fft.py b/Scripts/data_reduction/reduce_data_fft.py index 7abbe3fe..32de9f02 100755 --- a/Scripts/data_reduction/reduce_data_fft.py +++ b/Scripts/data_reduction/reduce_data_fft.py @@ -9,7 +9,8 @@ parser.add_argument("-o", "--output", type=str, default="reduced_data_fft.h5", help="Name of the output file (default: reduced_data_fft.h5)") args = parser.parse_args() -directories = sorted(glob.glob("plt?????")) +directories = glob.glob("plt*") +directories = sorted(directories, key=lambda x: int(x.lstrip("plt"))) t = [] From ee5aa8f37f51f4bc762f6b47d277c55ff2b2eaf0 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 27 Mar 2024 18:12:32 -0400 Subject: [PATCH 090/276] now reads plt files for time steps with more than 5 digits --- Scripts/data_reduction/reduce_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Scripts/data_reduction/reduce_data.py b/Scripts/data_reduction/reduce_data.py index 665f3866..a14669c9 100644 --- a/Scripts/data_reduction/reduce_data.py +++ b/Scripts/data_reduction/reduce_data.py @@ -385,7 +385,8 @@ def reduce_data(directory=".", nproc=4, do_average=True, do_fft=True, do_angular if(data_format=="Emu"): yt_descriptor = "boxlib" convert_N_to_inv_ccm = 1.0 - directories = sorted(glob.glob("plt?????")) + directories = glob.glob("plt*") + directories = sorted(directories, key=lambda x: int(x.lstrip("plt"))) # get NF eds = emu.EmuDataset(directories[0]) From 7638a4edfcecff5e5e8708e74bcdcc05938d633c Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Wed, 17 Apr 2024 05:04:18 -0400 Subject: [PATCH 091/276] print useful info for bipolar test --- Scripts/initial_conditions/st1_bipolar_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Scripts/initial_conditions/st1_bipolar_test.py b/Scripts/initial_conditions/st1_bipolar_test.py index f915303e..04a15e5f 100644 --- a/Scripts/initial_conditions/st1_bipolar_test.py +++ b/Scripts/initial_conditions/st1_bipolar_test.py @@ -23,8 +23,10 @@ # 10 dm2 c^4 / (2 sqrt(2) GF E) energy_erg = 50 * 1e6*amrex.eV dm2 = (m2-m1)**2 #g^2 +print(dm2*amrex.clight**4/(2.*energy_erg),"erg") # double omega = dm2*PhysConst::c4 / (2.*p.rdata(PIdx::pupt)); ndens = 10. * dm2*amrex.clight**4 / (2.*np.sqrt(2.) * amrex.GF * energy_erg) +print(ndens,"cm^-3") # double mu = sqrt(2.)*PhysConst::GF * ndens ndens_per_particle = ndens / nparticles # cm^-3 From 636b21118c99c5d2268583f7539fbc8f9337f705 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Wed, 17 Apr 2024 05:11:31 -0400 Subject: [PATCH 092/276] make a better babysitting plot --- Scripts/babysitting/avgfee_HDF5.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Scripts/babysitting/avgfee_HDF5.py b/Scripts/babysitting/avgfee_HDF5.py index 57c2f8eb..3cda6e23 100755 --- a/Scripts/babysitting/avgfee_HDF5.py +++ b/Scripts/babysitting/avgfee_HDF5.py @@ -53,7 +53,7 @@ # formatting # ############## ax.axhline(1./3., color="green") -ax.set_ylabel(r"$\langle N\rangle$ (cm$^{-3}$)") +ax.set_ylabel(r"$\langle N\rangle_{ee}$ (cm$^{-3}$)") ax.set_xlabel(r"$t\,(10^{-9}\,\mathrm{s})$") ax.tick_params(axis='both', which='both', direction='in', right=True,top=True) ax.xaxis.set_minor_locator(AutoMinorLocator()) @@ -61,16 +61,18 @@ ax.minorticks_on() ax.grid(which='both') -############# -# plot data # -############# ax.plot(t, N00) - -############ -# save pdf # -############ plt.savefig("avgfee.pdf", bbox_inches="tight") # same for f_e\mu +plt.cla() +ax.set_ylabel(r"$\langle N\rangle_\mathrm{offdiag}$ (cm$^{-3}$)") +ax.set_xlabel(r"$t\,(10^{-9}\,\mathrm{s})$") +ax.tick_params(axis='both', which='both', direction='in', right=True,top=True) +ax.xaxis.set_minor_locator(AutoMinorLocator()) +ax.yaxis.set_minor_locator(AutoMinorLocator()) +ax.minorticks_on() +ax.grid(which='both') + ax.semilogy(t, Noffdiag) plt.savefig("avgfemu.pdf", bbox_inches="tight") From f36f90f02d78453511cacacee81fbee0592524e7 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Wed, 17 Apr 2024 05:13:25 -0400 Subject: [PATCH 093/276] improve babysitting plots --- Scripts/babysitting/avgfee.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Scripts/babysitting/avgfee.py b/Scripts/babysitting/avgfee.py index 58f6c09e..40760a37 100755 --- a/Scripts/babysitting/avgfee.py +++ b/Scripts/babysitting/avgfee.py @@ -54,7 +54,7 @@ def plotdata(filename,a,b): # formatting # ############## ax.axhline(1./3., color="green") -ax.set_ylabel(r"$\langle N\rangle$ (cm$^{-3}$)") +ax.set_ylabel(r"$\langle N\rangle_{ee}$ (cm$^{-3}$)") ax.set_xlabel(r"$t\,(10^{-9}\,\mathrm{s})$") ax.tick_params(axis='both', which='both', direction='in', right=True,top=True) ax.xaxis.set_minor_locator(AutoMinorLocator()) @@ -62,19 +62,20 @@ def plotdata(filename,a,b): ax.minorticks_on() ax.grid(which='both') -############# -# plot data # -############# filename = "plt_reduced_data.h5" t,N = plotdata(filename,0,0) ax.plot(t, N) - -############ -# save pdf # -############ plt.savefig("avgfee.pdf", bbox_inches="tight") # same for f_e\mu +plt.cla() +ax.set_ylabel(r"$\langle N\rangle_\mathrm{offdiag}$ (cm$^{-3}$)") +ax.set_xlabel(r"$t\,(10^{-9}\,\mathrm{s})$") +ax.tick_params(axis='both', which='both', direction='in', right=True,top=True) +ax.xaxis.set_minor_locator(AutoMinorLocator()) +ax.yaxis.set_minor_locator(AutoMinorLocator()) +ax.minorticks_on() +ax.grid(which='both') t,N = plotdata(filename,0,1) ax.semilogy(t, N) plt.savefig("avgfemu.pdf", bbox_inches="tight") From c6365fe108999fcad9cd78f81c4b26b1d5ac74b0 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 19 Apr 2024 15:00:50 -0400 Subject: [PATCH 094/276] improve babysitting script to also plot the ELN --- Scripts/babysitting/avgfee_HDF5.py | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/Scripts/babysitting/avgfee_HDF5.py b/Scripts/babysitting/avgfee_HDF5.py index 3cda6e23..71ea17a8 100755 --- a/Scripts/babysitting/avgfee_HDF5.py +++ b/Scripts/babysitting/avgfee_HDF5.py @@ -24,8 +24,21 @@ # read averaged data # ###################### avgData = h5py.File("reduced0D.h5","r") + +NF = 0 +if "N00(1|ccm)" in avgData.keys(): + NF = 2 +if "N11(1|ccm)" in avgData.keys(): + NF = 3 +assert(NF==2 or NF==3) + t=np.array(avgData["time(s)"])*1e9 N00=np.array(avgData["N00(1|ccm)"]) +N11=np.array(avgData["N11(1|ccm)"]) +N22=np.array(avgData["N22(1|ccm)"]) +N00bar=np.array(avgData["N00bar(1|ccm)"]) +N11bar=np.array(avgData["N11bar(1|ccm)"]) +N22bar=np.array(avgData["N22bar(1|ccm)"]) Noffdiag = np.array(avgData["N_offdiag_mag(1|ccm)"]) avgData.close() @@ -76,3 +89,30 @@ ax.semilogy(t, Noffdiag) plt.savefig("avgfemu.pdf", bbox_inches="tight") + +# plot ELN +plt.cla() +ax.set_ylabel(r"$\delta$ELN/$N_\mathrm{tot}$") +ax.set_xlabel(r"$t\,(10^{-9}\,\mathrm{s})$") +ax.tick_params(axis='both', which='both', direction='in', right=True,top=True) +ax.xaxis.set_minor_locator(AutoMinorLocator()) +ax.yaxis.set_minor_locator(AutoMinorLocator()) +ax.minorticks_on() +ax.grid(which='both') + +Ntot = N00+N11+N00bar+N11bar +if NF==3: + Ntot += N22+N22bar + +emu = (N00-N00bar) - (N11-N11bar) +ax.plot(t, (emu-emu[0])/Ntot,label=r"$e\mu$") + +etau = (N00-N00bar) - (N22-N22bar) +ax.plot(t, (etau-etau[0])/Ntot,label=r"$e\tau$") + +if NF==3: + mutau = (N11-N11bar) - (N22-N22bar) + ax.plot(t, (mutau-mutau[0])/Ntot,label=r"$\mu\tau$") + +plt.legend() +plt.savefig("ELN.pdf", bbox_inches="tight") From bc0dcdfbbdf1f0be9af0ae28fc2bbd4c5dffedef Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 30 Apr 2024 15:08:09 -0400 Subject: [PATCH 095/276] upgraded cassowary to a V100 GPU --- makefiles/GNUmakefile_cassowary | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefiles/GNUmakefile_cassowary b/makefiles/GNUmakefile_cassowary index c4e1932f..fb131b0a 100644 --- a/makefiles/GNUmakefile_cassowary +++ b/makefiles/GNUmakefile_cassowary @@ -10,7 +10,7 @@ USE_MPI = TRUE USE_OMP = FALSE USE_ACC = FALSE USE_CUDA = FALSE -AMREX_CUDA_ARCH=60 +AMREX_CUDA_ARCH=72 USE_HDF5=FALSE HDF5_HOME=/usr/lib/x86_64-linux-gnu/hdf5/serial From 3737ee6f93430d37fdfb7cd5fac807d377d53bc2 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 30 Apr 2024 15:33:58 -0400 Subject: [PATCH 096/276] installed P100 GPU in jenkins server. Corrected cuda version on cassowary --- makefiles/GNUmakefile_cassowary | 2 +- makefiles/GNUmakefile_jenkins | 2 +- makefiles/GNUmakefile_jenkins_HDF5 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/makefiles/GNUmakefile_cassowary b/makefiles/GNUmakefile_cassowary index fb131b0a..72c88836 100644 --- a/makefiles/GNUmakefile_cassowary +++ b/makefiles/GNUmakefile_cassowary @@ -10,7 +10,7 @@ USE_MPI = TRUE USE_OMP = FALSE USE_ACC = FALSE USE_CUDA = FALSE -AMREX_CUDA_ARCH=72 +AMREX_CUDA_ARCH=70 USE_HDF5=FALSE HDF5_HOME=/usr/lib/x86_64-linux-gnu/hdf5/serial diff --git a/makefiles/GNUmakefile_jenkins b/makefiles/GNUmakefile_jenkins index 2a12488e..d8d1143d 100644 --- a/makefiles/GNUmakefile_jenkins +++ b/makefiles/GNUmakefile_jenkins @@ -10,7 +10,7 @@ USE_MPI = TRUE USE_OMP = FALSE USE_ACC = FALSE USE_CUDA = TRUE -AMREX_CUDA_ARCH=75 +AMREX_CUDA_ARCH=60 EMU_HOME = .. AMREX_HOME = ../submodules/amrex diff --git a/makefiles/GNUmakefile_jenkins_HDF5 b/makefiles/GNUmakefile_jenkins_HDF5 index c0dfe2a2..242f5e6b 100644 --- a/makefiles/GNUmakefile_jenkins_HDF5 +++ b/makefiles/GNUmakefile_jenkins_HDF5 @@ -10,7 +10,7 @@ USE_MPI = TRUE USE_OMP = FALSE USE_ACC = FALSE USE_CUDA = FALSE -AMREX_CUDA_ARCH=75 +AMREX_CUDA_ARCH=60 USE_HDF5=TRUE HDF5_HOME=/usr/lib/x86_64-linux-gnu/hdf5/openmpi From c33801b7b1f20d93a702929591b54e477e1f3c98 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 1 May 2024 14:33:51 -0400 Subject: [PATCH 097/276] add function to initialize the background enviroment Ye, rho and T --- Source/Evolve.H | 2 ++ Source/Evolve.cpp | 31 +++++++++++++++++++++++++++++++ Source/main.cpp | 5 ++++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/Source/Evolve.H b/Source/Evolve.H index 395634e7..eca88b07 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -28,6 +28,8 @@ namespace GIdx amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); +void init_background_to_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const MultiFab& state, const Geometry& geom, const TestParams* parms); + void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const amrex::MultiFab& state, const amrex::Geometry& geom, const TestParams* parms); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 2b6dcb73..bf8d22cd 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -103,6 +103,37 @@ Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutr return dt; } +void init_background_to_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const MultiFab& state, const Geometry& geom, const TestParams* parms) +{ + const auto plo = geom.ProbLoArray(); + const auto dxi = geom.InvCellSizeArray(); + + amrex::MeshToParticle(neutrinos_rhs, state, 0, + [=] AMREX_GPU_DEVICE (FlavoredNeutrinoContainer::ParticleType& p, + amrex::Array4 const& sarr) + { + double x,y,z; + + for (int k = 0; k < parms->ncell[2]; ++k) { + for (int j = 0; j < parms->ncell[1]; ++j) { + for (int i = 0; i < parms->ncell[0]; ++i) { + + x = plo[0] + dxi[0] * i + dxi[0] / 2.0; + y = plo[1] + dxi[1] * j + dxi[1] / 2.0; + z = plo[2] + dxi[2] * k + dxi[2] / 2.0; + + std::cout << "\n ibm x : " << x << std::endl; + std::cout << " ibm y : " << y << std::endl; + std::cout << " ibm z : " << z << std::endl; + std::cout << "\n ibm sarr("<do_restart) { @@ -148,6 +150,7 @@ void evolve_flavor(const TestParams* parms) // B) We only Redistribute the integrator new data at the end of the timestep, not all the RHS data. // Thus, this copy clears the old RHS particles and creates particles in the RHS container corresponding // to the current particles in neutrinos. + neutrinos_rhs.copyParticles(neutrinos, true); // Step 3: Interpolate Mesh to construct the neutrino RHS in place From 9d9268330b1c06f4dd276a2339b8a92ae5bb1c10 Mon Sep 17 00:00:00 2001 From: Javier Gomez <145796242+javierigm@users.noreply.github.com> Date: Mon, 6 May 2024 14:22:03 -0400 Subject: [PATCH 098/276] K plot (#91) * File with fixes for common errors * New babysitting tool to monitor fastest growing wave number * change cuda architecture to account for different gpu in Ganon --------- Co-authored-by: Sherwood Richers --- Emu_Fixes.md | 0 Scripts/babysitting/k_plot.py | 107 ++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 Emu_Fixes.md create mode 100644 Scripts/babysitting/k_plot.py diff --git a/Emu_Fixes.md b/Emu_Fixes.md new file mode 100644 index 00000000..e69de29b diff --git a/Scripts/babysitting/k_plot.py b/Scripts/babysitting/k_plot.py new file mode 100644 index 00000000..6f997b0c --- /dev/null +++ b/Scripts/babysitting/k_plot.py @@ -0,0 +1,107 @@ +import matplotlib.pyplot as mpl +import numpy as np + +############## +# parameters # +############## +direction = "z" #direction of domain size. +base_directory = "" +abskplt = "lin" #"log" #switch between log and linear plots for abs(k) + +################ +# reading data # +################ +Lx=0 +Ly=0 +Lz=0 +inputs_file = open(base_directory + "inputs", 'r') +for line in inputs_file.readlines(): + if "ncell" in line: + str = line.split("(")[1].split(")")[0].split(",") + n = [int(str[0]), int(str[1]), int(str[2])] + if line[:2] == "Lx": + Lx = float(line.split("=")[1].split("#")[0]) + if line[:2] == "Ly": + Ly = float(line.split("=")[1].split("#")[0]) + if line[:2] == "Lz": + Lz = float(line.split("=")[1].split("#")[0]) + + L = [Lx, Ly, Lz] + +inputs_file.close() + +kmax = np.array(np.genfromtxt(base_directory + "kmax_t_N01.dat", skip_header=1)) +amp = kmax[:,4] +kz = kmax[:,3] +ky = kmax[:,2] +kx = kmax[:,1] +t = kmax[:,0]*10**6 +imax = np.argmax(amp) + +# Changing the length of the domain and kmin and kmax according to axis +if direction == "x": + i = 0 + labels = ["kmin for Lx", "kmax for Lx"] +elif direction == "y": + i = 1 + labels = ["kmin for Ly", "kmax for Ly"] +elif direction == "z": + i = 2 + labels = ["kmin for Lz", "kmax for Lz"] +kmn = 2*np.pi/L[i] +kmx = 2*np.pi/(2*L[i]/n[i]) + +################ +# plot options # +################ +mpl.rcParams['font.size'] = 17 +mpl.rcParams['font.family'] = 'serif' +mpl.rcParams['xtick.major.size'] = 7 +mpl.rcParams['xtick.major.width'] = 2 +mpl.rcParams['xtick.major.pad'] = 8 +mpl.rcParams['xtick.minor.size'] = 4 +mpl.rcParams['xtick.minor.width'] = 2 +mpl.rcParams['ytick.major.size'] = 7 +mpl.rcParams['ytick.major.width'] = 2 +mpl.rcParams['ytick.minor.size'] = 4 +mpl.rcParams['ytick.minor.width'] = 2 +mpl.rcParams['axes.linewidth'] = 2 +mpl.rcParams['figure.figsize'] = (10,10) + +################################ +# generating + formatting plot # +################################ +fig, axs = mpl.subplots(2, 1, sharex=True) +fig.subplots_adjust(hspace=0) + +if abskplt == "log": + axs[0].semilogy(t, amp) +elif abskplt == "lin": + axs[0].plot(t, amp) +else: + axs[0].semilogy(t, amp) + +axs[0].tick_params(axis='both',which="both", direction="in",top=True,right=True) +axs[0].minorticks_on() + +kmag = np.sqrt(kz**2+ky**2+kx**2) + +axs[1].semilogy(t, kmag) +axs[1].axhline(kmx, color='r', linestyle='-', label=labels[1]) +axs[1].axhline(kmn, color='y', linestyle='-', label=labels[0]) +axs[1].tick_params(axis='both',which="both", direction="in",top=True,right=True) +axs[1].minorticks_on() + +for ax in axs: + ax.axvline(t[imax]) + ax.axvline(2*t[imax]) + ax.axvline(3*t[imax]) + +axs[1].set_xlabel(r'time ($\mu s$)') +axs[0].set_ylabel(r'Amplitude (cm$^{-3}$)') +axs[1].set_ylabel(r'$|k|$ (cm$^{-1}$)') + +mpl.legend(frameon=False, labelspacing=0.25,fontsize=12, loc=(0.75,0.75)) + + +mpl.savefig('kmax.png') From 9c067cfa6642c6bf44d11ad48415ee8e5d1ce9af Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 22 May 2024 10:31:24 -0400 Subject: [PATCH 099/276] insert chemical potential for neutrinos and antineutrino independently --- Source/Parameters.H | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index 3cc31a45..730e5288 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -131,8 +131,10 @@ struct TestParams : public amrex::Gpu::Managed pp.get(name.c_str(), IMFP_scat[1][i]); name = std::string("munu")+std::to_string(i)+std::string("_MeV"); pp.get(name.c_str(), munu[0][i]); + name = std::string("munu")+std::to_string(i)+std::string("bar_MeV"); + pp.get(name.c_str(), munu[1][i]); munu[0][i] *= 1e6*CGSUnitsConst::eV; - munu[1][i] = -munu[0][i]; + munu[1][i] *= 1e6*CGSUnitsConst::eV; } pp.get("delta_E", delta_E); delta_E*= 1e6*CGSUnitsConst::eV; // erg From bdb940552ec32b04fad0b45934a3ca2b6a1c41b0 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 22 May 2024 10:51:09 -0400 Subject: [PATCH 100/276] set python script that generate initial condition for collisional instability test --- Scripts/initial_conditions/st8_coll_inst_test.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Scripts/initial_conditions/st8_coll_inst_test.py b/Scripts/initial_conditions/st8_coll_inst_test.py index a5e58276..618d385f 100644 --- a/Scripts/initial_conditions/st8_coll_inst_test.py +++ b/Scripts/initial_conditions/st8_coll_inst_test.py @@ -12,13 +12,13 @@ # from ix=37 iy=86 iz=75 in Francois' data NF = 2 nphi_equator = 16 -nnue = 6.789813908916637e+33 -nnua = 7.733526735756642e+33 -nnux = 4.445468787928724e+33 -fnue = np.array([-7.54557877e+30, -5.43212748e+30, -7.69897358e+31]) / np.array(6.789813908916637e+33) -fnua = np.array([-7.54557877e+30, -5.43212748e+30, -2.94417496e+32]) / np.array(7.733526735756642e+33) -fnux = np.array([-9.35603382e+30, -2.95170204e+31, -2.04503596e+32]) / np.array(4.445468787928724e+33) -energy_erg = np.average([37.39393896350027, 38.51187069862436, 42.44995883633579, 42.44995883633579]) * 1e6*amrex.eV +nnue = 3.0e+33 +nnua = 2.4e+33 +nnux = 1.0e+33 +fnue = np.array([0.0 , 0.0 , 0.0]) +fnua = np.array([0.0 , 0.0 , 0.0]) +fnux = np.array([0.0 , 0.0 , 0.0]) +energy_erg = 20.0 * 1e6*amrex.eV nnu = np.zeros((2,NF)) nnu[0,0] = nnue From 5c8c3752a60774d2120247effbb07f1688e3bdcd Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 22 May 2024 10:51:51 -0400 Subject: [PATCH 101/276] set input file for collisional instability test --- .../inputs_collisional_instability_test | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test index 2f55554d..ee2085e1 100644 --- a/sample_inputs/inputs_collisional_instability_test +++ b/sample_inputs/inputs_collisional_instability_test @@ -2,9 +2,9 @@ perturbation_type = 0 perturbation_amplitude = 1e-6 # attenuation parameters to time derivative of N due to hamiltonians -attenuation_hamiltonians = 1 +attenuation_hamiltonians = 1.0 -collision_cfl_factor = 1e-3 +collision_cfl_factor = 1e-2 cfl_factor = 0.5 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 @@ -14,10 +14,10 @@ integration.type = 1 integration.rk.type = 4 # Domain size in 3D index space -ncell = (1, 1, 256) -Lx = 1.0 -Ly = 1.0 -Lz = 64.0 +ncell = (1, 1, 1) +Lx = 1 +Ly = 1 +Lz = 1 # Number of particles per cell nppc = (1, 1, 1) @@ -27,24 +27,24 @@ particle_data_filename = "particle_input.dat" max_grid_size = 16 # Number of steps to run -nsteps = 1000 +nsteps = 10000 # Simulation end time -end_time = 5.0e-9 +end_time = 5.0e19 # Make FPE signal errors so we get a Backtrace amrex.fpe_trap_invalid=1 # give background fluid conditions -rho_g_ccm = 2104856022650.468 -T_MeV = 92.30803950999913 -Ye = 0.09924428291081952 +rho_g_ccm = 0 +T_MeV = 7.0 +Ye = 0 # Write plotfiles -write_plot_every = 25 +write_plot_every = 100 # Write particle data in plotfiles -write_plot_particles_every = 25 +write_plot_particles_every = 100 # checkpointing do_restart = 0 @@ -62,16 +62,16 @@ mass1_eV = 0 #-0.008596511 mass2_eV = 0 # mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] -mass3_eV = 0.049487372 +mass3_eV = 0 # 1-2 mixing angle in degrees [NO/IO:33.82] -theta12_degrees = 1e-6 +theta12_degrees = 0 # 2-3 mixing angle in degrees [NO:8.61 IO:8.65] -theta23_degrees = 8.61 +theta23_degrees = 0 # 1-3 mixing angle in degrees [NO:48.3 IO:48.6] -theta13_degrees = 48.3 +theta13_degrees = 0 # Majorana angle 1 in degrees alpha1_degrees = 0 @@ -80,23 +80,26 @@ alpha1_degrees = 0 alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] -deltaCP_degrees = 222 +deltaCP_degrees = 0 ################# # opacity stuff # ################# IMFP_method = 1 -IMFP_abs0_cm = 1.403900319813648e-04 -IMFP_abs1_cm = 4.175459950002267e-08 +IMFP_abs0_cm = 2.398e-1 +IMFP_abs1_cm = 0 IMFP_abs2_cm = 0 -IMFP_abs0bar_cm = 7.950517300555263e-06 -IMFP_abs1bar_cm = 4.175459950002267e-08 +IMFP_abs0bar_cm = 2.294e-2 +IMFP_abs1bar_cm = 0 IMFP_abs2bar_cm = 0 -munu0_MeV=-4.171266344694622 -munu1_MeV=0 -munu2_MeV=0 +munu0_MeV = 20.0 +munu1_MeV = 0 +munu2_MeV = 0 +munu0bar_MeV = 16.082712484451488 +munu1bar_MeV = 0 +munu2bar_MeV = 0 IMFP_scat0_cm = 0 IMFP_scat1_cm = 0 @@ -105,4 +108,4 @@ IMFP_scat0bar_cm = 0 IMFP_scat1bar_cm = 0 IMFP_scat2bar_cm = 0 -delta_E = 10.444945789118993 # Mev \ No newline at end of file +delta_E = 3.404260193084921 # Mev \ No newline at end of file From 4e7cc86f8369eb16c5511920fd9739170e73aa4f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 22 May 2024 10:52:37 -0400 Subject: [PATCH 102/276] Create a Jenkins test for collisional flavor instability --- Jenkinsfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 5a2d6243..47648e7b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -90,6 +90,18 @@ pipeline { } }} + stage('Collisions flavor instability'){ steps{ + dir('Exec'){ + sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' + sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' + sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' + sh 'python ../Scripts/data_reduction/reduce_data.py' + sh 'python ../Scripts/tests/coll_equi_test.py' + sh 'rm -rf plt*' + } + }} + stage('Collisions to equilibrium'){ steps{ dir('Exec'){ sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' From 4da44925ec6e080ab4a72c078bcbe51fc9cadaea Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Wed, 22 May 2024 13:07:14 -0400 Subject: [PATCH 103/276] change cuda version to 7.0 --- makefiles/GNUmakefile_jenkins | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefiles/GNUmakefile_jenkins b/makefiles/GNUmakefile_jenkins index 2a12488e..2584f3c9 100644 --- a/makefiles/GNUmakefile_jenkins +++ b/makefiles/GNUmakefile_jenkins @@ -10,7 +10,7 @@ USE_MPI = TRUE USE_OMP = FALSE USE_ACC = FALSE USE_CUDA = TRUE -AMREX_CUDA_ARCH=75 +AMREX_CUDA_ARCH=70 EMU_HOME = .. AMREX_HOME = ../submodules/amrex From 417b0f3db6c266732fccc5ee74f4563ba81eb6c2 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 22 May 2024 13:13:44 -0400 Subject: [PATCH 104/276] delete std cout line --- Source/Evolve.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index bf8d22cd..16cc23a5 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -122,12 +122,6 @@ void init_background_to_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const Mul y = plo[1] + dxi[1] * j + dxi[1] / 2.0; z = plo[2] + dxi[2] * k + dxi[2] / 2.0; - std::cout << "\n ibm x : " << x << std::endl; - std::cout << " ibm y : " << y << std::endl; - std::cout << " ibm z : " << z << std::endl; - std::cout << "\n ibm sarr("< Date: Wed, 22 May 2024 14:20:08 -0400 Subject: [PATCH 105/276] correct cuda version for jenkins --- makefiles/GNUmakefile_jenkins | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefiles/GNUmakefile_jenkins b/makefiles/GNUmakefile_jenkins index 2584f3c9..d8d1143d 100644 --- a/makefiles/GNUmakefile_jenkins +++ b/makefiles/GNUmakefile_jenkins @@ -10,7 +10,7 @@ USE_MPI = TRUE USE_OMP = FALSE USE_ACC = FALSE USE_CUDA = TRUE -AMREX_CUDA_ARCH=70 +AMREX_CUDA_ARCH=60 EMU_HOME = .. AMREX_HOME = ../submodules/amrex From 6de21e6f03c7c6d9e52e27c7d49c161830742c3f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 22 May 2024 15:13:14 -0400 Subject: [PATCH 106/276] change order of commands first reduce_data_fft.py then reduce_data.py --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 47648e7b..27e2c663 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,8 +66,8 @@ pipeline { dir('Exec'){ sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_1d_fiducial' - sh 'python ../Scripts/data_reduction/reduce_data.py' sh 'python ../Scripts/data_reduction/reduce_data_fft.py' + sh 'python ../Scripts/data_reduction/reduce_data.py' sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data.h5' sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data_fft_power.h5' sh 'python ../Scripts/babysitting/avgfee.py' From eda852cb2826d7f480e024523fa063a97e711a4b Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 22 May 2024 15:23:12 -0400 Subject: [PATCH 107/276] delete function to initialize background quantities on this branch --- Source/Evolve.H | 2 -- Source/Evolve.cpp | 25 ------------------------- Source/main.cpp | 4 +--- 3 files changed, 1 insertion(+), 30 deletions(-) diff --git a/Source/Evolve.H b/Source/Evolve.H index eca88b07..395634e7 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -28,8 +28,6 @@ namespace GIdx amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); -void init_background_to_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const MultiFab& state, const Geometry& geom, const TestParams* parms); - void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const amrex::MultiFab& state, const amrex::Geometry& geom, const TestParams* parms); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 16cc23a5..2b6dcb73 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -103,31 +103,6 @@ Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutr return dt; } -void init_background_to_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const MultiFab& state, const Geometry& geom, const TestParams* parms) -{ - const auto plo = geom.ProbLoArray(); - const auto dxi = geom.InvCellSizeArray(); - - amrex::MeshToParticle(neutrinos_rhs, state, 0, - [=] AMREX_GPU_DEVICE (FlavoredNeutrinoContainer::ParticleType& p, - amrex::Array4 const& sarr) - { - double x,y,z; - - for (int k = 0; k < parms->ncell[2]; ++k) { - for (int j = 0; j < parms->ncell[1]; ++j) { - for (int i = 0; i < parms->ncell[0]; ++i) { - - x = plo[0] + dxi[0] * i + dxi[0] / 2.0; - y = plo[1] + dxi[1] * j + dxi[1] / 2.0; - z = plo[2] + dxi[2] * k + dxi[2] / 2.0; - - } - } - } - }); -} - void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, MultiFab& state, const Geometry& geom) { const auto plo = geom.ProbLoArray(); diff --git a/Source/main.cpp b/Source/main.cpp index c49639f9..d541c970 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -117,9 +117,7 @@ void evolve_flavor(const TestParams* parms) // Deposit particles to grid deposit_to_mesh(neutrinos_old, state, geom); - - init_background_to_mesh(neutrinos_old, state, geom, parms); - + // Write plotfile after initialization DataReducer rd; if (not parms->do_restart) { From 5d036cd34f2fcefb46e7cecce8a663834c21a2cd Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 22 May 2024 16:21:54 -0400 Subject: [PATCH 108/276] Solve mistake on collisional instability test. Now it read the right input file --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 27e2c663..e5e505b1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -95,7 +95,7 @@ pipeline { sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' sh 'python ../Scripts/data_reduction/reduce_data.py' sh 'python ../Scripts/tests/coll_equi_test.py' sh 'rm -rf plt*' From ea321e5f0106ef71b0c977218c6b79e65cd92f06 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 22 May 2024 16:38:47 -0400 Subject: [PATCH 109/276] Solve mistake on collisional instability test. Now it read the right python script --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e5e505b1..06fc598b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -97,7 +97,7 @@ pipeline { sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' sh 'python ../Scripts/data_reduction/reduce_data.py' - sh 'python ../Scripts/tests/coll_equi_test.py' + sh 'python ../Scripts/tests/coll_inst_test.py' sh 'rm -rf plt*' } }} From 400815b4ee75fe775287e1717cff1739d493d26c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 22 May 2024 18:38:16 -0400 Subject: [PATCH 110/276] add python script for collision instability test --- Scripts/tests/coll_inst_test.py | 79 +++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 Scripts/tests/coll_inst_test.py diff --git a/Scripts/tests/coll_inst_test.py b/Scripts/tests/coll_inst_test.py new file mode 100644 index 00000000..d557f06d --- /dev/null +++ b/Scripts/tests/coll_inst_test.py @@ -0,0 +1,79 @@ +import numpy as np +import argparse +import glob +import EmuReader +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__))+"/../data_reduction/" +sys.path.append(importpath) +import amrex_plot_tools as amrex +import numpy as np +import h5py +import glob + +parser = argparse.ArgumentParser() +parser.add_argument("-na", "--no_assert", action="store_true", help="If --no_assert is supplied, do not raise assertion errors if the test error > tolerance.") +args = parser.parse_args() + +if __name__ == "__main__": + + directories = glob.glob("plt*_reduced_data.h5") + directories = sorted(directories, key=lambda x: int(x.split("plt")[1].split("_")[0])) + + N_avg_mag = [] + Nbar_avg_mag = [] + F_avg_mag = [] + Fbar_avg_mag = [] + + t = [] + + for dire in directories: + with h5py.File(dire, 'r') as hf: + N_avg_mag.append(hf['N_avg_mag(1|ccm)'][:][0]) + Nbar_avg_mag.append(hf['Nbar_avg_mag(1|ccm)'][:][0]) + t.append(hf['t(s)'][:][0]) + + Nee = [] + Neu = [] + Nuu = [] + for N in N_avg_mag: + Nee.append(N[0][0]) + Neu.append(N[0][1]) + Nuu.append(N[1][1]) + + Neebar = [] + Neubar = [] + Nuubar = [] + for Nbar in Nbar_avg_mag: + Neebar.append(Nbar[0][0]) + Neubar.append(Nbar[0][1]) + Nuubar.append(Nbar[1][1]) + + # Fit the exponential function (y = ae^(bx)) to the data + + # neutrinos + l1 = 10 # initial point for fit + l2 = 55 # final point for fit + coefficients = np.polyfit(t[l1:l2], np.log(Neu[l1:l2]), 1) + a = np.exp(coefficients[1]) + b = coefficients[0] + print(f'Omega_eu = {b}') + + # antineutrinos + coefficients = np.polyfit(t[l1:l2], np.log(Neubar[l1:l2]), 1) + abar = np.exp(coefficients[1]) + bbar = coefficients[0] + print(f'Omega_eubar = {bbar}') + + b_lsa = 9.4e4 * 1e5 + print(f'LSA Omega_eu = {b_lsa}') + print(f'LSA Omega_eubar = {b_lsa}') + + def myassert(condition): + if not args.no_assert: + assert(condition) + + rel_error = 0.15 + + myassert( np.abs( b - b_lsa ) / np.abs( ( b + b_lsa ) / 2 ) < rel_error ) + myassert( np.abs( bbar - b_lsa ) / np.abs( ( bbar + b_lsa ) / 2 ) < rel_error ) \ No newline at end of file From 820c715f3a5fae67fee39f1b6f6ff4d740eed678 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 23 May 2024 12:27:36 -0400 Subject: [PATCH 111/276] solve problem of defining chemical potentials for antineutrinos in the input parameter file --- sample_inputs/inputs_coll_equi_test | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index acbf6502..2d8959f5 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -94,9 +94,12 @@ IMFP_abs0bar_cm = 5e-4 IMFP_abs1bar_cm = 5e-4 IMFP_abs2bar_cm = 5e-4 -munu0_MeV=0 -munu1_MeV=0 -munu2_MeV=0 +munu0_MeV = 0 +munu1_MeV = 0 +munu2_MeV = 0 +munu0bar_MeV = 0 +munu1bar_MeV = 0 +munu2bar_MeV = 0 IMFP_scat0_cm = 0 IMFP_scat1_cm = 0 From 38f885f7bb016add469306191adcb3b10238ad65 Mon Sep 17 00:00:00 2001 From: Sherwood Richers <5142652+srichers@users.noreply.github.com> Date: Fri, 24 May 2024 16:33:50 -0400 Subject: [PATCH 112/276] Kmax t hdf5 (#93) * check which version of HDF5 should be loaded * script for converting a reduced0D file to hdf5 --------- Co-authored-by: Sherwood Richers --- .../babysitting/convert_reduced0D_to_hdf5.py | 21 +++++++++++++++++++ makefiles/GNUmakefile_cassowary | 6 +++++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 Scripts/babysitting/convert_reduced0D_to_hdf5.py diff --git a/Scripts/babysitting/convert_reduced0D_to_hdf5.py b/Scripts/babysitting/convert_reduced0D_to_hdf5.py new file mode 100644 index 00000000..a8cf8047 --- /dev/null +++ b/Scripts/babysitting/convert_reduced0D_to_hdf5.py @@ -0,0 +1,21 @@ +import h5py +import numpy as np +import os + +def convert_reduced0D_to_hdf5(d): + infilename = d+"/reduced0D.dat" + with open(infilename,"r") as f: + labels = f.readline().split() + + data = np.genfromtxt(infilename, skip_header=1).transpose() + print(data.shape) + + outfilename = d+"/reduced0D.h5" + assert(not os.path.exists(outfilename)) + fout = h5py.File(outfilename,"w") + for i in range(len(labels)): + label = labels[i].split(":")[1] + fout[label] = data[i] + +if __name__ == '__main__': + convert_reduced0D_to_hdf5(".") diff --git a/makefiles/GNUmakefile_cassowary b/makefiles/GNUmakefile_cassowary index 72c88836..6ab77307 100644 --- a/makefiles/GNUmakefile_cassowary +++ b/makefiles/GNUmakefile_cassowary @@ -12,8 +12,12 @@ USE_ACC = FALSE USE_CUDA = FALSE AMREX_CUDA_ARCH=70 -USE_HDF5=FALSE +USE_HDF5=TRUE +ifeq ($(USE_MPI),TRUE) +HDF5_HOME=/usr/lib/x86_64-linux-gnu/hdf5/openmpi +else HDF5_HOME=/usr/lib/x86_64-linux-gnu/hdf5/serial +endif EMU_HOME = .. AMREX_HOME = ../submodules/amrex From 553702979818cd08fc97a6be09a35790031cda67 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 29 May 2024 13:07:18 -0400 Subject: [PATCH 113/276] rename f -> N for particles Beware: grid also has a Nij variable which is different --- Scripts/symbolic_hermitians/generate_code.py | 60 ++++++++++---------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 3bf26349..fcd27d12 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -94,12 +94,10 @@ def delete_generated_files(): #==================================# # FlavoredNeutrinoContainer.H_fill # #==================================# - vars = ["f"] + vars = ["N"] tails = ["","bar"] code = [] for t in tails: - code += ["N"+t] # number of neutrinos - code += ["L"+t] # length of isospin vector, units of number of neutrinos for v in vars: A = HermitianMatrix(args.N, v+"{}{}_{}"+t) code += A.header() @@ -170,7 +168,7 @@ def delete_generated_files(): code = [] for t in tails: string3 = ")" - flist = HermitianMatrix(args.N, "f{}{}_{}"+t).header() + flist = HermitianMatrix(args.N, "N{}{}_{}"+t).header() for ivar in range(len(deposit_vars)): deplist = HermitianMatrix(args.N, deposit_vars[ivar]+"{}{}_{}"+t).header() for icomp in range(len(flist)): @@ -184,7 +182,7 @@ def delete_generated_files(): code = [] for t in tails: # diagonal averages - N = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") + N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") Nlist = N.header_diagonals(); for i in range(len(Nlist)): code.append("Trf += "+Nlist[i]+";") @@ -439,10 +437,10 @@ def sgn(t,var): code = [] for t in tails: H = HermitianMatrix(args.N, "V{}{}_{}"+t) - F = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") + F = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") # G = Temporary variables for dFdt - G = HermitianMatrix(args.N, "dfdt{}{}_{}"+t) + G = HermitianMatrix(args.N, "dNdt{}{}_{}"+t) # Calculate C = i * [A,B] #Fnew.anticommutator(H,F).times(sympy.I * dt); @@ -453,52 +451,52 @@ def sgn(t,var): code.append(Gdeclare) # time derivative due to hamiltonians attenuation parameter - code.append(["dfdt00_Re"+t+" *= att_ham;"]) - code.append(["dfdt11_Re"+t+" *= att_ham;"]) - code.append(["dfdt01_Re"+t+" *= att_ham;"]) - code.append(["dfdt01_Im"+t+" *= att_ham;"]) + code.append(["dNdt00_Re"+t+" *= att_ham;"]) + code.append(["dNdt11_Re"+t+" *= att_ham;"]) + code.append(["dNdt01_Re"+t+" *= att_ham;"]) + code.append(["dNdt01_Im"+t+" *= att_ham;"]) if(args.N == 3): - code.append(["dfdt22_Re"+t+" *= att_ham;"]) - code.append(["dfdt02_Re"+t+" *= att_ham;"]) - code.append(["dfdt02_Im"+t+" *= att_ham;"]) - code.append(["dfdt12_Re"+t+" *= att_ham;"]) - code.append(["dfdt12_Im"+t+" *= att_ham;"]) + code.append(["dNdt22_Re"+t+" *= att_ham;"]) + code.append(["dNdt02_Re"+t+" *= att_ham;"]) + code.append(["dNdt02_Im"+t+" *= att_ham;"]) + code.append(["dNdt12_Re"+t+" *= att_ham;"]) + code.append(["dNdt12_Im"+t+" *= att_ham;"]) #collision term (emmission and apsortion) s=0 if(t == ""): s=0 if(t == "bar"): s=1 - code.append(["dfdt00_Re"+t+" += IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] * p.rdata(PIdx::Vphase) * pow( 1 / ( 2 * MathConst::pi * PhysConst::hbar ) , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][0] ) / 2 ) * p.rdata(PIdx::f00_Re"+t+");"]) - code.append(["dfdt11_Re"+t+" += IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] * p.rdata(PIdx::Vphase) * pow( 1 / ( 2 * MathConst::pi * PhysConst::hbar ) , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] + IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] ) / 2 + ( IMFP_abs["+str(s)+"][1] + IMFP_abs["+str(s)+"][1] ) / 2 ) * p.rdata(PIdx::f11_Re"+t+");"]) - code.append(["dfdt01_Re"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] ) / 2 ) * p.rdata(PIdx::f01_Re"+t+");"]) - code.append(["dfdt01_Im"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] ) / 2 ) * p.rdata(PIdx::f01_Im"+t+");"]) + code.append(["dNdt00_Re"+t+" += IMFP_abs["+str(s)+"][0] * N_eq["+str(s)+"][0] * p.rdata(PIdx::Vphase) * pow( 1 / ( 2 * MathConst::pi * PhysConst::hbar ) , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * N_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][0] * N_eq["+str(s)+"][0] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][0] ) / 2 ) * p.rdata(PIdx::N00_Re"+t+");"]) + code.append(["dNdt11_Re"+t+" += IMFP_abs["+str(s)+"][1] * N_eq["+str(s)+"][1] * p.rdata(PIdx::Vphase) * pow( 1 / ( 2 * MathConst::pi * PhysConst::hbar ) , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][1] * N_eq["+str(s)+"][1] + IMFP_abs["+str(s)+"][1] * N_eq["+str(s)+"][1] ) / 2 + ( IMFP_abs["+str(s)+"][1] + IMFP_abs["+str(s)+"][1] ) / 2 ) * p.rdata(PIdx::N11_Re"+t+");"]) + code.append(["dNdt01_Re"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * N_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] * N_eq["+str(s)+"][1] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] ) / 2 ) * p.rdata(PIdx::N01_Re"+t+");"]) + code.append(["dNdt01_Im"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * N_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] * N_eq["+str(s)+"][1] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][1] ) / 2 ) * p.rdata(PIdx::N01_Im"+t+");"]) if(args.N == 3): - code.append(["dfdt22_Re"+t+" += IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] * p.rdata(PIdx::Vphase) * pow( 1 / ( 2 * MathConst::pi * PhysConst::hbar ) , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][2] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f22_Re"+t+");"]) - code.append(["dfdt02_Re"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f02_Re"+t+");"]) - code.append(["dfdt02_Im"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * f_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f02_Im"+t+");"]) - code.append(["dfdt12_Re"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f02_Re"+t+");"]) - code.append(["dfdt12_Im"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][1] * f_eq["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] * f_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::f02_Im"+t+");"]) + code.append(["dNdt22_Re"+t+" += IMFP_abs["+str(s)+"][2] * N_eq["+str(s)+"][2] * p.rdata(PIdx::Vphase) * pow( 1 / ( 2 * MathConst::pi * PhysConst::hbar ) , 3 ) * ( 1 / PhysConst::c2 ) - PhysConst::c * ( ( IMFP_abs["+str(s)+"][2] * N_eq["+str(s)+"][2] + IMFP_abs["+str(s)+"][2] * N_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][2] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::N22_Re"+t+");"]) + code.append(["dNdt02_Re"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * N_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] * N_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::N02_Re"+t+");"]) + code.append(["dNdt02_Im"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][0] * N_eq["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] * N_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][0] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::N02_Im"+t+");"]) + code.append(["dNdt12_Re"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][1] * N_eq["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] * N_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::N02_Re"+t+");"]) + code.append(["dNdt12_Im"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][1] * N_eq["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] * N_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::N02_Im"+t+");"]) # Store dFdt back into the particle data for F - dFdt = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") - Gempty = HermitianMatrix(args.N, "dfdt{}{}_{}"+t) + dFdt = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") + Gempty = HermitianMatrix(args.N, "dNdt{}{}_{}"+t) dFdt.H = Gempty.H # Write out dFdt->F code.append(dFdt.code()) # evolution equations for N and Nbar, stored as dNdt-->N - line = "p.rdata(PIdx::N"+t+") = 0;" - code.append([line]) + #line = "p.rdata(PIdx::N"+t+") = 0;" + #code.append([line]) # evolution equations for L and Lbar, stored as dLdt-->L - line = "p.rdata(PIdx::L"+t+") = 0;" - code.append([line]) + #line = "p.rdata(PIdx::L"+t+") = 0;" + #code.append([line]) # store Tr(H*F) for estimating numerical errors TrHf = (H*F).trace(); From 12ffb90aa7835194c3714e48354c68bd0597047e Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 29 May 2024 13:23:02 -0400 Subject: [PATCH 114/276] finish renaming f -> N everywhere --- Scripts/symbolic_hermitians/generate_code.py | 53 +------------ Source/Evolve.cpp | 6 +- Source/FlavoredNeutrinoContainerInit.cpp | 80 ++++++++++---------- 3 files changed, 44 insertions(+), 95 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index fcd27d12..e2a0f4fc 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -505,55 +505,4 @@ def sgn(t,var): code = [line for sublist in code for line in sublist] write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill")) - #================================================# - # FlavoredNeutrinoContainer.cpp_Renormalize_fill # - #================================================# - code = [] - for t in tails: - # make sure the trace is 1 - code.append("sumP = 0;") - f = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") - fdlist = f.header_diagonals() - flist = f.header() - for fii in fdlist: - code.append("sumP += " + fii + ";") - code.append("error = sumP-1.0;") - code.append("if( std::abs(error) > 100.*parms->maxError) amrex::Abort();") - code.append("if( std::abs(error) > parms->maxError ) {") - for fii in fdlist: - code.append(fii + " -= error/"+str(args.N)+";") - code.append("}") - code.append("") - - # make sure diagonals are positive - for fii in fdlist: - code.append("if("+fii+"<-100.*parms->maxError) amrex::Abort();") - code.append("if("+fii+"<-parms->maxError) "+fii+"=0;") - code.append("") - - # make sure the flavor vector length is what it would be with a 1 in only one diagonal - length = sympy.symbols("length",real=True) - length = f.SU_vector_magnitude() - target_length = "p.rdata(PIdx::L"+t+")" - code.append("length = "+sympy.cxxcode(sympy.simplify(length))+";") - code.append("error = length-"+str(target_length)+";") - code.append("if( std::abs(error) > 100.*parms->maxError) amrex::Abort();") - code.append("if( std::abs(error) > parms->maxError) {") - for fii in flist: - code.append(fii+" /= length/"+str(target_length)+";") - code.append("}") - code.append("") - - write_code(code, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.cpp_Renormalize_fill")) - # Write code to output file, using a template if one is provided - # write_code(code, "code.cpp", args.output_template) - - - #====================================================# - # FlavoredNeutrinoContainerInit.cpp_set_trace_length # - #====================================================# - code = [] - for t in tails: - f = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") - code.append("p.rdata(PIdx::L"+t+") = "+sympy.cxxcode(sympy.simplify(f.SU_vector_magnitude()))+";" ) - write_code(code, os.path.join(args.emu_home, "Source/generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length")) + \ No newline at end of file diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 2b6dcb73..4b8d6335 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -178,7 +178,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M // and 2 x NF matrix to store equilibrium distribution values Real IMFP_abs[2][NUM_FLAVORS]; Real IMFP_scat[2][NUM_FLAVORS]; - Real f_eq[2][NUM_FLAVORS]; // equilibrium distribution function (dimensionless) + Real N_eq[2][NUM_FLAVORS]; // equilibrium distribution function (dimensionless) Real munu[2][NUM_FLAVORS]; // equilibrium chemical potential (erg) Real att_ham = parms->attenuation_hamiltonians; @@ -189,7 +189,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M for (int j=0; jkT_in; - f_eq[i][j] = 1. / (1. + exp(exponent)); + N_eq[i][j] = 1. / (1. + exp(exponent)); } } diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 26a75d34..67807ab5 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -255,17 +255,17 @@ InitParticles(const TestParams* parms) AMREX_ASSERT(p.rdata(PIdx::Nbar ) >= 0); AMREX_ASSERT(p.rdata(PIdx::L ) >= 0); AMREX_ASSERT(p.rdata(PIdx::Lbar ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f00_Re ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f11_Re ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f00_Rebar) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f11_Rebar) >= 0); - Real trace = p.rdata(PIdx::f00_Re ) + p.rdata(PIdx::f11_Re ); - Real tracebar = p.rdata(PIdx::f00_Rebar) + p.rdata(PIdx::f11_Rebar); + AMREX_ASSERT(p.rdata(PIdx::N00_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N11_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N00_Rebar) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N11_Rebar) >= 0); + Real trace = p.rdata(PIdx::N00_Re ) + p.rdata(PIdx::N11_Re ); + Real tracebar = p.rdata(PIdx::N00_Rebar) + p.rdata(PIdx::N11_Rebar); #if NUM_FLAVORS==3 - AMREX_ASSERT(p.rdata(PIdx::f22_Re ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f22_Rebar) >= 0); - trace += p.rdata(PIdx::f22_Re ); - tracebar += p.rdata(PIdx::f22_Rebar); + AMREX_ASSERT(p.rdata(PIdx::N22_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N22_Rebar) >= 0); + trace += p.rdata(PIdx::N22_Re ); + tracebar += p.rdata(PIdx::N22_Rebar); #endif AMREX_ASSERT(std::abs(trace -1)<1e-6); AMREX_ASSERT(std::abs(tracebar-1)<1e-6); @@ -282,30 +282,30 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::time) = 0; // scale particle numbers based on number of points per cell and the cell volume - p.rdata(PIdx::N ) *= scale_fac; - p.rdata(PIdx::Nbar) *= scale_fac; + //p.rdata(PIdx::N ) *= scale_fac; + //p.rdata(PIdx::Nbar) *= scale_fac; if(parms->IMFP_method == 1){ p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/(3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); } // set f ---> N*rho (before f ---> rho) - p.rdata(PIdx::f00_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::f00_Re); - p.rdata(PIdx::f01_Im) = p.rdata(PIdx::N )* p.rdata(PIdx::f01_Im); - p.rdata(PIdx::f01_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::f01_Re); - p.rdata(PIdx::f11_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::f11_Re); - p.rdata(PIdx::f00_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f00_Rebar); - p.rdata(PIdx::f01_Imbar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f01_Imbar); - p.rdata(PIdx::f01_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f01_Rebar); - p.rdata(PIdx::f11_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f11_Rebar); + /*p.rdata(PIdx::N00_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::N00_Re); + p.rdata(PIdx::N01_Im) = p.rdata(PIdx::N )* p.rdata(PIdx::N01_Im); + p.rdata(PIdx::N01_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::N01_Re); + p.rdata(PIdx::N11_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::N11_Re); + p.rdata(PIdx::N00_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::N00_Rebar); + p.rdata(PIdx::N01_Imbar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::N01_Imbar); + p.rdata(PIdx::N01_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::N01_Rebar); + p.rdata(PIdx::N11_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::N11_Rebar); #if NUM_FLAVORS==3 - p.rdata(PIdx::f02_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::f02_Re); - p.rdata(PIdx::f02_Im) = p.rdata(PIdx::N )* p.rdata(PIdx::f02_Im); + p.rdata(PIdx::N02_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::N02_Re); + p.rdata(PIdx::N02_Im) = p.rdata(PIdx::N )* p.rdata(PIdx::N02_Im); p.rdata(PIdx::f22_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::f22_Re); - p.rdata(PIdx::f02_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f02_Rebar); - p.rdata(PIdx::f02_Imbar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f02_Imbar); + p.rdata(PIdx::N02_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::N02_Rebar); + p.rdata(PIdx::N02_Imbar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::N02_Imbar); p.rdata(PIdx::f22_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f22_Rebar); -#endif +#endif*/ //=====================// // Apply Perturbations // @@ -314,41 +314,41 @@ InitParticles(const TestParams* parms) // random perturbations to the off-diagonals Real rand; symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); + p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); + p.rdata(PIdx::N01_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + p.rdata(PIdx::N01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); #if NUM_FLAVORS==3 symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::f22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::f22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::f22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::f22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::f22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::f22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::f22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::f22_Rebar)); #endif } if(parms->perturbation_type == 1){ // Perturb real part of e-mu component only sinusoidally in z Real nu_k = (2.*M_PI) / parms->perturbation_wavelength_cm; - p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); - p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); + p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); } -#include "generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length" +//#include "generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length" } // loop over direction } // loop over location From 4dede980b14989e6bc139d6caf43c6d71afef3e0 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 3 Jun 2024 14:53:31 -0400 Subject: [PATCH 115/276] comment lines that assert if N and Nbar are negatives, this lines will be deleted since now N and Nbar will no longer be used --- Source/FlavoredNeutrinoContainerInit.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 67807ab5..670beab0 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -251,10 +251,10 @@ InitParticles(const TestParams* parms) for(int i_attrib=0; i_attrib= 0); - AMREX_ASSERT(p.rdata(PIdx::Nbar ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::L ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::Lbar ) >= 0); + //AMREX_ASSERT(p.rdata(PIdx::N ) >= 0); + //AMREX_ASSERT(p.rdata(PIdx::Nbar ) >= 0); + //AMREX_ASSERT(p.rdata(PIdx::L ) >= 0); + //AMREX_ASSERT(p.rdata(PIdx::Lbar ) >= 0); AMREX_ASSERT(p.rdata(PIdx::N00_Re ) >= 0); AMREX_ASSERT(p.rdata(PIdx::N11_Re ) >= 0); AMREX_ASSERT(p.rdata(PIdx::N00_Rebar) >= 0); @@ -307,6 +307,8 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::f22_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f22_Rebar); #endif*/ + //printf("p.rdata(PIdx::N00_Re) = %15.6g\n", p.rdata(PIdx::N00_Re)); + //=====================// // Apply Perturbations // //=====================// From ff4fa1752f40e5a90ef1c21e264102d82863a10d Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 3 Jun 2024 23:04:31 -0400 Subject: [PATCH 116/276] change f for N in utility script to generate initial particles conditions --- Scripts/data_reduction/amrex_plot_tools.py | 60 ++++++++++------------ 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/Scripts/data_reduction/amrex_plot_tools.py b/Scripts/data_reduction/amrex_plot_tools.py index 94c8d094..ee717d97 100644 --- a/Scripts/data_reduction/amrex_plot_tools.py +++ b/Scripts/data_reduction/amrex_plot_tools.py @@ -26,18 +26,14 @@ def get_particle_keys(NF, ignore_pos=False, xp_only=False): "pupy", "pupz", "pupt", - "N", - "L", - "f00_Re", - "f01_Re", - "f01_Im", - "f11_Re", - "Nbar", - "Lbar", - "f00_Rebar", - "f01_Rebar", - "f01_Imbar", - "f11_Rebar"] + "N00_Re", + "N01_Re", + "N01_Im", + "N11_Re", + "N00_Rebar", + "N01_Rebar", + "N01_Imbar", + "N11_Rebar"] if(NF==3): real_quantities = ["pos_x", "pos_y", @@ -50,28 +46,24 @@ def get_particle_keys(NF, ignore_pos=False, xp_only=False): "pupy", "pupz", "pupt", - "N", - "L", - "f00_Re", - "f01_Re", - "f01_Im", - "f02_Re", - "f02_Im", - "f11_Re", - "f12_Re", - "f12_Im", - "f22_Re", - "Nbar", - "Lbar", - "f00_Rebar", - "f01_Rebar", - "f01_Imbar", - "f02_Rebar", - "f02_Imbar", - "f11_Rebar", - "f12_Rebar", - "f12_Imbar", - "f22_Rebar"] + "N00_Re", + "N01_Re", + "N01_Im", + "N02_Re", + "N02_Im", + "N11_Re", + "N12_Re", + "N12_Im", + "N22_Re", + "N00_Rebar", + "N01_Rebar", + "N01_Imbar", + "N02_Rebar", + "N02_Imbar", + "N11_Rebar", + "N12_Rebar", + "N12_Imbar", + "N22_Rebar"] if xp_only: real_quantities = real_quantities[:11] if ignore_pos: real_quantities = real_quantities[7:] From d21b1cfdbcc2866b8bc58e586cb58ea08250a425 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 3 Jun 2024 23:06:13 -0400 Subject: [PATCH 117/276] modify initial condition script for msw test. now it gives N_ab instead of N and f_ab as input --- Scripts/initial_conditions/st0_msw_test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Scripts/initial_conditions/st0_msw_test.py b/Scripts/initial_conditions/st0_msw_test.py index 2efafa1d..8330c3a4 100644 --- a/Scripts/initial_conditions/st0_msw_test.py +++ b/Scripts/initial_conditions/st0_msw_test.py @@ -3,6 +3,7 @@ import sys import os importpath = os.path.dirname(os.path.realpath(__file__)) +print(f'importpath = {importpath}') sys.path.append(importpath) sys.path.append(importpath+"/../visualization") from initial_condition_tools import uniform_sphere, write_particles @@ -23,7 +24,10 @@ # get variable keys rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) +print(f'rkey = {rkey}') + nelements = len(rkey) +print(f'nelements = {nelements}') # generate the grid of direction coordinates phat = uniform_sphere(nphi_equator) @@ -37,10 +41,8 @@ p[rkey["pupx"]] = phat[ip,0] * energy_erg p[rkey["pupy"]] = phat[ip,1] * energy_erg p[rkey["pupz"]] = phat[ip,2] * energy_erg - p[rkey["N"] ] = ndens_per_particle - p[rkey["Nbar"]] = ndens_per_particle - p[rkey["f00_Re"]] = 1 - p[rkey["f00_Rebar"]] = 1 + p[rkey["N00_Re"]] = 1 + p[rkey["N00_Rebar"]] = 1 write_particles(np.array(particles), NF, "particle_input.dat") From 8895d64cdf870dc7ad4071ec7072602abf3cf355 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 3 Jun 2024 23:14:40 -0400 Subject: [PATCH 118/276] add comments to the function that read the initial conditions of the particles --- Source/FlavoredNeutrinoContainerInit.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 670beab0..72a0405e 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -10,8 +10,13 @@ using namespace amrex; // Particle distribution in momentum space // //=========================================// -Gpu::ManagedVector > read_particle_data(std::string filename){ - Gpu::ManagedVector > particle_data; +Gpu::ManagedVector> read_particle_data(std::string filename){ + + // This function reads the input file containing the initial conditions of the particles. + // It reads the momentum, energy, and flavor occupation matrices for neutrinos and antineutrinos. + + // This array will save the particles information + Gpu::ManagedVector> particle_data; // open the file as a stream std::ifstream file(filename); @@ -32,9 +37,11 @@ Gpu::ManagedVector > read_particle_data(std::strin if(NF_in != NUM_FLAVORS) amrex::Print() << "Error: number of flavors in particle data file does not match the number of flavors Emu was compiled for." << std::endl; AMREX_ASSERT(NF_in == NUM_FLAVORS); + // Loop over every line in the initial condition file. + // This is equivalent to looping over every particle. + // Save every particle's information in the array particle_data. while(std::getline(file, line)){ ss = std::stringstream(line); - // skip over the first four attributes (x,y,z,t) for(int i=4; i> temp_particle[i]; particle_data.push_back(temp_particle); From 2f4a4cc3f1a32e0d746131cf2055d5ce57c230e4 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 4 Jun 2024 10:18:08 -0400 Subject: [PATCH 119/276] change msw test to include the change f_ab ---> N_ab --- Scripts/tests/msw_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Scripts/tests/msw_test.py b/Scripts/tests/msw_test.py index 0c4f24ac..d0976130 100644 --- a/Scripts/tests/msw_test.py +++ b/Scripts/tests/msw_test.py @@ -48,10 +48,10 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata[0] t.append(p[rkey["time"]]) - fee.append(p[rkey["f00_Re"]]) - fxx.append(p[rkey["f11_Re"]]) - feebar.append(p[rkey["f00_Rebar"]]) - fxxbar.append(p[rkey["f11_Rebar"]]) + fee.append(p[rkey["N00_Re"]]) + fxx.append(p[rkey["N11_Re"]]) + feebar.append(p[rkey["N00_Rebar"]]) + fxxbar.append(p[rkey["N11_Rebar"]]) pupt.append(p[rkey["pupt"]]) t = np.array(t) From a77c608837d2e0f3cc9835286eac910d6ee74c0c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 4 Jun 2024 10:20:41 -0400 Subject: [PATCH 120/276] reading N_ab for the particle initial condition file instead of N and f_ab independenly --- Source/FlavoredNeutrinoContainerInit.cpp | 40 +++++------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 72a0405e..46d69f94 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -258,24 +258,14 @@ InitParticles(const TestParams* parms) for(int i_attrib=0; i_attrib= 0); - //AMREX_ASSERT(p.rdata(PIdx::Nbar ) >= 0); - //AMREX_ASSERT(p.rdata(PIdx::L ) >= 0); - //AMREX_ASSERT(p.rdata(PIdx::Lbar ) >= 0); AMREX_ASSERT(p.rdata(PIdx::N00_Re ) >= 0); AMREX_ASSERT(p.rdata(PIdx::N11_Re ) >= 0); AMREX_ASSERT(p.rdata(PIdx::N00_Rebar) >= 0); AMREX_ASSERT(p.rdata(PIdx::N11_Rebar) >= 0); - Real trace = p.rdata(PIdx::N00_Re ) + p.rdata(PIdx::N11_Re ); - Real tracebar = p.rdata(PIdx::N00_Rebar) + p.rdata(PIdx::N11_Rebar); #if NUM_FLAVORS==3 AMREX_ASSERT(p.rdata(PIdx::N22_Re ) >= 0); AMREX_ASSERT(p.rdata(PIdx::N22_Rebar) >= 0); - trace += p.rdata(PIdx::N22_Re ); - tracebar += p.rdata(PIdx::N22_Rebar); #endif - AMREX_ASSERT(std::abs(trace -1)<1e-6); - AMREX_ASSERT(std::abs(tracebar-1)<1e-6); // Set particle position p.pos(0) = x; @@ -289,33 +279,19 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::time) = 0; // scale particle numbers based on number of points per cell and the cell volume - //p.rdata(PIdx::N ) *= scale_fac; - //p.rdata(PIdx::Nbar) *= scale_fac; + p.rdata(PIdx::N00_Re ) *= scale_fac; + p.rdata(PIdx::N11_Re ) *= scale_fac; + p.rdata(PIdx::N00_Rebar) *= scale_fac; + p.rdata(PIdx::N11_Rebar) *= scale_fac; +#if NUM_FLAVORS==3 + p.rdata(PIdx::N22_Re ) *= scale_fac; + p.rdata(PIdx::N22_Rebar) *= scale_fac; +#endif if(parms->IMFP_method == 1){ p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/(3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); } - // set f ---> N*rho (before f ---> rho) - /*p.rdata(PIdx::N00_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::N00_Re); - p.rdata(PIdx::N01_Im) = p.rdata(PIdx::N )* p.rdata(PIdx::N01_Im); - p.rdata(PIdx::N01_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::N01_Re); - p.rdata(PIdx::N11_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::N11_Re); - p.rdata(PIdx::N00_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::N00_Rebar); - p.rdata(PIdx::N01_Imbar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::N01_Imbar); - p.rdata(PIdx::N01_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::N01_Rebar); - p.rdata(PIdx::N11_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::N11_Rebar); -#if NUM_FLAVORS==3 - p.rdata(PIdx::N02_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::N02_Re); - p.rdata(PIdx::N02_Im) = p.rdata(PIdx::N )* p.rdata(PIdx::N02_Im); - p.rdata(PIdx::f22_Re) = p.rdata(PIdx::N )* p.rdata(PIdx::f22_Re); - p.rdata(PIdx::N02_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::N02_Rebar); - p.rdata(PIdx::N02_Imbar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::N02_Imbar); - p.rdata(PIdx::f22_Rebar) = p.rdata(PIdx::Nbar) * p.rdata(PIdx::f22_Rebar); -#endif*/ - - //printf("p.rdata(PIdx::N00_Re) = %15.6g\n", p.rdata(PIdx::N00_Re)); - //=====================// // Apply Perturbations // //=====================// From 50c6f8dfaefaae682b300582b89e6d32448c10ac Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 4 Jun 2024 11:56:17 -0400 Subject: [PATCH 121/276] removing f_ab variables for N_ab when setting perturbation --- Source/FlavoredNeutrinoContainerInit.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 46d69f94..f6c4bfea 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -308,21 +308,21 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::N01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); #if NUM_FLAVORS==3 symmetric_uniform(&rand, engine); - p.rdata(PIdx::N02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::N22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::N22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar)); #endif } if(parms->perturbation_type == 1){ From d8a29b96da574b4c679fdf50afc7ced5eba7a6a6 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 4 Jun 2024 11:59:25 -0400 Subject: [PATCH 122/276] delete no needed lines --- .../initial_condition_tools.py | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/Scripts/initial_conditions/initial_condition_tools.py b/Scripts/initial_conditions/initial_condition_tools.py index a3639f9f..99d9f548 100644 --- a/Scripts/initial_conditions/initial_condition_tools.py +++ b/Scripts/initial_conditions/initial_condition_tools.py @@ -200,6 +200,8 @@ def linear_interpolate(fluxfac, mu): def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_generator, interpolate_function): # number of neutrino flavors NF = nnu.shape[1] + + print(f'NF = {NF}') # flux magnitude and flux factor [nu/nubar, flavor] fluxmag = np.sqrt(np.sum(fnu**2, axis=2)) @@ -230,7 +232,7 @@ def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_g # get variable keys rkey, ikey = amrex.get_particle_keys(NF, ignore_pos=True) nelements = len(rkey) - + # generate the list of particle info particles = np.zeros((nparticles,nelements)) @@ -240,22 +242,10 @@ def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_g # save the total number density of neutrinos for each particle n_flavorsummed = np.sum(n_particle, axis=2) # [particle, nu/nubar] - for nu_nubar, suffix in zip(range(2), ["","bar"]): - nvarname = "N"+suffix - particles[:,rkey[nvarname]] = n_flavorsummed[:,nu_nubar] - + for nu_nubar, suffix in zip(range(2), ["","bar"]): for flavor in range(NF): - fvarname = "f"+str(flavor)+str(flavor)+"_Re"+suffix - particles[:,rkey[fvarname]] = n_particle[:,nu_nubar, flavor] / n_flavorsummed[:,nu_nubar] - particles[:,rkey[fvarname]][np.where(n_flavorsummed[:,nu_nubar]==0)] = 1./NF # ensure that trace stays equal to 1 - - # double check that the number densities are correct - particle_n = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]]) - particle_fmag = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]] * mu[:,nu_nubar, flavor]) - #print("nu/nubar,flavor =", nu_nubar, flavor) - #print("output/input ndens =",particle_n, nnu[nu_nubar,flavor]) - #print("output/input fluxfac =",particle_fmag / particle_n, fluxfac[nu_nubar,flavor]) - #print() + fvarname = "N"+str(flavor)+str(flavor)+"_Re"+suffix + particles[:,rkey[fvarname]] = n_particle[:,nu_nubar, flavor] return particles From 581e207e229a713dee8fc4f0735ee928a69984d1 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 4 Jun 2024 12:00:20 -0400 Subject: [PATCH 123/276] repare coll_equi_test.py test to consider N_ab instead of f_ab --- Scripts/tests/coll_equi_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Scripts/tests/coll_equi_test.py b/Scripts/tests/coll_equi_test.py index 75a80f7a..089feef5 100644 --- a/Scripts/tests/coll_equi_test.py +++ b/Scripts/tests/coll_equi_test.py @@ -34,12 +34,12 @@ for i in range(len(rdata)): p = rdata[i] - N_ee.append(p[rkey["f00_Re"]]) - N_uu.append(p[rkey["f11_Re"]]) - N_tt.append(p[rkey["f22_Re"]]) - N_eebar.append(p[rkey["f00_Rebar"]]) - N_uubar.append(p[rkey["f11_Rebar"]]) - N_ttbar.append(p[rkey["f22_Rebar"]]) + N_ee.append(p[rkey["N00_Re"]]) + N_uu.append(p[rkey["N11_Re"]]) + N_tt.append(p[rkey["N22_Re"]]) + N_eebar.append(p[rkey["N00_Rebar"]]) + N_uubar.append(p[rkey["N11_Rebar"]]) + N_ttbar.append(p[rkey["N22_Rebar"]]) print(f'average N_ee {np.average(N_ee)}') print(f'average N_uu {np.average(N_uu)}') From 2a8905165862380645520c7d3eb3d61deaad5791 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 4 Jun 2024 12:09:43 -0400 Subject: [PATCH 124/276] making bypolar test work with the changes f_ab ---> N_ab --- Scripts/initial_conditions/st1_bipolar_test.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Scripts/initial_conditions/st1_bipolar_test.py b/Scripts/initial_conditions/st1_bipolar_test.py index 04a15e5f..4f8948c1 100644 --- a/Scripts/initial_conditions/st1_bipolar_test.py +++ b/Scripts/initial_conditions/st1_bipolar_test.py @@ -34,7 +34,6 @@ rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) nelements = len(rkey) - # generate the list of particle info particles = np.zeros((nparticles,nelements)) for ip in range(len(phat)): @@ -43,10 +42,7 @@ p[rkey["pupx"]] = phat[ip,0] * energy_erg p[rkey["pupy"]] = phat[ip,1] * energy_erg p[rkey["pupz"]] = phat[ip,2] * energy_erg - p[rkey["N"] ] = ndens_per_particle - p[rkey["Nbar"]] = ndens_per_particle - p[rkey["f00_Re"]] = 1 - p[rkey["f00_Rebar"]] = 1 + p[rkey["N00_Re"]] = ndens_per_particle + p[rkey["N00_Rebar"]] = ndens_per_particle - -write_particles(np.array(particles), NF, "particle_input.dat") +write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file From e4e4c4f140d4a3c6eb2e8c1f859cbaa44c4c0813 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 4 Jun 2024 12:16:59 -0400 Subject: [PATCH 125/276] making fast flavor test work for f_ab ---> N_ab --- Scripts/initial_conditions/st2_2beam_fast_flavor.py | 7 ++----- Scripts/tests/fast_flavor_test.py | 8 ++++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/Scripts/initial_conditions/st2_2beam_fast_flavor.py b/Scripts/initial_conditions/st2_2beam_fast_flavor.py index 740b5de1..6839e9d5 100644 --- a/Scripts/initial_conditions/st2_2beam_fast_flavor.py +++ b/Scripts/initial_conditions/st2_2beam_fast_flavor.py @@ -43,10 +43,7 @@ p[rkey["pupx"]] = u[0] * energy_erg p[rkey["pupy"]] = u[1] * energy_erg p[rkey["pupz"]] = u[2] * energy_erg - p[rkey["N"] ] = ndens_per_particle * (1. + u[2]) - p[rkey["Nbar"]] = ndens_per_particle * (1. - u[2]) - p[rkey["f00_Re"]] = 1 - p[rkey["f00_Rebar"]] = 1 - + p[rkey["N00_Re"]] = ndens_per_particle * (1. + u[2]) + p[rkey["N00_Rebar"]] = ndens_per_particle * (1. - u[2]) write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/tests/fast_flavor_test.py b/Scripts/tests/fast_flavor_test.py index 557ab791..c1e4bdbc 100644 --- a/Scripts/tests/fast_flavor_test.py +++ b/Scripts/tests/fast_flavor_test.py @@ -39,11 +39,11 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata[0] t.append(p[rkey["time"]]) - fexR.append(p[rkey["f01_Re"]]) - fexI.append(p[rkey["f01_Im"]]) + fexR.append(p[rkey["N01_Re"]]) + fexI.append(p[rkey["N01_Im"]]) p = rdata[1] - fexRbar.append(p[rkey["f01_Rebar"]]) - fexIbar.append(p[rkey["f01_Imbar"]]) + fexRbar.append(p[rkey["N01_Rebar"]]) + fexIbar.append(p[rkey["N01_Imbar"]]) pupt.append(p[rkey["pupt"]]) t = np.array(t) From 44409f0b9db372abb17cd2abb9d7b795a03e8437 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 4 Jun 2024 13:07:55 -0400 Subject: [PATCH 126/276] making fast_flavor_k_test works for f_ab ---> N_ab --- .../st3_2beam_fast_flavor_nonzerok.py | 10 +++------- Scripts/tests/fast_flavor_k_test.py | 8 ++++---- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py b/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py index bbcdfab4..c2fd2da8 100644 --- a/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py +++ b/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py @@ -35,7 +35,6 @@ rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) nelements = len(rkey) - # generate the list of particle info particles = np.zeros((nparticles,nelements)) for ip in range(len(phat)): @@ -45,10 +44,7 @@ p[rkey["pupx"]] = u[0] * energy_erg p[rkey["pupy"]] = u[1] * energy_erg p[rkey["pupz"]] = u[2] * energy_erg - p[rkey["N"] ] = ndens_per_particle * (1. + u[2]) - p[rkey["Nbar"]] = ndens_per_particle * (1. - u[2]) - p[rkey["f00_Re"]] = 1 - p[rkey["f00_Rebar"]] = 1 - + p[rkey["N00_Re"]] = ndens_per_particle * (1. + u[2]) + p[rkey["N00_Rebar"]] = ndens_per_particle * (1. - u[2]) -write_particles(np.array(particles), NF, "particle_input.dat") +write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file diff --git a/Scripts/tests/fast_flavor_k_test.py b/Scripts/tests/fast_flavor_k_test.py index 5703e49c..0e885ef1 100644 --- a/Scripts/tests/fast_flavor_k_test.py +++ b/Scripts/tests/fast_flavor_k_test.py @@ -43,10 +43,10 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata t.append(p[0][rkey["time"]]) - fexR.append(np.max(np.abs(p[:,rkey["f01_Re"]]))) - fexI.append(np.max(np.abs(p[:,rkey["f01_Im"]]))) - fexRbar.append(np.max(np.abs(p[:,rkey["f01_Rebar"]]))) - fexIbar.append(np.max(np.abs(p[:,rkey["f01_Imbar"]]))) + fexR.append(np.max(np.abs(p[:,rkey["N01_Re"]]))) + fexI.append(np.max(np.abs(p[:,rkey["N01_Im"]]))) + fexRbar.append(np.max(np.abs(p[:,rkey["N01_Rebar"]]))) + fexIbar.append(np.max(np.abs(p[:,rkey["N01_Imbar"]]))) pupt.append(p[0][rkey["pupt"]]) t = np.array(t) From 61c1649a382d78d2ad914bb27fd7ce6a67bb8c52 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 4 Jun 2024 13:25:43 -0400 Subject: [PATCH 127/276] delete print line --- Scripts/initial_conditions/initial_condition_tools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Scripts/initial_conditions/initial_condition_tools.py b/Scripts/initial_conditions/initial_condition_tools.py index 99d9f548..d0885bdd 100644 --- a/Scripts/initial_conditions/initial_condition_tools.py +++ b/Scripts/initial_conditions/initial_condition_tools.py @@ -200,8 +200,6 @@ def linear_interpolate(fluxfac, mu): def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_generator, interpolate_function): # number of neutrino flavors NF = nnu.shape[1] - - print(f'NF = {NF}') # flux magnitude and flux factor [nu/nubar, flavor] fluxmag = np.sqrt(np.sum(fnu**2, axis=2)) From 843dfde9c9e2c3fc801c8b844694ac68c52a36a4 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 4 Jun 2024 13:26:17 -0400 Subject: [PATCH 128/276] delete print lines --- Scripts/initial_conditions/st0_msw_test.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Scripts/initial_conditions/st0_msw_test.py b/Scripts/initial_conditions/st0_msw_test.py index 8330c3a4..1c6767d5 100644 --- a/Scripts/initial_conditions/st0_msw_test.py +++ b/Scripts/initial_conditions/st0_msw_test.py @@ -3,7 +3,6 @@ import sys import os importpath = os.path.dirname(os.path.realpath(__file__)) -print(f'importpath = {importpath}') sys.path.append(importpath) sys.path.append(importpath+"/../visualization") from initial_condition_tools import uniform_sphere, write_particles @@ -24,10 +23,7 @@ # get variable keys rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) -print(f'rkey = {rkey}') - nelements = len(rkey) -print(f'nelements = {nelements}') # generate the grid of direction coordinates phat = uniform_sphere(nphi_equator) @@ -43,6 +39,5 @@ p[rkey["pupz"]] = phat[ip,2] * energy_erg p[rkey["N00_Re"]] = 1 p[rkey["N00_Rebar"]] = 1 - -write_particles(np.array(particles), NF, "particle_input.dat") +write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file From c1988dca554a56afece7c1ca38e3ec0cdac858d5 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 4 Jun 2024 13:54:44 -0400 Subject: [PATCH 129/276] make attenuation parameter symbolic in code generation scripts --- Scripts/symbolic_hermitians/generate_code.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index e2a0f4fc..8864b187 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -434,6 +434,7 @@ def sgn(t,var): # Set up Hermitian matrices A, B, C hbar = sympy.symbols("PhysConst\:\:hbar",real=True) + attenuation = sympy.symbols("att_ham", real=True) code = [] for t in tails: H = HermitianMatrix(args.N, "V{}{}_{}"+t) @@ -444,26 +445,12 @@ def sgn(t,var): # Calculate C = i * [A,B] #Fnew.anticommutator(H,F).times(sympy.I * dt); - G.H = ((H*F - F*H).times(-sympy.I/hbar)).H + G.H = ((H*F - F*H).times(-sympy.I/hbar)).H * attenuation # Write the temporary variables for dFdt Gdeclare = ["amrex::Real {}".format(line) for line in G.code()] code.append(Gdeclare) - # time derivative due to hamiltonians attenuation parameter - code.append(["dNdt00_Re"+t+" *= att_ham;"]) - code.append(["dNdt11_Re"+t+" *= att_ham;"]) - code.append(["dNdt01_Re"+t+" *= att_ham;"]) - code.append(["dNdt01_Im"+t+" *= att_ham;"]) - - if(args.N == 3): - - code.append(["dNdt22_Re"+t+" *= att_ham;"]) - code.append(["dNdt02_Re"+t+" *= att_ham;"]) - code.append(["dNdt02_Im"+t+" *= att_ham;"]) - code.append(["dNdt12_Re"+t+" *= att_ham;"]) - code.append(["dNdt12_Im"+t+" *= att_ham;"]) - #collision term (emmission and apsortion) s=0 if(t == ""): s=0 @@ -505,4 +492,4 @@ def sgn(t,var): code = [line for sublist in code for line in sublist] write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill")) - \ No newline at end of file + From e0395160fcbf8ee0f62162bdf6e569933fd3db7c Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 4 Jun 2024 14:02:47 -0400 Subject: [PATCH 130/276] move f-->N in a few more places --- Scripts/symbolic_hermitians/generate_code.py | 28 +++++++------------- Source/DataReducer.cpp | 16 +++++------ 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 8864b187..4d90137d 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -101,7 +101,7 @@ def delete_generated_files(): for v in vars: A = HermitianMatrix(args.N, v+"{}{}_{}"+t) code += A.header() - code += ["TrHf"] + code += ["TrHN"] code += ["Vphase"] code_lines = [code[i]+"," for i in range(len(code))] @@ -438,14 +438,14 @@ def sgn(t,var): code = [] for t in tails: H = HermitianMatrix(args.N, "V{}{}_{}"+t) - F = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") + N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") - # G = Temporary variables for dFdt + # G = Temporary variables for dNdt G = HermitianMatrix(args.N, "dNdt{}{}_{}"+t) # Calculate C = i * [A,B] #Fnew.anticommutator(H,F).times(sympy.I * dt); - G.H = ((H*F - F*H).times(-sympy.I/hbar)).H * attenuation + G.H = ((H*N - N*H).times(-sympy.I/hbar)).H * attenuation # Write the temporary variables for dFdt Gdeclare = ["amrex::Real {}".format(line) for line in G.code()] @@ -470,24 +470,16 @@ def sgn(t,var): code.append(["dNdt12_Im"+t+" += -1 * PhysConst::c * ( ( IMFP_abs["+str(s)+"][1] * N_eq["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] * N_eq["+str(s)+"][2] ) / 2 + ( IMFP_abs["+str(s)+"][1] + IMFP_abs["+str(s)+"][2] ) / 2 ) * p.rdata(PIdx::N02_Im"+t+");"]) # Store dFdt back into the particle data for F - dFdt = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") + dNdt = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") Gempty = HermitianMatrix(args.N, "dNdt{}{}_{}"+t) - dFdt.H = Gempty.H - - # Write out dFdt->F - code.append(dFdt.code()) - - # evolution equations for N and Nbar, stored as dNdt-->N - #line = "p.rdata(PIdx::N"+t+") = 0;" - #code.append([line]) + dNdt.H = Gempty.H - # evolution equations for L and Lbar, stored as dLdt-->L - #line = "p.rdata(PIdx::L"+t+") = 0;" - #code.append([line]) + # Write out dNdt->N + code.append(dNdt.code()) # store Tr(H*F) for estimating numerical errors - TrHf = (H*F).trace(); - code.append(["p.rdata(PIdx::TrHf) += ("+sympy.cxxcode(sympy.simplify(TrHf))+");"]) + TrHN = (H*N).trace(); + code.append(["p.rdata(PIdx::TrHN) += ("+sympy.cxxcode(sympy.simplify(TrHN))+");"]) code = [line for sublist in code for line in sublist] write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill")) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index d339beb6..b65489dd 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -65,7 +65,7 @@ void DataReducer::InitializeFiles() } file0D.createDataSet("N_offdiag_mag(1|ccm)", dataspace, create_datatype(), props); file0D.createDataSet("sumTrf", dataspace, create_datatype(), props); - file0D.createDataSet("sumTrHf", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrHN", dataspace, create_datatype(), props); #else @@ -106,7 +106,7 @@ void DataReducer::InitializeFiles() j++; outfile << j << ":sumTrf\t"; j++; - outfile << j << ":sumTrHf\t"; + outfile << j << ":sumTrHN\t"; outfile << std::endl; outfile.close(); @@ -130,15 +130,15 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, amrex::ReduceOps reduce_ops; auto particleResult = amrex::ParticleReduce< ReduceData >(neutrinos, [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { - Real TrHf = p.rdata(PIdx::TrHf); + Real TrHN = p.rdata(PIdx::TrHN); Real Trf = 0; #include "generated_files/DataReducer.cpp_fill_particles" - return GpuTuple{Trf,TrHf}; + return GpuTuple{Trf,TrHN}; }, reduce_ops); Real Trf = amrex::get<0>(particleResult); - Real TrHf = amrex::get<1>(particleResult); + Real TrHN = amrex::get<1>(particleResult); ParallelDescriptor::ReduceRealSum(Trf); - ParallelDescriptor::ReduceRealSum(TrHf); + ParallelDescriptor::ReduceRealSum(TrHN); //=============================// // Do reductions over the grid // @@ -291,7 +291,7 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, } append_0D(file0D, "N_offdiag_mag(1|ccm)", N_offdiag_mag); append_0D(file0D, "sumTrf", Trf); - append_0D(file0D, "sumTrHf", TrHf); + append_0D(file0D, "sumTrHN", TrHN); #else std::ofstream outfile; outfile.open(filename0D, std::ofstream::app); @@ -325,7 +325,7 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, } outfile << N_offdiag_mag << "\t"; outfile << Trf << "\t"; - outfile << TrHf << "\t"; + outfile << TrHN << "\t"; outfile << std::endl; outfile.close(); #endif From 28af3b5d382c28c9abe7a13933ceef1def1ced06 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 4 Jun 2024 14:09:20 -0400 Subject: [PATCH 131/276] change f to N in test scripts --- Scripts/tests/convergence.sh | 2 +- Scripts/tests/fast_flavor_k_test.py | 54 +++++++++++++-------------- Scripts/tests/fast_flavor_test.py | 58 ++++++++++++++--------------- Scripts/tests/msw_test.py | 44 +++++++++++----------- Scripts/tests/plot_convergence.py | 4 +- 5 files changed, 81 insertions(+), 81 deletions(-) diff --git a/Scripts/tests/convergence.sh b/Scripts/tests/convergence.sh index 3816a379..bcd6dfee 100644 --- a/Scripts/tests/convergence.sh +++ b/Scripts/tests/convergence.sh @@ -57,7 +57,7 @@ do_convergence () { INTPARAMS=" integration.type=0" -INTNAME="fe" +INTNAME="Ne" do_convergence diff --git a/Scripts/tests/fast_flavor_k_test.py b/Scripts/tests/fast_flavor_k_test.py index 0e885ef1..8b661152 100644 --- a/Scripts/tests/fast_flavor_k_test.py +++ b/Scripts/tests/fast_flavor_k_test.py @@ -30,10 +30,10 @@ rkey, ikey = amrex.get_particle_keys(NF) t = [] - fexR = [] - fexI = [] - fexRbar = [] - fexIbar = [] + NexR = [] + NexI = [] + NexRbar = [] + NexIbar = [] pupt = [] nfiles = len(glob.glob("plt[0-9][0-9][0-9][0-9][0-9]")) @@ -43,18 +43,18 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata t.append(p[0][rkey["time"]]) - fexR.append(np.max(np.abs(p[:,rkey["N01_Re"]]))) - fexI.append(np.max(np.abs(p[:,rkey["N01_Im"]]))) - fexRbar.append(np.max(np.abs(p[:,rkey["N01_Rebar"]]))) - fexIbar.append(np.max(np.abs(p[:,rkey["N01_Imbar"]]))) + NexR.append(np.max(np.abs(p[:,rkey["N01_Re"]]))) + NexI.append(np.max(np.abs(p[:,rkey["N01_Im"]]))) + NexRbar.append(np.max(np.abs(p[:,rkey["N01_Rebar"]]))) + NexIbar.append(np.max(np.abs(p[:,rkey["N01_Imbar"]]))) pupt.append(p[0][rkey["pupt"]]) t = np.array(t) - fexR = np.array(fexR) - fexI = np.array(fexI) - fexRbar = np.array(fexRbar) - fexIbar = np.array(fexIbar) - print(fexR) + NexR = np.array(NexR) + NexI = np.array(NexI) + NexRbar = np.array(NexRbar) + NexIbar = np.array(NexIbar) + print(NexR) # The neutrino energy we set E = 50. * 1e6*amrex.eV @@ -70,27 +70,27 @@ # get growth rate from each diagonal component dt = t[i1]-t[i0] - fexRomega = np.log(np.abs(fexR[i1]/fexR[i0])) / dt - fexIomega = np.log(np.abs(fexI[i1]/fexI[i0])) / dt - fexRbaromega = np.log(np.abs(fexRbar[i1]/fexRbar[i0])) / dt - fexIbaromega = np.log(np.abs(fexIbar[i1]/fexIbar[i0])) / dt + NexRomega = np.log(np.abs(NexR[i1]/NexR[i0])) / dt + NexIomega = np.log(np.abs(NexI[i1]/NexI[i0])) / dt + NexRbaromega = np.log(np.abs(NexRbar[i1]/NexRbar[i0])) / dt + NexIbaromega = np.log(np.abs(NexIbar[i1]/NexIbar[i0])) / dt - print("growth rates:",fexRomega,fexIomega,fexRbaromega,fexIbaromega) - print("growth rates / theoretical:",fexRomega/ImOmega,fexIomega/ImOmega,fexRbaromega/ImOmega,fexIbaromega/ImOmega) + print("growth rates:",NexRomega,NexIomega,NexRbaromega,NexIbaromega) + print("growth rates / theoretical:",NexRomega/ImOmega,NexIomega/ImOmega,NexRbaromega/ImOmega,NexIbaromega/ImOmega) def myassert(condition): if not args.no_assert: assert(condition) - fexRerror = np.abs(ImOmega - fexRomega) / ImOmega - myassert( fexRerror < tolerance ) + NexRerror = np.abs(ImOmega - NexRomega) / ImOmega + myassert( NexRerror < tolerance ) - fexIerror = np.abs(ImOmega - fexIomega) / ImOmega - myassert( fexIerror < tolerance ) + NexIerror = np.abs(ImOmega - NexIomega) / ImOmega + myassert( NexIerror < tolerance ) - fexRbarerror = np.abs(ImOmega - fexRbaromega) / ImOmega - myassert( fexRbarerror < tolerance ) + NexRbarerror = np.abs(ImOmega - NexRbaromega) / ImOmega + myassert( NexRbarerror < tolerance ) - fexIbarerror = np.abs(ImOmega - fexIbaromega) / ImOmega - myassert( fexIbarerror < tolerance ) + NexIbarerror = np.abs(ImOmega - NexIbaromega) / ImOmega + myassert( NexIbarerror < tolerance ) diff --git a/Scripts/tests/fast_flavor_test.py b/Scripts/tests/fast_flavor_test.py index c1e4bdbc..cd50fd75 100644 --- a/Scripts/tests/fast_flavor_test.py +++ b/Scripts/tests/fast_flavor_test.py @@ -26,10 +26,10 @@ rkey, ikey = amrex.get_particle_keys(NF) t = [] - fexR = [] - fexI = [] - fexRbar = [] - fexIbar = [] + NexR = [] + NexI = [] + NexRbar = [] + NexIbar = [] pupt = [] nfiles = len(glob.glob("plt[0-9][0-9][0-9][0-9][0-9]")) @@ -39,18 +39,18 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata[0] t.append(p[rkey["time"]]) - fexR.append(p[rkey["N01_Re"]]) - fexI.append(p[rkey["N01_Im"]]) + NexR.append(p[rkey["N01_Re"]]) + NexI.append(p[rkey["N01_Im"]]) p = rdata[1] - fexRbar.append(p[rkey["N01_Rebar"]]) - fexIbar.append(p[rkey["N01_Imbar"]]) + NexRbar.append(p[rkey["N01_Rebar"]]) + NexIbar.append(p[rkey["N01_Imbar"]]) pupt.append(p[rkey["pupt"]]) t = np.array(t) - fexR = np.array(fexR) - fexI = np.array(fexI) - fexRbar = np.array(fexRbar) - fexIbar = np.array(fexIbar) + NexR = np.array(NexR) + NexI = np.array(NexI) + NexRbar = np.array(NexRbar) + NexIbar = np.array(NexIbar) # The neutrino energy we set E = 50. * 1e6*amrex.eV @@ -62,32 +62,32 @@ # get growth rate from each diagonal component dt = t[i1]-t[i0] - fexRomega = np.log(fexR[i1]/fexR[i0]) / dt - fexIomega = np.log(fexI[i1]/fexI[i0]) / dt - fexRbaromega = np.log(fexRbar[i1]/fexRbar[i0]) / dt - fexIbaromega = np.log(fexIbar[i1]/fexIbar[i0]) / dt + NexRomega = np.log(NexR[i1]/NexR[i0]) / dt + NexIomega = np.log(NexI[i1]/NexI[i0]) / dt + NexRbaromega = np.log(NexRbar[i1]/NexRbar[i0]) / dt + NexIbaromega = np.log(NexIbar[i1]/NexIbar[i0]) / dt def myassert(condition): if not args.no_assert: assert(condition) - print("growth rates:",fexRomega,fexIomega,fexRbaromega,fexIbaromega) + print("growth rates:",NexRomega,NexIomega,NexRbaromega,NexIbaromega) print(dt,t[i0],t[i1]) - print(fexR[i1],fexR[i0]) - print(fexI[i1],fexI[i0]) - print(fexRbar[i1],fexRbar[i0]) - print(fexIbar[i1],fexIbar[i0]) + print(NexR[i1],NexR[i0]) + print(NexI[i1],NexI[i0]) + print(NexRbar[i1],NexRbar[i0]) + print(NexIbar[i1],NexIbar[i0]) - fexRerror = np.abs(ImOmega - fexRomega) / ImOmega - myassert( fexRerror < tolerance ) + NexRerror = np.abs(ImOmega - NexRomega) / ImOmega + myassert( NexRerror < tolerance ) - fexIerror = np.abs(ImOmega - fexIomega) / ImOmega - myassert( fexIerror < tolerance ) + NexIerror = np.abs(ImOmega - NexIomega) / ImOmega + myassert( NexIerror < tolerance ) - fexRbarerror = np.abs(ImOmega - fexRbaromega) / ImOmega - myassert( fexRbarerror < tolerance ) + NexRbarerror = np.abs(ImOmega - NexRbaromega) / ImOmega + myassert( NexRbarerror < tolerance ) - fexIbarerror = np.abs(ImOmega - fexIbaromega) / ImOmega - myassert( fexIbarerror < tolerance ) + NexIbarerror = np.abs(ImOmega - NexIbaromega) / ImOmega + myassert( NexIbarerror < tolerance ) diff --git a/Scripts/tests/msw_test.py b/Scripts/tests/msw_test.py index d0976130..aab770cb 100644 --- a/Scripts/tests/msw_test.py +++ b/Scripts/tests/msw_test.py @@ -35,10 +35,10 @@ rkey, ikey = amrex.get_particle_keys(NF) t = [] - fee = [] - fxx = [] - feebar = [] - fxxbar = [] + Nee = [] + Nxx = [] + Neebar = [] + Nxxbar = [] pupt = [] nfiles = len(glob.glob("plt[0-9][0-9][0-9][0-9][0-9]")) @@ -48,17 +48,17 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata[0] t.append(p[rkey["time"]]) - fee.append(p[rkey["N00_Re"]]) - fxx.append(p[rkey["N11_Re"]]) - feebar.append(p[rkey["N00_Rebar"]]) - fxxbar.append(p[rkey["N11_Rebar"]]) + Nee.append(p[rkey["N00_Re"]]) + Nxx.append(p[rkey["N11_Re"]]) + Neebar.append(p[rkey["N00_Rebar"]]) + Nxxbar.append(p[rkey["N11_Rebar"]]) pupt.append(p[rkey["pupt"]]) t = np.array(t) - fee = np.array(fee) - fxx = np.array(fxx) - feebar = np.array(feebar) - fxxbar = np.array(fxxbar) + Nee = np.array(Nee) + Nxx = np.array(Nxx) + Neebar = np.array(Neebar) + Nxxbar = np.array(Nxxbar) # The neutrino energy we set #E = dm21c4 * np.sin(2.*theta12) / (8.*np.pi*hbar*clight) @@ -83,31 +83,31 @@ def myassert(condition): assert(condition) # calculate errors - fee_analytic = Psurv(dm2_eff, sin2_eff, E) - error_ee = np.max(np.abs( fee - fee_analytic ) ) + Nee_analytic = Psurv(dm2_eff, sin2_eff, E) + error_ee = np.max(np.abs( Nee - Nee_analytic ) ) print("f_ee error:", error_ee) myassert( error_ee < tolerance ) - fxx_analytic = 1. - Psurv(dm2_eff, sin2_eff, E) - error_xx = np.max(np.abs( fxx - fxx_analytic ) ) + Nxx_analytic = 1. - Psurv(dm2_eff, sin2_eff, E) + error_xx = np.max(np.abs( Nxx - Nxx_analytic ) ) print("f_xx error:", error_xx) myassert( error_xx < tolerance ) - feebar_analytic = Psurv(dm2_effbar, sin2_effbar, E) - error_eebar = np.max(np.abs( feebar - feebar_analytic ) ) + Neebar_analytic = Psurv(dm2_effbar, sin2_effbar, E) + error_eebar = np.max(np.abs( Neebar - Neebar_analytic ) ) print("f_eebar error:", error_eebar) myassert( error_eebar < tolerance ) - fxxbar_analytic = 1. - Psurv(dm2_effbar, sin2_effbar, E) - error_xxbar = np.max(np.abs( fxxbar - fxxbar_analytic ) ) + Nxxbar_analytic = 1. - Psurv(dm2_effbar, sin2_effbar, E) + error_xxbar = np.max(np.abs( Nxxbar - Nxxbar_analytic ) ) print("f_xxbar error:", error_xxbar) myassert( error_xxbar < tolerance ) - conservation_error = np.max(np.abs( (fee+fxx) -1. )) + conservation_error = np.max(np.abs( (Nee+Nxx) -1. )) print("conservation_error:", conservation_error) myassert(conservation_error < tolerance) - conservation_errorbar = np.max(np.abs( (feebar+fxxbar) -1. )) + conservation_errorbar = np.max(np.abs( (Neebar+Nxxbar) -1. )) print("conservation_errorbar:", conservation_errorbar) myassert(conservation_errorbar < tolerance) diff --git a/Scripts/tests/plot_convergence.py b/Scripts/tests/plot_convergence.py index 4be2b8c3..2012f587 100644 --- a/Scripts/tests/plot_convergence.py +++ b/Scripts/tests/plot_convergence.py @@ -69,12 +69,12 @@ def plot_on_axis(self, axis, key, label, color): cdata = {} -cdata["fe"] = ConvergenceData("msw_test_fe.txt") +cdata["Ne"] = ConvergenceData("msw_test_Ne.txt") cdata["trapz"] = ConvergenceData("msw_test_trapz.txt") cdata["ssprk3"] = ConvergenceData("msw_test_ssprk3.txt") cdata["rk4"] = ConvergenceData("msw_test_rk4.txt") -variables = cdata["fe"].error_keys() +variables = cdata["Ne"].error_keys() for v in variables: fig, ax = plt.subplots() From 7f90f423c8ca11701078b4d86d0bf18278c20b9c Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 4 Jun 2024 14:12:09 -0400 Subject: [PATCH 132/276] change Trf to TrN --- Scripts/symbolic_hermitians/generate_code.py | 2 +- Source/DataReducer.cpp | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 4d90137d..a36da9a3 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -185,7 +185,7 @@ def delete_generated_files(): N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") Nlist = N.header_diagonals(); for i in range(len(Nlist)): - code.append("Trf += "+Nlist[i]+";") + code.append("TrN += "+Nlist[i]+";") write_code(code, os.path.join(args.emu_home, "Source/generated_files", "DataReducer.cpp_fill_particles")) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index b65489dd..4c3c15ab 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -64,7 +64,7 @@ void DataReducer::InitializeFiles() #endif } file0D.createDataSet("N_offdiag_mag(1|ccm)", dataspace, create_datatype(), props); - file0D.createDataSet("sumTrf", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrN", dataspace, create_datatype(), props); file0D.createDataSet("sumTrHN", dataspace, create_datatype(), props); #else @@ -104,7 +104,7 @@ void DataReducer::InitializeFiles() j++; outfile << j << ":N_offdiag_mag(1|ccm)\t"; j++; - outfile << j << ":sumTrf\t"; + outfile << j << ":sumTrN\t"; j++; outfile << j << ":sumTrHN\t"; outfile << std::endl; @@ -131,13 +131,13 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, auto particleResult = amrex::ParticleReduce< ReduceData >(neutrinos, [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { Real TrHN = p.rdata(PIdx::TrHN); - Real Trf = 0; + Real TrN = 0; #include "generated_files/DataReducer.cpp_fill_particles" - return GpuTuple{Trf,TrHN}; + return GpuTuple{TrN,TrHN}; }, reduce_ops); - Real Trf = amrex::get<0>(particleResult); + Real TrN = amrex::get<0>(particleResult); Real TrHN = amrex::get<1>(particleResult); - ParallelDescriptor::ReduceRealSum(Trf); + ParallelDescriptor::ReduceRealSum(TrN); ParallelDescriptor::ReduceRealSum(TrHN); //=============================// @@ -290,7 +290,7 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, #endif } append_0D(file0D, "N_offdiag_mag(1|ccm)", N_offdiag_mag); - append_0D(file0D, "sumTrf", Trf); + append_0D(file0D, "sumTrN", TrN); append_0D(file0D, "sumTrHN", TrHN); #else std::ofstream outfile; @@ -324,7 +324,7 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, #endif } outfile << N_offdiag_mag << "\t"; - outfile << Trf << "\t"; + outfile << TrN << "\t"; outfile << TrHN << "\t"; outfile << std::endl; outfile.close(); From 4022724b12aac070bb22a35f8a943fd199fa5b25 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 4 Jun 2024 14:17:33 -0400 Subject: [PATCH 133/276] remove unused line --- Source/FlavoredNeutrinoContainerInit.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index f6c4bfea..97cce76f 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -331,9 +331,6 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); } - - -//#include "generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length" } // loop over direction } // loop over location From 8859292fbf37b370e80715cec1a3a827d2b493e0 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Thu, 6 Jun 2024 02:04:59 -0400 Subject: [PATCH 134/276] Add file to read EoS table This is the beginning of implementation of HDF5 EoS table support. --- Source/EosTable.H | 71 +++++++++ Source/Evolve.cpp | 7 + Source/Make.package | 4 +- Source/ReadEosTable.cpp | 321 ++++++++++++++++++++++++++++++++++++++++ Source/main.cpp | 7 +- 5 files changed, 408 insertions(+), 2 deletions(-) create mode 100644 Source/EosTable.H create mode 100644 Source/ReadEosTable.cpp diff --git a/Source/EosTable.H b/Source/EosTable.H new file mode 100644 index 00000000..13bc5299 --- /dev/null +++ b/Source/EosTable.H @@ -0,0 +1,71 @@ +#ifndef EOS_TABLE_H +#define EOS_TABLE_H + +void ReadEosTable(); + +// OLD TODO: remove hard coded constants +// OLD TODO: introduce defines for table index of variables +#define HAVEGR 1 +#define MAX2(x, y) (((x) > (y)) ? (x) : (y)) +#define MIN2(x, y) (((x) < (y)) ? (x) : (y)) +#define NTABLES 19 +#define LENGTHGF 6.77269222552442e-06 +#define TIMEGF 2.03040204956746e05 +#define RHOGF 1.61887093132742e-18 +#define PRESSGF 1.80123683248503e-39 +#define EPSGF 1.11265005605362e-21 +#define INVRHOGF 6.17714470405638e17 +#define INVEPSGF 8.98755178736818e20 +#define INVPRESSGF 5.55174079257738e38 +#define CLIGHT 1 + +// table key +// 0 logpress +// 1 logenergy +// 2 entropy +// 3 munu +// 4 cs2 +// 5 dedt +// 6 dpdrhoe +// 7 dpderho +// 8 muhat +// 9 mu_e +// 10 mu_p +// 11 mu_n +// 12 Xa +// 13 Xh +// 14 Xn +// 15 Xp +// 16 Abar +// 17 Zbar +// 18 Gamma +// enum eos_var {i_logpress=0, i_logenergy, i_entropy, i_munu, i_cs2, i_dedt, + //i_dpdrhoe, i_dpderho, i_muhat, i_mu_e, i_mu_p, i_mu_n, i_Xa, + //i_Xh, i_Xn, i_Xp, i_Abar, i_Zbar, i_Gamma}; +//} + +namespace nuc_eos_private { + +// table data + extern double *alltables; + extern double *epstable; + extern double *logrho; + extern double *logtemp; + extern double *yes; + +#define EOSVAR(VAR) helperVarsReal[helperVarsEnumReal::VAR] +//WARNING: If the number of variables is increased here, then memory allocation for helperVarsReal/helperVarsInt pointer in GRHydroX_ReadEOSTable.cxx will also need to be increased. + enum helperVarsEnumReal { eos_rhomin, eos_rhomax, eos_tempmin, eos_tempmax, eos_yemin, eos_yemax, + energy_shift, dlintemp, dlintempi, drholintempi, dlintempyei, drholintempyei, + dtemp, dtempi, drho, drhoi, dye, dyei, drhotempi, drhoyei, dtempyei, + drhotempyei, eos_epsmin, eos_epsmax }; + extern double *helperVarsReal; + +#define EOSVAR_INT(VAR) helperVarsInt[helperVarsEnumInt::VAR] + enum helperVarsEnumInt { nrho, ntemp, nye } ; + extern int *helperVarsInt; + + +} + +#endif // EOS_TABLE_H diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 4b8d6335..f1b69273 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -205,6 +205,13 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } } else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0 or 1"); + + //---------------------------- EoS call ----------------------------------------------- + printf("(Evolve.cpp) munu[0][0] = %f\n", munu[0][0]); + double rho = 1.0e10; //g/cm^3 + double temperature = 1.0; //MeV + double Ye = 0.3; + //--------------------------------------------------------------------------------------- // calculate the equilibrium distribution. Really munu and temperature should be interpolated from the grid. for(int i=0; i<2; i++){ diff --git a/Source/Make.package b/Source/Make.package index b038507d..4f46334f 100644 --- a/Source/Make.package +++ b/Source/Make.package @@ -4,6 +4,7 @@ CEXE_sources += IO.cpp CEXE_sources += FlavoredNeutrinoContainerInit.cpp CEXE_sources += Evolve.cpp CEXE_sources += FlavoredNeutrinoContainer.cpp +CEXE_sources += ReadEosTable.cpp CEXE_headers += Evolve.H CEXE_headers += FlavoredNeutrinoContainer.H @@ -13,4 +14,5 @@ CEXE_headers += IO.H CEXE_headers += Parameters.H CEXE_headers += ParticleInterpolator.H CEXE_headers += DataReducer.H -CEXE_headers += ArithmeticArray.H \ No newline at end of file +CEXE_headers += ArithmeticArray.H +CEXE_headers += EosTable.H \ No newline at end of file diff --git a/Source/ReadEosTable.cpp b/Source/ReadEosTable.cpp new file mode 100644 index 00000000..2b873026 --- /dev/null +++ b/Source/ReadEosTable.cpp @@ -0,0 +1,321 @@ +#include + +#include + +//#include "GRHydroX_EOS.hxx" +#define H5_USE_16_API 1 +#include "hdf5.h" + +#include "EosTable.H" + +// mini NoMPI +#define HAVE_CAPABILITY_MPI //FIXME: This should be defined only when USE_MPI = TRUE +#ifdef HAVE_CAPABILITY_MPI +#include +#define BCAST(buffer, size) MPI_Bcast(buffer, size, MPI_BYTE, my_reader_process, MPI_COMM_WORLD) +#else +#define BCAST(buffer, size) do { /* do nothing */ } while(0) +#endif + +// Catch HDF5 errors +#define HDF5_ERROR(fn_call) \ + if(doIO) { \ + int _error_code = fn_call; \ + if (_error_code < 0) { \ + AMREX_ASSERT_WITH_MESSAGE(false, "HDF5 call failed"); \ + } \ + } + +using namespace amrex; + +static int file_is_readable(std::string filename); +static int file_is_readable(std::string filename) +{ + FILE* fp = NULL; + fp = fopen(filename.c_str(), "r"); + if(fp != NULL) + { + fclose(fp); + return 1; + } + return 0; +} + +namespace nuc_eos_private { + double *alltables; + double *epstable; + double *logrho; + double *logtemp; + double *yes; + double *helperVarsReal; + int *helperVarsInt; +} + +//TODO: Pass the /path/to/table here in the function argument +void ReadEosTable() { + using namespace nuc_eos_private; + + std::string nuceos_table_name = "/home/sshanka/000_UTK_projects/Emu/Exec/SFHo.h5"; //FIXME: Read from parameter file + amrex::Print() << "Using table: " << nuceos_table_name << std::endl; + + //TODO: + int my_reader_process = 0; //reader_process; + /*if (my_reader_process < 0 || my_reader_process >= CCTK_nProcs(cctkGH)) + { + CCTK_VWarn(CCTK_WARN_COMPLAIN, __LINE__, __FILE__, CCTK_THORNSTRING, + "Requested IO process %d out of range. Reverting to process 0.", my_reader_process); + my_reader_process = 0; + }*/ + + const int read_table_on_single_process = 1; + //const int doIO = !read_table_on_single_process || CCTK_MyProc(cctkGH) == my_reader_process; //TODO: + const int doIO = 1; + + hid_t file; + if (doIO && !file_is_readable(nuceos_table_name)) { + AMREX_ASSERT_WITH_MESSAGE(false, "Could not read nuceos_table_name"); + } + + HDF5_ERROR(file = H5Fopen(nuceos_table_name.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); + +// Use these two defines to easily read in a lot of variables in the same way +// The first reads in one variable of a given type completely +#define READ_BCAST_EOS_HDF5(NAME,VAR,TYPE,MEM,NELEMS) \ + do { \ + hid_t dataset; \ + HDF5_ERROR(dataset = H5Dopen(file, NAME)); \ + HDF5_ERROR(H5Dread(dataset, TYPE, MEM, H5S_ALL, H5P_DEFAULT, VAR)); \ + if (read_table_on_single_process) \ + BCAST (VAR, sizeof(*(VAR))*(NELEMS)); \ + HDF5_ERROR(H5Dclose(dataset)); \ + } while (0) +// The second reads a given variable into a hyperslab of the alltables_temp array +#define READ_BCAST_EOSTABLE_HDF5(NAME,OFF,DIMS) \ + do { \ + READ_BCAST_EOS_HDF5(NAME,&alltables_temp[(OFF)*(DIMS)[1]],H5T_NATIVE_DOUBLE,H5S_ALL,(DIMS)[1]); \ + } while (0) + + int nrho_; + int ntemp_; + int nye_; + // Read size of tables + READ_BCAST_EOS_HDF5("pointsrho", &nrho_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("pointstemp", &ntemp_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("pointsye", &nye_, H5T_NATIVE_INT, H5S_ALL, 1); + + printf("nrho = %d, ntemp = %d, nye = %d\n", nrho_, ntemp_, nye_); + +#if 0 + //Allocate managed memory arena on unified memory + ManagedArenaAllocator myManagedArena; + ManagedArenaAllocator myManagedArena_Int; + //EOS_array = myManagedArena.allocate(nrho_ * ntemp_ * nye_ * NTABLES * sizeof(double)); + + // Allocate memory for tables + CCTK_REAL *alltables_temp; + if (!(alltables_temp = myManagedArena.allocate(nrho_ * ntemp_ * nye_ * NTABLES) )) { + CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING, + "Cannot allocate memory for EOS table"); + } + if (!(logrho = myManagedArena.allocate(nrho_) )) { + CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING, + "Cannot allocate memory for EOS table"); + } + if (!(logtemp = myManagedArena.allocate(ntemp_) )) { + CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING, + "Cannot allocate memory for EOS table"); + } + if (!(yes = myManagedArena.allocate(nye_) )) { + CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING, + "Cannot allocate memory for EOS table"); + } + + // Prepare HDF5 to read hyperslabs into alltables_temp + hsize_t table_dims[2] = {NTABLES, (hsize_t)nrho_ * ntemp_ * nye_}; + hsize_t var3[2] = { 1, (hsize_t)nrho_ * ntemp_ * nye_}; + hid_t mem3 = H5Screate_simple(2, table_dims, NULL); + + // Read alltables_temp + READ_BCAST_EOSTABLE_HDF5("logpress", 0, table_dims); + READ_BCAST_EOSTABLE_HDF5("logenergy", 1, table_dims); + READ_BCAST_EOSTABLE_HDF5("entropy", 2, table_dims); + READ_BCAST_EOSTABLE_HDF5("munu", 3, table_dims); + READ_BCAST_EOSTABLE_HDF5("cs2", 4, table_dims); + READ_BCAST_EOSTABLE_HDF5("dedt", 5, table_dims); + READ_BCAST_EOSTABLE_HDF5("dpdrhoe", 6, table_dims); + READ_BCAST_EOSTABLE_HDF5("dpderho", 7, table_dims); + // chemical potentials + READ_BCAST_EOSTABLE_HDF5("muhat", 8, table_dims); + READ_BCAST_EOSTABLE_HDF5("mu_e", 9, table_dims); + READ_BCAST_EOSTABLE_HDF5("mu_p", 10, table_dims); + READ_BCAST_EOSTABLE_HDF5("mu_n", 11, table_dims); + // compositions + READ_BCAST_EOSTABLE_HDF5("Xa", 12, table_dims); + READ_BCAST_EOSTABLE_HDF5("Xh", 13, table_dims); + READ_BCAST_EOSTABLE_HDF5("Xn", 14, table_dims); + READ_BCAST_EOSTABLE_HDF5("Xp", 15, table_dims); + // average nucleus + READ_BCAST_EOSTABLE_HDF5("Abar", 16, table_dims); + READ_BCAST_EOSTABLE_HDF5("Zbar", 17, table_dims); + // Gamma + READ_BCAST_EOSTABLE_HDF5("gamma", 18, table_dims); + + double energy_shift_; + // Read additional tables and variables + READ_BCAST_EOS_HDF5("logrho", logrho, H5T_NATIVE_DOUBLE, H5S_ALL, nrho_); + READ_BCAST_EOS_HDF5("logtemp", logtemp, H5T_NATIVE_DOUBLE, H5S_ALL, ntemp_); + READ_BCAST_EOS_HDF5("ye", yes, H5T_NATIVE_DOUBLE, H5S_ALL, nye_); + READ_BCAST_EOS_HDF5("energy_shift", &energy_shift_, H5T_NATIVE_DOUBLE, H5S_ALL, 1); + + HDF5_ERROR(H5Sclose(mem3)); + HDF5_ERROR(H5Fclose(file)); + + // change ordering of alltables array so that + // the table kind is the fastest changing index + if (!(alltables = myManagedArena.allocate(nrho_ * ntemp_ * nye_ * NTABLES) )) { + CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING, + "Cannot allocate memory for EOS table"); + } + for(int iv = 0;iv epsmax) && (epstable[i] < 1.0e150)){ + epsmax = epstable[i]; + } + if (epstable[i] < epsmin){ + epsmin = epstable[i]; + } + } + + //TODO: Is it correct to subtract energy_shift here? + EOSVAR(eos_epsmin) = epsmin - energy_shift_; + EOSVAR(eos_epsmax) = epsmax - energy_shift_; + + printf(" EOS:rhomin = %.5e\n", EOSVAR(eos_rhomin)); + printf(" EOS:rhomax = %.5e\n", EOSVAR(eos_rhomax)); + printf(" EOS:tempmin = %.5e\n", EOSVAR(eos_tempmin)); + printf(" EOS:tempmax = %.5e\n", EOSVAR(eos_tempmax)); + printf(" EOS:yemin = %.6f\n", EOSVAR(eos_yemin)); + printf(" EOS:yemax = %.6f\n", EOSVAR(eos_yemax)); + +#endif + +} // ReadEOSTable + + + diff --git a/Source/main.cpp b/Source/main.cpp index d541c970..5077725c 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -32,6 +32,7 @@ #include "Constants.H" #include "IO.H" #include "DataReducer.H" +#include "EosTable.H" using namespace amrex; @@ -88,8 +89,12 @@ void evolve_flavor(const TestParams* parms) // initialize the grid variable names GIdx::Initialize(); + // read the EoS table + amrex::Print() << "Reading EoS table... " << std::endl; + ReadEosTable(); + // Initialize particles on the domain - amrex::Print() << "Initializing particles... "; + amrex::Print() << "Initializing particles... " << std::endl; // We store old-time and new-time data FlavoredNeutrinoContainer neutrinos_old(geom, dm, ba); From 11b6e4c9329851a879d450cfc499370e293d353c Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Thu, 6 Jun 2024 12:30:29 -0400 Subject: [PATCH 135/276] Finish reading EoS table onto memory --- Source/ReadEosTable.cpp | 293 ++++++++++++++++++++-------------------- 1 file changed, 149 insertions(+), 144 deletions(-) diff --git a/Source/ReadEosTable.cpp b/Source/ReadEosTable.cpp index 2b873026..040dee67 100644 --- a/Source/ReadEosTable.cpp +++ b/Source/ReadEosTable.cpp @@ -56,7 +56,7 @@ void ReadEosTable() { using namespace nuc_eos_private; std::string nuceos_table_name = "/home/sshanka/000_UTK_projects/Emu/Exec/SFHo.h5"; //FIXME: Read from parameter file - amrex::Print() << "Using table: " << nuceos_table_name << std::endl; + amrex::Print() << "(ReadEosTable.cpp) Using table: " << nuceos_table_name << std::endl; //TODO: int my_reader_process = 0; //reader_process; @@ -103,156 +103,161 @@ void ReadEosTable() { READ_BCAST_EOS_HDF5("pointstemp", &ntemp_, H5T_NATIVE_INT, H5S_ALL, 1); READ_BCAST_EOS_HDF5("pointsye", &nye_, H5T_NATIVE_INT, H5S_ALL, 1); - printf("nrho = %d, ntemp = %d, nye = %d\n", nrho_, ntemp_, nye_); + printf("(ReadEosTable.cpp) nrho = %d, ntemp = %d, nye = %d\n", nrho_, ntemp_, nye_); -#if 0 - //Allocate managed memory arena on unified memory - ManagedArenaAllocator myManagedArena; - ManagedArenaAllocator myManagedArena_Int; - //EOS_array = myManagedArena.allocate(nrho_ * ntemp_ * nye_ * NTABLES * sizeof(double)); + //Allocate managed memory arena on unified memory + ManagedArenaAllocator myManagedArena; + ManagedArenaAllocator myManagedArena_Int; + + // Allocate memory for tables + double *alltables_temp; + if (!(alltables_temp = myManagedArena.allocate(nrho_ * ntemp_ * nye_ * NTABLES) )) { + printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); + assert(0); + } + if (!(logrho = myManagedArena.allocate(nrho_) )) { + printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); + assert(0); + } + if (!(logtemp = myManagedArena.allocate(ntemp_) )) { + printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); + assert(0); + } + if (!(yes = myManagedArena.allocate(nye_) )) { + printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); + assert(0); + } - // Allocate memory for tables - CCTK_REAL *alltables_temp; - if (!(alltables_temp = myManagedArena.allocate(nrho_ * ntemp_ * nye_ * NTABLES) )) { - CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING, - "Cannot allocate memory for EOS table"); - } - if (!(logrho = myManagedArena.allocate(nrho_) )) { - CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING, - "Cannot allocate memory for EOS table"); - } - if (!(logtemp = myManagedArena.allocate(ntemp_) )) { - CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING, - "Cannot allocate memory for EOS table"); - } - if (!(yes = myManagedArena.allocate(nye_) )) { - CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING, - "Cannot allocate memory for EOS table"); - } + // Prepare HDF5 to read hyperslabs into alltables_temp + hsize_t table_dims[2] = {NTABLES, (hsize_t)nrho_ * ntemp_ * nye_}; + hsize_t var3[2] = { 1, (hsize_t)nrho_ * ntemp_ * nye_}; + hid_t mem3 = H5Screate_simple(2, table_dims, NULL); + + // Read alltables_temp + READ_BCAST_EOSTABLE_HDF5("logpress", 0, table_dims); + READ_BCAST_EOSTABLE_HDF5("logenergy", 1, table_dims); + READ_BCAST_EOSTABLE_HDF5("entropy", 2, table_dims); + READ_BCAST_EOSTABLE_HDF5("munu", 3, table_dims); + READ_BCAST_EOSTABLE_HDF5("cs2", 4, table_dims); + READ_BCAST_EOSTABLE_HDF5("dedt", 5, table_dims); + READ_BCAST_EOSTABLE_HDF5("dpdrhoe", 6, table_dims); + READ_BCAST_EOSTABLE_HDF5("dpderho", 7, table_dims); + // chemical potentials + READ_BCAST_EOSTABLE_HDF5("muhat", 8, table_dims); + READ_BCAST_EOSTABLE_HDF5("mu_e", 9, table_dims); + READ_BCAST_EOSTABLE_HDF5("mu_p", 10, table_dims); + READ_BCAST_EOSTABLE_HDF5("mu_n", 11, table_dims); + // compositions + READ_BCAST_EOSTABLE_HDF5("Xa", 12, table_dims); + READ_BCAST_EOSTABLE_HDF5("Xh", 13, table_dims); + READ_BCAST_EOSTABLE_HDF5("Xn", 14, table_dims); + READ_BCAST_EOSTABLE_HDF5("Xp", 15, table_dims); + // average nucleus + READ_BCAST_EOSTABLE_HDF5("Abar", 16, table_dims); + READ_BCAST_EOSTABLE_HDF5("Zbar", 17, table_dims); + // Gamma + READ_BCAST_EOSTABLE_HDF5("gamma", 18, table_dims); + + double energy_shift_; + // Read additional tables and variables + READ_BCAST_EOS_HDF5("logrho", logrho, H5T_NATIVE_DOUBLE, H5S_ALL, nrho_); + READ_BCAST_EOS_HDF5("logtemp", logtemp, H5T_NATIVE_DOUBLE, H5S_ALL, ntemp_); + READ_BCAST_EOS_HDF5("ye", yes, H5T_NATIVE_DOUBLE, H5S_ALL, nye_); + READ_BCAST_EOS_HDF5("energy_shift", &energy_shift_, H5T_NATIVE_DOUBLE, H5S_ALL, 1); + + HDF5_ERROR(H5Sclose(mem3)); + HDF5_ERROR(H5Fclose(file)); + + + // change ordering of alltables array so that + // the table kind is the fastest changing index + if (!(alltables = myManagedArena.allocate(nrho_ * ntemp_ * nye_ * NTABLES) )) { + printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); + assert(0); + } - // Prepare HDF5 to read hyperslabs into alltables_temp - hsize_t table_dims[2] = {NTABLES, (hsize_t)nrho_ * ntemp_ * nye_}; - hsize_t var3[2] = { 1, (hsize_t)nrho_ * ntemp_ * nye_}; - hid_t mem3 = H5Screate_simple(2, table_dims, NULL); - - // Read alltables_temp - READ_BCAST_EOSTABLE_HDF5("logpress", 0, table_dims); - READ_BCAST_EOSTABLE_HDF5("logenergy", 1, table_dims); - READ_BCAST_EOSTABLE_HDF5("entropy", 2, table_dims); - READ_BCAST_EOSTABLE_HDF5("munu", 3, table_dims); - READ_BCAST_EOSTABLE_HDF5("cs2", 4, table_dims); - READ_BCAST_EOSTABLE_HDF5("dedt", 5, table_dims); - READ_BCAST_EOSTABLE_HDF5("dpdrhoe", 6, table_dims); - READ_BCAST_EOSTABLE_HDF5("dpderho", 7, table_dims); - // chemical potentials - READ_BCAST_EOSTABLE_HDF5("muhat", 8, table_dims); - READ_BCAST_EOSTABLE_HDF5("mu_e", 9, table_dims); - READ_BCAST_EOSTABLE_HDF5("mu_p", 10, table_dims); - READ_BCAST_EOSTABLE_HDF5("mu_n", 11, table_dims); - // compositions - READ_BCAST_EOSTABLE_HDF5("Xa", 12, table_dims); - READ_BCAST_EOSTABLE_HDF5("Xh", 13, table_dims); - READ_BCAST_EOSTABLE_HDF5("Xn", 14, table_dims); - READ_BCAST_EOSTABLE_HDF5("Xp", 15, table_dims); - // average nucleus - READ_BCAST_EOSTABLE_HDF5("Abar", 16, table_dims); - READ_BCAST_EOSTABLE_HDF5("Zbar", 17, table_dims); - // Gamma - READ_BCAST_EOSTABLE_HDF5("gamma", 18, table_dims); - - double energy_shift_; - // Read additional tables and variables - READ_BCAST_EOS_HDF5("logrho", logrho, H5T_NATIVE_DOUBLE, H5S_ALL, nrho_); - READ_BCAST_EOS_HDF5("logtemp", logtemp, H5T_NATIVE_DOUBLE, H5S_ALL, ntemp_); - READ_BCAST_EOS_HDF5("ye", yes, H5T_NATIVE_DOUBLE, H5S_ALL, nye_); - READ_BCAST_EOS_HDF5("energy_shift", &energy_shift_, H5T_NATIVE_DOUBLE, H5S_ALL, 1); - - HDF5_ERROR(H5Sclose(mem3)); - HDF5_ERROR(H5Fclose(file)); - - // change ordering of alltables array so that - // the table kind is the fastest changing index - if (!(alltables = myManagedArena.allocate(nrho_ * ntemp_ * nye_ * NTABLES) )) { - CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING, - "Cannot allocate memory for EOS table"); - } - for(int iv = 0;iv epsmax) && (epstable[i] < 1.0e150)){ epsmax = epstable[i]; @@ -306,14 +311,14 @@ void ReadEosTable() { EOSVAR(eos_epsmin) = epsmin - energy_shift_; EOSVAR(eos_epsmax) = epsmax - energy_shift_; - printf(" EOS:rhomin = %.5e\n", EOSVAR(eos_rhomin)); - printf(" EOS:rhomax = %.5e\n", EOSVAR(eos_rhomax)); - printf(" EOS:tempmin = %.5e\n", EOSVAR(eos_tempmin)); - printf(" EOS:tempmax = %.5e\n", EOSVAR(eos_tempmax)); - printf(" EOS:yemin = %.6f\n", EOSVAR(eos_yemin)); - printf(" EOS:yemax = %.6f\n", EOSVAR(eos_yemax)); - -#endif + printf("(ReadEosTable.cpp) EOS:rhomin = %.5e g/cm^3\n", EOSVAR(eos_rhomin)); + printf("(ReadEosTable.cpp) EOS:rhomax = %.5e g/cm^3\n", EOSVAR(eos_rhomax)); + printf("(ReadEosTable.cpp) EOS:tempmin = %.4f MeV\n", EOSVAR(eos_tempmin)); + printf("(ReadEosTable.cpp) EOS:tempmax = %.4f MeV\n", EOSVAR(eos_tempmax)); + printf("(ReadEosTable.cpp) EOS:yemin = %.4f\n", EOSVAR(eos_yemin)); + printf("(ReadEosTable.cpp) EOS:yemax = %.4f\n", EOSVAR(eos_yemax)); + + printf("(ReadEosTable.cpp) Finished reading EoS table!\n"); } // ReadEOSTable From eb31a55e6266358ac8416299b09ec6d104cb5e7f Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Thu, 6 Jun 2024 14:27:23 -0400 Subject: [PATCH 136/276] Add infrastructure to get the interolated quantities of Tabulated EoS --- Source/EosTableFunctions.H | 68 ++++++++++++++++++++++++++++++++++++++ Source/Evolve.cpp | 15 ++++++++- Source/Make.package | 3 +- 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 Source/EosTableFunctions.H diff --git a/Source/EosTableFunctions.H b/Source/EosTableFunctions.H new file mode 100644 index 00000000..d17d3f12 --- /dev/null +++ b/Source/EosTableFunctions.H @@ -0,0 +1,68 @@ +#ifndef EOSTABLEFUCNTIONS_HXX +#define EOSTABLEFUCNTIONS_HXX + +struct EOS_tabulated { + double *alltables, *epstable, *logrho, *logtemp, *yes, *helperVarsReal; + int *helperVarsInt; + + //constructor for Tabulated EOS + AMREX_GPU_DEVICE AMREX_GPU_HOST EOS_tabulated() = default;//Need to keep it + AMREX_GPU_DEVICE AMREX_GPU_HOST EOS_tabulated(double *alltables, double *epstable, double *logrho, + double *logtemp, double *yes, double *helperVarsReal, int *helperVarsInt): + alltables(alltables), epstable(epstable), logrho(logrho), logtemp(logtemp), + yes(yes), helperVarsReal(helperVarsReal), helperVarsInt(helperVarsInt) {} + + //--------------- get helperVarsReal pointer --------------------------------------------- + AMREX_GPU_DEVICE AMREX_GPU_HOST double *get_helperVarsReal() const { + return helperVarsReal; + };//get_helperVarsReal + + + //--------------- get entropy/munu for tabulated EOS --------------------------------------------- + AMREX_GPU_DEVICE AMREX_GPU_HOST void get_entropy_munu(double rho, double temperature, double Ye, + double &entropy, double &munu, + int &keyerr, int &anyerr) const { + + /*nuc_eos_m_kt1_entropy_munu(rho, temperature, Ye, entropy, munu, keyerr, anyerr, + alltables, logrho, logtemp, yes, helperVarsReal, helperVarsInt);*/ + keyerr = -404; + anyerr = -404; + entropy = -404.22; + munu = -404.22; + return; + + /*Actual steps: + (1) check bounds + (2) get interp spots + (3) do interpolation to get press and eps + Let's wrap this up in a funtion calleds nuc_eos_press_eps_kT1 (~nuc_eos_press.c) + This will further require another include ~helpers.hh + Pass errors such as anyerr or keyerr to the calling routine. + */ + };//get_entropy_munu + + + //--------------- get pressure for tabulated EOS --------------------------------------------- + AMREX_GPU_DEVICE AMREX_GPU_HOST void get_press(double rho, double temperature, double Ye, + double &press, int &keyerr, int &anyerr) const { + + //int keyerr, anyerr; + /*nuc_eos_m_kt1_press(rho, temperature, Ye, press, keyerr, anyerr, + alltables, logrho, logtemp, yes, helperVarsReal, helperVarsInt);*/ + return; + };//get_press + + + //--------------- get composition xh for tabulated EOS --------------------------------------------- + AMREX_GPU_DEVICE AMREX_GPU_HOST void get_xh(double rho, double temperature, double Ye, + double &xh, int &keyerr, int &anyerr) const { + + //int keyerr, anyerr; + /*nuc_eos_m_kt1_xh(rho, temperature, Ye, xh, keyerr, anyerr, + alltables, logrho, logtemp, yes, helperVarsReal, helperVarsInt);*/ + return; + };//get_xh + +}; //struct EOS_tabulated + +#endif // EOSTABLEFUCNTIONS_HXX \ No newline at end of file diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index f1b69273..e38b8f49 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -3,6 +3,9 @@ #include "ParticleInterpolator.H" #include +#include "EosTableFunctions.H" +#include "EosTable.H" + using namespace amrex; namespace GIdx @@ -150,6 +153,11 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M const int shape_factor_order_y = geom.Domain().length(1) > 1 ? SHAPE_FACTOR_ORDER : 0; const int shape_factor_order_z = geom.Domain().length(2) > 1 ? SHAPE_FACTOR_ORDER : 0; + //Create the EoS table object + using namespace nuc_eos_private; + EOS_tabulated EOS_tabulated_obj(alltables, epstable, logrho, logtemp, + yes, helperVarsReal, helperVarsInt); + amrex::MeshToParticle(neutrinos_rhs, state, 0, [=] AMREX_GPU_DEVICE (FlavoredNeutrinoContainer::ParticleType& p, amrex::Array4 const& sarr) @@ -207,10 +215,15 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0 or 1"); //---------------------------- EoS call ----------------------------------------------- - printf("(Evolve.cpp) munu[0][0] = %f\n", munu[0][0]); double rho = 1.0e10; //g/cm^3 double temperature = 1.0; //MeV double Ye = 0.3; + + double entropy_out, munu_out; + int keyerr, anyerr; + EOS_tabulated_obj.get_entropy_munu(rho, temperature, Ye, entropy_out, munu_out, keyerr, anyerr); + printf("(Evolve.cpp) munu interpolated = %f\n", munu_out); + //munu[0][0] = munu; //--------------------------------------------------------------------------------------- // calculate the equilibrium distribution. Really munu and temperature should be interpolated from the grid. diff --git a/Source/Make.package b/Source/Make.package index 4f46334f..a00b9eeb 100644 --- a/Source/Make.package +++ b/Source/Make.package @@ -15,4 +15,5 @@ CEXE_headers += Parameters.H CEXE_headers += ParticleInterpolator.H CEXE_headers += DataReducer.H CEXE_headers += ArithmeticArray.H -CEXE_headers += EosTable.H \ No newline at end of file +CEXE_headers += EosTable.H +CEXE_headers += EosTableFunctions.H \ No newline at end of file From 14ef26f13cc4613e1ac8a957e01ede120e3b9e24 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Thu, 6 Jun 2024 15:36:41 -0400 Subject: [PATCH 137/276] Finish interpolation routine for munu/entropy Routines to interpolate other variables have the infrastructure but not yet fully implemented. --- Source/EosTableFunctions.H | 15 ++-- Source/EosTableHelpers.H | 176 +++++++++++++++++++++++++++++++++++++ Source/Evolve.cpp | 6 +- Source/Make.package | 3 +- 4 files changed, 189 insertions(+), 11 deletions(-) create mode 100644 Source/EosTableHelpers.H diff --git a/Source/EosTableFunctions.H b/Source/EosTableFunctions.H index d17d3f12..daa23377 100644 --- a/Source/EosTableFunctions.H +++ b/Source/EosTableFunctions.H @@ -1,6 +1,9 @@ #ifndef EOSTABLEFUCNTIONS_HXX #define EOSTABLEFUCNTIONS_HXX + +#include "EosTableHelpers.H" + struct EOS_tabulated { double *alltables, *epstable, *logrho, *logtemp, *yes, *helperVarsReal; int *helperVarsInt; @@ -23,14 +26,10 @@ struct EOS_tabulated { double &entropy, double &munu, int &keyerr, int &anyerr) const { - /*nuc_eos_m_kt1_entropy_munu(rho, temperature, Ye, entropy, munu, keyerr, anyerr, - alltables, logrho, logtemp, yes, helperVarsReal, helperVarsInt);*/ - keyerr = -404; - anyerr = -404; - entropy = -404.22; - munu = -404.22; + nuc_eos_entropy_munu(rho, temperature, Ye, entropy, munu, keyerr, anyerr, + alltables, logrho, logtemp, yes, helperVarsReal, helperVarsInt); return; - + /*Actual steps: (1) check bounds (2) get interp spots @@ -49,6 +48,7 @@ struct EOS_tabulated { //int keyerr, anyerr; /*nuc_eos_m_kt1_press(rho, temperature, Ye, press, keyerr, anyerr, alltables, logrho, logtemp, yes, helperVarsReal, helperVarsInt);*/ + assert(0); //FIXME: Add actual routine. return; };//get_press @@ -60,6 +60,7 @@ struct EOS_tabulated { //int keyerr, anyerr; /*nuc_eos_m_kt1_xh(rho, temperature, Ye, xh, keyerr, anyerr, alltables, logrho, logtemp, yes, helperVarsReal, helperVarsInt);*/ + assert(0); //FIXME: Add actual routine. return; };//get_xh diff --git a/Source/EosTableHelpers.H b/Source/EosTableHelpers.H new file mode 100644 index 00000000..13a259a1 --- /dev/null +++ b/Source/EosTableHelpers.H @@ -0,0 +1,176 @@ +#ifndef EOSTABLEHELPERS_HXX +#define EOSTABLEHELPERS_HXX + +#include "EosTable.H" + +//Check whether input (rho, temperature, Ye) are within the table bounds. +static inline AMREX_GPU_DEVICE AMREX_GPU_HOST int +checkbounds(const double xrho, const double xtemp, const double xye, double *helperVarsReal) { + + using namespace nuc_eos_private; + // keyerr codes: + // 101 -- Y_e too high + // 102 -- Y_e too low + // 103 -- temp too high (if keytemp = 1) + // 104 -- temp too low (if keytemp = 1) + // 105 -- rho too high + // 106 -- rho too low + + if(xrho > EOSVAR(eos_rhomax)) { + return 105; + } + if(xrho < EOSVAR(eos_rhomin)) { + return 106; + } + if(xye > EOSVAR(eos_yemax)) { + return 101; + } + if(xye < EOSVAR(eos_yemin)) { + return 102; + } + if(xtemp > EOSVAR(eos_tempmax)) { + return 103; + } + if(xtemp < EOSVAR(eos_tempmin)) { + return 104; + } + return 0; + +}//function checkbounds + +//Get the indices to prepare for interpolation. +static inline AMREX_GPU_DEVICE AMREX_GPU_HOST void +get_interp_spots(const double x, const double y, const double z, double &delx, + double &dely, double &delz, int *idx, double *logrho, + double *logtemp, double *yes, double *helperVarsReal, int *helperVarsInt) +{ + using namespace nuc_eos_private; + + int ix = 1 + (int)( (x - logrho[0] - 1.0e-10) * EOSVAR(drhoi) ); + int iy = 1 + (int)( (y - logtemp[0] - 1.0e-10) * EOSVAR(dtempi) ); + int iz = 1 + (int)( (z - yes[0] - 1.0e-10) * EOSVAR(dyei) ); + + const int nrho = EOSVAR_INT(nrho); + const int ntemp = EOSVAR_INT(ntemp); + const int nye = EOSVAR_INT(nye); + + ix = MAX2( 1, MIN2( ix, nrho-1 ) ); + iy = MAX2( 1, MIN2( iy, ntemp-1 ) ); + iz = MAX2( 1, MIN2( iz, nye-1 ) ); + + idx[0] = NTABLES*(ix + nrho*(iy + ntemp*iz)); + idx[1] = NTABLES*((ix-1) + nrho*(iy + ntemp*iz)); + idx[2] = NTABLES*(ix + nrho*((iy-1) + ntemp*iz)); + idx[3] = NTABLES*(ix + nrho*(iy + ntemp*(iz-1))); + idx[4] = NTABLES*((ix-1) + nrho*((iy-1) + ntemp*iz)); + idx[5] = NTABLES*((ix-1) + nrho*(iy + ntemp*(iz-1))); + idx[6] = NTABLES*(ix + nrho*((iy-1) + ntemp*(iz-1))); + idx[7] = NTABLES*((ix-1) + nrho*((iy-1) + ntemp*(iz-1))); + + // set up aux vars for interpolation + delx = logrho[ix] - x; + dely = logtemp[iy] - y; + delz = yes[iz] - z; + + return; +} // function get_interp_spots + +//Perform the interpolation +static inline AMREX_GPU_DEVICE AMREX_GPU_HOST void +nuc_eos_C_linterp_one(const int *idx, const double delx, const double dely, + const double delz, double &f, const int iv, + double *alltables, double *helperVarsReal) +{ + using namespace nuc_eos_private; + + // helper variables + double fh[8], a[8]; + + fh[0] = alltables[iv+idx[0]]; + fh[1] = alltables[iv+idx[1]]; + fh[2] = alltables[iv+idx[2]]; + fh[3] = alltables[iv+idx[3]]; + fh[4] = alltables[iv+idx[4]]; + fh[5] = alltables[iv+idx[5]]; + fh[6] = alltables[iv+idx[6]]; + fh[7] = alltables[iv+idx[7]]; + + // set up coeffs of interpolation polynomical and + // evaluate function values + a[0] = fh[0]; + a[1] = EOSVAR(drhoi) * ( fh[1] - fh[0] ); + a[2] = EOSVAR(dtempi) * ( fh[2] - fh[0] ); + a[3] = EOSVAR(dyei) * ( fh[3] - fh[0] ); + a[4] = EOSVAR(drhotempi) * ( fh[4] - fh[1] - fh[2] + fh[0] ); + a[5] = EOSVAR(drhoyei) * ( fh[5] - fh[1] - fh[3] + fh[0] ); + a[6] = EOSVAR(dtempyei) * ( fh[6] - fh[2] - fh[3] + fh[0] ); + a[7] = EOSVAR(drhotempyei) * ( fh[7] - fh[0] + fh[1] + fh[2] + + fh[3] - fh[4] - fh[5] - fh[6] ); + + f = a[0] + a[1] * delx + + a[2] * dely + + a[3] * delz + + a[4] * delx * dely + + a[5] * delx * delz + + a[6] * dely * delz + + a[7] * delx * dely * delz; + + return; +} // function nuc_eos_C_linterp_one + + + +//Main function for entropy and munu given (rho, temperature, Ye) +static inline AMREX_GPU_DEVICE AMREX_GPU_HOST void +nuc_eos_entropy_munu(const double rho, const double temperature, + const double Ye, double &entropy, double &munu, + int &keyerr, int &anyerr, double *alltables, + double *logrho, double *logtemp, double *yes, + double *helperVarsReal, int *helperVarsInt) +{ + using namespace nuc_eos_private; + + int anyerr_ = 0; + // check if we are fine + int keyerr_ = checkbounds(rho, temperature, Ye, helperVarsReal); + if(keyerr_ != 0) anyerr_ = 1; + + keyerr = keyerr_ ; anyerr = anyerr_; + // Abort if there is any error in checkbounds. + // This should never happen and the program should abort with + // a fatal error anyway. No point in doing any further EOS calculations. + if(anyerr_){ + printf("Error in checkbounds::%s::%d::%s, keyerr = %d\n", __FILE__, __LINE__, __func__, keyerr); + printf(" rho = %.5e, temperature = %.5e, Ye = %.6f \n", rho, temperature, Ye); + return; + } + + int idx[8]; + double delx,dely,delz; + const double lrho = log(rho); + const double ltemp = log(temperature); + + get_interp_spots(lrho, ltemp, Ye, delx, dely, delz, idx, + logrho, logtemp, yes, helperVarsReal, helperVarsInt); + + double entropy_ , munu_; + { + const int iv = 2; + nuc_eos_C_linterp_one(idx, delx, dely, delz, entropy_, iv, alltables, helperVarsReal); + } + + { + const int iv = 3; + nuc_eos_C_linterp_one(idx, delx, dely, delz, munu_, iv, alltables, helperVarsReal); + } + + // Assign values to reference variables: + entropy = entropy_; + munu = munu_; + + return; +}//nuc_eos_entropy_munu + + + +#endif //EOSTABLEHELPERS_HXX \ No newline at end of file diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index e38b8f49..932d16fe 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -215,9 +215,9 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0 or 1"); //---------------------------- EoS call ----------------------------------------------- - double rho = 1.0e10; //g/cm^3 - double temperature = 1.0; //MeV - double Ye = 0.3; + double rho = 1.7e2; //g/cm^3 + double temperature = 0.01; //MeV + double Ye = 0.01; double entropy_out, munu_out; int keyerr, anyerr; diff --git a/Source/Make.package b/Source/Make.package index a00b9eeb..20ecfc9d 100644 --- a/Source/Make.package +++ b/Source/Make.package @@ -16,4 +16,5 @@ CEXE_headers += ParticleInterpolator.H CEXE_headers += DataReducer.H CEXE_headers += ArithmeticArray.H CEXE_headers += EosTable.H -CEXE_headers += EosTableFunctions.H \ No newline at end of file +CEXE_headers += EosTableFunctions.H +CEXE_headers += EosTableHelpers.H \ No newline at end of file From a019f8e1a94096e1bd06d3f18fa4e8d8e2b74515 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Thu, 6 Jun 2024 21:36:23 -0400 Subject: [PATCH 138/276] Disable AMREX_USE_HDF5 use of AMREX_USE_HDF5 leads tp errors when we try to use HDF5 with CUDA --- Source/DataReducer.H | 5 +++++ Source/DataReducer.cpp | 5 +++++ Source/IO.cpp | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/Source/DataReducer.H b/Source/DataReducer.H index 008ad833..c3f2209e 100644 --- a/Source/DataReducer.H +++ b/Source/DataReducer.H @@ -7,6 +7,11 @@ #include #include #include + +//We use the AMReX binary format to write data for now +//That's because the HDF5 write format gives errors when running with CUDA. +#undef AMREX_USE_HDF5 + #ifdef AMREX_USE_HDF5 #include <../submodules/HighFive/include/highfive/H5File.hpp> #include <../submodules/HighFive/include/highfive/H5DataSpace.hpp> diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index 4c3c15ab..b4da473c 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -4,6 +4,11 @@ #include "ArithmeticArray.H" #include #include + +//We use the AMReX binary format to write data for now +//That's because the HDF5 write format gives errors when running with CUDA. +#undef AMREX_USE_HDF5 + #ifdef AMREX_USE_HDF5 #include <../submodules/HighFive/include/highfive/H5File.hpp> #include <../submodules/HighFive/include/highfive/H5DataSpace.hpp> diff --git a/Source/IO.cpp b/Source/IO.cpp index 8d5bd473..2a839ef8 100644 --- a/Source/IO.cpp +++ b/Source/IO.cpp @@ -2,6 +2,10 @@ #include #include +//We use the AMReX binary format to write data for now +//That's because the HDF5 write format gives errors when running with CUDA. +#undef AMREX_USE_HDF5 + #ifdef AMREX_USE_HDF5 #include "hdf5.h" #endif From d6c797adf3c05e6f6ad71adb2e68547cd3ef1c26 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 11 Jun 2024 15:37:39 -0400 Subject: [PATCH 139/276] give Jenkins access to the tables stored on the workstation --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 06fc598b..10c7fa4b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ pipeline { triggers { pollSCM('') } // Run tests whenever a new commit is detected. - agent { dockerfile {args '--gpus all'}} // Use the Dockerfile defined in the root Flash-X directory + agent { dockerfile {args '--gpus all -v /mnt/scratch/tables:/tables:ro'}} // Use the Dockerfile defined in the root Flash-X directory environment { // Get rid of Read -1, expected , errno =1 error // See https://github.com/open-mpi/ompi/issues/4948 From 7da9496f5a10da6119b3e49a9f9b0b4a40e8fa22 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Mon, 17 Jun 2024 12:44:00 -0400 Subject: [PATCH 140/276] Evolve.cpp: write comments --- Source/Evolve.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 932d16fe..da7fd1ed 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -204,15 +204,18 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } else if(parms->IMFP_method==1){ // use the IMFPs from the input file - for(int i=0; i<2; i++){ - for(int j=0; jneutrino or 1->antineutrino + for(int j=0; jelectron, 1->heavy(muon), 2->heavy(tau); all heavy same for current table IMFP_abs[i][j] = parms->IMFP_abs[i][j]; IMFP_scat[i][j] = parms->IMFP_scat[i][j]; - munu[i][j] = parms->munu[i][j]; + munu[i][j] = parms->munu[i][j]; //munu -> "mu_e" - "muhat" } } } else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0 or 1"); + + //for energy, we are specifying neutrino energy right now. + // double neutrino_energy = p.rdata(PIdx::pupt); locate energy bin using this. //---------------------------- EoS call ----------------------------------------------- double rho = 1.7e2; //g/cm^3 From 0428f1d1db16f43723815374dbadd5cc2a6e12f8 Mon Sep 17 00:00:00 2001 From: Swapnil Shankar <55387159+shankar-1729@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:31:21 -0400 Subject: [PATCH 141/276] Fix initial huge memory allocation issue (#96) --- Source/main.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Source/main.cpp b/Source/main.cpp index c19a282f..5bbb2118 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -231,6 +231,15 @@ void evolve_flavor(const TestParams* parms) int main(int argc, char* argv[]) { + //In amrex::Initialize, a large amount of GPU device memory is allocated and is kept in The_Arena(). + //The default is 3/4 of the total device memory. + //It can be changed with a ParmParse parameter, amrex.the_arena_init_size, in the unit of bytes. + //The default initial size for other arenas is 8388608 (i.e., 8 MB). + ParmParse pp; + pp.add("amrex.the_arena_init_size", 8388608); + pp.add("amrex.the_managed_arena_init_size", 8388608); + pp.add("amrex.the_device_arena_init_size", 8388608); + amrex::Initialize(argc,argv); MFIter::allowMultipleMFIters(true); From 6d5f0166726b037b287a975a5828912dd719a3e0 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Tue, 18 Jun 2024 22:15:34 -0400 Subject: [PATCH 142/276] Implement support for NuLib HDF5 tables --- Source/Evolve.cpp | 44 +++++- Source/Make.package | 6 +- Source/NuLibTable.H | 37 +++++ Source/NuLibTableFunctions.H | 47 ++++++ Source/NuLibTableHelpers.H | 191 +++++++++++++++++++++++++ Source/ReadEosTable.cpp | 1 - Source/ReadNuLibTable.cpp | 268 +++++++++++++++++++++++++++++++++++ Source/main.cpp | 5 + 8 files changed, 591 insertions(+), 8 deletions(-) create mode 100644 Source/NuLibTable.H create mode 100644 Source/NuLibTableFunctions.H create mode 100644 Source/NuLibTableHelpers.H create mode 100644 Source/ReadNuLibTable.cpp diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index da7fd1ed..85046863 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -6,6 +6,9 @@ #include "EosTableFunctions.H" #include "EosTable.H" +#include "NuLibTableFunctions.H" +#include "NuLibTable.H" + using namespace amrex; namespace GIdx @@ -153,11 +156,16 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M const int shape_factor_order_y = geom.Domain().length(1) > 1 ? SHAPE_FACTOR_ORDER : 0; const int shape_factor_order_z = geom.Domain().length(2) > 1 ? SHAPE_FACTOR_ORDER : 0; - //Create the EoS table object + //Create EoS table object using namespace nuc_eos_private; EOS_tabulated EOS_tabulated_obj(alltables, epstable, logrho, logtemp, yes, helperVarsReal, helperVarsInt); + //Create NuLib table object + using namespace nulib_private; + NuLib_tabulated NuLib_tabulated_obj(alltables_nulib, logrho_nulib, logtemp_nulib, + yes_nulib, helperVarsReal_nulib, helperVarsInt_nulib); + amrex::MeshToParticle(neutrinos_rhs, state, 0, [=] AMREX_GPU_DEVICE (FlavoredNeutrinoContainer::ParticleType& p, amrex::Array4 const& sarr) @@ -217,16 +225,40 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M //for energy, we are specifying neutrino energy right now. // double neutrino_energy = p.rdata(PIdx::pupt); locate energy bin using this. - //---------------------------- EoS call ----------------------------------------------- - double rho = 1.7e2; //g/cm^3 - double temperature = 0.01; //MeV - double Ye = 0.01; + //---------------------------- HDF5 table calls ----------------------------------------------- + //EoS table + double rho = 1.0e6; //g/cm^3 + double temperature = 0.6103379806197231; //0.05 //MeV + double Ye = 0.035; double entropy_out, munu_out; int keyerr, anyerr; EOS_tabulated_obj.get_entropy_munu(rho, temperature, Ye, entropy_out, munu_out, keyerr, anyerr); printf("(Evolve.cpp) munu interpolated = %f\n", munu_out); - //munu[0][0] = munu; + //if (anyerr) assert(0); + + //NuLib table + int idx_group = 10; + + int idx_species = 0; + double absorption_opacity, scattering_opacity; + NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, + keyerr, anyerr, idx_species, idx_group); + printf("(Evolve.cpp) absorption_opacity[e] interpolated = %17.6g\n", absorption_opacity); + printf("(Evolve.cpp) scattering_opacity[e] interpolated = %17.6g\n", scattering_opacity); + + idx_species = 1; + NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, + keyerr, anyerr, idx_species, idx_group); + printf("(Evolve.cpp) absorption_opacity[a] interpolated = %17.6g\n", absorption_opacity); + printf("(Evolve.cpp) scattering_opacity[a] interpolated = %17.6g\n", scattering_opacity); + + idx_species = 2; + NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, + keyerr, anyerr, idx_species, idx_group); + printf("(Evolve.cpp) absorption_opacity[x] interpolated = %17.6g\n", absorption_opacity); + printf("(Evolve.cpp) scattering_opacity[x] interpolated = %17.6g\n", scattering_opacity); + //if (anyerr) assert(0); //--------------------------------------------------------------------------------------- // calculate the equilibrium distribution. Really munu and temperature should be interpolated from the grid. diff --git a/Source/Make.package b/Source/Make.package index 20ecfc9d..03cb310b 100644 --- a/Source/Make.package +++ b/Source/Make.package @@ -5,6 +5,7 @@ CEXE_sources += FlavoredNeutrinoContainerInit.cpp CEXE_sources += Evolve.cpp CEXE_sources += FlavoredNeutrinoContainer.cpp CEXE_sources += ReadEosTable.cpp +CEXE_sources += ReadNuLibTable.cpp CEXE_headers += Evolve.H CEXE_headers += FlavoredNeutrinoContainer.H @@ -17,4 +18,7 @@ CEXE_headers += DataReducer.H CEXE_headers += ArithmeticArray.H CEXE_headers += EosTable.H CEXE_headers += EosTableFunctions.H -CEXE_headers += EosTableHelpers.H \ No newline at end of file +CEXE_headers += EosTableHelpers.H +CEXE_headers += NuLibTable.H +CEXE_headers += NuLibTableFunctions.H +CEXE_headers += NuLibTableHelpers.H \ No newline at end of file diff --git a/Source/NuLibTable.H b/Source/NuLibTable.H new file mode 100644 index 00000000..094e0ee0 --- /dev/null +++ b/Source/NuLibTable.H @@ -0,0 +1,37 @@ +#ifndef NULIB_TABLE_H +#define NULIB_TABLE_H + +void ReadNuLibTable(); + +#define NTABLES_NULIB 2 + +// table key +// 0 absoprtion_opacity +// 1 scattering opacity + + +namespace nulib_private { + +// table data + extern double *alltables_nulib; + //extern double *epstable; + extern double *logrho_nulib; + extern double *logtemp_nulib; + extern double *yes_nulib; + extern double *species_nulib; + extern double *group_nulib; + +#define NULIBVAR(VAR) helperVarsReal_nulib[helperVarsEnumReal_nulib::VAR] +//WARNING: If the number of variables is increased here, then memory allocation for helperVarsReal/helperVarsInt pointer in GRHydroX_ReadEOSTable.cxx will also need to be increased. + enum helperVarsEnumReal_nulib { eos_rhomin, eos_rhomax, eos_tempmin, eos_tempmax, eos_yemin, eos_yemax, + dlintemp, dlintempi, drholintempi, dlintempyei, drholintempyei, + dtemp, dtempi, drho, drhoi, dye, dyei, drhotempi, drhoyei, dtempyei, + drhotempyei, eos_epsmin, eos_epsmax }; + extern double *helperVarsReal_nulib; + +#define NULIBVAR_INT(VAR) helperVarsInt_nulib[helperVarsEnumInt_nulib::VAR] + enum helperVarsEnumInt_nulib { nrho, ntemp, nye, nspecies, ngroup } ; + extern int *helperVarsInt_nulib; +} + +#endif // NULIB_TABLE_H diff --git a/Source/NuLibTableFunctions.H b/Source/NuLibTableFunctions.H new file mode 100644 index 00000000..4faf2ee8 --- /dev/null +++ b/Source/NuLibTableFunctions.H @@ -0,0 +1,47 @@ +#ifndef NULIBTABLEFUCNTIONS_HXX +#define NULIBTABLEFUCNTIONS_HXX + + +#include "NuLibTableHelpers.H" + +struct NuLib_tabulated { + double *alltables_nulib, *logrho_nulib, *logtemp_nulib, *yes_nulib, *helperVarsReal_nulib; + int *helperVarsInt_nulib; + + //constructor for NuLib_tabulated + AMREX_GPU_DEVICE AMREX_GPU_HOST NuLib_tabulated() = default;//Need to keep it + AMREX_GPU_DEVICE AMREX_GPU_HOST NuLib_tabulated(double *alltables_nulib, double *logrho_nulib, + double *logtemp_nulib, double *yes_nulib, double *helperVarsReal_nulib, int *helperVarsInt_nulib): + alltables_nulib(alltables_nulib), logrho_nulib(logrho_nulib), logtemp_nulib(logtemp_nulib), + yes_nulib(yes_nulib), helperVarsReal_nulib(helperVarsReal_nulib), helperVarsInt_nulib(helperVarsInt_nulib) {} + + //--------------- get helperVarsReal pointer --------------------------------------------- + AMREX_GPU_DEVICE AMREX_GPU_HOST double *get_helperVarsReal_nulib() const { + return helperVarsReal_nulib; + };//get_helperVarsReal_nulib + + + //--------------- get opacaties --------------------------------------------- + AMREX_GPU_DEVICE AMREX_GPU_HOST void get_opacities(double rho, double temperature, double Ye, + double &absorption_opacity, double &scattering_opacity, + int &keyerr, int &anyerr, const int idx_species, const int idx_group) const { + + nulib_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, keyerr, anyerr, + alltables_nulib, logrho_nulib, logtemp_nulib, yes_nulib, helperVarsReal_nulib, helperVarsInt_nulib, + idx_species, idx_group); + return; + + /*Actual steps: + (1) check bounds + (2) get interp spots + (3) do interpolation to get press and eps + Let's wrap this up in a funtion calleds nuc_eos_press_eps_kT1 (~nuc_eos_press.c) + This will further require another include ~helpers.hh + Pass errors such as anyerr or keyerr to the calling routine. + */ + };//get_opacities + + +}; //struct NuLib_tabulated + +#endif // NULIBTABLEFUCNTIONS_HXX \ No newline at end of file diff --git a/Source/NuLibTableHelpers.H b/Source/NuLibTableHelpers.H new file mode 100644 index 00000000..bac7ea71 --- /dev/null +++ b/Source/NuLibTableHelpers.H @@ -0,0 +1,191 @@ +#ifndef NULIBTABLEHELPERS_HXX +#define NULIBTABLEHELPERS_HXX + +#include "NuLibTable.H" + +//Check whether input (rho, temperature, Ye) are within the table bounds. +static inline AMREX_GPU_DEVICE AMREX_GPU_HOST int +checkbounds_nulib(const double xrho, const double xtemp, const double xye, double *helperVarsReal_nulib) { + + using namespace nulib_private; + // keyerr codes: + // 101 -- Y_e too high + // 102 -- Y_e too low + // 103 -- temp too high (if keytemp = 1) + // 104 -- temp too low (if keytemp = 1) + // 105 -- rho too high + // 106 -- rho too low + + if(xrho > NULIBVAR(eos_rhomax)) { + return 105; + } + if(xrho < NULIBVAR(eos_rhomin)) { + return 106; + } + if(xye > NULIBVAR(eos_yemax)) { + return 101; + } + if(xye < NULIBVAR(eos_yemin)) { + return 102; + } + if(xtemp > NULIBVAR(eos_tempmax)) { + return 103; + } + if(xtemp < NULIBVAR(eos_tempmin)) { + return 104; + } + return 0; + +}//function checkbounds_nulib + +//Get the indices to prepare for interpolation. +static inline AMREX_GPU_DEVICE AMREX_GPU_HOST void +get_interp_spots_nulib(const double x, const double y, const double z, double &delx, + double &dely, double &delz, int *idx, double *logrho_nulib, + double *logtemp_nulib, double *yes_nulib, double *helperVarsReal_nulib, int *helperVarsInt_nulib, + const int idx_species, const int idx_group) +{ + using namespace nulib_private; + + int ix = 1 + (int)( (x - logrho_nulib[0] - 1.0e-10) * NULIBVAR(drhoi) ); + int iy = 1 + (int)( (y - logtemp_nulib[0] - 1.0e-10) * NULIBVAR(dtempi) ); + int iz = 1 + (int)( (z - yes_nulib[0] - 1.0e-10) * NULIBVAR(dyei) ); + + const int nrho = NULIBVAR_INT(nrho); + const int ntemp = NULIBVAR_INT(ntemp); + const int nye = NULIBVAR_INT(nye); + const int nspecies = NULIBVAR_INT(nspecies); + + ix = MAX2( 1, MIN2( ix, nrho-1 ) ); + iy = MAX2( 1, MIN2( iy, ntemp-1 ) ); + iz = MAX2( 1, MIN2( iz, nye-1 ) ); + + //int indnew = iv + NTABLES_NULIB*(i + nrho_*(j + ntemp_*k)); //indexing in EoS table + //int indnew = iv + NTABLES_NULIB*(i + nrho_*(j + ntemp_*(k + nye_*(l + nspecies_*m)))); //indexing in NuLib table + + idx[0] = NTABLES_NULIB*(ix + nrho*(iy + ntemp*(iz + nye*(idx_species + nspecies*idx_group)))); + idx[1] = NTABLES_NULIB*((ix-1) + nrho*(iy + ntemp*(iz + nye*(idx_species + nspecies*idx_group)))); + idx[2] = NTABLES_NULIB*(ix + nrho*((iy-1) + ntemp*(iz + nye*(idx_species + nspecies*idx_group)))); + idx[3] = NTABLES_NULIB*(ix + nrho*(iy + ntemp*((iz-1) + nye*(idx_species + nspecies*idx_group)))); + idx[4] = NTABLES_NULIB*((ix-1) + nrho*((iy-1) + ntemp*(iz + nye*(idx_species + nspecies*idx_group)))); + idx[5] = NTABLES_NULIB*((ix-1) + nrho*(iy + ntemp*((iz-1) + nye*(idx_species + nspecies*idx_group)))); + idx[6] = NTABLES_NULIB*(ix + nrho*((iy-1) + ntemp*((iz-1) + nye*(idx_species + nspecies*idx_group)))); + idx[7] = NTABLES_NULIB*((ix-1) + nrho*((iy-1) + ntemp*((iz-1) + nye*(idx_species + nspecies*idx_group)))); + + //idx[0] = NTABLES_NULIB*(ix + nrho*(iy + ntemp*iz)); + //idx[1] = NTABLES_NULIB*((ix-1) + nrho*(iy + ntemp*iz)); + //idx[2] = NTABLES_NULIB*(ix + nrho*((iy-1) + ntemp*iz)); + //idx[3] = NTABLES_NULIB*(ix + nrho*(iy + ntemp*(iz-1))); + //idx[4] = NTABLES_NULIB*((ix-1) + nrho*((iy-1) + ntemp*iz)); + //idx[5] = NTABLES_NULIB*((ix-1) + nrho*(iy + ntemp*(iz-1))); + //idx[6] = NTABLES_NULIB*(ix + nrho*((iy-1) + ntemp*(iz-1))); + //idx[7] = NTABLES_NULIB*((ix-1) + nrho*((iy-1) + ntemp*(iz-1))); + + // set up aux vars for interpolation + delx = logrho_nulib[ix] - x; + dely = logtemp_nulib[iy] - y; + delz = yes_nulib[iz] - z; + + return; +} // function get_interp_spots_nulib + +//Perform the interpolation +static inline AMREX_GPU_DEVICE AMREX_GPU_HOST void +nuc_eos_C_linterp_one_nulib(const int *idx, const double delx, const double dely, + const double delz, double &f, const int iv, + double *alltables_nulib, double *helperVarsReal_nulib) +{ + using namespace nulib_private; + + // helper variables + double fh[8], a[8]; + + fh[0] = alltables_nulib[iv+idx[0]]; + fh[1] = alltables_nulib[iv+idx[1]]; + fh[2] = alltables_nulib[iv+idx[2]]; + fh[3] = alltables_nulib[iv+idx[3]]; + fh[4] = alltables_nulib[iv+idx[4]]; + fh[5] = alltables_nulib[iv+idx[5]]; + fh[6] = alltables_nulib[iv+idx[6]]; + fh[7] = alltables_nulib[iv+idx[7]]; + + // set up coeffs of interpolation polynomical and + // evaluate function values + a[0] = fh[0]; + a[1] = NULIBVAR(drhoi) * ( fh[1] - fh[0] ); + a[2] = NULIBVAR(dtempi) * ( fh[2] - fh[0] ); + a[3] = NULIBVAR(dyei) * ( fh[3] - fh[0] ); + a[4] = NULIBVAR(drhotempi) * ( fh[4] - fh[1] - fh[2] + fh[0] ); + a[5] = NULIBVAR(drhoyei) * ( fh[5] - fh[1] - fh[3] + fh[0] ); + a[6] = NULIBVAR(dtempyei) * ( fh[6] - fh[2] - fh[3] + fh[0] ); + a[7] = NULIBVAR(drhotempyei) * ( fh[7] - fh[0] + fh[1] + fh[2] + + fh[3] - fh[4] - fh[5] - fh[6] ); + + f = a[0] + a[1] * delx + + a[2] * dely + + a[3] * delz + + a[4] * delx * dely + + a[5] * delx * delz + + a[6] * dely * delz + + a[7] * delx * dely * delz; + + return; +} // function nuc_eos_C_linterp_one_nulib + + + +//Main function for entropy and munu given (rho, temperature, Ye) +static inline AMREX_GPU_DEVICE AMREX_GPU_HOST void +nulib_opacities(const double rho, const double temperature, + const double Ye, double &absorption_opacity, double &scattering_opacity, + int &keyerr, int &anyerr, double *alltables_nulib, + double *logrho_nulib, double *logtemp_nulib, double *yes_nulib, + double *helperVarsReal_nulib, int *helperVarsInt_nulib, + const int idx_species, const int idx_group) +{ + using namespace nulib_private; + + int anyerr_ = 0; + // check if we are fine + int keyerr_ = checkbounds_nulib(rho, temperature, Ye, helperVarsReal_nulib); + if(keyerr_ != 0) anyerr_ = 1; + + keyerr = keyerr_ ; anyerr = anyerr_; + // Abort if there is any error in checkbounds. + // This should never happen and the program should abort with + // a fatal error anyway. No point in doing any further EOS calculations. + if(anyerr_){ + printf("Error in checkbounds::%s::%d::%s, keyerr = %d\n", __FILE__, __LINE__, __func__, keyerr); + printf(" rho = %.5e, temperature = %.5e, Ye = %.6f \n", rho, temperature, Ye); + return; + } + + int idx[8]; + double delx,dely,delz; + const double lrho = log(rho); + const double ltemp = log(temperature); + + get_interp_spots_nulib(lrho, ltemp, Ye, delx, dely, delz, idx, + logrho_nulib, logtemp_nulib, yes_nulib, helperVarsReal_nulib, helperVarsInt_nulib, + idx_species, idx_group); + + double absorption_opacity_ , scattering_opacity_; + { + const int iv = 0; + nuc_eos_C_linterp_one_nulib(idx, delx, dely, delz, absorption_opacity_, iv, alltables_nulib, helperVarsReal_nulib); + } + + { + const int iv = 1; + nuc_eos_C_linterp_one_nulib(idx, delx, dely, delz, scattering_opacity_, iv, alltables_nulib, helperVarsReal_nulib); + } + + // Assign values to reference variables: + absorption_opacity = absorption_opacity_; + scattering_opacity = scattering_opacity_; + + return; +}//nulib_opacities + + +#endif //NULIBTABLEHELPERS_HXX \ No newline at end of file diff --git a/Source/ReadEosTable.cpp b/Source/ReadEosTable.cpp index 040dee67..4f814bad 100644 --- a/Source/ReadEosTable.cpp +++ b/Source/ReadEosTable.cpp @@ -2,7 +2,6 @@ #include -//#include "GRHydroX_EOS.hxx" #define H5_USE_16_API 1 #include "hdf5.h" diff --git a/Source/ReadNuLibTable.cpp b/Source/ReadNuLibTable.cpp new file mode 100644 index 00000000..17036323 --- /dev/null +++ b/Source/ReadNuLibTable.cpp @@ -0,0 +1,268 @@ +#include + +#include + +#define H5_USE_16_API 1 +#include "hdf5.h" + +#include "NuLibTable.H" + +// mini NoMPI +#define HAVE_CAPABILITY_MPI //FIXME: This should be defined only when USE_MPI = TRUE +#ifdef HAVE_CAPABILITY_MPI +#include +#define BCAST(buffer, size) MPI_Bcast(buffer, size, MPI_BYTE, my_reader_process, MPI_COMM_WORLD) +#else +#define BCAST(buffer, size) do { /* do nothing */ } while(0) +#endif + +// Catch HDF5 errors +#define HDF5_ERROR(fn_call) \ + if(doIO) { \ + int _error_code = fn_call; \ + if (_error_code < 0) { \ + AMREX_ASSERT_WITH_MESSAGE(false, "HDF5 call failed"); \ + } \ + } + +using namespace amrex; + +static int file_is_readable(std::string filename); +static int file_is_readable(std::string filename) +{ + FILE* fp = NULL; + fp = fopen(filename.c_str(), "r"); + if(fp != NULL) + { + fclose(fp); + return 1; + } + return 0; +} + +namespace nulib_private { + double *alltables_nulib; + //double *epstable; + double *logrho_nulib; + double *logtemp_nulib; + double *yes_nulib; + double *species_nulib; //TODO: Get rid of this? + double *group_nulib; + double *helperVarsReal_nulib; + int *helperVarsInt_nulib; +} + +//TODO: Pass the /path/to/table here in the function argument +void ReadNuLibTable() { + using namespace nulib_private; + + std::string nulib_table_name = "/mnt/scratch/tables/NuLib/NuLib_SFHo.h5"; //FIXME: Read from parameter file + amrex::Print() << "(ReadNuLibTable.cpp) Using table: " << nulib_table_name << std::endl; + + //TODO: + int my_reader_process = 0; //reader_process; + /*if (my_reader_process < 0 || my_reader_process >= CCTK_nProcs(cctkGH)) + { + CCTK_VWarn(CCTK_WARN_COMPLAIN, __LINE__, __FILE__, CCTK_THORNSTRING, + "Requested IO process %d out of range. Reverting to process 0.", my_reader_process); + my_reader_process = 0; + }*/ + + const int read_table_on_single_process = 1; + //const int doIO = !read_table_on_single_process || CCTK_MyProc(cctkGH) == my_reader_process; //TODO: + const int doIO = 1; + + hid_t file; + if (doIO && !file_is_readable(nulib_table_name)) { + AMREX_ASSERT_WITH_MESSAGE(false, "Could not read nulib_table_name"); + } + + HDF5_ERROR(file = H5Fopen(nulib_table_name.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); + +// Use these two defines to easily read in a lot of variables in the same way +// The first reads in one variable of a given type completely +#define READ_BCAST_EOS_HDF5(NAME,VAR,TYPE,MEM,NELEMS) \ + do { \ + hid_t dataset; \ + HDF5_ERROR(dataset = H5Dopen(file, NAME)); \ + HDF5_ERROR(H5Dread(dataset, TYPE, MEM, H5S_ALL, H5P_DEFAULT, VAR)); \ + if (read_table_on_single_process) \ + BCAST (VAR, sizeof(*(VAR))*(NELEMS)); \ + HDF5_ERROR(H5Dclose(dataset)); \ + } while (0) +// The second reads a given variable into a hyperslab of the alltables_temp array +#define READ_BCAST_EOSTABLE_HDF5(NAME,OFF,DIMS) \ + do { \ + READ_BCAST_EOS_HDF5(NAME,&alltables_temp[(OFF)*(DIMS)[1]],H5T_NATIVE_DOUBLE,H5S_ALL,(DIMS)[1]); \ + } while (0) + + int nrho_; + int ntemp_; + int nye_; + int nspecies_; + int ngroup_; + + // Read size of tables + READ_BCAST_EOS_HDF5("nrho", &nrho_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ntemp", &ntemp_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("nye", &nye_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("number_species", &nspecies_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("number_groups", &ngroup_, H5T_NATIVE_INT, H5S_ALL, 1); + + assert(nspecies_ == 2); //For now, the code only works when NuLib table has (e,a,x) values. + + printf("(ReadNuLibTable.cpp) nrho = %d, ntemp = %d, nye = %d, nspecies=%d, ngroup=%d\n", nrho_, ntemp_, nye_, nspecies_, ngroup_); + + //Allocate managed memory arena on unified memory + ManagedArenaAllocator myManagedArena; + ManagedArenaAllocator myManagedArena_Int; + + // Allocate memory for tables + double *alltables_temp; + if (!(alltables_temp = myManagedArena.allocate(nrho_ * ntemp_ * nye_ * nspecies_ * ngroup_ * NTABLES_NULIB) )) { + printf("(ReadNuLibTable.cpp) Cannot allocate memory for NuLib table"); + assert(0); + } + if (!(logrho_nulib = myManagedArena.allocate(nrho_) )) { + printf("(ReadNuLibTable.cpp) Cannot allocate memory for NuLib table"); + assert(0); + } + if (!(logtemp_nulib = myManagedArena.allocate(ntemp_) )) { + printf("(ReadNuLibTable.cpp) Cannot allocate memory for NuLib table"); + assert(0); + } + if (!(yes_nulib = myManagedArena.allocate(nye_) )) { + printf("(ReadNuLibTable.cpp) Cannot allocate memory for NuLib table"); + assert(0); + } + if (!(species_nulib = myManagedArena.allocate(nspecies_) )) { + printf("(ReadNuLibTable.cpp) Cannot allocate memory for NuLib table"); + assert(0); + } + if (!(group_nulib = myManagedArena.allocate(ngroup_) )) { + printf("(ReadNuLibTable.cpp) Cannot allocate memory for NuLib table"); + assert(0); + } + + // Prepare HDF5 to read hyperslabs into alltables_temp + hsize_t table_dims[2] = {NTABLES_NULIB, (hsize_t)nrho_ * ntemp_ * nye_ * nspecies_ * ngroup_}; + //hsize_t var3[2] = { 1, (hsize_t)nrho_ * ntemp_ * nye_ * nspecies_ * ngroup_}; + hid_t mem3 = H5Screate_simple(2, table_dims, NULL); + + // Read alltables_temp + READ_BCAST_EOSTABLE_HDF5("absorption_opacity", 0, table_dims); + READ_BCAST_EOSTABLE_HDF5("scattering_opacity", 1, table_dims); + + // Read additional tables and variables + //This is not log yet. + READ_BCAST_EOS_HDF5("rho_points", logrho_nulib, H5T_NATIVE_DOUBLE, H5S_ALL, nrho_); + READ_BCAST_EOS_HDF5("temp_points", logtemp_nulib, H5T_NATIVE_DOUBLE, H5S_ALL, ntemp_); + READ_BCAST_EOS_HDF5("ye_points", yes_nulib, H5T_NATIVE_DOUBLE, H5S_ALL, nye_); + READ_BCAST_EOS_HDF5("neutrino_energies", group_nulib, H5T_NATIVE_DOUBLE, H5S_ALL, ngroup_); + + HDF5_ERROR(H5Sclose(mem3)); + HDF5_ERROR(H5Fclose(file)); + + // change ordering of alltables_nulib array so that + // the table kind is the fastest changing index + if (!(alltables_nulib = myManagedArena.allocate(nrho_ * ntemp_ * nye_ * nspecies_ * ngroup_ * NTABLES_NULIB) )) { + printf("(ReadNuLibTable.cpp) Cannot allocate memory for NuLib table"); + assert(0); + } + + for(int iv = 0;iv Date: Tue, 18 Jun 2024 23:45:43 -0400 Subject: [PATCH 143/276] Determine energy bin automatically from given neutrino energy --- Source/Evolve.cpp | 3 ++- Source/NuLibTable.H | 2 +- Source/ReadNuLibTable.cpp | 36 +++++++++++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 85046863..3b219cc7 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -238,7 +238,8 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M //if (anyerr) assert(0); //NuLib table - int idx_group = 10; + double *helperVarsReal_nulib = NuLib_tabulated_obj.get_helperVarsReal_nulib(); + int idx_group = NULIBVAR(idx_group); int idx_species = 0; double absorption_opacity, scattering_opacity; diff --git a/Source/NuLibTable.H b/Source/NuLibTable.H index 094e0ee0..9af3dba7 100644 --- a/Source/NuLibTable.H +++ b/Source/NuLibTable.H @@ -26,7 +26,7 @@ namespace nulib_private { enum helperVarsEnumReal_nulib { eos_rhomin, eos_rhomax, eos_tempmin, eos_tempmax, eos_yemin, eos_yemax, dlintemp, dlintempi, drholintempi, dlintempyei, drholintempyei, dtemp, dtempi, drho, drhoi, dye, dyei, drhotempi, drhoyei, dtempyei, - drhotempyei, eos_epsmin, eos_epsmax }; + drhotempyei, eos_epsmin, eos_epsmax, idx_group}; extern double *helperVarsReal_nulib; #define NULIBVAR_INT(VAR) helperVarsInt_nulib[helperVarsEnumInt_nulib::VAR] diff --git a/Source/ReadNuLibTable.cpp b/Source/ReadNuLibTable.cpp index 17036323..e15d0dc6 100644 --- a/Source/ReadNuLibTable.cpp +++ b/Source/ReadNuLibTable.cpp @@ -144,6 +144,18 @@ void ReadNuLibTable() { assert(0); } + //Allocate memory for energy bin determination. + double *energy_bottom; + double *energy_top; + if (!(energy_bottom = myManagedArena.allocate(ngroup_) )) { + printf("(ReadNuLibTable.cpp) Cannot allocate memory for NuLib table"); + assert(0); + } + if (!(energy_top = myManagedArena.allocate(ngroup_) )) { + printf("(ReadNuLibTable.cpp) Cannot allocate memory for NuLib table"); + assert(0); + } + // Prepare HDF5 to read hyperslabs into alltables_temp hsize_t table_dims[2] = {NTABLES_NULIB, (hsize_t)nrho_ * ntemp_ * nye_ * nspecies_ * ngroup_}; //hsize_t var3[2] = { 1, (hsize_t)nrho_ * ntemp_ * nye_ * nspecies_ * ngroup_}; @@ -160,6 +172,9 @@ void ReadNuLibTable() { READ_BCAST_EOS_HDF5("ye_points", yes_nulib, H5T_NATIVE_DOUBLE, H5S_ALL, nye_); READ_BCAST_EOS_HDF5("neutrino_energies", group_nulib, H5T_NATIVE_DOUBLE, H5S_ALL, ngroup_); + READ_BCAST_EOS_HDF5("bin_bottom", energy_bottom, H5T_NATIVE_DOUBLE, H5S_ALL, ngroup_); + READ_BCAST_EOS_HDF5("bin_top", energy_top, H5T_NATIVE_DOUBLE, H5S_ALL, ngroup_); + HDF5_ERROR(H5Sclose(mem3)); HDF5_ERROR(H5Fclose(file)); @@ -196,6 +211,23 @@ void ReadNuLibTable() { logtemp_nulib[i] = log(logtemp_nulib[i]); } + //---------------------------- Energy bin determeination -------------------------------- + //FIXME: FIXME: Set from parameter file. + double given_energy = 55.0; //TODO: Is this log or linear in table? + int idx_group_; + + //Decide which energy bin to use (i.e. determine 'idx_group') + for (int i=0; i= energy_bottom[i] && given_energy <= energy_top[i]){ + idx_group_ = i; + break; + } + } + + printf("Given neutrino energy = %f, selected bin index = %d\n", given_energy, idx_group); + myManagedArena.deallocate(energy_bottom, ngroup_); + myManagedArena.deallocate(energy_top, ngroup_); + //---------------------------------------------------------------------------------------------- // convert any other quantities to log. /*for(int i=0;i Date: Wed, 19 Jun 2024 00:52:03 -0400 Subject: [PATCH 144/276] add mue,muhat interpolation function + general cleanup --- Source/EosTableFunctions.H | 39 ++++++------------------------------ Source/EosTableHelpers.H | 22 ++++++++++---------- Source/Evolve.cpp | 7 ++++--- Source/NuLibTableFunctions.H | 5 +---- Source/NuLibTableHelpers.H | 2 +- 5 files changed, 23 insertions(+), 52 deletions(-) diff --git a/Source/EosTableFunctions.H b/Source/EosTableFunctions.H index daa23377..fdd6aba9 100644 --- a/Source/EosTableFunctions.H +++ b/Source/EosTableFunctions.H @@ -21,48 +21,21 @@ struct EOS_tabulated { };//get_helperVarsReal - //--------------- get entropy/munu for tabulated EOS --------------------------------------------- - AMREX_GPU_DEVICE AMREX_GPU_HOST void get_entropy_munu(double rho, double temperature, double Ye, - double &entropy, double &munu, + //--------------- get mu_e and muhat for tabulated EOS --------------------------------------------- + AMREX_GPU_DEVICE AMREX_GPU_HOST void get_mue_muhat(double rho, double temperature, double Ye, + double &mue, double &muhat, int &keyerr, int &anyerr) const { - nuc_eos_entropy_munu(rho, temperature, Ye, entropy, munu, keyerr, anyerr, + nuc_eos_mue_muhat(rho, temperature, Ye, mue, muhat, keyerr, anyerr, alltables, logrho, logtemp, yes, helperVarsReal, helperVarsInt); return; /*Actual steps: (1) check bounds (2) get interp spots - (3) do interpolation to get press and eps - Let's wrap this up in a funtion calleds nuc_eos_press_eps_kT1 (~nuc_eos_press.c) - This will further require another include ~helpers.hh - Pass errors such as anyerr or keyerr to the calling routine. + (3) do interpolation to get mu_e and muhat */ - };//get_entropy_munu - - - //--------------- get pressure for tabulated EOS --------------------------------------------- - AMREX_GPU_DEVICE AMREX_GPU_HOST void get_press(double rho, double temperature, double Ye, - double &press, int &keyerr, int &anyerr) const { - - //int keyerr, anyerr; - /*nuc_eos_m_kt1_press(rho, temperature, Ye, press, keyerr, anyerr, - alltables, logrho, logtemp, yes, helperVarsReal, helperVarsInt);*/ - assert(0); //FIXME: Add actual routine. - return; - };//get_press - - - //--------------- get composition xh for tabulated EOS --------------------------------------------- - AMREX_GPU_DEVICE AMREX_GPU_HOST void get_xh(double rho, double temperature, double Ye, - double &xh, int &keyerr, int &anyerr) const { - - //int keyerr, anyerr; - /*nuc_eos_m_kt1_xh(rho, temperature, Ye, xh, keyerr, anyerr, - alltables, logrho, logtemp, yes, helperVarsReal, helperVarsInt);*/ - assert(0); //FIXME: Add actual routine. - return; - };//get_xh + };//get_mue_muhat }; //struct EOS_tabulated diff --git a/Source/EosTableHelpers.H b/Source/EosTableHelpers.H index 13a259a1..70ac0637 100644 --- a/Source/EosTableHelpers.H +++ b/Source/EosTableHelpers.H @@ -120,10 +120,10 @@ nuc_eos_C_linterp_one(const int *idx, const double delx, const double dely, -//Main function for entropy and munu given (rho, temperature, Ye) +//Main function for mu_e and muhat given (rho, temperature, Ye) static inline AMREX_GPU_DEVICE AMREX_GPU_HOST void -nuc_eos_entropy_munu(const double rho, const double temperature, - const double Ye, double &entropy, double &munu, +nuc_eos_mue_muhat(const double rho, const double temperature, + const double Ye, double &mue, double &muhat, int &keyerr, int &anyerr, double *alltables, double *logrho, double *logtemp, double *yes, double *helperVarsReal, int *helperVarsInt) @@ -153,23 +153,23 @@ nuc_eos_entropy_munu(const double rho, const double temperature, get_interp_spots(lrho, ltemp, Ye, delx, dely, delz, idx, logrho, logtemp, yes, helperVarsReal, helperVarsInt); - double entropy_ , munu_; + double mue_ , muhat_; { - const int iv = 2; - nuc_eos_C_linterp_one(idx, delx, dely, delz, entropy_, iv, alltables, helperVarsReal); + const int iv = 9; + nuc_eos_C_linterp_one(idx, delx, dely, delz, mue_, iv, alltables, helperVarsReal); } { - const int iv = 3; - nuc_eos_C_linterp_one(idx, delx, dely, delz, munu_, iv, alltables, helperVarsReal); + const int iv = 8; + nuc_eos_C_linterp_one(idx, delx, dely, delz, muhat_, iv, alltables, helperVarsReal); } // Assign values to reference variables: - entropy = entropy_; - munu = munu_; + mue = mue_; + muhat = muhat_; return; -}//nuc_eos_entropy_munu +}//nuc_eos_mue_muhat diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 3b219cc7..5fed92b5 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -231,10 +231,11 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M double temperature = 0.6103379806197231; //0.05 //MeV double Ye = 0.035; - double entropy_out, munu_out; + double mue_out, muhat_out; int keyerr, anyerr; - EOS_tabulated_obj.get_entropy_munu(rho, temperature, Ye, entropy_out, munu_out, keyerr, anyerr); - printf("(Evolve.cpp) munu interpolated = %f\n", munu_out); + EOS_tabulated_obj.get_mue_muhat(rho, temperature, Ye, mue_out, muhat_out, keyerr, anyerr); + printf("(Evolve.cpp) mu_e interpolated = %f\n", mue_out); + printf("(Evolve.cpp) muhat interpolated = %f\n", muhat_out); //if (anyerr) assert(0); //NuLib table diff --git a/Source/NuLibTableFunctions.H b/Source/NuLibTableFunctions.H index 4faf2ee8..95f2c198 100644 --- a/Source/NuLibTableFunctions.H +++ b/Source/NuLibTableFunctions.H @@ -34,10 +34,7 @@ struct NuLib_tabulated { /*Actual steps: (1) check bounds (2) get interp spots - (3) do interpolation to get press and eps - Let's wrap this up in a funtion calleds nuc_eos_press_eps_kT1 (~nuc_eos_press.c) - This will further require another include ~helpers.hh - Pass errors such as anyerr or keyerr to the calling routine. + (3) do interpolation to get absorption and scattering opacity */ };//get_opacities diff --git a/Source/NuLibTableHelpers.H b/Source/NuLibTableHelpers.H index bac7ea71..cc41bc1c 100644 --- a/Source/NuLibTableHelpers.H +++ b/Source/NuLibTableHelpers.H @@ -134,7 +134,7 @@ nuc_eos_C_linterp_one_nulib(const int *idx, const double delx, const double dely -//Main function for entropy and munu given (rho, temperature, Ye) +//Main function for absorption and scattering opacities (rho, temperature, Ye) static inline AMREX_GPU_DEVICE AMREX_GPU_HOST void nulib_opacities(const double rho, const double temperature, const double Ye, double &absorption_opacity, double &scattering_opacity, From 4b57f8bdaf46d2e012e7a074ae166f0a709823a1 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Wed, 19 Jun 2024 01:44:15 -0400 Subject: [PATCH 145/276] Read EoS and NuLib table names from parfile --- Source/EosTable.H | 2 +- Source/NuLibTable.H | 2 +- Source/Parameters.H | 8 ++++++++ Source/ReadEosTable.cpp | 4 ++-- Source/ReadNuLibTable.cpp | 4 ++-- Source/main.cpp | 4 ++-- 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Source/EosTable.H b/Source/EosTable.H index 13bc5299..7f32acfe 100644 --- a/Source/EosTable.H +++ b/Source/EosTable.H @@ -1,7 +1,7 @@ #ifndef EOS_TABLE_H #define EOS_TABLE_H -void ReadEosTable(); +void ReadEosTable(const std::string nuceos_table_name); // OLD TODO: remove hard coded constants // OLD TODO: introduce defines for table index of variables diff --git a/Source/NuLibTable.H b/Source/NuLibTable.H index 9af3dba7..262460ed 100644 --- a/Source/NuLibTable.H +++ b/Source/NuLibTable.H @@ -1,7 +1,7 @@ #ifndef NULIB_TABLE_H #define NULIB_TABLE_H -void ReadNuLibTable(); +void ReadNuLibTable(const std::string nulib_table_name); #define NTABLES_NULIB 2 diff --git a/Source/Parameters.H b/Source/Parameters.H index 730e5288..c6fc1531 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -53,6 +53,10 @@ struct TestParams : public amrex::Gpu::Managed // attenuation to hamiltonians Real attenuation_hamiltonians; + //HDF5 table names (with full path) for EoS and NuLib tables + std::string nuceos_table_name; + std::string nulib_table_name; + void Initialize(){ ParmParse pp; pp.get("ncell", ncell); @@ -113,6 +117,10 @@ struct TestParams : public amrex::Gpu::Managed // attenuation to hamiltonians pp.get("attenuation_hamiltonians", attenuation_hamiltonians); + //HDF5 table names (with full path) for EoS and NuLib tables + pp.get("nuceos_table_name", nuceos_table_name); + pp.get("nulib_table_name", nulib_table_name); + // absorption opacities and equilibrium neutrino chemical potentials pp.get("IMFP_method", IMFP_method); if(IMFP_method==0){ diff --git a/Source/ReadEosTable.cpp b/Source/ReadEosTable.cpp index 4f814bad..eba72b95 100644 --- a/Source/ReadEosTable.cpp +++ b/Source/ReadEosTable.cpp @@ -51,10 +51,10 @@ namespace nuc_eos_private { } //TODO: Pass the /path/to/table here in the function argument -void ReadEosTable() { +void ReadEosTable(const std::string nuceos_table_name) { using namespace nuc_eos_private; - std::string nuceos_table_name = "/home/sshanka/000_UTK_projects/Emu/Exec/SFHo.h5"; //FIXME: Read from parameter file + //std::string nuceos_table_name = "/home/sshanka/000_UTK_projects/Emu/Exec/SFHo.h5"; amrex::Print() << "(ReadEosTable.cpp) Using table: " << nuceos_table_name << std::endl; //TODO: diff --git a/Source/ReadNuLibTable.cpp b/Source/ReadNuLibTable.cpp index e15d0dc6..62ab8817 100644 --- a/Source/ReadNuLibTable.cpp +++ b/Source/ReadNuLibTable.cpp @@ -53,10 +53,10 @@ namespace nulib_private { } //TODO: Pass the /path/to/table here in the function argument -void ReadNuLibTable() { +void ReadNuLibTable(const std::string nulib_table_name) { using namespace nulib_private; - std::string nulib_table_name = "/mnt/scratch/tables/NuLib/NuLib_SFHo.h5"; //FIXME: Read from parameter file + //std::string nulib_table_name = "/mnt/scratch/tables/NuLib/NuLib_SFHo.h5"; amrex::Print() << "(ReadNuLibTable.cpp) Using table: " << nulib_table_name << std::endl; //TODO: diff --git a/Source/main.cpp b/Source/main.cpp index 9404c97d..d7eebeda 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -92,11 +92,11 @@ void evolve_flavor(const TestParams* parms) // read the EoS table amrex::Print() << "Reading EoS table... " << std::endl; - ReadEosTable(); + ReadEosTable(parms->nuceos_table_name); // read the NuLib table amrex::Print() << "Reading NuLib table... " << std::endl; - ReadNuLibTable(); + ReadNuLibTable(parms->nulib_table_name); // Initialize particles on the domain amrex::Print() << "Initializing particles... " << std::endl; From cf78f14b447173d8f5ffc22a02098655bacd5d6d Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Wed, 19 Jun 2024 02:34:37 -0400 Subject: [PATCH 146/276] Finish application of EoS/NuLib tables in evolve.cpp --- Source/Evolve.cpp | 127 ++++++++++++++++++++++++++++++---------------- Source/main.cpp | 17 ++++--- 2 files changed, 94 insertions(+), 50 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 5fed92b5..4f4ac572 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -155,7 +155,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M const int shape_factor_order_x = geom.Domain().length(0) > 1 ? SHAPE_FACTOR_ORDER : 0; const int shape_factor_order_y = geom.Domain().length(1) > 1 ? SHAPE_FACTOR_ORDER : 0; const int shape_factor_order_z = geom.Domain().length(2) > 1 ? SHAPE_FACTOR_ORDER : 0; - + //Create EoS table object using namespace nuc_eos_private; EOS_tabulated EOS_tabulated_obj(alltables, epstable, logrho, logtemp, @@ -220,48 +220,89 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } } } - else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0 or 1"); - - //for energy, we are specifying neutrino energy right now. - // double neutrino_energy = p.rdata(PIdx::pupt); locate energy bin using this. - - //---------------------------- HDF5 table calls ----------------------------------------------- - //EoS table - double rho = 1.0e6; //g/cm^3 - double temperature = 0.6103379806197231; //0.05 //MeV - double Ye = 0.035; - - double mue_out, muhat_out; - int keyerr, anyerr; - EOS_tabulated_obj.get_mue_muhat(rho, temperature, Ye, mue_out, muhat_out, keyerr, anyerr); - printf("(Evolve.cpp) mu_e interpolated = %f\n", mue_out); - printf("(Evolve.cpp) muhat interpolated = %f\n", muhat_out); - //if (anyerr) assert(0); - - //NuLib table - double *helperVarsReal_nulib = NuLib_tabulated_obj.get_helperVarsReal_nulib(); - int idx_group = NULIBVAR(idx_group); - - int idx_species = 0; - double absorption_opacity, scattering_opacity; - NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, - keyerr, anyerr, idx_species, idx_group); - printf("(Evolve.cpp) absorption_opacity[e] interpolated = %17.6g\n", absorption_opacity); - printf("(Evolve.cpp) scattering_opacity[e] interpolated = %17.6g\n", scattering_opacity); - - idx_species = 1; - NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, - keyerr, anyerr, idx_species, idx_group); - printf("(Evolve.cpp) absorption_opacity[a] interpolated = %17.6g\n", absorption_opacity); - printf("(Evolve.cpp) scattering_opacity[a] interpolated = %17.6g\n", scattering_opacity); - - idx_species = 2; - NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, - keyerr, anyerr, idx_species, idx_group); - printf("(Evolve.cpp) absorption_opacity[x] interpolated = %17.6g\n", absorption_opacity); - printf("(Evolve.cpp) scattering_opacity[x] interpolated = %17.6g\n", scattering_opacity); - //if (anyerr) assert(0); - //--------------------------------------------------------------------------------------- + else if(parms->IMFP_method==2){ + // use the IMFPs from NuLib table and munu from EoS table. + double rho = 1.0e6; //g/cm^3 + double temperature = 0.6103379806197231; //0.05 //MeV + double Ye = 0.035; + + //-------------------- Values from EoS table ------------------------------ + double mue_out, muhat_out; + int keyerr, anyerr; + EOS_tabulated_obj.get_mue_muhat(rho, temperature, Ye, mue_out, muhat_out, keyerr, anyerr); + if (anyerr) assert(0); //If there is an error in interpolation call, stop execution. + +//#define DEBUG_INTERPOLATION_TABLES +#ifdef DEBUG_INTERPOLATION_TABLES + printf("(Evolve.cpp) mu_e interpolated = %f\n", mue_out); + printf("(Evolve.cpp) muhat interpolated = %f\n", muhat_out); +#endif + + const double munu_val = mue_out - muhat_out; //munu -> "mu_e" - "muhat" + + for(int i=0; i<2; i++){ + for(int j=0; jneutrino or 1->antineutrino + for(int j=1; jelectron, 1->heavy(muon), 2->heavy(tau); all heavy same for current table + IMFP_abs[i][j] = absorption_opacity; + IMFP_scat[i][j] = scattering_opacity; + } + } + //----------------------------------------------------------------------- + + } + else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0, 1 or 2"); // calculate the equilibrium distribution. Really munu and temperature should be interpolated from the grid. for(int i=0; i<2; i++){ diff --git a/Source/main.cpp b/Source/main.cpp index d7eebeda..b43238b7 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -90,13 +90,16 @@ void evolve_flavor(const TestParams* parms) // initialize the grid variable names GIdx::Initialize(); - // read the EoS table - amrex::Print() << "Reading EoS table... " << std::endl; - ReadEosTable(parms->nuceos_table_name); - - // read the NuLib table - amrex::Print() << "Reading NuLib table... " << std::endl; - ReadNuLibTable(parms->nulib_table_name); + //We only need HDF5 tables if IMFP_method is 2. + if(parms->IMFP_method==2){ + // read the EoS table + amrex::Print() << "Reading EoS table... " << std::endl; + ReadEosTable(parms->nuceos_table_name); + + // read the NuLib table + amrex::Print() << "Reading NuLib table... " << std::endl; + ReadNuLibTable(parms->nulib_table_name); + } // Initialize particles on the domain amrex::Print() << "Initializing particles... " << std::endl; From 8937f1110ab0899e595b6bd98072c5f8ac11d4a9 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Mon, 24 Jun 2024 16:24:20 -0400 Subject: [PATCH 147/276] Add support routine set_rho_T_Ye that reads values from HDF5 table GPU-arrays and sets them to corresponding MultiFabs --- Source/Make.package | 4 ++- Source/ReadInput_RhoTempYe.H | 14 ++++++++++ Source/ReadInput_RhoTempYe.cpp | 50 ++++++++++++++++++++++++++++++++++ Source/ReadNuLibTable.cpp | 2 ++ Source/main.cpp | 20 +++++++++++--- 5 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 Source/ReadInput_RhoTempYe.H create mode 100644 Source/ReadInput_RhoTempYe.cpp diff --git a/Source/Make.package b/Source/Make.package index 03cb310b..f40235ed 100644 --- a/Source/Make.package +++ b/Source/Make.package @@ -6,6 +6,7 @@ CEXE_sources += Evolve.cpp CEXE_sources += FlavoredNeutrinoContainer.cpp CEXE_sources += ReadEosTable.cpp CEXE_sources += ReadNuLibTable.cpp +CEXE_sources += ReadInput_RhoTempYe.cpp CEXE_headers += Evolve.H CEXE_headers += FlavoredNeutrinoContainer.H @@ -21,4 +22,5 @@ CEXE_headers += EosTableFunctions.H CEXE_headers += EosTableHelpers.H CEXE_headers += NuLibTable.H CEXE_headers += NuLibTableFunctions.H -CEXE_headers += NuLibTableHelpers.H \ No newline at end of file +CEXE_headers += NuLibTableHelpers.H +CEXE_headers += ReadInput_RhoTempYe.H \ No newline at end of file diff --git a/Source/ReadInput_RhoTempYe.H b/Source/ReadInput_RhoTempYe.H new file mode 100644 index 00000000..25414e10 --- /dev/null +++ b/Source/ReadInput_RhoTempYe.H @@ -0,0 +1,14 @@ +#ifndef READINPUT_RHOTEMPYE_H +#define READINPUT_RHOTEMPYE_H + +#include +#include +#include +#include +#include +#include + +void set_rho_T_Ye(MultiFab& state, const Geometry& geom); + + +#endif diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp new file mode 100644 index 00000000..413a5aa4 --- /dev/null +++ b/Source/ReadInput_RhoTempYe.cpp @@ -0,0 +1,50 @@ +#include "Evolve.H" +#include "Constants.H" + +#include + + +void set_rho_T_Ye(MultiFab& state, const Geometry& geom) +{ + // Create an alias of the MultiFab so set_rho_T_Ye only sets rho, T and Ye. + int start_comp = GIdx::rho; + int num_comps = 3; //We only want to set GIdx::rho, GIdx::T and GIdx::Ye + MultiFab rho_T_ye_state(state, amrex::make_alias, start_comp, num_comps); + + amrex::GpuArray dx = geom.CellSizeArray(); + //const auto plo = geom.ProbLoArray(); + //const auto dxi = geom.InvCellSizeArray(); + //const Real inv_cell_volume = dxi[0]*dxi[1]*dxi[2]; + + //const int shape_factor_order_x = geom.Domain().length(0) > 1 ? SHAPE_FACTOR_ORDER : 0; + //const int shape_factor_order_y = geom.Domain().length(1) > 1 ? SHAPE_FACTOR_ORDER : 0; + //const int shape_factor_order_z = geom.Domain().length(2) > 1 ? SHAPE_FACTOR_ORDER : 0; + + //always access mf comp index as (GIdx::rho - start_comp) + //Example: Amrex tutorials -> ExampleCodes/MPMD/Case-2/main.cpp. + + for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ + const amrex::Box& bx = mfi.validbox(); + const amrex::Array4& mf_array = rho_T_ye_state.array(mfi); + + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k){ + + //x, y and z are the coordinates. + //This is not really needed. Cook up some assert statement to make sure we are at the same (x, y, z) that the table is vlaue is referring to. + amrex::Real x = (i+0.5) * dx[0]; + amrex::Real y = (j+0.5) * dx[1]; + amrex::Real z = (k+0.5) * dx[2]; + + //printf("Inside MFIter: x=%f, y=%f, z=%f\n", x, y, z); + + //TODO: Find the global (i, j, k) from the amrex domain. call them (ig, jg, kg). + //TODO: Then get the values from GPU-array for (ig, jg, kg) and set them to corresponding MultiFabs here. + mf_array(i, j, k, GIdx::rho - start_comp) = -404.0; //FIXME: + mf_array(i, j, k, GIdx::T - start_comp) = -404.0; //FIXME: + mf_array(i, j, k, GIdx::Ye - start_comp) = -404.0; //FIXME: + }); + } + +} + + diff --git a/Source/ReadNuLibTable.cpp b/Source/ReadNuLibTable.cpp index 62ab8817..6947ad3f 100644 --- a/Source/ReadNuLibTable.cpp +++ b/Source/ReadNuLibTable.cpp @@ -211,6 +211,8 @@ void ReadNuLibTable(const std::string nulib_table_name) { logtemp_nulib[i] = log(logtemp_nulib[i]); } + //FIXME: FIXME: Make an assert that rho, temp and Ye are uniformally spaced. + //---------------------------- Energy bin determeination -------------------------------- //FIXME: FIXME: Set from parameter file. double given_energy = 55.0; //TODO: Is this log or linear in table? diff --git a/Source/main.cpp b/Source/main.cpp index b43238b7..392f50c0 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -34,6 +34,7 @@ #include "DataReducer.H" #include "EosTable.H" #include "NuLibTable.H" +#include "ReadInput_RhoTempYe.H" using namespace amrex; @@ -80,13 +81,24 @@ void evolve_flavor(const TestParams* parms) // Create a MultiFab to hold our grid state data and initialize to 0.0 MultiFab state(ba, dm, ncomp, ngrow); + //FIXME: FIXME: Define this in paramete file. + const int read_rho_T_Ye_from_table = 1; + // initialize with NaNs ... state.setVal(0.0); - state.setVal(parms->rho_in,GIdx::rho,1); // g/ccm - state.setVal(parms->Ye_in,GIdx::Ye,1); - state.setVal(parms->kT_in,GIdx::T,1); // erg - state.FillBoundary(geom.periodicity()); + //If reading from table, call function "set_rho_T_Ye". + //Else set rho, T and Ye to constant value throughout the grid using values from parameter file. + if (read_rho_T_Ye_from_table){ + set_rho_T_Ye(state, geom); + } else { + state.setVal(parms->rho_in,GIdx::rho,1); // g/ccm + state.setVal(parms->Ye_in,GIdx::Ye,1); + state.setVal(parms->kT_in,GIdx::T,1); // erg + } + + state.FillBoundary(geom.periodicity()); + // initialize the grid variable names GIdx::Initialize(); From 482da204d6f8549a7fc1cbba50b90d09c2384ba5 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Tue, 25 Jun 2024 15:01:54 -0400 Subject: [PATCH 148/276] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 10c7fa4b..1ac920af 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,7 +16,7 @@ pipeline { sh 'nvidia-smi' sh 'nvcc -V' sh 'git submodule update --init' - sh 'cp makefiles/GNUmakefile_jenkins Exec/GNUmakefile' + sh 'cp makefiles/GNUmakefile_jenkins_HDF5 Exec/GNUmakefile' dir('Exec'){ sh 'make generate; make -j' } From f5bc8b0a736ab22fabd0e8dea74013f5bd830f52 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Tue, 25 Jun 2024 15:21:45 -0400 Subject: [PATCH 149/276] Read table names only when IMFP_method is 2 --- Source/Parameters.H | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index c6fc1531..8b4e76cb 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -117,10 +117,6 @@ struct TestParams : public amrex::Gpu::Managed // attenuation to hamiltonians pp.get("attenuation_hamiltonians", attenuation_hamiltonians); - //HDF5 table names (with full path) for EoS and NuLib tables - pp.get("nuceos_table_name", nuceos_table_name); - pp.get("nulib_table_name", nulib_table_name); - // absorption opacities and equilibrium neutrino chemical potentials pp.get("IMFP_method", IMFP_method); if(IMFP_method==0){ @@ -146,8 +142,11 @@ struct TestParams : public amrex::Gpu::Managed } pp.get("delta_E", delta_E); delta_E*= 1e6*CGSUnitsConst::eV; // erg - } - else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 1"); + } else if (IMFP_method==2){ + //HDF5 table names (with full path) for EoS and NuLib tables + pp.get("nuceos_table_name", nuceos_table_name); + pp.get("nulib_table_name", nulib_table_name); + } else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 1"); } }; From 96dff1433b638194920e3dea4ac3e8e8e064da81 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Tue, 25 Jun 2024 15:49:15 -0400 Subject: [PATCH 150/276] Add new makefile GNUmakefile_jenkins_HDF5_CUDA and update Jenkinsfile --- Jenkinsfile | 2 +- makefiles/GNUmakefile_jenkins_HDF5_CUDA | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 makefiles/GNUmakefile_jenkins_HDF5_CUDA diff --git a/Jenkinsfile b/Jenkinsfile index 1ac920af..a4493ab0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,7 +16,7 @@ pipeline { sh 'nvidia-smi' sh 'nvcc -V' sh 'git submodule update --init' - sh 'cp makefiles/GNUmakefile_jenkins_HDF5 Exec/GNUmakefile' + sh 'cp makefiles/GNUmakefile_jenkins_HDF5_CUDA Exec/GNUmakefile' dir('Exec'){ sh 'make generate; make -j' } diff --git a/makefiles/GNUmakefile_jenkins_HDF5_CUDA b/makefiles/GNUmakefile_jenkins_HDF5_CUDA new file mode 100644 index 00000000..4919c22e --- /dev/null +++ b/makefiles/GNUmakefile_jenkins_HDF5_CUDA @@ -0,0 +1,20 @@ +NUM_FLAVORS = 3 +SHAPE_FACTOR_ORDER = 2 +NUM_MOMENTS = 3 + +COMP = gnu + +DEBUG = FALSE + +USE_MPI = TRUE +USE_OMP = FALSE +USE_ACC = FALSE +USE_CUDA = TRUE +AMREX_CUDA_ARCH=60 + +USE_HDF5=TRUE +HDF5_HOME=/usr/lib/x86_64-linux-gnu/hdf5/openmpi + +EMU_HOME = .. +AMREX_HOME = ../submodules/amrex +include ../Make.Emu From f5cfa00a5b2ebf022193d032aaec3e6648846a87 Mon Sep 17 00:00:00 2001 From: Javier Gomez <145796242+javierigm@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:44:47 -0400 Subject: [PATCH 151/276] Improve k plot (#95) * Improving k_plot * changed line endings to unix --------- Co-authored-by: Sherwood Richers --- Scripts/babysitting/k_plot.py | 225 ++++++++++++++++++---------------- 1 file changed, 118 insertions(+), 107 deletions(-) diff --git a/Scripts/babysitting/k_plot.py b/Scripts/babysitting/k_plot.py index 6f997b0c..97f56e2a 100644 --- a/Scripts/babysitting/k_plot.py +++ b/Scripts/babysitting/k_plot.py @@ -1,107 +1,118 @@ -import matplotlib.pyplot as mpl -import numpy as np - -############## -# parameters # -############## -direction = "z" #direction of domain size. -base_directory = "" -abskplt = "lin" #"log" #switch between log and linear plots for abs(k) - -################ -# reading data # -################ -Lx=0 -Ly=0 -Lz=0 -inputs_file = open(base_directory + "inputs", 'r') -for line in inputs_file.readlines(): - if "ncell" in line: - str = line.split("(")[1].split(")")[0].split(",") - n = [int(str[0]), int(str[1]), int(str[2])] - if line[:2] == "Lx": - Lx = float(line.split("=")[1].split("#")[0]) - if line[:2] == "Ly": - Ly = float(line.split("=")[1].split("#")[0]) - if line[:2] == "Lz": - Lz = float(line.split("=")[1].split("#")[0]) - - L = [Lx, Ly, Lz] - -inputs_file.close() - -kmax = np.array(np.genfromtxt(base_directory + "kmax_t_N01.dat", skip_header=1)) -amp = kmax[:,4] -kz = kmax[:,3] -ky = kmax[:,2] -kx = kmax[:,1] -t = kmax[:,0]*10**6 -imax = np.argmax(amp) - -# Changing the length of the domain and kmin and kmax according to axis -if direction == "x": - i = 0 - labels = ["kmin for Lx", "kmax for Lx"] -elif direction == "y": - i = 1 - labels = ["kmin for Ly", "kmax for Ly"] -elif direction == "z": - i = 2 - labels = ["kmin for Lz", "kmax for Lz"] -kmn = 2*np.pi/L[i] -kmx = 2*np.pi/(2*L[i]/n[i]) - -################ -# plot options # -################ -mpl.rcParams['font.size'] = 17 -mpl.rcParams['font.family'] = 'serif' -mpl.rcParams['xtick.major.size'] = 7 -mpl.rcParams['xtick.major.width'] = 2 -mpl.rcParams['xtick.major.pad'] = 8 -mpl.rcParams['xtick.minor.size'] = 4 -mpl.rcParams['xtick.minor.width'] = 2 -mpl.rcParams['ytick.major.size'] = 7 -mpl.rcParams['ytick.major.width'] = 2 -mpl.rcParams['ytick.minor.size'] = 4 -mpl.rcParams['ytick.minor.width'] = 2 -mpl.rcParams['axes.linewidth'] = 2 -mpl.rcParams['figure.figsize'] = (10,10) - -################################ -# generating + formatting plot # -################################ -fig, axs = mpl.subplots(2, 1, sharex=True) -fig.subplots_adjust(hspace=0) - -if abskplt == "log": - axs[0].semilogy(t, amp) -elif abskplt == "lin": - axs[0].plot(t, amp) -else: - axs[0].semilogy(t, amp) - -axs[0].tick_params(axis='both',which="both", direction="in",top=True,right=True) -axs[0].minorticks_on() - -kmag = np.sqrt(kz**2+ky**2+kx**2) - -axs[1].semilogy(t, kmag) -axs[1].axhline(kmx, color='r', linestyle='-', label=labels[1]) -axs[1].axhline(kmn, color='y', linestyle='-', label=labels[0]) -axs[1].tick_params(axis='both',which="both", direction="in",top=True,right=True) -axs[1].minorticks_on() - -for ax in axs: - ax.axvline(t[imax]) - ax.axvline(2*t[imax]) - ax.axvline(3*t[imax]) - -axs[1].set_xlabel(r'time ($\mu s$)') -axs[0].set_ylabel(r'Amplitude (cm$^{-3}$)') -axs[1].set_ylabel(r'$|k|$ (cm$^{-1}$)') - -mpl.legend(frameon=False, labelspacing=0.25,fontsize=12, loc=(0.75,0.75)) - - -mpl.savefig('kmax.png') +import matplotlib.pyplot as mpl +import numpy as np +from scipy.signal import argrelextrema + +############## +# parameters # +############## +direction = "z" #direction of domain size. +base_directory = "" +abskplt = "lin" #"log" #switch between log and linear plots for abs(k) + +################ +# reading data # +################ +Lx=0 +Ly=0 +Lz=0 +inputs_file = open(base_directory + "inputs", 'r') +for line in inputs_file.readlines(): + if "ncell" in line: + str = line.split("(")[1].split(")")[0].split(",") + n = [int(str[0]), int(str[1]), int(str[2])] + if line[:2] == "Lx": + Lx = float(line.split("=")[1].split("#")[0]) + if line[:2] == "Ly": + Ly = float(line.split("=")[1].split("#")[0]) + if line[:2] == "Lz": + Lz = float(line.split("=")[1].split("#")[0]) + + L = [Lx, Ly, Lz] + +inputs_file.close() + +kmax = np.array(np.genfromtxt(base_directory + "kmax_t_N01.dat", skip_header=1)) +amp = kmax[:,4] +kz = kmax[:,3] +ky = kmax[:,2] +kx = kmax[:,1] +t = kmax[:,0]*10**6 + +# Obtaining the index at which exponential growth stops +flag = True # Flag to prevent any errors if the following method fails +indexes = np.concatenate([[0],argrelextrema(amp, np.greater)[0]]) # Take all the indexes at which there is a local maxima, as well as the first point in the data + +for i in range(len(indexes)-1): # Loop over all the indexes to check if there are two adjacent points that have a difference of three orders of magnitude + if abs(round((np.log10(amp[indexes[i]]/amp[indexes[i+1]])))) >= 3: + imax = indexes[i+1] + flag = False +if flag == True: # If there previous method does not work, the following is used + imax = np.argmax(amp) + +# Changing the length of the domain and kmin and kmax according to axis +if direction == "x": + i = 0 + labels = ["kmin for Lx", "kmax for Lx"] +elif direction == "y": + i = 1 + labels = ["kmin for Ly", "kmax for Ly"] +elif direction == "z": + i = 2 + labels = ["kmin for Lz", "kmax for Lz"] +kmn = 2*np.pi/L[i] +kmx = 2*np.pi/(2*L[i]/n[i]) + +################ +# plot options # +################ +mpl.rcParams['font.size'] = 17 +mpl.rcParams['font.family'] = 'serif' +mpl.rcParams['xtick.major.size'] = 7 +mpl.rcParams['xtick.major.width'] = 2 +mpl.rcParams['xtick.major.pad'] = 8 +mpl.rcParams['xtick.minor.size'] = 4 +mpl.rcParams['xtick.minor.width'] = 2 +mpl.rcParams['ytick.major.size'] = 7 +mpl.rcParams['ytick.major.width'] = 2 +mpl.rcParams['ytick.minor.size'] = 4 +mpl.rcParams['ytick.minor.width'] = 2 +mpl.rcParams['axes.linewidth'] = 2 +mpl.rcParams['figure.figsize'] = (10,10) + +################################ +# generating + formatting plot # +################################ +fig, axs = mpl.subplots(2, 1, sharex=True) +fig.subplots_adjust(hspace=0) + +axs[0].semilogy(t, amp) +axs[0].tick_params(axis='both',which="both", direction="in",top=True,right=True) +axs[0].minorticks_on() + +kmag = np.sqrt(kz**2+ky**2+kx**2) + +if abskplt == "log": + axs[1].semilogy(t, kmag) +elif abskplt == "lin": + axs[1].plot(t, kmag) +else: + axs[1].semilogy(t, kmag) + +axs[1].axhline(kmx, color='r', linestyle='-', label=labels[1]) +axs[1].axhline(kmn, color='g', linestyle='-', label=labels[0]) +axs[1].tick_params(axis='both',which="both", direction="in",top=True,right=True) +axs[1].minorticks_on() + +for ax in axs: + ax.axvline(t[imax]) + ax.axvline(2*t[imax]) + ax.axvline(3*t[imax]) + +axs[1].set_xlabel(r'time ($\mu s$)') +axs[0].set_ylabel(r'Amplitude (cm$^{-3}$)') +axs[1].set_ylabel(r'$|k|$ (cm$^{-1}$)') + +mpl.legend(frameon=False, labelspacing=0.25,fontsize=12, loc=(0.75,0.75)) + + +mpl.savefig('kmax.png') From 3961f887f7d74f3fc18d6a0a2cf5a8516508e740 Mon Sep 17 00:00:00 2001 From: Sherwood Richers <5142652+srichers@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:47:30 -0400 Subject: [PATCH 152/276] script to check for an ELN crossing in a particle input file (#92) Co-authored-by: Sherwood Richers --- Scripts/babysitting/has_crossing.py | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 Scripts/babysitting/has_crossing.py diff --git a/Scripts/babysitting/has_crossing.py b/Scripts/babysitting/has_crossing.py new file mode 100644 index 00000000..d97fc1f2 --- /dev/null +++ b/Scripts/babysitting/has_crossing.py @@ -0,0 +1,50 @@ +import numpy as np +import sys +sys.path.append("/mnt/scratch/srichers/software/Emu/Scripts/data_reduction") +import amrex_plot_tools as amrex + +if len(sys.argv) != 2: + print("Usage: has_crossing.py particle_input.dat") + exit() + +filename = sys.argv[1] +print(filename) + +# read the number of flavors +f = open(filename,"r") +NF = int(f.readline()) +print(NF,"flavors") +f.close() + +# get variable keys +rkey, ikey = amrex.get_particle_keys(NF, ignore_pos=True) + +# get the ELN info +data = np.genfromtxt(filename, skip_header=1) +nparticles = data.shape[0] + +N = data[:,rkey["N"]] +Nbar = data[:,rkey["Nbar"]] + +ndens = np.zeros((nparticles, 2,NF)) +suffixes = ["","bar"] +for i in range(2): + for j in range(NF): + Nname = "N"+suffixes[i] + fname = "f"+str(j)+str(j)+"_Re"+suffixes[i] + ndens[:,i,j] = data[:,rkey[Nname]] * data[:,rkey[fname]] + +for i in range(NF): + for j in range(i+1,NF): + lepdens_i = ndens[:,0,i] - ndens[:,1,i] + lepdens_j = ndens[:,0,j] - ndens[:,1,j] + eln = lepdens_i - lepdens_j + print(i,j,"crossing:") + mineln = np.min(eln) + maxeln = np.max(eln) + print(" min eln =",mineln) + print(" max eln =",maxeln) + if mineln*maxeln<0: + print('\033[92m UNSTABLE\x1b[0m') + else: + print(' stable') From 4b660a3bf9b9bcfd4463b996a06ab65f32e79a43 Mon Sep 17 00:00:00 2001 From: Sherwood Richers <5142652+srichers@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:48:02 -0400 Subject: [PATCH 153/276] add option to perturb diagonals instead of off-diagonals (#94) Co-authored-by: Sherwood Richers --- Source/FlavoredNeutrinoContainerInit.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index e47d042a..6885b413 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -324,6 +324,24 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); } + if(parms->perturbation_type == 2){ + // random perturbations of the diagonals + Real rand; + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f00_Re) *= 1. + parms->perturbation_amplitude*rand; + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f00_Rebar) *= 1. + parms->perturbation_amplitude*rand; + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f11_Re) *= 1. + parms->perturbation_amplitude*rand; + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f11_Rebar) *= 1. + parms->perturbation_amplitude*rand; +#if NUM_FLAVORS==3 + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f22_Re) *= 1. + parms->perturbation_amplitude*rand; + symmetric_uniform(&rand, engine); + p.rdata(PIdx::f22_Rebar) *= 1. + parms->perturbation_amplitude*rand; +#endif + } #include "generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length" From 9276ecea39e8b4fa60daef538c0eb6fbe2f6cfe7 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Sat, 27 Jul 2024 14:58:07 -0400 Subject: [PATCH 154/276] Implement function to continuously add particles at outer boundary --- Source/FlavoredNeutrinoContainer.H | 2 + Source/FlavoredNeutrinoContainerInit.cpp | 290 +++++++++++++++++++++++ Source/Make.package | 2 +- Source/main.cpp | 8 +- 4 files changed, 300 insertions(+), 2 deletions(-) diff --git a/Source/FlavoredNeutrinoContainer.H b/Source/FlavoredNeutrinoContainer.H index 42e13c2a..e16273e9 100644 --- a/Source/FlavoredNeutrinoContainer.H +++ b/Source/FlavoredNeutrinoContainer.H @@ -76,6 +76,8 @@ public: void InitParticles(const TestParams* parms); + void CreateParticlesAtBoundary(const TestParams* parms); + void SyncLocation(int type); void UpdateLocationFrom(FlavoredNeutrinoContainer& Other); diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 97cce76f..6fc27184 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -342,3 +342,293 @@ InitParticles(const TestParams* parms) ParallelDescriptor::ReduceRealMin(pupt_min); #include "generated_files/FlavoredNeutrinoContainerInit.cpp_Vvac_fill" } // InitParticles() + + + +//==================================================================================================================// +//========================================= CreateParticlesAtBoundary ==============================================// +//==================================================================================================================// +void FlavoredNeutrinoContainer:: +CreateParticlesAtBoundary(const TestParams* parms) +{ + BL_PROFILE("FlavoredNeutrinoContainer::CreateParticlesAtBoundary"); + + const int lev = 0; + const auto dx = Geom(lev).CellSizeArray(); + const auto plo = Geom(lev).ProbLoArray(); + const auto& a_bounds = Geom(lev).ProbDomain(); + + //printf("plo = [%f, %f, %f] \n", plo[0], plo[1], plo[2]); + + const int nlocs_per_cell = AMREX_D_TERM( parms->nppc[0], + *parms->nppc[1], + *parms->nppc[2]); + + // array of direction vectors + //TODO: We can use a different custom file to set particle data at boundary points. + //FIXME: The first line of particle_input.dat is just NFLAVORS (and not an array). Is it correct? + Gpu::ManagedVector > particle_data = read_particle_data(parms->particle_data_filename);; + auto* particle_data_p = particle_data.dataPtr(); + + // determine the number of directions per location + int ndirs_per_loc = particle_data.size(); + amrex::Print() << "Using " << ndirs_per_loc << " directions." << std::endl; + const Real scale_fac = dx[0]*dx[1]*dx[2]/nlocs_per_cell; + + + // Loop over multifabs // +#ifdef _OPENMP +#pragma omp parallel +#endif + for (MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) + { + + //Box tilebox () const noexcept: Return the tile Box at the current index. + const Box& tile_box = mfi.tilebox(); + + const int ncellx = parms->ncell[0]; + const int ncelly = parms->ncell[1]; + const int ncellz = parms->ncell[2]; + + //These actually represent the global indices of the tilebox. + const auto lo = amrex::lbound(tile_box); + const auto hi = amrex::ubound(tile_box); + + printf("tile_box = [%d, %d, %d] x [%d, %d, %d] \n", lo.x, lo.y, lo.z, hi.x, hi.y, hi.z); + + Gpu::ManagedVector counts(tile_box.numPts(), 0); //PODVector > counts(n, 0) + + unsigned int* pcount = counts.dataPtr(); + + Gpu::ManagedVector offsets(tile_box.numPts()); + unsigned int* poffset = offsets.dataPtr(); + + // Determine how many particles to add to the particle tile per cell + //This loops runs over all the particles in a given box. + //For each particle, it calculates a unique "cellid". + //It then adds the pcount for that cell by adding ndirs_per_loc value to it (which is number of particles per location emitted). + //From amrex documentation: Tiling is turned off if GPU is enabled so that more parallelism is exposed to GPU kernels. + //Also note that when tiling is off, tilebox returns validbox. + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + //assert(0); + //amrex::Abort("This function is not used."); + for (int i_part=0; i_partnppc, i_part); + + Real x = plo[0] + (i + r[0])*dx[0]; + Real y = plo[1] + (j + r[1])*dx[1]; + Real z = plo[2] + (k + r[2])*dx[2]; + + //printf("a_bounds.lo = [%f, %f, %f] \n", a_bounds.lo(0), a_bounds.lo(1), a_bounds.lo(2)); + //printf("a_bounds.hi = [%f, %f, %f] \n", a_bounds.hi(0), a_bounds.hi(1), a_bounds.hi(2)); + //printf("x = %f, y = %f, z = %f \n", x, y, z); + //printf("i = %d, j = %d, k = %d \n", i, j, k); + //printf("ncellx = %d, ncelly = %d, ncellz = %d \n", ncellx, ncelly, ncellz); + + /*if (x >= a_bounds.hi(0) || x < a_bounds.lo(0) || + y >= a_bounds.hi(1) || y < a_bounds.lo(1) || + z >= a_bounds.hi(2) || z < a_bounds.lo(2) ) continue;*/ + + + //Only create particles in the outermost interior cells. + bool create_particle_this_cell = (i==0 || i==ncellx-1 || + j==0 || j==ncelly-1 || + k==0 || k==ncellz-1); + if (!create_particle_this_cell) continue; + //printf("CREATE PARTRICLE AT: i = %d, j = %d, k = %d \n", i, j, k); + + int ix = i - lo.x; + int iy = j - lo.y; + int iz = k - lo.z; + int nx = hi.x-lo.x+1; + int ny = hi.y-lo.y+1; + int nz = hi.z-lo.z+1; + unsigned int uix = amrex::min(nx-1,amrex::max(0,ix)); //Forces the value of 'uix' to be in the range of 0 to nx-1. + unsigned int uiy = amrex::min(ny-1,amrex::max(0,iy)); + unsigned int uiz = amrex::min(nz-1,amrex::max(0,iz)); + unsigned int cellid = (uix * ny + uiy) * nz + uiz; + pcount[cellid] += ndirs_per_loc; + } + }); + + // Determine total number of particles to add to the particle tile + Gpu::inclusive_scan(counts.begin(), counts.end(), offsets.begin()); //This sets the value of "offsets" + + //Print the value of pcount for each cell. + /*printf("pcount for each cell (ncells=%d):\n", tile_box.numPts()); + for (int i=0; inppc, i_loc); + + Real x = plo[0] + (i + r[0])*dx[0]; + Real y = plo[1] + (j + r[1])*dx[1]; + Real z = plo[2] + (k + r[2])*dx[2]; + + /*if (x >= a_bounds.hi(0) || x < a_bounds.lo(0) || + y >= a_bounds.hi(1) || y < a_bounds.lo(1) || + z >= a_bounds.hi(2) || z < a_bounds.lo(2) ) continue;*/ + + //Only create particles in the outermost interior cells. + bool create_particle_this_cell = (i==0 || i==ncellx-1 || + j==0 || j==ncelly-1 || + k==0 || k==ncellz-1); + if (!create_particle_this_cell) continue; + //printf("CREATE PARTRICLE AT: i = %d, j = %d, k = %d \n", i, j, k); + + for(int i_direction=0; i_direction= 0); + AMREX_ASSERT(p.rdata(PIdx::N11_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N00_Rebar) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N11_Rebar) >= 0); +#if NUM_FLAVORS==3 + AMREX_ASSERT(p.rdata(PIdx::N22_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N22_Rebar) >= 0); +#endif + + // Set particle position + p.pos(0) = x; + p.pos(1) = y; + p.pos(2) = z; + + // Set particle integrated position + p.rdata(PIdx::x) = x; + p.rdata(PIdx::y) = y; + p.rdata(PIdx::z) = z; + p.rdata(PIdx::time) = 0; + + // scale particle numbers based on number of points per cell and the cell volume + p.rdata(PIdx::N00_Re ) *= scale_fac; + p.rdata(PIdx::N11_Re ) *= scale_fac; + p.rdata(PIdx::N00_Rebar) *= scale_fac; + p.rdata(PIdx::N11_Rebar) *= scale_fac; +#if NUM_FLAVORS==3 + p.rdata(PIdx::N22_Re ) *= scale_fac; + p.rdata(PIdx::N22_Rebar) *= scale_fac; +#endif + + if(parms->IMFP_method == 1){ + p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/(3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); + } + + //=====================// + // Apply Perturbations // + //=====================// + if(parms->perturbation_type == 0){ + // random perturbations to the off-diagonals + Real rand; + symmetric_uniform(&rand, engine); + p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::N01_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::N01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); +#if NUM_FLAVORS==3 + symmetric_uniform(&rand, engine); + p.rdata(PIdx::N02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::N02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::N12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::N22_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::N12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::N22_Re )); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::N02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N22_Rebar)); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::N02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N22_Rebar)); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::N12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar)); + symmetric_uniform(&rand, engine); + p.rdata(PIdx::N12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar)); +#endif + } + if(parms->perturbation_type == 1){ + // Perturb real part of e-mu component only sinusoidally in z + Real nu_k = (2.*M_PI) / parms->perturbation_wavelength_cm; + p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); + p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); + } + + } // loop over direction + } // loop over location + }); // loop over grid cells + } // loop over multifabs + +} // CreateParticlesAtBoundary() + diff --git a/Source/Make.package b/Source/Make.package index f40235ed..4941a84f 100644 --- a/Source/Make.package +++ b/Source/Make.package @@ -23,4 +23,4 @@ CEXE_headers += EosTableHelpers.H CEXE_headers += NuLibTable.H CEXE_headers += NuLibTableFunctions.H CEXE_headers += NuLibTableHelpers.H -CEXE_headers += ReadInput_RhoTempYe.H \ No newline at end of file +CEXE_headers += ReadInput_RhoTempYe.H diff --git a/Source/main.cpp b/Source/main.cpp index 392f50c0..a5c59fb3 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -75,13 +75,15 @@ void evolve_flavor(const TestParams* parms) const IntVect ngrow(1 + (1+shape_factor_order_vec)/2); for(int i=0; incell[i] >= ngrow[i]); + printf("ngrow = [%d, %d, %d] \n", ngrow[0], ngrow[1], ngrow[2]); + // We want 1 component (this is one real scalar field on the domain) const int ncomp = GIdx::ncomp; // Create a MultiFab to hold our grid state data and initialize to 0.0 MultiFab state(ba, dm, ncomp, ngrow); - //FIXME: FIXME: Define this in paramete file. + //FIXME: FIXME: Define this in parameter file. const int read_rho_T_Ye_from_table = 1; // initialize with NaNs ... @@ -187,6 +189,10 @@ void evolve_flavor(const TestParams* parms) // Use the latest-time neutrino data auto& neutrinos = neutrinos_new; + //FIXME: Think carefully where to call this function. + //Create particles at outer boundary + neutrinos.CreateParticlesAtBoundary(parms); + // Update the new time particle locations in the domain with their // integrated coordinates. neutrinos.SyncLocation(Sync::CoordinateToPosition); From c043dd55bc7312d703816fd673a80cae551b7134 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Sat, 27 Jul 2024 15:30:34 -0400 Subject: [PATCH 155/276] Make the function to add particles at boundary templated The function does either outer_boundary or inner_boundary depending on which template parameter is true and which is false --- Source/FlavoredNeutrinoContainer.H | 2 + Source/FlavoredNeutrinoContainerInit.cpp | 50 ++++++++++++++++++++---- Source/main.cpp | 5 ++- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/Source/FlavoredNeutrinoContainer.H b/Source/FlavoredNeutrinoContainer.H index e16273e9..2892a917 100644 --- a/Source/FlavoredNeutrinoContainer.H +++ b/Source/FlavoredNeutrinoContainer.H @@ -61,6 +61,7 @@ public: } }; + class FlavoredNeutrinoContainer : public amrex::ParticleContainer { @@ -76,6 +77,7 @@ public: void InitParticles(const TestParams* parms); + template void CreateParticlesAtBoundary(const TestParams* parms); void SyncLocation(int type); diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 6fc27184..4ee031b1 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -348,6 +348,7 @@ InitParticles(const TestParams* parms) //==================================================================================================================// //========================================= CreateParticlesAtBoundary ==============================================// //==================================================================================================================// +template void FlavoredNeutrinoContainer:: CreateParticlesAtBoundary(const TestParams* parms) { @@ -433,11 +434,24 @@ CreateParticlesAtBoundary(const TestParams* parms) y >= a_bounds.hi(1) || y < a_bounds.lo(1) || z >= a_bounds.hi(2) || z < a_bounds.lo(2) ) continue;*/ + bool create_particle_this_cell = false; - //Only create particles in the outermost interior cells. - bool create_particle_this_cell = (i==0 || i==ncellx-1 || - j==0 || j==ncelly-1 || - k==0 || k==ncellz-1); + //Create particles at outer boundary + if(outer_boundary && !inner_boundary){ + //Only create particles in the outermost interior cells. + if (i==0 || i==ncellx-1 || j==0 || j==ncelly-1 || k==0 || k==ncellz-1) create_particle_this_cell = true; + } + + //Create particles at inner boundary + if (inner_boundary && !outer_boundary){ + //RUNTIME ERROR: Inner boundary is not implemented yet. + //Not giving a compile time error due to explicit instantiation of this function. + printf("ERROR: Inner boundary is not implemented yet.\n"); + assert(0); + //TODO: Implement this. + } + + if (!create_particle_this_cell) continue; //printf("CREATE PARTRICLE AT: i = %d, j = %d, k = %d \n", i, j, k); @@ -528,10 +542,23 @@ CreateParticlesAtBoundary(const TestParams* parms) y >= a_bounds.hi(1) || y < a_bounds.lo(1) || z >= a_bounds.hi(2) || z < a_bounds.lo(2) ) continue;*/ - //Only create particles in the outermost interior cells. - bool create_particle_this_cell = (i==0 || i==ncellx-1 || - j==0 || j==ncelly-1 || - k==0 || k==ncellz-1); + bool create_particle_this_cell = false; + + //Create particles at outer boundary + if(outer_boundary && !inner_boundary){ + //Only create particles in the outermost interior cells. + if (i==0 || i==ncellx-1 || j==0 || j==ncelly-1 || k==0 || k==ncellz-1) create_particle_this_cell = true; + } + + //Create particles at inner boundary + if (inner_boundary && !outer_boundary){ + //RUNTIME ERROR: Inner boundary is not implemented yet. + //Not giving a compile time error due to explicit instantiation of this function. + printf("ERROR: Inner boundary is not implemented yet.\n"); + assert(0); + //TODO: Implement this. + } + if (!create_particle_this_cell) continue; //printf("CREATE PARTRICLE AT: i = %d, j = %d, k = %d \n", i, j, k); @@ -632,3 +659,10 @@ CreateParticlesAtBoundary(const TestParams* parms) } // CreateParticlesAtBoundary() +//We need to explicitly instantiate the template function for different use cases. +//outer_boundary = true, inner_boundary = false +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); +//outer_boundary = false, inner_boundary = true +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); + + diff --git a/Source/main.cpp b/Source/main.cpp index a5c59fb3..de330554 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -191,7 +191,10 @@ void evolve_flavor(const TestParams* parms) //FIXME: Think carefully where to call this function. //Create particles at outer boundary - neutrinos.CreateParticlesAtBoundary(parms); + neutrinos.CreateParticlesAtBoundary(parms); + + //Create particles at inner boundary + //neutrinos.CreateParticlesAtBoundary(parms); //TODO: This needs to be implemented. // Update the new time particle locations in the domain with their // integrated coordinates. From 8f842f19ab9b2d5ebac12b7f75eb6c6e6509724f Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Wed, 31 Jul 2024 01:08:23 -0400 Subject: [PATCH 156/276] Reimplement particle creation at outer boundary We set (x,y,z) coordinates face-centered based on direction, and also set Vphase based on direction. --- Source/FlavoredNeutrinoContainer.H | 5 +- Source/FlavoredNeutrinoContainerInit.cpp | 227 ++++++++++++++++------- Source/main.cpp | 9 +- 3 files changed, 174 insertions(+), 67 deletions(-) diff --git a/Source/FlavoredNeutrinoContainer.H b/Source/FlavoredNeutrinoContainer.H index 2892a917..7454af59 100644 --- a/Source/FlavoredNeutrinoContainer.H +++ b/Source/FlavoredNeutrinoContainer.H @@ -61,6 +61,8 @@ public: } }; +//Create an enum to pass as a template argument to the particle creation function. +enum BoundaryParticleCreationDirection {I_PLUS, I_MINUS, J_PLUS, J_MINUS, K_PLUS, K_MINUS}; class FlavoredNeutrinoContainer : public amrex::ParticleContainer @@ -77,7 +79,7 @@ public: void InitParticles(const TestParams* parms); - template + template void CreateParticlesAtBoundary(const TestParams* parms); void SyncLocation(int type); @@ -102,4 +104,5 @@ public: static ApplyFlavoredNeutrinoRHS particle_apply_rhs; }; + #endif diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 4ee031b1..71f1b921 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -348,7 +348,7 @@ InitParticles(const TestParams* parms) //==================================================================================================================// //========================================= CreateParticlesAtBoundary ==============================================// //==================================================================================================================// -template +template void FlavoredNeutrinoContainer:: CreateParticlesAtBoundary(const TestParams* parms) { @@ -376,6 +376,8 @@ CreateParticlesAtBoundary(const TestParams* parms) amrex::Print() << "Using " << ndirs_per_loc << " directions." << std::endl; const Real scale_fac = dx[0]*dx[1]*dx[2]/nlocs_per_cell; + //FIXME: We need to use outflow boundary condition, not periodic boundary condition. Put an assert(!periodic) here. + // Loop over multifabs // #ifdef _OPENMP @@ -413,45 +415,49 @@ CreateParticlesAtBoundary(const TestParams* parms) amrex::ParallelFor(tile_box, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { - //assert(0); - //amrex::Abort("This function is not used."); for (int i_part=0; i_partnppc, i_part); - - Real x = plo[0] + (i + r[0])*dx[0]; - Real y = plo[1] + (j + r[1])*dx[1]; - Real z = plo[2] + (k + r[2])*dx[2]; - - //printf("a_bounds.lo = [%f, %f, %f] \n", a_bounds.lo(0), a_bounds.lo(1), a_bounds.lo(2)); - //printf("a_bounds.hi = [%f, %f, %f] \n", a_bounds.hi(0), a_bounds.hi(1), a_bounds.hi(2)); - //printf("x = %f, y = %f, z = %f \n", x, y, z); - //printf("i = %d, j = %d, k = %d \n", i, j, k); - //printf("ncellx = %d, ncelly = %d, ncellz = %d \n", ncellx, ncelly, ncellz); - - /*if (x >= a_bounds.hi(0) || x < a_bounds.lo(0) || - y >= a_bounds.hi(1) || y < a_bounds.lo(1) || - z >= a_bounds.hi(2) || z < a_bounds.lo(2) ) continue;*/ bool create_particle_this_cell = false; //Create particles at outer boundary - if(outer_boundary && !inner_boundary){ - //Only create particles in the outermost interior cells. - if (i==0 || i==ncellx-1 || j==0 || j==ncelly-1 || k==0 || k==ncellz-1) create_particle_this_cell = true; - } + switch (DIRECTION) + { + //Create particles in +ve x direction at lower x boundary. + case BoundaryParticleCreationDirection::I_PLUS: + if (i==0) create_particle_this_cell = true; + break; + + //Create particles in -ve x direction at upper x boundary. + case BoundaryParticleCreationDirection::I_MINUS: + if (i==ncellx-1) create_particle_this_cell = true; + break; - //Create particles at inner boundary - if (inner_boundary && !outer_boundary){ - //RUNTIME ERROR: Inner boundary is not implemented yet. - //Not giving a compile time error due to explicit instantiation of this function. - printf("ERROR: Inner boundary is not implemented yet.\n"); - assert(0); - //TODO: Implement this. + //Create particles in +ve y direction at lower y boundary. + case BoundaryParticleCreationDirection::J_PLUS: + if (j==0) create_particle_this_cell = true; + break; + + //Create particles in -ve y direction at upper y boundary. + case BoundaryParticleCreationDirection::J_MINUS: + if (j==ncelly-1) create_particle_this_cell = true; + break; + + //Create particles in +ve z direction at lower z boundary. + case BoundaryParticleCreationDirection::K_PLUS: + if (k==0 ) create_particle_this_cell = true; + break; + + //Create particles in -ve z direction at upper z boundary. + case BoundaryParticleCreationDirection::K_MINUS: + if (k==ncellz-1) create_particle_this_cell = true; + break; + + default: + printf("Invalid direction specified. \n"); + assert(0); + break; } - if (!create_particle_this_cell) continue; //printf("CREATE PARTRICLE AT: i = %d, j = %d, k = %d \n", i, j, k); @@ -472,13 +478,6 @@ CreateParticlesAtBoundary(const TestParams* parms) // Determine total number of particles to add to the particle tile Gpu::inclusive_scan(counts.begin(), counts.end(), offsets.begin()); //This sets the value of "offsets" - //Print the value of pcount for each cell. - /*printf("pcount for each cell (ncells=%d):\n", tile_box.numPts()); - for (int i=0; inppc, i_loc); - Real x = plo[0] + (i + r[0])*dx[0]; - Real y = plo[1] + (j + r[1])*dx[1]; - Real z = plo[2] + (k + r[2])*dx[2]; + //Real x = plo[0] + (i + r[0])*dx[0]; + //Real x = plo[0] for i_plus direction i.e VC, (y,z) remain same CC. + //Real x = plo[0] + (i+1)*dx[0] for i_minus direction i.e. VC, (y,z) remain same CC. + //Real y = plo[1] + (j + r[1])*dx[1]; + //Real z = plo[2] + (k + r[2])*dx[2]; + Real x, y, z; - /*if (x >= a_bounds.hi(0) || x < a_bounds.lo(0) || - y >= a_bounds.hi(1) || y < a_bounds.lo(1) || - z >= a_bounds.hi(2) || z < a_bounds.lo(2) ) continue;*/ - bool create_particle_this_cell = false; - //Create particles at outer boundary - if(outer_boundary && !inner_boundary){ - //Only create particles in the outermost interior cells. - if (i==0 || i==ncellx-1 || j==0 || j==ncelly-1 || k==0 || k==ncellz-1) create_particle_this_cell = true; - } + //Create particles at outer boundary and set face centered coordinates. + //VC=vertex-centered; CC=cell-centered; + switch (DIRECTION) + { + //Create particles in +ve x direction at lower x boundary. + case BoundaryParticleCreationDirection::I_PLUS: + if (i==0) create_particle_this_cell = true; + x = plo[0]; //VC, lower x boundary + y = plo[1] + (j + r[1])*dx[1]; //CC + z = plo[2] + (k + r[2])*dx[2]; //CC + break; + + //Create particles in -ve x direction at upper x boundary. + case BoundaryParticleCreationDirection::I_MINUS: + if (i==ncellx-1) create_particle_this_cell = true; + x = plo[0] + ncellx*dx[0]; //VC, upper x boundary + y = plo[1] + (j + r[1])*dx[1]; //CC + z = plo[2] + (k + r[2])*dx[2]; //CC + break; - //Create particles at inner boundary - if (inner_boundary && !outer_boundary){ - //RUNTIME ERROR: Inner boundary is not implemented yet. - //Not giving a compile time error due to explicit instantiation of this function. - printf("ERROR: Inner boundary is not implemented yet.\n"); - assert(0); - //TODO: Implement this. + //Create particles in +ve y direction at lower y boundary. + case BoundaryParticleCreationDirection::J_PLUS: + if (j==0) create_particle_this_cell = true; + y = plo[1]; //VC, lower y boundary + x = plo[0] + (i + r[0])*dx[0]; //CC + z = plo[2] + (k + r[2])*dx[2]; //CC + break; + + //Create particles in -ve y direction at upper y boundary. + case BoundaryParticleCreationDirection::J_MINUS: + if (j==ncelly-1) create_particle_this_cell = true; + y = plo[1] + ncelly*dx[1]; //VC, upper y boundary + x = plo[0] + (i + r[0])*dx[0]; //CC + z = plo[2] + (k + r[2])*dx[2]; //CC + break; + + //Create particles in +ve z direction at lower z boundary. + case BoundaryParticleCreationDirection::K_PLUS: + if (k==0 ) create_particle_this_cell = true; + z = plo[2]; //VC, lower z boundary + x = plo[0] + (i + r[0])*dx[0]; //CC + y = plo[1] + (j + r[1])*dx[1]; //CC + break; + + //Create particles in -ve z direction at upper z boundary. + case BoundaryParticleCreationDirection::K_MINUS: + if (k==ncellz-1) create_particle_this_cell = true; + z = plo[2] + ncellz*dx[2]; //VC, upper z boundary + x = plo[0] + (i + r[0])*dx[0]; //CC + y = plo[1] + (j + r[1])*dx[1]; //CC + break; + + default: + printf("Invalid direction specified. \n"); + assert(0); + break; } + printf("x = %f, y = %f, z = %f \n", x, y, z); + if (!create_particle_this_cell) continue; //printf("CREATE PARTRICLE AT: i = %d, j = %d, k = %d \n", i, j, k); @@ -609,7 +652,55 @@ CreateParticlesAtBoundary(const TestParams* parms) #endif if(parms->IMFP_method == 1){ - p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/(3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); + const Real V_momentum = 4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/ + (3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); + + //p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*V_momentum; + const Real dt = 0.1; //FIXME: FIXME: This is a dummy value. Set correct value from time integrator. + const Real clight = 1.0; //FIXME: FIXME: This is a dummy value. Set correct value. + const Real pupx_ = 1.0; //FIXME: FIXME: This is a dummy value. Set correct value. + const Real pupy_ = 1.0; //FIXME: FIXME: This is a dummy value. Set correct value. + const Real pupz_ = 1.0; //FIXME: FIXME: This is a dummy value. Set correct value. + const Real pupt_ = 1.0; //FIXME: FIXME: This is a dummy value. Set correct value. + printf("(WARNING) Using dummy values: dt = %f, clight = %f, pupt = %f etc.\n", dt, clight, pupt_); + + switch (DIRECTION) + { + //Create particles in +ve x direction at lower x boundary. + case BoundaryParticleCreationDirection::I_PLUS: + p.rdata(PIdx::Vphase) = dx[1]*dx[2]*clight*dt*V_momentum*pupx_/pupt_; + break; + + //Create particles in -ve x direction at upper x boundary. + case BoundaryParticleCreationDirection::I_MINUS: + p.rdata(PIdx::Vphase) = dx[1]*dx[2]*clight*dt*V_momentum*pupx_/pupt_; + break; + + //Create particles in +ve y direction at lower y boundary. + case BoundaryParticleCreationDirection::J_PLUS: + p.rdata(PIdx::Vphase) = dx[0]*dx[2]*clight*dt*V_momentum*pupy_/pupt_; + break; + + //Create particles in -ve y direction at upper y boundary. + case BoundaryParticleCreationDirection::J_MINUS: + p.rdata(PIdx::Vphase) = dx[0]*dx[2]*clight*dt*V_momentum*pupy_/pupt_; + break; + + //Create particles in +ve z direction at lower z boundary. + case BoundaryParticleCreationDirection::K_PLUS: + p.rdata(PIdx::Vphase) = dx[0]*dx[1]*clight*dt*V_momentum*pupz_/pupt_; + break; + + //Create particles in -ve z direction at upper z boundary. + case BoundaryParticleCreationDirection::K_MINUS: + p.rdata(PIdx::Vphase) = dx[0]*dx[1]*clight*dt*V_momentum*pupz_/pupt_; + break; + + default: + printf("Invalid direction specified. \n"); + assert(0); + break; + } } //=====================// @@ -660,9 +751,17 @@ CreateParticlesAtBoundary(const TestParams* parms) } // CreateParticlesAtBoundary() //We need to explicitly instantiate the template function for different use cases. -//outer_boundary = true, inner_boundary = false -template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); -//outer_boundary = false, inner_boundary = true -template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); +//DIRECTION == BoundaryParticleCreationDirection::I_PLUS (+ve x direction at lower x boundary.) +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); +//DIRECTION == BoundaryParticleCreationDirection::I_MINUS (-ve x direction at upper x boundary.) +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); +//DIRECTION == BoundaryParticleCreationDirection::J_PLUS (+ve y direction at lower y boundary.) +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); +//DIRECTION == BoundaryParticleCreationDirection::J_MINUS (-ve y direction at upper y boundary.) +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); +//DIRECTION == BoundaryParticleCreationDirection::K_PLUS (+ve z direction at lower z boundary.) +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); +//DIRECTION == BoundaryParticleCreationDirection::K_MINUS (-ve z direction at upper z boundary.) +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); diff --git a/Source/main.cpp b/Source/main.cpp index de330554..ed125cdb 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -191,10 +191,15 @@ void evolve_flavor(const TestParams* parms) //FIXME: Think carefully where to call this function. //Create particles at outer boundary - neutrinos.CreateParticlesAtBoundary(parms); + neutrinos.CreateParticlesAtBoundary(parms); + neutrinos.CreateParticlesAtBoundary(parms); + neutrinos.CreateParticlesAtBoundary(parms); + neutrinos.CreateParticlesAtBoundary(parms); + neutrinos.CreateParticlesAtBoundary(parms); + neutrinos.CreateParticlesAtBoundary(parms); //Create particles at inner boundary - //neutrinos.CreateParticlesAtBoundary(parms); //TODO: This needs to be implemented. + //TODO: This needs to be implemented. // Update the new time particle locations in the domain with their // integrated coordinates. From 44339c55c41699ed7bfbc3a7260f0f15aac85d4f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 <101488516+erickurquilla1999@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:56:17 -0400 Subject: [PATCH 157/276] Collisions (#90) * Particles store the phase space volume they occupy * set evolution equations for N and L to be zero in code generation script in anticipation of changing them based on opacities. Set Vphase to not evolve in time. * whitespace * infrastructure to set IMFPs and chemical potentials in the parameter file * redefining f --> Nrho (before was f ---> Nrho) * computing volume of phase space * deleting lines that convert twice mu to erg * adding delta E as an input parameter to compute phase space volume * adding new input parameters to input files to run collitions * adding collition term in QKE * moving line that compute derivates, now it set the phase volume to zero meaning its time derivative its zero after it is used in collision term * considering the particle that carry antineutrinos in the fast flavor test * Fixing an error in the collision term * including adaptative time step for absortion and emission * creating vacuum paticles initial condition script * adding attenuation parameter to the time derivative of N_Ab due to the hamiltonians * solving mistake to compute time step due to collisions * adding collision test input file * testing if the particles converge to the expected equilibrium distribution * solving issue to copy GNUmakefile_jenkins to Exec directory * doing fast compilation in collision test * clarifying better the collision time steps, now delta t collision is collision_CFL / ( c * max_IMFP ) * deletting renomarlization function. It will not be used anymore * solving mistake in compilation comand to compile the code for 3 flavors * Setting default collision time step to infinite. It will change if the code do collisions. * setting right indentation * including attenuation parameter to compute the time steps due to potentials * computing minimum between time steps due to potentials and collision even if max_adaptive_speedup>1 is not satisfy * solve mistake in chemical potential units in coment * add coment on the energy units * delete the individual parms arguments * creating collisional flavor instability test * delete print line * now reads plt* files for time steps of more than 5 digits * now reads plt files for time steps with more than 5 digits * add function to initialize the background enviroment Ye, rho and T * insert chemical potential for neutrinos and antineutrino independently * set python script that generate initial condition for collisional instability test * set input file for collisional instability test * Create a Jenkins test for collisional flavor instability * change cuda version to 7.0 * delete std cout line * correct cuda version for jenkins * change order of commands first reduce_data_fft.py then reduce_data.py * delete function to initialize background quantities on this branch * Solve mistake on collisional instability test. Now it read the right input file * Solve mistake on collisional instability test. Now it read the right python script * add python script for collision instability test * solve problem of defining chemical potentials for antineutrinos in the input parameter file * rename f -> N for particles Beware: grid also has a Nij variable which is different * finish renaming f -> N everywhere * comment lines that assert if N and Nbar are negatives, this lines will be deleted since now N and Nbar will no longer be used * change f for N in utility script to generate initial particles conditions * modify initial condition script for msw test. now it gives N_ab instead of N and f_ab as input * add comments to the function that read the initial conditions of the particles * change msw test to include the change f_ab ---> N_ab * reading N_ab for the particle initial condition file instead of N and f_ab independenly * removing f_ab variables for N_ab when setting perturbation * delete no needed lines * repare coll_equi_test.py test to consider N_ab instead of f_ab * making bypolar test work with the changes f_ab ---> N_ab * making fast flavor test work for f_ab ---> N_ab * making fast_flavor_k_test works for f_ab ---> N_ab * delete print line * delete print lines * make attenuation parameter symbolic in code generation scripts * move f-->N in a few more places * change f to N in test scripts * change Trf to TrN * remove unused line * setting right initial conditions for collisional instability test * inteporlate T, Ye, rho from grid to particles, but return right value multiplied by 4 * solve mistake in interpolating rho, Ye and T from grid to particles posision, now the generation code script does not repeate the code 4 times * delete print lines * using the interpolated value of T for the calculation of the equlibrium distribution * collisional instability test do not include a random perturbation on the off diagonal component of the density matrix. Not it use vacuum potentials. * upload julien nsm input script * solve mistake in energy average * set right initial conditions for julien nsm simulation * Modify collision instability test The collision instaiblity test now includes the LSA in equation 14 of L. Johns [2104.11369] Julien Frousley script to solve isotropic QKE is used to compare EMU solution Plots are included all over the test but were commented * Add collisions directory to store usefull script for simulations containing collisions The script compute_dE.py is used to compute the energy bin size base on the expected equilibrium number of neutrinos * Comments added to compute_dE.py script * reordered scripts in jenkinsfile to clean up PR * added line in data reduction scripts to prevent glob from picking up generated hdf5 files * Deleting unnecessary variables in the script that generates initial conditions for the collision to equilibrium test (coll_equi_test.py). This commit reduces code size by removing redundant variables in the st7_empty_particles.py script (Scripts/tests/coll_equi_test.py), which generates a set of particles with zero neutrinos and antineutrinos. * Delete Julien's NSM input file from the initial conditions script directory. * Removed opacities and chemical potential entries from input files that do not include collision simulations. To avoid future confusion, I deleted the collision-related input options (opacities, chemical potential, and energy bin size) from input files used for tests without collisions. * Added a more accurate description of the assert error for opacity methods. The assert error now clearly indicates that the only available opacity methods are 0 (no collisions) and 1 (with collisions). * Change variable type from double to Real to keep consistency * Adding more comments to the script that computes the energy bin size for collision tests. * Delete commented lines that are under development. I deleted commented lines containing Julien's script that computes the collisional flavor instability evolution. This could potentially be included in the future to solidify the collisional instability test. * Delete Python script of a previous test that will no longer be used. * Delete Python script that generates the initial conditions of particles in Julien's NSM simulation. * Delete old script used for testing convergence. It will no longer be used. * Change N_eq to f_eq since this better represents the collision term. f_eq is the Fermi-Dirac neutrino distribution for a given particle energy. * Include option to multiply the inverse mean free paths by the Pauli blocking term. A new input parameter was created (Do_Pauli_blocking). If set to 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if set to 0, it will do nothing. * Adding a semicolon * Adding the input parameter Do_Pauli_blocking to all input files. I added the input parameter Do_Pauli_blocking to all input files to avoid errors caused by not reading this parameter during execution. If this parameter is set to 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if set to 0, it will do nothing. In the tests that don't involve collisions, it will not have any effect. * Correction in the collision term of QKE The collision term implemented in this commit is C = { gamma , N_eq - N }. Here Gamma = diag( k*_e , k*_x ) / 2 . k*_a is the inverse mean free path for flavor a, including Pauli blocking term. * means that Pauli blocking term is already in the inverse mean free path values. Equilibrium neutrino number matrix N_eq equals the integral of f_eq, where the integral is over the phase space that the particle represents. * Modify the script to compute the energy bin size and antineutrino chemical potential for the collision instability test. This change emerged from the correction made to the collision term and the necessity of obtaining the antineutrino chemical potential to ensure the test runs properly. * Setting up the collisional instability test Set the appropriate input parameters for the collisional instability test, generate plots to visualize the results, and add comments to the Python script that generates the initial conditions for this test. --------- Co-authored-by: Sherwood Richers Co-authored-by: erickurquilla1999 Co-authored-by: Sherwood Richers <5142652+srichers@users.noreply.github.com> --- Jenkinsfile | 23 ++ Scripts/collisions/compute_dE.py | 55 ++++ Scripts/data_reduction/amrex_plot_tools.py | 60 ++-- Scripts/data_reduction/reduce_data.py | 4 +- Scripts/data_reduction/reduce_data_fft.py | 4 +- .../initial_condition_tools.py | 20 +- Scripts/initial_conditions/st0_msw_test.py | 9 +- .../initial_conditions/st1_bipolar_test.py | 10 +- .../st2_2beam_fast_flavor.py | 7 +- .../st3_2beam_fast_flavor_nonzerok.py | 10 +- .../initial_conditions/st7_empty_particles.py | 22 ++ .../initial_conditions/st8_coll_inst_test.py | 36 +++ Scripts/symbolic_hermitians/generate_code.py | 164 +++++------ Scripts/tests/coll_equi_test.py | 60 ++++ Scripts/tests/coll_inst_test.py | 276 ++++++++++++++++++ Scripts/tests/convergence.sh | 89 ------ Scripts/tests/fast_flavor_k_test.py | 54 ++-- Scripts/tests/fast_flavor_test.py | 59 ++-- Scripts/tests/msw_test.py | 44 +-- Scripts/tests/plot_convergence.py | 98 ------- Source/Constants.H | 1 + Source/DataReducer.cpp | 30 +- Source/Evolve.H | 2 +- Source/Evolve.cpp | 114 ++++++-- Source/FlavoredNeutrinoContainer.H | 2 - Source/FlavoredNeutrinoContainer.cpp | 25 +- Source/FlavoredNeutrinoContainerInit.cpp | 80 ++--- Source/Parameters.H | 94 ++++-- Source/main.cpp | 12 +- sample_inputs/inputs_1d_fiducial | 9 + sample_inputs/inputs_bipolar_test | 10 + sample_inputs/inputs_coll_equi_test | 113 +++++++ .../inputs_collisional_instability_test | 113 +++++++ sample_inputs/inputs_fast_flavor | 10 + sample_inputs/inputs_fast_flavor_nonzerok | 9 + sample_inputs/inputs_msw_test | 10 + 36 files changed, 1164 insertions(+), 574 deletions(-) create mode 100644 Scripts/collisions/compute_dE.py create mode 100644 Scripts/initial_conditions/st7_empty_particles.py create mode 100644 Scripts/initial_conditions/st8_coll_inst_test.py create mode 100644 Scripts/tests/coll_equi_test.py create mode 100644 Scripts/tests/coll_inst_test.py delete mode 100644 Scripts/tests/convergence.sh delete mode 100644 Scripts/tests/plot_convergence.py create mode 100644 sample_inputs/inputs_coll_equi_test create mode 100644 sample_inputs/inputs_collisional_instability_test diff --git a/Jenkinsfile b/Jenkinsfile index 84c48283..be2611aa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -90,6 +90,29 @@ pipeline { } }} + stage('Collisions flavor instability'){ steps{ + dir('Exec'){ + sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' + sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' + sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' + sh 'python ../Scripts/data_reduction/reduce_data.py' + sh 'python ../Scripts/tests/coll_inst_test.py' + sh 'rm -rf plt*' + } + }} + + stage('Collisions to equilibrium'){ steps{ + dir('Exec'){ + sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' + sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3' + sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' + sh 'python ../Scripts/tests/coll_equi_test.py' + sh 'rm -rf plt*' + } + }} + } // stages{ post { diff --git a/Scripts/collisions/compute_dE.py b/Scripts/collisions/compute_dE.py new file mode 100644 index 00000000..4faf5ab4 --- /dev/null +++ b/Scripts/collisions/compute_dE.py @@ -0,0 +1,55 @@ +''' +This script compute the energy bin size ( /Delta E ) for monoenergetic neutrino simulations and the antineutrino chemical potential, given: +- Number of neutrinos at equilibrium +- Volume of a cell +- Number of momentum beams isotropically distributed per cell +- Neutrinos energy bin center +- Neutrino chemical potential +- Background matter temperature +This script was used to compute the energy bin size in the test scripts and antineutrino chemical potential: coll_equi_test.py and coll_inst_test.py. +''' + +import numpy as np + +# constants +hbar = 1.05457266e-27 # erg s +h = hbar * 2 * np.pi # erg s +c = 2.99792458e10 # cm/s +hc = h * c # erg cm + +# Simulation parameters +V = 1 # Volume of a cell ( ccm ) +Ndir = 92 # Number of momentum beams isotropically distributed per cell +E = 20.0 # Neutrinos and antineutrinos energy bin center ( Mev ) +T = 7.0 # Background matter temperature ( Mev ) + +N_eq_electron_neutrino = 3.260869565e+31 # Number of electron neutrinos at equilibrium +u_electron_neutrino = 20.0 # Electron neutrino chemical potential ( Mev ) + +# Fermi-dirac distribution factor for electron neutrinos +f_eq_electron_neutrinos = 1 / ( 1 + np.exp( ( E - u_electron_neutrino ) / T ) ) # adimentional + +# We know : +# dE^3 = 3 * Neq * ( hc )^ 3 / ( dV * dOmega * feq ) +delta_E_cubic = 3 * N_eq_electron_neutrino * ( hc )**3 / ( V * ( 4 * np.pi / Ndir ) * f_eq_electron_neutrinos ) # cubic erg +# dOmega = 4 * pi / ( number directions ) + +# We know polinomial of delta E in term of delta E cubic and E ( deltaE**3 = ( E + dE / 2)**3 - ( E - dE / 2)**3 ) +coeff = [ 0.25 , 0 , 3 * E**2 , -1.0 * delta_E_cubic / ( 1.60218e-6**3 ) ] +# Solving for this polinomial +deltaE = np.roots(coeff) + +# Extracting just the real root +dE=0 +for complex_deltaE in deltaE: + if (np.imag(complex_deltaE)==0): + print(f'Delta energy bin in MeV = {np.real(complex_deltaE)}') + dE=np.real(complex_deltaE) + +# Electron neutrino flavor +N_eq_electron_antineutrino = 2.717391304e+31 # Number of electron antineutrinos at equilibrium + +# Computing electron antineutrino chemical potential +f_eq_electron_antineutrino = 3 * N_eq_electron_antineutrino * ( hc )**3 / ( V * ( 4 * np.pi / Ndir ) * delta_E_cubic ) +u_electron_antineutrino = E - T * np.log( 1 / f_eq_electron_antineutrino - 1 ) +print(f'Electron neutrino chemical potential in MeV = {u_electron_antineutrino}') \ No newline at end of file diff --git a/Scripts/data_reduction/amrex_plot_tools.py b/Scripts/data_reduction/amrex_plot_tools.py index 94c8d094..ee717d97 100644 --- a/Scripts/data_reduction/amrex_plot_tools.py +++ b/Scripts/data_reduction/amrex_plot_tools.py @@ -26,18 +26,14 @@ def get_particle_keys(NF, ignore_pos=False, xp_only=False): "pupy", "pupz", "pupt", - "N", - "L", - "f00_Re", - "f01_Re", - "f01_Im", - "f11_Re", - "Nbar", - "Lbar", - "f00_Rebar", - "f01_Rebar", - "f01_Imbar", - "f11_Rebar"] + "N00_Re", + "N01_Re", + "N01_Im", + "N11_Re", + "N00_Rebar", + "N01_Rebar", + "N01_Imbar", + "N11_Rebar"] if(NF==3): real_quantities = ["pos_x", "pos_y", @@ -50,28 +46,24 @@ def get_particle_keys(NF, ignore_pos=False, xp_only=False): "pupy", "pupz", "pupt", - "N", - "L", - "f00_Re", - "f01_Re", - "f01_Im", - "f02_Re", - "f02_Im", - "f11_Re", - "f12_Re", - "f12_Im", - "f22_Re", - "Nbar", - "Lbar", - "f00_Rebar", - "f01_Rebar", - "f01_Imbar", - "f02_Rebar", - "f02_Imbar", - "f11_Rebar", - "f12_Rebar", - "f12_Imbar", - "f22_Rebar"] + "N00_Re", + "N01_Re", + "N01_Im", + "N02_Re", + "N02_Im", + "N11_Re", + "N12_Re", + "N12_Im", + "N22_Re", + "N00_Rebar", + "N01_Rebar", + "N01_Imbar", + "N02_Rebar", + "N02_Imbar", + "N11_Rebar", + "N12_Rebar", + "N12_Imbar", + "N22_Rebar"] if xp_only: real_quantities = real_quantities[:11] if ignore_pos: real_quantities = real_quantities[7:] diff --git a/Scripts/data_reduction/reduce_data.py b/Scripts/data_reduction/reduce_data.py index 665f3866..3b4fbbe2 100644 --- a/Scripts/data_reduction/reduce_data.py +++ b/Scripts/data_reduction/reduce_data.py @@ -385,7 +385,9 @@ def reduce_data(directory=".", nproc=4, do_average=True, do_fft=True, do_angular if(data_format=="Emu"): yt_descriptor = "boxlib" convert_N_to_inv_ccm = 1.0 - directories = sorted(glob.glob("plt?????")) + directories = glob.glob("plt*") + directories = [d for d in directories if ".h5" not in d] + directories = sorted(directories, key=lambda x: int(x.lstrip("plt"))) # get NF eds = emu.EmuDataset(directories[0]) diff --git a/Scripts/data_reduction/reduce_data_fft.py b/Scripts/data_reduction/reduce_data_fft.py index 7abbe3fe..15949757 100755 --- a/Scripts/data_reduction/reduce_data_fft.py +++ b/Scripts/data_reduction/reduce_data_fft.py @@ -9,7 +9,9 @@ parser.add_argument("-o", "--output", type=str, default="reduced_data_fft.h5", help="Name of the output file (default: reduced_data_fft.h5)") args = parser.parse_args() -directories = sorted(glob.glob("plt?????")) +directories = glob.glob("plt*") +directories = [d for d in directories if ".h5" not in d] +directories = sorted(directories, key=lambda x: int(x.lstrip("plt"))) t = [] diff --git a/Scripts/initial_conditions/initial_condition_tools.py b/Scripts/initial_conditions/initial_condition_tools.py index a3639f9f..d0885bdd 100644 --- a/Scripts/initial_conditions/initial_condition_tools.py +++ b/Scripts/initial_conditions/initial_condition_tools.py @@ -230,7 +230,7 @@ def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_g # get variable keys rkey, ikey = amrex.get_particle_keys(NF, ignore_pos=True) nelements = len(rkey) - + # generate the list of particle info particles = np.zeros((nparticles,nelements)) @@ -240,22 +240,10 @@ def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_g # save the total number density of neutrinos for each particle n_flavorsummed = np.sum(n_particle, axis=2) # [particle, nu/nubar] - for nu_nubar, suffix in zip(range(2), ["","bar"]): - nvarname = "N"+suffix - particles[:,rkey[nvarname]] = n_flavorsummed[:,nu_nubar] - + for nu_nubar, suffix in zip(range(2), ["","bar"]): for flavor in range(NF): - fvarname = "f"+str(flavor)+str(flavor)+"_Re"+suffix - particles[:,rkey[fvarname]] = n_particle[:,nu_nubar, flavor] / n_flavorsummed[:,nu_nubar] - particles[:,rkey[fvarname]][np.where(n_flavorsummed[:,nu_nubar]==0)] = 1./NF # ensure that trace stays equal to 1 - - # double check that the number densities are correct - particle_n = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]]) - particle_fmag = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]] * mu[:,nu_nubar, flavor]) - #print("nu/nubar,flavor =", nu_nubar, flavor) - #print("output/input ndens =",particle_n, nnu[nu_nubar,flavor]) - #print("output/input fluxfac =",particle_fmag / particle_n, fluxfac[nu_nubar,flavor]) - #print() + fvarname = "N"+str(flavor)+str(flavor)+"_Re"+suffix + particles[:,rkey[fvarname]] = n_particle[:,nu_nubar, flavor] return particles diff --git a/Scripts/initial_conditions/st0_msw_test.py b/Scripts/initial_conditions/st0_msw_test.py index 2efafa1d..1c6767d5 100644 --- a/Scripts/initial_conditions/st0_msw_test.py +++ b/Scripts/initial_conditions/st0_msw_test.py @@ -37,10 +37,7 @@ p[rkey["pupx"]] = phat[ip,0] * energy_erg p[rkey["pupy"]] = phat[ip,1] * energy_erg p[rkey["pupz"]] = phat[ip,2] * energy_erg - p[rkey["N"] ] = ndens_per_particle - p[rkey["Nbar"]] = ndens_per_particle - p[rkey["f00_Re"]] = 1 - p[rkey["f00_Rebar"]] = 1 - + p[rkey["N00_Re"]] = 1 + p[rkey["N00_Rebar"]] = 1 -write_particles(np.array(particles), NF, "particle_input.dat") +write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file diff --git a/Scripts/initial_conditions/st1_bipolar_test.py b/Scripts/initial_conditions/st1_bipolar_test.py index 04a15e5f..4f8948c1 100644 --- a/Scripts/initial_conditions/st1_bipolar_test.py +++ b/Scripts/initial_conditions/st1_bipolar_test.py @@ -34,7 +34,6 @@ rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) nelements = len(rkey) - # generate the list of particle info particles = np.zeros((nparticles,nelements)) for ip in range(len(phat)): @@ -43,10 +42,7 @@ p[rkey["pupx"]] = phat[ip,0] * energy_erg p[rkey["pupy"]] = phat[ip,1] * energy_erg p[rkey["pupz"]] = phat[ip,2] * energy_erg - p[rkey["N"] ] = ndens_per_particle - p[rkey["Nbar"]] = ndens_per_particle - p[rkey["f00_Re"]] = 1 - p[rkey["f00_Rebar"]] = 1 + p[rkey["N00_Re"]] = ndens_per_particle + p[rkey["N00_Rebar"]] = ndens_per_particle - -write_particles(np.array(particles), NF, "particle_input.dat") +write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file diff --git a/Scripts/initial_conditions/st2_2beam_fast_flavor.py b/Scripts/initial_conditions/st2_2beam_fast_flavor.py index 740b5de1..6839e9d5 100644 --- a/Scripts/initial_conditions/st2_2beam_fast_flavor.py +++ b/Scripts/initial_conditions/st2_2beam_fast_flavor.py @@ -43,10 +43,7 @@ p[rkey["pupx"]] = u[0] * energy_erg p[rkey["pupy"]] = u[1] * energy_erg p[rkey["pupz"]] = u[2] * energy_erg - p[rkey["N"] ] = ndens_per_particle * (1. + u[2]) - p[rkey["Nbar"]] = ndens_per_particle * (1. - u[2]) - p[rkey["f00_Re"]] = 1 - p[rkey["f00_Rebar"]] = 1 - + p[rkey["N00_Re"]] = ndens_per_particle * (1. + u[2]) + p[rkey["N00_Rebar"]] = ndens_per_particle * (1. - u[2]) write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py b/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py index bbcdfab4..c2fd2da8 100644 --- a/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py +++ b/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py @@ -35,7 +35,6 @@ rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) nelements = len(rkey) - # generate the list of particle info particles = np.zeros((nparticles,nelements)) for ip in range(len(phat)): @@ -45,10 +44,7 @@ p[rkey["pupx"]] = u[0] * energy_erg p[rkey["pupy"]] = u[1] * energy_erg p[rkey["pupz"]] = u[2] * energy_erg - p[rkey["N"] ] = ndens_per_particle * (1. + u[2]) - p[rkey["Nbar"]] = ndens_per_particle * (1. - u[2]) - p[rkey["f00_Re"]] = 1 - p[rkey["f00_Rebar"]] = 1 - + p[rkey["N00_Re"]] = ndens_per_particle * (1. + u[2]) + p[rkey["N00_Rebar"]] = ndens_per_particle * (1. - u[2]) -write_particles(np.array(particles), NF, "particle_input.dat") +write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file diff --git a/Scripts/initial_conditions/st7_empty_particles.py b/Scripts/initial_conditions/st7_empty_particles.py new file mode 100644 index 00000000..dca380ad --- /dev/null +++ b/Scripts/initial_conditions/st7_empty_particles.py @@ -0,0 +1,22 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../data_analysis") +from initial_condition_tools import uniform_sphere, write_particles, moment_interpolate_particles, linear_interpolate +import amrex_plot_tools as amrex + +# generation parameters +# MUST MATCH THE INPUTS IN THE EMU INPUT FILE! +NF = 3 +nphi_equator = 16 +energy_erg = 50 * 1e6*amrex.eV + +nnu = np.zeros((2,NF)) +fnu = np.zeros((2,NF,3)) + +particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, linear_interpolate) # [particle, variable] + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st8_coll_inst_test.py b/Scripts/initial_conditions/st8_coll_inst_test.py new file mode 100644 index 00000000..0b925b21 --- /dev/null +++ b/Scripts/initial_conditions/st8_coll_inst_test.py @@ -0,0 +1,36 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../data_reduction") +from initial_condition_tools import uniform_sphere, moment_interpolate_particles, minerbo_interpolate, write_particles +import amrex_plot_tools as amrex + +# These initial conditions are intended to replicate the collisional instability outputs in "Collisional Flavor Instabilities of Supernova Neutrinos" by L. Johns [2104.11369]. +# Simulation parameters +NF = 2 +nphi_equator = 16 +nnue = 3.0e+33 # 1/ccm +nnua = 2.5e+33 # 1/ccm +nnux = 1.0e+33 # 1/ccm +fnue = np.array([0.0 , 0.0 , 0.0]) +fnua = np.array([0.0 , 0.0 , 0.0]) +fnux = np.array([0.0 , 0.0 , 0.0]) +energy_erg = 20.0 # MeV +energy_erg *= 1e6*amrex.eV # erg + +nnu = np.zeros((2,NF)) +nnu[0,0] = nnue +nnu[1,0] = nnua +nnu[:,1:] = nnux + +fnu = np.zeros((2,NF,3)) +fnu[0,0,:] = nnue * fnue +fnu[1,0,:] = nnua * fnua +fnu[:,1:,:] = nnu[:,1:,np.newaxis] * fnux[np.newaxis,np.newaxis,:] + +particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, minerbo_interpolate) # [particle, variable] + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index ebb8d6db..5207b5f7 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -94,33 +94,22 @@ def delete_generated_files(): #==================================# # FlavoredNeutrinoContainer.H_fill # #==================================# - vars = ["f"] + vars = ["N"] tails = ["","bar"] code = [] for t in tails: - code += ["N"+t] # number of neutrinos - code += ["L"+t] # length of isospin vector, units of number of neutrinos for v in vars: A = HermitianMatrix(args.N, v+"{}{}_{}"+t) code += A.header() - code += ["TrHf"] + code += ["TrHN"] + code += ["Vphase"] - code = [code[i]+"," for i in range(len(code))] - write_code(code, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.H_fill")) + code_lines = [code[i]+"," for i in range(len(code))] + write_code(code_lines, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.H_fill")) #========================================================# # FlavoredNeutrinoContainerInit.H_particle_varnames_fill # #========================================================# - vars = ["f"] - tails = ["","bar"] - code = [] - for t in tails: - code += ["N"+t] - code += ["L"+t] - for v in vars: - A = HermitianMatrix(args.N, v+"{}{}_{}"+t) - code += A.header() - code += ["TrHf"] code_string = 'attribute_names = {"time", "x", "y", "z", "pupx", "pupy", "pupz", "pupt", ' code = ['"{}"'.format(c) for c in code] code_string = code_string + ", ".join(code) + "};" @@ -178,8 +167,8 @@ def delete_generated_files(): "*p.rdata(PIdx::pupz)*p.rdata(PIdx::pupz)/p.rdata(PIdx::pupt)/p.rdata(PIdx::pupt));"]) code = [] for t in tails: - string3 = ")*p.rdata(PIdx::N"+t+")" - flist = HermitianMatrix(args.N, "f{}{}_{}"+t).header() + string3 = ")" + flist = HermitianMatrix(args.N, "N{}{}_{}"+t).header() for ivar in range(len(deposit_vars)): deplist = HermitianMatrix(args.N, deposit_vars[ivar]+"{}{}_{}"+t).header() for icomp in range(len(flist)): @@ -193,10 +182,10 @@ def delete_generated_files(): code = [] for t in tails: # diagonal averages - N = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") + N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") Nlist = N.header_diagonals(); for i in range(len(Nlist)): - code.append("Trf += "+Nlist[i]+";") + code.append("TrN += "+Nlist[i]+";") write_code(code, os.path.join(args.emu_home, "Source/generated_files", "DataReducer.cpp_fill_particles")) @@ -437,94 +426,85 @@ def sgn(t,var): line += "sqrt(2.) * PhysConst::GF * sx(i) * sy(j) * sz(k) * (inside_parentheses);" code.append(line) code.append("") + + code.append("T_pp += sx(i) * sy(j) * sz(k) * sarr(i, j, k, GIdx::T);") + code.append("Ye_pp += sx(i) * sy(j) * sz(k) * sarr(i, j, k, GIdx::Ye);") + code.append("rho_pp += sx(i) * sy(j) * sz(k) * sarr(i, j, k, GIdx::rho);") + code.append("") + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_interpolate_from_mesh_fill")) #========================# # Evolve.cpp_dfdt_fill # #========================# - # Set up Hermitian matrices A, B, C + # Generate quantum kinetic equations + + # Define useful constants hbar = sympy.symbols("PhysConst\:\:hbar",real=True) - code = [] - for t in tails: - H = HermitianMatrix(args.N, "V{}{}_{}"+t) - F = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") + attenuation_to_hamiltonian = sympy.symbols("parms->attenuation_hamiltonians", real=True) + V_phase = sympy.symbols("p.rdata(PIdx\:\:Vphase)", real=True) + pi = sympy.symbols("MathConst\:\:pi", real=True) + c = sympy.symbols("PhysConst\:\:c", real=True) - # G = Temporary variables for dFdt - G = HermitianMatrix(args.N, "dfdt{}{}_{}"+t) + # List that will store the QKE code. + code = [] - # Calculate C = i * [A,B] - #Fnew.anticommutator(H,F).times(sympy.I * dt); - G.H = ((H*F - F*H).times(-sympy.I/hbar)).H + # Looping over neutrinos(tail: no tail) and antineutrinos(tail: bar) + for t in tails: - # Write the temporary variables for dFdt - Gdeclare = ["amrex::Real {}".format(line) for line in G.code()] - code.append(Gdeclare) + # Define Fermi-Dirac distribution matrix f_eq = diag( f_e , f_x ) from input parameters + f_eq = HermitianMatrix(args.N, "f_eq_{}{}_{}"+t) # Fermi-dirac distribution matrix ----> To be used in calculation of QKE in sympy format + f_eq_cpp = HermitianMatrix(args.N, "f_eq"+t+"[{}][{}]") # Fermi-dirac distribution matrix ----> Using the systaxis of line 183 of the Evolve.cpp file + f_eq.H = f_eq_cpp.H # Assigning input mean free paths to SymPy matrix + f_eq_temp_declare = ["amrex::Real {}".format(line) for line in f_eq.code()] # + code.append(f_eq_temp_declare) + + # Define Gamma matrix from input parameters : Gamma = diag( k*_e , k*_x ) / 2 . ka is the inverse mean free path for flavor a, including Pauli blocking term. * means that Pauli blocking term is already in the inverse mean free path values. + Gamma = HermitianMatrix(args.N, "Gamma_{}{}_{}"+t) # Inverse mean free path matrix. Gamma = diag( k*e , k*x ) / 2. ----> To be used in calculation of QKE in sympy format + IMFP_abs = HermitianMatrix(args.N, "IMFP_abs"+t+"[{}][{}]") # Inverse mean free path matrix IMFP_abs = diag( k*e , k*x ) ----> Using the systaxis of line 181 of the Evolve.cpp file + Gamma.H = IMFP_abs.H / 2 # Compute Gamma + Gamma_temp_declare = ["amrex::Real {}".format(line) for line in Gamma.code()] + code.append(Gamma_temp_declare) + + # Define N_eq matrix + f_eq = HermitianMatrix(args.N, "f_eq_{}{}_{}"+t) # Fermi-dirac distribution matrix f_eq = diag( fe , fx ) + N_eq = HermitianMatrix(args.N, "N_eq_{}{}_{}"+t) # Equilibrium neutrino number matrix N_eq equals the integral of f_eq, where the integral is over the phase space that the particle represents. + N_eq.H = f_eq.H * V_phase / ( 2 * pi * hbar * c )**3 + N_eq_temp_declare = ["amrex::Real {}".format(line) for line in N_eq.code()] + code.append(N_eq_temp_declare) + + # Define collision term + Gamma = HermitianMatrix(args.N, "Gamma_{}{}_{}"+t) # Inverse mean free path matrix. Gamma = diag( k*e , k*x ) / 2. ka is the inverse mean free path for flavor a, including Pauli blocking term. + N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") # Neutrino number matrix + N_eq = HermitianMatrix(args.N, "N_eq_{}{}_{}"+t) # Equilibrium neutrino number matrix N_eq equals the integral of f_eq, where the integral is over the phase space that the particle represents. + C = HermitianMatrix(args.N, "C_{}{}_{}"+t) # Collision term C = { gamma , N_eq - N }, {} means anticonmutator + C.H = Gamma.H * ( N_eq.H - N.H ) + ( N_eq.H - N.H ) * Gamma.H # Compute collision term + C_temp_declare = ["amrex::Real {}".format(line) for line in C.code()] + code.append(C_temp_declare) + + # Writing QKE + C = HermitianMatrix(args.N, "C_{}{}_{}"+t) # Collision term C = { gamma , N_eq - N }, {} means anticonmutator + H = HermitianMatrix(args.N, "V{}{}_{}"+t) # Hamiltonian + N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") # Neutrino number matrix + dNdt_temp = HermitianMatrix(args.N, "dNdt{}{}_{}"+t) # Temporary matrix for dNdt + dNdt_temp.H = C.H * c + ((H*N - N*H).times(-sympy.I/hbar)).H * attenuation_to_hamiltonian # Compute quantum kinetic equation + dNdt_temp_declare = ["amrex::Real {}".format(line) for line in dNdt_temp.code()] + code.append(dNdt_temp_declare) # Store dFdt back into the particle data for F - dFdt = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") - Gempty = HermitianMatrix(args.N, "dfdt{}{}_{}"+t) - dFdt.H = Gempty.H + dNdt = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") + dNdt_empty = HermitianMatrix(args.N, "dNdt{}{}_{}"+t) + dNdt.H = dNdt_empty.H - # Write out dFdt->F - code.append(dFdt.code()) + # Write out dNdt->N + code.append(dNdt.code()) # store Tr(H*F) for estimating numerical errors - TrHf = (H*F).trace(); - code.append(["p.rdata(PIdx::TrHf) += p.rdata(PIdx::N"+t+") * ("+sympy.cxxcode(sympy.simplify(TrHf))+");"]) + TrHN = (H*N).trace(); + code.append(["p.rdata(PIdx::TrHN) += ("+sympy.cxxcode(sympy.simplify(TrHN))+");"]) code = [line for sublist in code for line in sublist] write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill")) - #================================================# - # FlavoredNeutrinoContainer.cpp_Renormalize_fill # - #================================================# - code = [] - for t in tails: - # make sure the trace is 1 - code.append("sumP = 0;") - f = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") - fdlist = f.header_diagonals() - flist = f.header() - for fii in fdlist: - code.append("sumP += " + fii + ";") - code.append("error = sumP-1.0;") - code.append("if( std::abs(error) > 100.*parms->maxError) amrex::Abort();") - code.append("if( std::abs(error) > parms->maxError ) {") - for fii in fdlist: - code.append(fii + " -= error/"+str(args.N)+";") - code.append("}") - code.append("") - - # make sure diagonals are positive - for fii in fdlist: - code.append("if("+fii+"<-100.*parms->maxError) amrex::Abort();") - code.append("if("+fii+"<-parms->maxError) "+fii+"=0;") - code.append("") - - # make sure the flavor vector length is what it would be with a 1 in only one diagonal - length = sympy.symbols("length",real=True) - length = f.SU_vector_magnitude() - target_length = "p.rdata(PIdx::L"+t+")" - code.append("length = "+sympy.cxxcode(sympy.simplify(length))+";") - code.append("error = length-"+str(target_length)+";") - code.append("if( std::abs(error) > 100.*parms->maxError) amrex::Abort();") - code.append("if( std::abs(error) > parms->maxError) {") - for fii in flist: - code.append(fii+" /= length/"+str(target_length)+";") - code.append("}") - code.append("") - - write_code(code, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.cpp_Renormalize_fill")) - # Write code to output file, using a template if one is provided - # write_code(code, "code.cpp", args.output_template) - - - #====================================================# - # FlavoredNeutrinoContainerInit.cpp_set_trace_length # - #====================================================# - code = [] - for t in tails: - f = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") - code.append("p.rdata(PIdx::L"+t+") = "+sympy.cxxcode(sympy.simplify(f.SU_vector_magnitude()))+";" ) - write_code(code, os.path.join(args.emu_home, "Source/generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length")) + diff --git a/Scripts/tests/coll_equi_test.py b/Scripts/tests/coll_equi_test.py new file mode 100644 index 00000000..089feef5 --- /dev/null +++ b/Scripts/tests/coll_equi_test.py @@ -0,0 +1,60 @@ +import numpy as np +import argparse +import glob +import EmuReader +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__))+"/../data_reduction/" +sys.path.append(importpath) +import amrex_plot_tools as amrex + +parser = argparse.ArgumentParser() +parser.add_argument("-na", "--no_assert", action="store_true", help="If --no_assert is supplied, do not raise assertion errors if the test error > tolerance.") +args = parser.parse_args() + +# physical constants +theta12 = 33.82*np.pi/180. # radians +dm21c4 = 7.39e-5 * amrex.eV**2 # erg^2 + +tolerance = 2e-2 +NF=3 + +if __name__ == "__main__": + + rkey, ikey = amrex.get_particle_keys(NF) + + N_ee = [] + N_uu = [] + N_tt = [] + N_eebar = [] + N_uubar = [] + N_ttbar = [] + + idata, rdata = EmuReader.read_particle_data('plt01000', ptype="neutrinos") + + for i in range(len(rdata)): + p = rdata[i] + N_ee.append(p[rkey["N00_Re"]]) + N_uu.append(p[rkey["N11_Re"]]) + N_tt.append(p[rkey["N22_Re"]]) + N_eebar.append(p[rkey["N00_Rebar"]]) + N_uubar.append(p[rkey["N11_Rebar"]]) + N_ttbar.append(p[rkey["N22_Rebar"]]) + + print(f'average N_ee {np.average(N_ee)}') + print(f'average N_uu {np.average(N_uu)}') + print(f'average N_tt {np.average(N_tt)}') + print(f'average N_eebar {np.average(N_eebar)}') + print(f'average N_uubar {np.average(N_uubar)}') + print(f'average N_ttbar {np.average(N_ttbar)}') + + def myassert(condition): + if not args.no_assert: + assert(condition) + + myassert( np.all(np.isclose(N_ee, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_uu, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_tt, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_eebar, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_uubar, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_ttbar, np.array(1e33), atol=1e33/100)) ) \ No newline at end of file diff --git a/Scripts/tests/coll_inst_test.py b/Scripts/tests/coll_inst_test.py new file mode 100644 index 00000000..e5f0ad49 --- /dev/null +++ b/Scripts/tests/coll_inst_test.py @@ -0,0 +1,276 @@ +''' +This test script is used to reproduce the isotropic 2-flavor simulation in "Collisional Flavor Instabilities of Supernova Neutrinos" by L. Johns [2104.11369]. +The points of comparison are the LSA conducted in this paper (equation 14) and Julien's script that reproduces the same results (script received via private communication). +Created by Erick Urquilla. University of Tennessee Knoxville, USA. +''' + +import numpy as np +import argparse +import glob +import EmuReader +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__))+"/../data_reduction/" +sys.path.append(importpath) +import amrex_plot_tools as amrex +import numpy as np +import h5py +import glob + +parser = argparse.ArgumentParser() +parser.add_argument("-na", "--no_assert", action="store_true", help="If --no_assert is supplied, do not raise assertion errors if the test error > tolerance.") +args = parser.parse_args() + +if __name__ == "__main__": + + # Create a list of data files to read + directories = glob.glob("plt*_reduced_data.h5") + # Sort the data file names by time step number + directories = sorted(directories, key=lambda x: int(x.split("plt")[1].split("_")[0])) + + N_avg_mag = np.zeros((len(directories),2,2)) + Nbar_avg_mag = np.zeros((len(directories),2,2)) + F_avg_mag = np.zeros((len(directories),3,2,2)) + Fbar_avg_mag = np.zeros((len(directories),3,2,2)) + t = np.zeros(len(directories)) + + for i in range(len(directories)): + with h5py.File(directories[i], 'r') as hf: + N_avg_mag[i] = np.array(hf['N_avg_mag(1|ccm)'][:][0]) + Nbar_avg_mag[i] = np.array(hf['Nbar_avg_mag(1|ccm)'][:][0]) + F_avg_mag[i] = np.array(hf['F_avg_mag(1|ccm)'][:][0]) + Fbar_avg_mag[i] = np.array(hf['Fbar_avg_mag(1|ccm)'][:][0]) + t[i] = np.array(hf['t(s)'][:][0]) + + # Fit the exponential function ( y = a e ^ ( b x ) ) to the data + l1 = 50 # initial item for fit + l2 = 200 # last item for fit + coefficients = np.polyfit(t[l1:l2], np.log(N_avg_mag[:,0,1][l1:l2]), 1) + coefficients_bar = np.polyfit(t[l1:l2], np.log(Nbar_avg_mag[:,0,1][l1:l2]), 1) + a = np.exp(coefficients[1]) + b = coefficients[0] + abar = np.exp(coefficients_bar[1]) + bbar = coefficients_bar[0] + print(f'{b} ---> EMU : Im Omega') + print(f'{bbar} ---> EMU : Im Omegabar') + + # Plots + import matplotlib.pyplot as plt + + # Plots N and Nbar + plt.plot(t, N_avg_mag[:,0,0], label = r'$N_{ee}$') + plt.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$') + plt.plot(t, N_avg_mag[:,1,1], label = r'$N_{uu}$') + plt.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{N}_{ee}$') + plt.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') + plt.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') + plt.legend() + plt.xlabel(r'$t$ (s)') + plt.ylabel(r'$N$ and $\bar{N}$') + plt.savefig('N_and_Nbar.pdf') + plt.clf() + + # Plots N and F + plt.plot(t, N_avg_mag[:,0,0], label = r'$N_{ee}$') + plt.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$') + plt.plot(t, N_avg_mag[:,1,1], label = r'$N_{uu}$') + plt.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {b}') + plt.plot(t, F_avg_mag[:,0,0,1], label = r'$F^x_{eu}$') + plt.plot(t, F_avg_mag[:,1,0,1], label = r'$F^y_{eu}$') + plt.plot(t, F_avg_mag[:,2,0,1], label = r'$F^z_{eu}$') + plt.legend() + plt.xlabel(r'$t$ (s)') + plt.ylabel(r'$N$ and $\vec{F}$') + plt.yscale('log') + plt.savefig('N_and_F.pdf') + plt.clf() + + # Plots Nbar and Fbar + plt.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{N}_{ee}$') + plt.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') + plt.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') + plt.plot(t[l1:l2], Nbar_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {bbar}') + plt.plot(t, Fbar_avg_mag[:,0,0,1], label = r'$\bar{F}^x_{eu}$') + plt.plot(t, Fbar_avg_mag[:,1,0,1], label = r'$\bar{F}^y_{eu}$') + plt.plot(t, Fbar_avg_mag[:,2,0,1], label = r'$\bar{F}^z_{eu}$') + plt.legend() + plt.xlabel(r'$t$ (s)') + plt.ylabel(r'$\bar{N}$ and $\vec{\bar{F}}$') + plt.yscale('log') + plt.savefig('Nbar_and_Fbar.pdf') + plt.clf() + + ###################################################################################### + ###################################################################################### + # LSA in "Collisional flavor instabilities of supernova neutrinos", L. Johns [2104.11369] + + h = 6.6260755e-27 # erg s + hbar = h/(2.*np.pi) # erg s + c = 2.99792458e10 # cm/s + MeV = 1.60218e-6 # erg + eV = MeV/1e6 # erg + GF_GeV2 = 1.1663787e-5 # GeV^-2 + GF = GF_GeV2 / (1000*MeV)**2 * (hbar*c)**3 # erg cm^3 + + Nee = 3e33 # cm^-3 + Neebar = 2.5e33 # cm^-3 + Nxx = 1e33 # cm^-3 + + opac_rescale = 1e4 + + kappa_e = 1/(0.417*1e5)*opac_rescale # cm^-1 + kappa_ebar = 1/(4.36*1e5)*opac_rescale # cm^-1 + kappa_x = 0.*opac_rescale # cm^-1 + + # Collision rates (in s^-1) + Gamma_plus = (kappa_e+kappa_x)/2 * c + Gamma_minus = (kappa_e-kappa_x)/2 * c + Gammabar_plus = (kappa_ebar+kappa_x)/2 * c + Gammabar_minus= (kappa_ebar - kappa_x)/2 * c + + omega = 0.304*1e-5 * c # Delta m^2/2E, in s^-1 + mu = np.sqrt(2)*GF/hbar # s^-1.cm^3 + + S = Nee - Nxx + Neebar - Nxx + D = Nee - Nxx - Neebar + Nxx + + ImOmega_Lucas_LSA = ( ( Gamma_plus - Gammabar_plus ) / 2 ) * ( mu * S / np.sqrt( ( mu * D )**2 + 4 * omega * mu * S ) ) - ( Gamma_plus + Gammabar_plus ) / 2 + + print(f'{ImOmega_Lucas_LSA} ---> Im Omega and Omegabar : LSA in equation 14 of L. Johns [2104.11369]') + + ###################################################################################### + ###################################################################################### + + def myassert(condition): + if not args.no_assert: + assert(condition) + + b_lsa = ImOmega_Lucas_LSA + rel_error = np.abs( b - b_lsa ) / np.abs( ( b + b_lsa ) / 2 ) + rel_error_bar = np.abs( bbar - b_lsa ) / np.abs( ( bbar + b_lsa ) / 2 ) + rel_error_max = 0.05 + + print(f"{rel_error} ---> relative error in ImOmega : EMU") + print(f"{rel_error_bar} ---> relative error in ImOmegabar : EMU") + + myassert( rel_error < rel_error_max ) + myassert( rel_error_bar < rel_error_max ) + + ###################################################################################### + ###################################################################################### + + """ + Created on Wed Jun 5 13:11:50 2024 + Solves the isotropic QKE following "Collisional flavor instabilities of supernova neutrinos", L. Johns [2104.11369] + + @author: jfroustey + """ + + import numpy as np + import matplotlib.pyplot as plt + from scipy.integrate import solve_ivp + + h = 6.6260755e-27 # erg s + hbar = h/(2.*np.pi) # erg s + c = 2.99792458e10 # cm/s + MeV = 1.60218e-6 # erg + eV = MeV/1e6 # erg + GF_GeV2 = 1.1663787e-5 # GeV^-2 + GF = GF_GeV2 / (1000*MeV)**2 * (hbar*c)**3 # erg cm^3 + + Nee = 3e33 # cm^-3 + Neebar = 2.5e33 # cm^-3 + Nxx = 1e33 # cm^-3 + + opac_rescale = 1e4 + + kappa_e = 1/(0.417*1e5)*opac_rescale # cm^-1 + kappa_ebar = 1/(4.36*1e5)*opac_rescale # cm^-1 + kappa_x = 0.*opac_rescale # cm^-1 + + # Collision rates (in s^-1) + Gamma_plus = (kappa_e+kappa_x)/2 * c + Gamma_minus = (kappa_e-kappa_x)/2 * c + Gammabar_plus = (kappa_ebar+kappa_x)/2 * c + Gammabar_minus= (kappa_ebar - kappa_x)/2 * c + + # Vacuum term + + theta = 1e-6 + c2t = np.cos(2*theta) + s2t = np.sin(2*theta) + omega = 0.304*1e-5 * c # Delta m^2/2E, in s^-1 + + P0_AE = (Nee+Nxx)/2 + Pz_AE = (Nee-Nxx)/2 + Pbar0_AE = (Neebar+Nxx)/2 + Pbarz_AE = (Neebar-Nxx)/2 + + mu = np.sqrt(2)*GF/hbar # s^-1.cm^3 + + def QKE(t,y): + P0, Px, Py, Pz, Pbar0, Pbarx, Pbary, Pbarz = y + deriv = np.zeros(8) + + # Variation of P0, Pbar0 + deriv[0] = Gamma_plus*(P0_AE-P0) + Gamma_minus*(Pz_AE-Pz) + deriv[4] = Gammabar_plus*(Pbar0_AE-Pbar0) + Gammabar_minus*(Pbarz_AE - Pbarz) + + # Spatial parts + deriv[1] = omega*c2t*Py + mu*((Py-Pbary)*Pz - (Pz-Pbarz)*Py) - Gamma_plus*Px + deriv[2] = omega*(-s2t*Pz-c2t*Px) + mu*((Pz-Pbarz)*Px - (Px-Pbarx)*Pz) - Gamma_plus*Py + deriv[3] = omega*s2t*Py + mu*((Px-Pbarx)*Py - (Py-Pbary)*Px) + Gamma_plus*(Pz_AE-Pz) + Gamma_minus*(P0_AE-P0) + + deriv[5] = -omega*c2t*Pbary + mu*((Py-Pbary)*Pbarz - (Pz-Pbarz)*Pbary) - Gammabar_plus*Pbarx + deriv[6] = -omega*(-s2t*Pbarz - c2t*Pbarx) + mu*((Pz-Pbarz)*Pbarx - (Px-Pbarx)*Pbarz) - Gammabar_plus*Pbary + deriv[7] = -omega*s2t*Pbary + mu*((Px-Pbarx)*Pbary - (Py-Pbary)*Pbarx) + Gammabar_plus*(Pbarz_AE-Pbarz) + Gammabar_minus*(Pbar0_AE-Pbar0) + + return deriv + + time = np.linspace(0,90e-6/opac_rescale,2000) + y0 = np.array([P0_AE, 0., 0., Pz_AE, Pbar0_AE, 0., 0., Pbarz_AE]) + + myrtol, myatol = 1e-5, 1e-8 + sol = solve_ivp(QKE, (time[0],time[-1]), y0, t_eval=time, rtol=myrtol, atol=myatol) + + # PLOTS + plt.plot(time, sol.y[0,:]+sol.y[3,:], color='k', lw=2, label=r'$n_{\nu_e}$') + plt.plot(time, sol.y[4,:]+sol.y[7,:], color='k', lw=1.5, label=r'$n_{\bar{\nu}_e}$') + plt.plot(time, sol.y[0,:]-sol.y[3,:], color='k', lw=1, label=r'$n_{\nu_x}$') + + plt.plot(time,np.sqrt(sol.y[1,:]**2+sol.y[2,:]**2), lw=2, color='teal',label=r'$\nu_e - \nu_x$'+" coherence density") + plt.legend() + plt.grid(ls=':',color='C7') + plt.xlabel(r'$t \ (\mathrm{s})$') + plt.xlim(time[0],time[-1]) + plt.title(f"Opacities scaled by {opac_rescale:.1e}, with rtol={myrtol:.1e}, atol={myatol:.1e}") + plt.tight_layout() + plt.savefig(f"Johns_CFI_rescale_{opac_rescale:.0e}_rtol_{myrtol:.0e}_atol_{myatol:.0e}.pdf") + plt.close() + + ###################################################################################### + ###################################################################################### + + p1 = 150 # initial point for fit + p2 = 500 # final point for fit + N_eu_julien = np.sqrt(sol.y[1,:]**2+sol.y[2,:]**2) + coefficients = np.polyfit(time[p1:p2], np.log(N_eu_julien[p1:p2]), 1) + aj = np.exp(coefficients[1]) + bj = coefficients[0] + print(f'{bj} ---> Im Omega Julien') + rel_error_j = np.abs( bj - b_lsa ) / np.abs( ( bj + b_lsa ) / 2 ) + print(f"{rel_error_j} ---> relative erroror in Julien script") + + # Plotting Julien, EMU and Lucas LSA data + plt.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$ EMU') + plt.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega EMU = {b}', linestyle = 'dashed') + plt.plot(time,N_eu_julien, label = r'$N_{eu}$ Julien script') + plt.plot(time[p1:p2],N_eu_julien[p1:p2], label = f'Im Omega Julien = {bj}', linestyle = 'dashed') + plt.plot(time[p1:p2], 1e23*np.exp(ImOmega_Lucas_LSA*time[p1:p2]), label = f'Im Omega Lucas LSA = {ImOmega_Lucas_LSA}', linestyle = 'dashed') + plt.xlabel(r'$t \ (\mathrm{s})$') + plt.ylabel(r'$N_{eu}$') + plt.title(f"Collisional flavor instability test") + plt.yscale('log') + plt.legend() + plt.savefig('EMU_Julien_LucasLSA_Neu.pdf') + plt.close() \ No newline at end of file diff --git a/Scripts/tests/convergence.sh b/Scripts/tests/convergence.sh deleted file mode 100644 index 3816a379..00000000 --- a/Scripts/tests/convergence.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -# echo the commands -set -x - -# All runs will use these -DIM=3 -EXEC=./main${DIM}d.gnu.TPROF.MPI.ex -MPINUM=4 - -RUNPARAMS=" -cfl_factor=-1 -nsteps=1000000 -end_time=5.0e-11" - -# Each integrator will set these -INTPARAMS="" -INTNAME="" - -# Clean up any existing output files before we start -rm -rf plt* -rm -rf single_neutrino*.png -rm -rf msw_test_*.txt - -# Define a function for a single run -do_single () { - mpiexec -n ${MPINUM} ${EXEC} inputs_msw_test ${RUNPARAMS} flavor_cfl_factor=${FCFL} ${INTPARAMS} - echo "cfl: ${FCFL}" >> msw_test_${INTNAME}.txt - python3 msw_test.py -na >> msw_test_${INTNAME}.txt - python3 plot_first_particle.py - mv single_neutrino.png single_neutrino_fcfl_${FCFL}_${INTNAME}.png - rm -rf plt* -} - -# Define a function for running convergence -do_convergence () { - FCFL=0.1 - do_single - - FCFL=0.05 - do_single - - FCFL=0.025 - do_single - - FCFL=0.0125 - do_single - - FCFL=0.00625 - do_single - - FCFL=0.003125 - do_single -} - -# Forward Euler convergence -INTPARAMS=" -integration.type=0" - -INTNAME="fe" - -do_convergence - -# Trapezoid convergence -INTPARAMS=" -integration.type=1 -integration.rk.type=2" - -INTNAME="trapz" - -do_convergence - -# SSPRK3 Convergence -INTPARAMS=" -integration.type=1 -integration.rk.type=3" - -INTNAME="ssprk3" - -do_convergence - -# RK4 Convergence -INTPARAMS=" -integration.type=1 -integration.rk.type=4" - -INTNAME="rk4" - -do_convergence diff --git a/Scripts/tests/fast_flavor_k_test.py b/Scripts/tests/fast_flavor_k_test.py index 5703e49c..8b661152 100644 --- a/Scripts/tests/fast_flavor_k_test.py +++ b/Scripts/tests/fast_flavor_k_test.py @@ -30,10 +30,10 @@ rkey, ikey = amrex.get_particle_keys(NF) t = [] - fexR = [] - fexI = [] - fexRbar = [] - fexIbar = [] + NexR = [] + NexI = [] + NexRbar = [] + NexIbar = [] pupt = [] nfiles = len(glob.glob("plt[0-9][0-9][0-9][0-9][0-9]")) @@ -43,18 +43,18 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata t.append(p[0][rkey["time"]]) - fexR.append(np.max(np.abs(p[:,rkey["f01_Re"]]))) - fexI.append(np.max(np.abs(p[:,rkey["f01_Im"]]))) - fexRbar.append(np.max(np.abs(p[:,rkey["f01_Rebar"]]))) - fexIbar.append(np.max(np.abs(p[:,rkey["f01_Imbar"]]))) + NexR.append(np.max(np.abs(p[:,rkey["N01_Re"]]))) + NexI.append(np.max(np.abs(p[:,rkey["N01_Im"]]))) + NexRbar.append(np.max(np.abs(p[:,rkey["N01_Rebar"]]))) + NexIbar.append(np.max(np.abs(p[:,rkey["N01_Imbar"]]))) pupt.append(p[0][rkey["pupt"]]) t = np.array(t) - fexR = np.array(fexR) - fexI = np.array(fexI) - fexRbar = np.array(fexRbar) - fexIbar = np.array(fexIbar) - print(fexR) + NexR = np.array(NexR) + NexI = np.array(NexI) + NexRbar = np.array(NexRbar) + NexIbar = np.array(NexIbar) + print(NexR) # The neutrino energy we set E = 50. * 1e6*amrex.eV @@ -70,27 +70,27 @@ # get growth rate from each diagonal component dt = t[i1]-t[i0] - fexRomega = np.log(np.abs(fexR[i1]/fexR[i0])) / dt - fexIomega = np.log(np.abs(fexI[i1]/fexI[i0])) / dt - fexRbaromega = np.log(np.abs(fexRbar[i1]/fexRbar[i0])) / dt - fexIbaromega = np.log(np.abs(fexIbar[i1]/fexIbar[i0])) / dt + NexRomega = np.log(np.abs(NexR[i1]/NexR[i0])) / dt + NexIomega = np.log(np.abs(NexI[i1]/NexI[i0])) / dt + NexRbaromega = np.log(np.abs(NexRbar[i1]/NexRbar[i0])) / dt + NexIbaromega = np.log(np.abs(NexIbar[i1]/NexIbar[i0])) / dt - print("growth rates:",fexRomega,fexIomega,fexRbaromega,fexIbaromega) - print("growth rates / theoretical:",fexRomega/ImOmega,fexIomega/ImOmega,fexRbaromega/ImOmega,fexIbaromega/ImOmega) + print("growth rates:",NexRomega,NexIomega,NexRbaromega,NexIbaromega) + print("growth rates / theoretical:",NexRomega/ImOmega,NexIomega/ImOmega,NexRbaromega/ImOmega,NexIbaromega/ImOmega) def myassert(condition): if not args.no_assert: assert(condition) - fexRerror = np.abs(ImOmega - fexRomega) / ImOmega - myassert( fexRerror < tolerance ) + NexRerror = np.abs(ImOmega - NexRomega) / ImOmega + myassert( NexRerror < tolerance ) - fexIerror = np.abs(ImOmega - fexIomega) / ImOmega - myassert( fexIerror < tolerance ) + NexIerror = np.abs(ImOmega - NexIomega) / ImOmega + myassert( NexIerror < tolerance ) - fexRbarerror = np.abs(ImOmega - fexRbaromega) / ImOmega - myassert( fexRbarerror < tolerance ) + NexRbarerror = np.abs(ImOmega - NexRbaromega) / ImOmega + myassert( NexRbarerror < tolerance ) - fexIbarerror = np.abs(ImOmega - fexIbaromega) / ImOmega - myassert( fexIbarerror < tolerance ) + NexIbarerror = np.abs(ImOmega - NexIbaromega) / ImOmega + myassert( NexIbarerror < tolerance ) diff --git a/Scripts/tests/fast_flavor_test.py b/Scripts/tests/fast_flavor_test.py index ff9d7647..cd50fd75 100644 --- a/Scripts/tests/fast_flavor_test.py +++ b/Scripts/tests/fast_flavor_test.py @@ -26,10 +26,10 @@ rkey, ikey = amrex.get_particle_keys(NF) t = [] - fexR = [] - fexI = [] - fexRbar = [] - fexIbar = [] + NexR = [] + NexI = [] + NexRbar = [] + NexIbar = [] pupt = [] nfiles = len(glob.glob("plt[0-9][0-9][0-9][0-9][0-9]")) @@ -39,17 +39,18 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata[0] t.append(p[rkey["time"]]) - fexR.append(p[rkey["f01_Re"]]) - fexI.append(p[rkey["f01_Im"]]) - fexRbar.append(p[rkey["f01_Rebar"]]) - fexIbar.append(p[rkey["f01_Imbar"]]) + NexR.append(p[rkey["N01_Re"]]) + NexI.append(p[rkey["N01_Im"]]) + p = rdata[1] + NexRbar.append(p[rkey["N01_Rebar"]]) + NexIbar.append(p[rkey["N01_Imbar"]]) pupt.append(p[rkey["pupt"]]) t = np.array(t) - fexR = np.array(fexR) - fexI = np.array(fexI) - fexRbar = np.array(fexRbar) - fexIbar = np.array(fexIbar) + NexR = np.array(NexR) + NexI = np.array(NexI) + NexRbar = np.array(NexRbar) + NexIbar = np.array(NexIbar) # The neutrino energy we set E = 50. * 1e6*amrex.eV @@ -61,32 +62,32 @@ # get growth rate from each diagonal component dt = t[i1]-t[i0] - fexRomega = np.log(fexR[i1]/fexR[i0]) / dt - fexIomega = np.log(fexI[i1]/fexI[i0]) / dt - fexRbaromega = np.log(fexRbar[i1]/fexRbar[i0]) / dt - fexIbaromega = np.log(fexIbar[i1]/fexIbar[i0]) / dt + NexRomega = np.log(NexR[i1]/NexR[i0]) / dt + NexIomega = np.log(NexI[i1]/NexI[i0]) / dt + NexRbaromega = np.log(NexRbar[i1]/NexRbar[i0]) / dt + NexIbaromega = np.log(NexIbar[i1]/NexIbar[i0]) / dt def myassert(condition): if not args.no_assert: assert(condition) - print("growth rates:",fexRomega,fexIomega,fexRbaromega,fexIbaromega) + print("growth rates:",NexRomega,NexIomega,NexRbaromega,NexIbaromega) print(dt,t[i0],t[i1]) - print(fexR[i1],fexR[i0]) - print(fexI[i1],fexI[i0]) - print(fexRbar[i1],fexRbar[i0]) - print(fexIbar[i1],fexIbar[i0]) + print(NexR[i1],NexR[i0]) + print(NexI[i1],NexI[i0]) + print(NexRbar[i1],NexRbar[i0]) + print(NexIbar[i1],NexIbar[i0]) - fexRerror = np.abs(ImOmega - fexRomega) / ImOmega - myassert( fexRerror < tolerance ) + NexRerror = np.abs(ImOmega - NexRomega) / ImOmega + myassert( NexRerror < tolerance ) - fexIerror = np.abs(ImOmega - fexIomega) / ImOmega - myassert( fexIerror < tolerance ) + NexIerror = np.abs(ImOmega - NexIomega) / ImOmega + myassert( NexIerror < tolerance ) - fexRbarerror = np.abs(ImOmega - fexRbaromega) / ImOmega - myassert( fexRbarerror < tolerance ) + NexRbarerror = np.abs(ImOmega - NexRbaromega) / ImOmega + myassert( NexRbarerror < tolerance ) - fexIbarerror = np.abs(ImOmega - fexIbaromega) / ImOmega - myassert( fexIbarerror < tolerance ) + NexIbarerror = np.abs(ImOmega - NexIbaromega) / ImOmega + myassert( NexIbarerror < tolerance ) diff --git a/Scripts/tests/msw_test.py b/Scripts/tests/msw_test.py index 0c4f24ac..aab770cb 100644 --- a/Scripts/tests/msw_test.py +++ b/Scripts/tests/msw_test.py @@ -35,10 +35,10 @@ rkey, ikey = amrex.get_particle_keys(NF) t = [] - fee = [] - fxx = [] - feebar = [] - fxxbar = [] + Nee = [] + Nxx = [] + Neebar = [] + Nxxbar = [] pupt = [] nfiles = len(glob.glob("plt[0-9][0-9][0-9][0-9][0-9]")) @@ -48,17 +48,17 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata[0] t.append(p[rkey["time"]]) - fee.append(p[rkey["f00_Re"]]) - fxx.append(p[rkey["f11_Re"]]) - feebar.append(p[rkey["f00_Rebar"]]) - fxxbar.append(p[rkey["f11_Rebar"]]) + Nee.append(p[rkey["N00_Re"]]) + Nxx.append(p[rkey["N11_Re"]]) + Neebar.append(p[rkey["N00_Rebar"]]) + Nxxbar.append(p[rkey["N11_Rebar"]]) pupt.append(p[rkey["pupt"]]) t = np.array(t) - fee = np.array(fee) - fxx = np.array(fxx) - feebar = np.array(feebar) - fxxbar = np.array(fxxbar) + Nee = np.array(Nee) + Nxx = np.array(Nxx) + Neebar = np.array(Neebar) + Nxxbar = np.array(Nxxbar) # The neutrino energy we set #E = dm21c4 * np.sin(2.*theta12) / (8.*np.pi*hbar*clight) @@ -83,31 +83,31 @@ def myassert(condition): assert(condition) # calculate errors - fee_analytic = Psurv(dm2_eff, sin2_eff, E) - error_ee = np.max(np.abs( fee - fee_analytic ) ) + Nee_analytic = Psurv(dm2_eff, sin2_eff, E) + error_ee = np.max(np.abs( Nee - Nee_analytic ) ) print("f_ee error:", error_ee) myassert( error_ee < tolerance ) - fxx_analytic = 1. - Psurv(dm2_eff, sin2_eff, E) - error_xx = np.max(np.abs( fxx - fxx_analytic ) ) + Nxx_analytic = 1. - Psurv(dm2_eff, sin2_eff, E) + error_xx = np.max(np.abs( Nxx - Nxx_analytic ) ) print("f_xx error:", error_xx) myassert( error_xx < tolerance ) - feebar_analytic = Psurv(dm2_effbar, sin2_effbar, E) - error_eebar = np.max(np.abs( feebar - feebar_analytic ) ) + Neebar_analytic = Psurv(dm2_effbar, sin2_effbar, E) + error_eebar = np.max(np.abs( Neebar - Neebar_analytic ) ) print("f_eebar error:", error_eebar) myassert( error_eebar < tolerance ) - fxxbar_analytic = 1. - Psurv(dm2_effbar, sin2_effbar, E) - error_xxbar = np.max(np.abs( fxxbar - fxxbar_analytic ) ) + Nxxbar_analytic = 1. - Psurv(dm2_effbar, sin2_effbar, E) + error_xxbar = np.max(np.abs( Nxxbar - Nxxbar_analytic ) ) print("f_xxbar error:", error_xxbar) myassert( error_xxbar < tolerance ) - conservation_error = np.max(np.abs( (fee+fxx) -1. )) + conservation_error = np.max(np.abs( (Nee+Nxx) -1. )) print("conservation_error:", conservation_error) myassert(conservation_error < tolerance) - conservation_errorbar = np.max(np.abs( (feebar+fxxbar) -1. )) + conservation_errorbar = np.max(np.abs( (Neebar+Nxxbar) -1. )) print("conservation_errorbar:", conservation_errorbar) myassert(conservation_errorbar < tolerance) diff --git a/Scripts/tests/plot_convergence.py b/Scripts/tests/plot_convergence.py deleted file mode 100644 index 4be2b8c3..00000000 --- a/Scripts/tests/plot_convergence.py +++ /dev/null @@ -1,98 +0,0 @@ -import numpy as np -import matplotlib.pyplot as plt - -# Read forward Euler data -class ConvergenceData(object): - def __init__(self, filename=None): - self.data = {"cfl": [], - "f_ee error": [], - "f_xx error": [], - "f_eebar error": [], - "f_xxbar error": []} - - if filename: - self.readfrom(filename) - - def readfrom(self, filename): - f = open(filename, "r") - - while True: - entry = [f.readline().strip() for i in range(9)] - if not entry[0]: - break - for line in entry: - ls = line.split(":") - name = ls[0].strip() - value = ls[-1].strip() - for k in self.data.keys(): - if name == k: - self.data[k].append(float(value)) - - f.close() - - for k in self.data.keys(): - self.data[k] = np.array(self.data[k]) - - def get(self, key): - return self.data[key] - - def keys(self): - return self.data.keys() - - def error_keys(self): - return [k for k in self.data.keys() if k != "cfl"] - - def average_convergence(self, key): - # get the average convergence order for the keyed quantity - err = self.get(key) - cfl = self.get("cfl") - - orders = [] - for i in range(len(err)-1): - order = np.log10(err[i+1]/err[i]) / np.log10(cfl[i+1]/cfl[i]) - orders.append(order) - orders = np.array(orders) - - order_average = np.average(orders) - return order_average - - def plot_on_axis(self, axis, key, label, color): - log_cfl = np.log10(self.get("cfl")) - log_err = np.log10(self.get(key)) - axis.plot(log_cfl, log_err, label=label, marker="o", linestyle="None", color=color) - - order = self.average_convergence(key) - iMaxErr = np.argmax(log_err) - intercept = log_err[iMaxErr] - order * log_cfl[iMaxErr] - log_order_err = intercept + order * log_cfl - axis.plot(log_cfl, log_order_err, label="$O({}) = {:0.2f}$".format(label, order), marker="None", linestyle="--", color=color) - - -cdata = {} -cdata["fe"] = ConvergenceData("msw_test_fe.txt") -cdata["trapz"] = ConvergenceData("msw_test_trapz.txt") -cdata["ssprk3"] = ConvergenceData("msw_test_ssprk3.txt") -cdata["rk4"] = ConvergenceData("msw_test_rk4.txt") - -variables = cdata["fe"].error_keys() - -for v in variables: - fig, ax = plt.subplots() - - ax.set_xlabel("log10 flavor CFL") - ax.set_ylabel("log10 {}".format(v)) - - colors = ["red", "blue", "green", "magenta"] - - for k, c in zip(cdata.keys(), colors): - cd = cdata[k] - cd.plot_on_axis(ax, v, k, c) - - ax.invert_xaxis() - - ax.legend(loc=(1.05, 0.0)) - fig.tight_layout() - - plt.savefig("convergence_{}.eps".format(v.replace(" ","_"))) - plt.savefig("convergence_{}.png".format(v.replace(" ","_")), dpi=300) - plt.clf() diff --git a/Source/Constants.H b/Source/Constants.H index 1e9eeab6..21883969 100644 --- a/Source/Constants.H +++ b/Source/Constants.H @@ -18,6 +18,7 @@ namespace PhysConst static constexpr amrex::Real GF = 1.1663787e-5/*GeV^-2*//(1e9*1e9*CGSUnitsConst::eV*CGSUnitsConst::eV) * hbarc*hbarc*hbarc; //erg cm^3 static constexpr amrex::Real Mp = 1.6726219e-24; // g static constexpr amrex::Real sin2thetaW = 0.23122; + static constexpr amrex::Real kB = 1.380658e-16; // erg/K } namespace MathConst diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index d339beb6..4c3c15ab 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -64,8 +64,8 @@ void DataReducer::InitializeFiles() #endif } file0D.createDataSet("N_offdiag_mag(1|ccm)", dataspace, create_datatype(), props); - file0D.createDataSet("sumTrf", dataspace, create_datatype(), props); - file0D.createDataSet("sumTrHf", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrN", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrHN", dataspace, create_datatype(), props); #else @@ -104,9 +104,9 @@ void DataReducer::InitializeFiles() j++; outfile << j << ":N_offdiag_mag(1|ccm)\t"; j++; - outfile << j << ":sumTrf\t"; + outfile << j << ":sumTrN\t"; j++; - outfile << j << ":sumTrHf\t"; + outfile << j << ":sumTrHN\t"; outfile << std::endl; outfile.close(); @@ -130,15 +130,15 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, amrex::ReduceOps reduce_ops; auto particleResult = amrex::ParticleReduce< ReduceData >(neutrinos, [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { - Real TrHf = p.rdata(PIdx::TrHf); - Real Trf = 0; + Real TrHN = p.rdata(PIdx::TrHN); + Real TrN = 0; #include "generated_files/DataReducer.cpp_fill_particles" - return GpuTuple{Trf,TrHf}; + return GpuTuple{TrN,TrHN}; }, reduce_ops); - Real Trf = amrex::get<0>(particleResult); - Real TrHf = amrex::get<1>(particleResult); - ParallelDescriptor::ReduceRealSum(Trf); - ParallelDescriptor::ReduceRealSum(TrHf); + Real TrN = amrex::get<0>(particleResult); + Real TrHN = amrex::get<1>(particleResult); + ParallelDescriptor::ReduceRealSum(TrN); + ParallelDescriptor::ReduceRealSum(TrHN); //=============================// // Do reductions over the grid // @@ -290,8 +290,8 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, #endif } append_0D(file0D, "N_offdiag_mag(1|ccm)", N_offdiag_mag); - append_0D(file0D, "sumTrf", Trf); - append_0D(file0D, "sumTrHf", TrHf); + append_0D(file0D, "sumTrN", TrN); + append_0D(file0D, "sumTrHN", TrHN); #else std::ofstream outfile; outfile.open(filename0D, std::ofstream::app); @@ -324,8 +324,8 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, #endif } outfile << N_offdiag_mag << "\t"; - outfile << Trf << "\t"; - outfile << TrHf << "\t"; + outfile << TrN << "\t"; + outfile << TrHN << "\t"; outfile << std::endl; outfile.close(); #endif diff --git a/Source/Evolve.H b/Source/Evolve.H index 55f23dec..395634e7 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -26,7 +26,7 @@ namespace GIdx void Initialize(); } -amrex::Real compute_dt(const amrex::Geometry& geom, const amrex::Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor, const Real max_adaptive_speedup); +amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index f6f63d4e..62011ae7 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -19,19 +19,19 @@ namespace GIdx } } -Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const Real flavor_cfl_factor, const Real max_adaptive_speedup) +Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const TestParams* parms) { - AMREX_ASSERT(cfl_factor > 0.0 || flavor_cfl_factor > 0.0); + AMREX_ASSERT(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0); // translation part of timestep limit const auto dxi = geom.CellSizeArray(); Real dt_translation = 0.0; - if (cfl_factor > 0.0) { - dt_translation = std::min(std::min(dxi[0],dxi[1]), dxi[2]) / PhysConst::c * cfl_factor; + if (parms->cfl_factor > 0.0) { + dt_translation = std::min(std::min(dxi[0],dxi[1]), dxi[2]) / PhysConst::c * parms->cfl_factor; } Real dt_flavor = 0.0; - if (flavor_cfl_factor > 0.0) { + if (parms->flavor_cfl_factor > 0.0 && parms->collision_cfl_factor > 0.0) { // define the reduction operator to get the max contribution to // the potential from matter and neutrinos // compute "effective" potential (ergs) that produces characteristic timescale @@ -39,10 +39,9 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta ReduceOps reduce_op; ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; - for (MFIter mfi(state); mfi.isValid(); ++mfi) - { + for (MFIter mfi(state); mfi.isValid(); ++mfi) { const Box& bx = mfi.fabbox(); - auto const& fab = state.array(mfi); + auto const& fab = state.array(mfi); reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { @@ -50,25 +49,44 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta #include "generated_files/Evolve.cpp_compute_dt_fill" return {V_adaptive, V_stupid}; }); - } + } - // extract the reduced values from the combined reduced data structure - auto rv = reduce_data.value(); - Real Vmax_adaptive = amrex::get<0>(rv) + FlavoredNeutrinoContainer::Vvac_max; - Real Vmax_stupid = amrex::get<1>(rv) + FlavoredNeutrinoContainer::Vvac_max; + // extract the reduced values from the combined reduced data structure + auto rv = reduce_data.value(); + Real Vmax_adaptive = amrex::get<0>(rv) + FlavoredNeutrinoContainer::Vvac_max; + Real Vmax_stupid = amrex::get<1>(rv) + FlavoredNeutrinoContainer::Vvac_max; - // reduce across MPI ranks - ParallelDescriptor::ReduceRealMax(Vmax_adaptive); - ParallelDescriptor::ReduceRealMax(Vmax_stupid ); + // reduce across MPI ranks + ParallelDescriptor::ReduceRealMax(Vmax_adaptive); + ParallelDescriptor::ReduceRealMax(Vmax_stupid ); - // define the dt associated with each method - Real dt_flavor_adaptive = PhysConst::hbar/Vmax_adaptive*flavor_cfl_factor; - Real dt_flavor_stupid = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; + // define the dt associated with each method + Real dt_flavor_adaptive = std::numeric_limits::max(); + Real dt_flavor_stupid = std::numeric_limits::max(); + Real dt_flavor_absorption = std::numeric_limits::max(); // Initialize with infinity - // pick the appropriate timestep - dt_flavor = dt_flavor_stupid; - if(max_adaptive_speedup>1) - dt_flavor = min(dt_flavor_stupid*max_adaptive_speedup, dt_flavor_adaptive); + if (parms->attenuation_hamiltonians != 0) { + dt_flavor_adaptive = PhysConst::hbar / Vmax_adaptive * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; + dt_flavor_stupid = PhysConst::hbar / Vmax_stupid * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; + } + + if (parms->IMFP_method == 1) { + // Use the IMFPs from the input file and find the maximum absorption IMFP + double max_IMFP_abs = std::numeric_limits::lowest(); // Initialize max to lowest possible value + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < NUM_FLAVORS; ++j) { + max_IMFP_abs = std::max(max_IMFP_abs, parms->IMFP_abs[i][j]); + } + } + // Calculate dt_flavor_absorption + dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor; + } + + // pick the appropriate timestep + dt_flavor = min(dt_flavor_stupid, dt_flavor_adaptive, dt_flavor_absorption); + if(parms->max_adaptive_speedup>1) { + dt_flavor = min(dt_flavor_stupid*parms->max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); + } } Real dt = 0.0; @@ -146,6 +164,11 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M const ParticleInterpolator sy(delta_y, shape_factor_order_y); const ParticleInterpolator sz(delta_z, shape_factor_order_z); + // The following variables contains temperature, electron fraction, and density interpolated from grid quantities to particle positions + Real T_pp = 0; + Real Ye_pp = 0; + Real rho_pp = 0; + for (int k = sz.first(); k <= sz.last(); ++k) { for (int j = sy.first(); j <= sy.last(); ++j) { for (int i = sx.first(); i <= sx.last(); ++i) { @@ -154,6 +177,44 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } } + // Declare matrices to be used in quantum kinetic equation calculation + Real IMFP_abs[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino inverse mean free path matrix: diag( k_e , k_u , k_t ) + Real IMFP_absbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino inverse mean free path matrix: diag( kbar_e , kbar_u , kbar_t ) + Real f_eq[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino equilibrium Fermi-dirac distribution matrix: f_eq = diag( f_e , f_u , f_t ) + Real f_eqbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino equilibrium Fermi-dirac distribution matrix: f_eq = diag( fbar_e , fbar_u , fbar_t ) + + // Initialize matrices with zeros + for (int i=0; iIMFP_method==1){ + for (int i=0; iIMFP_abs[0][i]; // Read absorption inverse mean free path from input parameters file. + IMFP_absbar[i][i] = parms->IMFP_abs[1][i]; // Read absorption inverse mean free path from input parameters file. + + // Calculate the Fermi-Dirac distribution for neutrinos and antineutrinos. + f_eq[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - parms->munu[0][i] ) / T_pp ) ); + f_eqbar[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - parms->munu[1][i] ) / T_pp ) ); + + // Include the Pauli blocking term + if (parms->Do_Pauli_blocking == 1){ + IMFP_abs[i][i] = IMFP_abs[i][i] / ( 1 - f_eq[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). + IMFP_absbar[i][i] = IMFP_absbar[i][i] / ( 1 - f_eqbar[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). + } + } + } + else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0 or 1"); + + #include "generated_files/Evolve.cpp_dfdt_fill" + // set the dfdt values into p.rdata p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; @@ -163,11 +224,6 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M p.rdata(PIdx::pupy) = 0; p.rdata(PIdx::pupz) = 0; p.rdata(PIdx::pupt) = 0; - p.rdata(PIdx::N) = 0; - p.rdata(PIdx::Nbar) = 0; - p.rdata(PIdx::L) = 0; - p.rdata(PIdx::Lbar) = 0; - - #include "generated_files/Evolve.cpp_dfdt_fill" + p.rdata(PIdx::Vphase) = 0; }); } diff --git a/Source/FlavoredNeutrinoContainer.H b/Source/FlavoredNeutrinoContainer.H index cb6bcd68..42e13c2a 100644 --- a/Source/FlavoredNeutrinoContainer.H +++ b/Source/FlavoredNeutrinoContainer.H @@ -89,8 +89,6 @@ public: Redistribute(lev_min, lev_max, nGrow, local); } - void Renormalize(const TestParams* parms); - amrex::Vector get_attribute_names() const { return attribute_names; diff --git a/Source/FlavoredNeutrinoContainer.cpp b/Source/FlavoredNeutrinoContainer.cpp index cc15a333..6abd1907 100644 --- a/Source/FlavoredNeutrinoContainer.cpp +++ b/Source/FlavoredNeutrinoContainer.cpp @@ -82,27 +82,4 @@ UpdateLocationFrom(FlavoredNeutrinoContainer& Ploc) } }); } -} - -void FlavoredNeutrinoContainer:: -Renormalize(const TestParams* parms) -{ - BL_PROFILE("FlavoredNeutrinoContainer::Renormalize"); - - const int lev = 0; - -#ifdef _OPENMP -#pragma omp parallel -#endif - for (FNParIter pti(*this, lev); pti.isValid(); ++pti) - { - const int np = pti.numParticles(); - ParticleType * pstruct = &(pti.GetArrayOfStructs()[0]); - - amrex::ParallelFor (np, [=] AMREX_GPU_DEVICE (int i) { - ParticleType& p = pstruct[i]; - Real sumP, length, error; - #include "generated_files/FlavoredNeutrinoContainer.cpp_Renormalize_fill" - }); - } -} +} \ No newline at end of file diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 6885b413..356fa612 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -10,8 +10,13 @@ using namespace amrex; // Particle distribution in momentum space // //=========================================// -Gpu::ManagedVector > read_particle_data(std::string filename){ - Gpu::ManagedVector > particle_data; +Gpu::ManagedVector> read_particle_data(std::string filename){ + + // This function reads the input file containing the initial conditions of the particles. + // It reads the momentum, energy, and flavor occupation matrices for neutrinos and antineutrinos. + + // This array will save the particles information + Gpu::ManagedVector> particle_data; // open the file as a stream std::ifstream file(filename); @@ -32,9 +37,11 @@ Gpu::ManagedVector > read_particle_data(std::strin if(NF_in != NUM_FLAVORS) amrex::Print() << "Error: number of flavors in particle data file does not match the number of flavors Emu was compiled for." << std::endl; AMREX_ASSERT(NF_in == NUM_FLAVORS); + // Loop over every line in the initial condition file. + // This is equivalent to looping over every particle. + // Save every particle's information in the array particle_data. while(std::getline(file, line)){ ss = std::stringstream(line); - // skip over the first four attributes (x,y,z,t) for(int i=4; i> temp_particle[i]; particle_data.push_back(temp_particle); @@ -251,24 +258,14 @@ InitParticles(const TestParams* parms) for(int i_attrib=0; i_attrib= 0); - AMREX_ASSERT(p.rdata(PIdx::Nbar ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::L ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::Lbar ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f00_Re ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f11_Re ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f00_Rebar) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f11_Rebar) >= 0); - Real trace = p.rdata(PIdx::f00_Re ) + p.rdata(PIdx::f11_Re ); - Real tracebar = p.rdata(PIdx::f00_Rebar) + p.rdata(PIdx::f11_Rebar); + AMREX_ASSERT(p.rdata(PIdx::N00_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N11_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N00_Rebar) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N11_Rebar) >= 0); #if NUM_FLAVORS==3 - AMREX_ASSERT(p.rdata(PIdx::f22_Re ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f22_Rebar) >= 0); - trace += p.rdata(PIdx::f22_Re ); - tracebar += p.rdata(PIdx::f22_Rebar); + AMREX_ASSERT(p.rdata(PIdx::N22_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N22_Rebar) >= 0); #endif - AMREX_ASSERT(std::abs(trace -1)<1e-6); - AMREX_ASSERT(std::abs(tracebar-1)<1e-6); // Set particle position p.pos(0) = x; @@ -282,8 +279,18 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::time) = 0; // scale particle numbers based on number of points per cell and the cell volume - p.rdata(PIdx::N ) *= scale_fac; - p.rdata(PIdx::Nbar) *= scale_fac; + p.rdata(PIdx::N00_Re ) *= scale_fac; + p.rdata(PIdx::N11_Re ) *= scale_fac; + p.rdata(PIdx::N00_Rebar) *= scale_fac; + p.rdata(PIdx::N11_Rebar) *= scale_fac; +#if NUM_FLAVORS==3 + p.rdata(PIdx::N22_Re ) *= scale_fac; + p.rdata(PIdx::N22_Rebar) *= scale_fac; +#endif + + if(parms->IMFP_method == 1){ + p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/(3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); + } //=====================// // Apply Perturbations // @@ -292,37 +299,37 @@ InitParticles(const TestParams* parms) // random perturbations to the off-diagonals Real rand; symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); + p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); + p.rdata(PIdx::N01_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + p.rdata(PIdx::N01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); #if NUM_FLAVORS==3 symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::N22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::N22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar)); #endif } if(parms->perturbation_type == 1){ // Perturb real part of e-mu component only sinusoidally in z Real nu_k = (2.*M_PI) / parms->perturbation_wavelength_cm; - p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); - p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); + p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); } if(parms->perturbation_type == 2){ // random perturbations of the diagonals @@ -342,9 +349,6 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::f22_Rebar) *= 1. + parms->perturbation_amplitude*rand; #endif } - - -#include "generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length" } // loop over direction } // loop over location diff --git a/Source/Parameters.H b/Source/Parameters.H index 4ee8121a..d8ce7ee6 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -22,8 +22,8 @@ struct TestParams : public amrex::Gpu::Managed Real end_time; int write_plot_every; int write_plot_particles_every; - Real rho_in, Ye_in, T_in; // g/ccm, 1, MeV - Real cfl_factor, flavor_cfl_factor; + Real rho_in, Ye_in, kT_in; // g/ccm, 1, erg + Real cfl_factor, flavor_cfl_factor,collision_cfl_factor; Real max_adaptive_speedup; bool do_restart; std::string restart_dir; @@ -42,7 +42,18 @@ struct TestParams : public amrex::Gpu::Managed int perturbation_type; Real perturbation_wavelength_cm; Real perturbation_amplitude; - + + // absorption opacities and equilibrium neutrino chemical potentials + int IMFP_method; + int Do_Pauli_blocking = 0; // If 1, it will multiply the opacities by 1 / (1 - f_eq); if 0, do nothing. + Real IMFP_abs[2][NUM_FLAVORS]; + Real IMFP_scat[2][NUM_FLAVORS]; + Real munu[2][NUM_FLAVORS]; // equilibrium electron neutrino chemical potential in erg (CGS unist) + Real delta_E; // erg (CGS unist) + + // attenuation to hamiltonians + Real attenuation_hamiltonians; + void Initialize(){ ParmParse pp; pp.get("ncell", ncell); @@ -55,9 +66,10 @@ struct TestParams : public amrex::Gpu::Managed pp.get("end_time", end_time); pp.get("rho_g_ccm", rho_in); pp.get("Ye", Ye_in); - pp.get("T_MeV", T_in); + pp.get("T_MeV", kT_in); pp.get("cfl_factor", cfl_factor); pp.get("flavor_cfl_factor", flavor_cfl_factor); + pp.get("collision_cfl_factor", collision_cfl_factor); pp.get("max_adaptive_speedup", max_adaptive_speedup); pp.get("write_plot_every", write_plot_every); pp.get("write_plot_particles_every", write_plot_particles_every); @@ -65,6 +77,9 @@ struct TestParams : public amrex::Gpu::Managed pp.get("restart_dir", restart_dir); pp.get("maxError", maxError); + // convert to cgs + kT_in *= 1e6 * CGSUnitsConst::eV; // erg + // angular grid pp.get("particle_data_filename",particle_data_filename); @@ -74,28 +89,59 @@ struct TestParams : public amrex::Gpu::Managed if(perturbation_type == 1) pp.get("perturbation_wavelength_cm", perturbation_wavelength_cm); - // neutrino physics parameters for 2-flavor - pp.get("mass1_eV", mass1); - pp.get("mass2_eV", mass2); - pp.get("theta12_degrees", theta12); - pp.get("alpha1_degrees", alpha1); - mass1 *= CGSUnitsConst::eV/PhysConst::c2; - mass2 *= CGSUnitsConst::eV/PhysConst::c2; - theta12 *= M_PI/180.; - alpha1 *= M_PI/180.; + // neutrino physics parameters for 2-flavor + pp.get("mass1_eV", mass1); + pp.get("mass2_eV", mass2); + pp.get("theta12_degrees", theta12); + pp.get("alpha1_degrees", alpha1); + mass1 *= CGSUnitsConst::eV/PhysConst::c2; + mass2 *= CGSUnitsConst::eV/PhysConst::c2; + theta12 *= M_PI/180.; + alpha1 *= M_PI/180.; + if(NUM_FLAVORS>=2){ + pp.get("mass3_eV", mass3); + pp.get("theta13_degrees", theta13); + pp.get("theta23_degrees", theta23); + pp.get("alpha2_degrees", alpha2); + pp.get("deltaCP_degrees", deltaCP); + mass3 *= CGSUnitsConst::eV/PhysConst::c2; + theta13 *= M_PI/180.; + theta23 *= M_PI/180.; + alpha2 *= M_PI/180.; + deltaCP *= M_PI/180.; + } + + // attenuation to hamiltonians + pp.get("attenuation_hamiltonians", attenuation_hamiltonians); - if(NUM_FLAVORS>=2){ - pp.get("mass3_eV", mass3); - pp.get("theta13_degrees", theta13); - pp.get("theta23_degrees", theta23); - pp.get("alpha2_degrees", alpha2); - pp.get("deltaCP_degrees", deltaCP); - mass3 *= CGSUnitsConst::eV/PhysConst::c2; - theta13 *= M_PI/180.; - theta23 *= M_PI/180.; - alpha2 *= M_PI/180.; - deltaCP *= M_PI/180.; + // absorption opacities and equilibrium neutrino chemical potentials + pp.get("IMFP_method", IMFP_method); + pp.get("Do_Pauli_blocking", Do_Pauli_blocking); // If 1, it will multiply the opacities by 1 / (1 - f_eq); if 0, do nothing. + if(IMFP_method==0){ + // do nothing - all values will be set to zero + } + else if(IMFP_method==1){ + for(int i=0;irho_in,GIdx::rho,1); // g/ccm state.setVal(parms->Ye_in,GIdx::Ye,1); - state.setVal(parms->T_in,GIdx::T,1); // MeV + state.setVal(parms->kT_in,GIdx::T,1); // erg state.FillBoundary(geom.periodicity()); // initialize the grid variable names @@ -117,7 +117,7 @@ void evolve_flavor(const TestParams* parms) // Deposit particles to grid deposit_to_mesh(neutrinos_old, state, geom); - + // Write plotfile after initialization DataReducer rd; if (not parms->do_restart) { @@ -148,6 +148,7 @@ void evolve_flavor(const TestParams* parms) // B) We only Redistribute the integrator new data at the end of the timestep, not all the RHS data. // Thus, this copy clears the old RHS particles and creates particles in the RHS container corresponding // to the current particles in neutrinos. + neutrinos_rhs.copyParticles(neutrinos, true); // Step 3: Interpolate Mesh to construct the neutrino RHS in place @@ -172,9 +173,6 @@ void evolve_flavor(const TestParams* parms) // since Redistribute() applies periodic boundary conditions. neutrinos.SyncLocation(Sync::PositionToCoordinate); - // Renormalize the neutrino state - neutrinos.Renormalize(parms); - // Get which step the integrator is on const int step = integrator.get_step_number(); const Real time = integrator.get_time(); @@ -197,7 +195,7 @@ void evolve_flavor(const TestParams* parms) // Note: this won't be the same as the new-time grid data // because the last deposit_to_mesh call was at either the old time (forward Euler) // or the final RK stage, if using Runge-Kutta. - const Real dt = compute_dt(geom,parms->cfl_factor,state,neutrinos,parms->flavor_cfl_factor,parms->max_adaptive_speedup); + const Real dt = compute_dt(geom, state, neutrinos, parms); integrator.set_timestep(dt); }; @@ -206,7 +204,7 @@ void evolve_flavor(const TestParams* parms) integrator.set_post_timestep(post_timestep_fun); // Get a starting timestep - const Real starting_dt = compute_dt(geom,parms->cfl_factor,state,neutrinos_old,parms->flavor_cfl_factor, parms->max_adaptive_speedup); + const Real starting_dt = compute_dt(geom, state, neutrinos_old, parms); // Do all the science! amrex::Print() << "Starting timestepping loop... " << std::endl; diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index c3e89d37..5413306c 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -1,6 +1,10 @@ perturbation_type = 0 perturbation_amplitude = 1e-6 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + +collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 @@ -78,3 +82,8 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 +################# +# opacity stuff # +################# +IMFP_method = 0 +Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. \ No newline at end of file diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index e4d58c56..4414c0aa 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -1,8 +1,12 @@ +collision_cfl_factor = 1e-3 cfl_factor = 0.5 max_adaptive_speedup = 0 flavor_cfl_factor = .5 maxError = 1e-6 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + perturbation_type = 0 perturbation_amplitude = 0 @@ -78,3 +82,9 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 0 +Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. \ No newline at end of file diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test new file mode 100644 index 00000000..49f5ea92 --- /dev/null +++ b/sample_inputs/inputs_coll_equi_test @@ -0,0 +1,113 @@ +perturbation_type = 0 +perturbation_amplitude = 1e-6 + +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 0 + +collision_cfl_factor = 0.04 +cfl_factor = 1e10 +flavor_cfl_factor = 0.5 +max_adaptive_speedup = 0 +maxError = 1e-6 + +integration.type = 1 +integration.rk.type = 4 + +# Domain size in 3D index space +ncell = (2, 2, 2) +Lx = 20 +Ly = 20 +Lz = 20 + +# Number of particles per cell +nppc = (1, 1, 1) +particle_data_filename = "particle_input.dat" + +# Maximum size of each grid in the domain +max_grid_size = 16 + +# Number of steps to run +nsteps = 1000 + +# Simulation end time +end_time = 5.0e9 + +# Make FPE signal errors so we get a Backtrace +amrex.fpe_trap_invalid=1 + +# give background fluid conditions +rho_g_ccm = 0 +T_MeV = 10 +Ye = 1 + +# Write plotfiles +write_plot_every = 1000 + +# Write particle data in plotfiles +write_plot_particles_every = 1000 + +# checkpointing +do_restart = 0 +restart_dir = "" + +############################### +# NEUTRINO PHYSICS PARAMETERS # +############################### +# see first column of table 14.7 in http://pdg.lbl.gov/2019/reviews/rpp2019-rev-neutrino-mixing.pdf + +# mass state 1 mass in eV [NO/IO:-sqrt(7.39e-5)] +mass1_eV = 0 #-0.008596511 + +# mass state 2 mass in eV (define at 0 arbitrarily because oscillations only sensitive to deltaM^2) +mass2_eV = 0 + +# mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] +mass3_eV = 0.049487372 + +# 1-2 mixing angle in degrees [NO/IO:33.82] +theta12_degrees = 1e-6 + +# 2-3 mixing angle in degrees [NO:8.61 IO:8.65] +theta23_degrees = 8.61 + +# 1-3 mixing angle in degrees [NO:48.3 IO:48.6] +theta13_degrees = 48.3 + +# Majorana angle 1 in degrees +alpha1_degrees = 0 + +# Majorana angle 2 in degrees +alpha2_degrees = 0 + +# CP-violating phase in degrees [NO:222 IO:285] +deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 1 + +Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. + +IMFP_abs0_cm = 5e-4 +IMFP_abs1_cm = 5e-4 +IMFP_abs2_cm = 5e-4 +IMFP_abs0bar_cm = 5e-4 +IMFP_abs1bar_cm = 5e-4 +IMFP_abs2bar_cm = 5e-4 + +munu0_MeV = 0 +munu1_MeV = 0 +munu2_MeV = 0 +munu0bar_MeV = 0 +munu1bar_MeV = 0 +munu2bar_MeV = 0 + +IMFP_scat0_cm = 0 +IMFP_scat1_cm = 0 +IMFP_scat2_cm = 0 +IMFP_scat0bar_cm = 0 +IMFP_scat1bar_cm = 0 +IMFP_scat2bar_cm = 0 + +delta_E = 0.8394810651882109 # Mev diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test new file mode 100644 index 00000000..decf1ec7 --- /dev/null +++ b/sample_inputs/inputs_collisional_instability_test @@ -0,0 +1,113 @@ +perturbation_type = 0 +perturbation_amplitude = 0.0 + +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1.0 + +collision_cfl_factor = 1e-3 +cfl_factor = 0.5 +flavor_cfl_factor = 0.5 +max_adaptive_speedup = 0 +maxError = 1e-6 + +integration.type = 1 +integration.rk.type = 4 + +# Domain size in 3D index space +ncell = (1, 1, 1) +Lx = 1 # cm +Ly = 1 # cm +Lz = 1 # cm + +# Number of particles per cell +nppc = (1, 1, 1) +particle_data_filename = "particle_input.dat" + +# Maximum size of each grid in the domain +max_grid_size = 16 + +# Number of steps to run +nsteps = 100000 + +# Simulation end time +end_time = 5.0e19 + +# Make FPE signal errors so we get a Backtrace +amrex.fpe_trap_invalid=1 + +# give background fluid conditions +rho_g_ccm = 0 +T_MeV = 7.0 +Ye = 0 + +# Write plotfiles +write_plot_every = 100 + +# Write particle data in plotfiles +write_plot_particles_every = 100 + +# checkpointing +do_restart = 0 +restart_dir = "" + +############################### +# NEUTRINO PHYSICS PARAMETERS # +############################### +# see first column of table 14.7 in http://pdg.lbl.gov/2019/reviews/rpp2019-rev-neutrino-mixing.pdf + +# mass state 1 mass in eV [NO/IO:-sqrt(7.39e-5)] +mass1_eV = 0.04866 #-0.008596511 + +# mass state 2 mass in eV (define at 0 arbitrarily because oscillations only sensitive to deltaM^2) +mass2_eV = 0 + +# mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] +mass3_eV = 0 + +# 1-2 mixing angle in degrees [NO/IO:33.82] +theta12_degrees = 1e-6 + +# 2-3 mixing angle in degrees [NO:8.61 IO:8.65] +theta23_degrees = 0 + +# 1-3 mixing angle in degrees [NO:48.3 IO:48.6] +theta13_degrees = 0 + +# Majorana angle 1 in degrees +alpha1_degrees = 0 + +# Majorana angle 2 in degrees +alpha2_degrees = 0 + +# CP-violating phase in degrees [NO:222 IO:285] +deltaCP_degrees = 0 + +################# +# opacity stuff # +################# +IMFP_method = 1 + +Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. + +IMFP_abs0_cm = 2.398082e-1 +IMFP_abs1_cm = 0 +IMFP_abs2_cm = 0 +IMFP_abs0bar_cm = 2.29358e-2 +IMFP_abs1bar_cm = 0 +IMFP_abs2bar_cm = 0 + +munu0_MeV = 20.0 +munu1_MeV = 0 +munu2_MeV = 0 +munu0bar_MeV = 17.644694342915507 +munu1bar_MeV = 0 +munu2bar_MeV = 0 + +IMFP_scat0_cm = 0 +IMFP_scat1_cm = 0 +IMFP_scat2_cm = 0 +IMFP_scat0bar_cm = 0 +IMFP_scat1bar_cm = 0 +IMFP_scat2bar_cm = 0 + +delta_E = 2.272540842052914 # Mev diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index 2406fbeb..c07c6936 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -1,8 +1,12 @@ +collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = .5 max_adaptive_speedup = 0 maxError = 1e-6 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + perturbation_type = 0 perturbation_amplitude = 0 @@ -78,3 +82,9 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 0 +Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index dc858a8e..c4e877ad 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -2,6 +2,10 @@ perturbation_type = 1 perturbation_amplitude = 1e-6 perturbation_wavelength_cm = 1 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + +collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 @@ -79,3 +83,8 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 +################# +# opacity stuff # +################# +IMFP_method = 0 +Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. \ No newline at end of file diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index cdd14ac7..17cd8dfd 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -1,8 +1,12 @@ +collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = 0.1 max_adaptive_speedup = 0 maxError = 1e-6 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + perturbation_type = 0 perturbation_amplitude = 0 @@ -78,3 +82,9 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 0 +Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. \ No newline at end of file From e8eb7773f654d9c38a1141332d10a0ef9db47731 Mon Sep 17 00:00:00 2001 From: Sherwood Richers <5142652+srichers@users.noreply.github.com> Date: Thu, 22 Aug 2024 11:13:37 -0400 Subject: [PATCH 158/276] Revert "Collisions (#90)" (#97) This reverts commit 44339c55c41699ed7bfbc3a7260f0f15aac85d4f. --- Jenkinsfile | 23 -- Scripts/collisions/compute_dE.py | 55 ---- Scripts/data_reduction/amrex_plot_tools.py | 60 ++-- Scripts/data_reduction/reduce_data.py | 4 +- Scripts/data_reduction/reduce_data_fft.py | 4 +- .../initial_condition_tools.py | 20 +- Scripts/initial_conditions/st0_msw_test.py | 9 +- .../initial_conditions/st1_bipolar_test.py | 10 +- .../st2_2beam_fast_flavor.py | 7 +- .../st3_2beam_fast_flavor_nonzerok.py | 10 +- .../initial_conditions/st7_empty_particles.py | 22 -- .../initial_conditions/st8_coll_inst_test.py | 36 --- Scripts/symbolic_hermitians/generate_code.py | 164 ++++++----- Scripts/tests/coll_equi_test.py | 60 ---- Scripts/tests/coll_inst_test.py | 276 ------------------ Scripts/tests/convergence.sh | 89 ++++++ Scripts/tests/fast_flavor_k_test.py | 54 ++-- Scripts/tests/fast_flavor_test.py | 59 ++-- Scripts/tests/msw_test.py | 44 +-- Scripts/tests/plot_convergence.py | 98 +++++++ Source/Constants.H | 1 - Source/DataReducer.cpp | 30 +- Source/Evolve.H | 2 +- Source/Evolve.cpp | 114 ++------ Source/FlavoredNeutrinoContainer.H | 2 + Source/FlavoredNeutrinoContainer.cpp | 25 +- Source/FlavoredNeutrinoContainerInit.cpp | 80 +++-- Source/Parameters.H | 94 ++---- Source/main.cpp | 12 +- sample_inputs/inputs_1d_fiducial | 9 - sample_inputs/inputs_bipolar_test | 10 - sample_inputs/inputs_coll_equi_test | 113 ------- .../inputs_collisional_instability_test | 113 ------- sample_inputs/inputs_fast_flavor | 10 - sample_inputs/inputs_fast_flavor_nonzerok | 9 - sample_inputs/inputs_msw_test | 10 - 36 files changed, 574 insertions(+), 1164 deletions(-) delete mode 100644 Scripts/collisions/compute_dE.py delete mode 100644 Scripts/initial_conditions/st7_empty_particles.py delete mode 100644 Scripts/initial_conditions/st8_coll_inst_test.py delete mode 100644 Scripts/tests/coll_equi_test.py delete mode 100644 Scripts/tests/coll_inst_test.py create mode 100644 Scripts/tests/convergence.sh create mode 100644 Scripts/tests/plot_convergence.py delete mode 100644 sample_inputs/inputs_coll_equi_test delete mode 100644 sample_inputs/inputs_collisional_instability_test diff --git a/Jenkinsfile b/Jenkinsfile index be2611aa..84c48283 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -90,29 +90,6 @@ pipeline { } }} - stage('Collisions flavor instability'){ steps{ - dir('Exec'){ - sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' - sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' - sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' - sh 'python ../Scripts/data_reduction/reduce_data.py' - sh 'python ../Scripts/tests/coll_inst_test.py' - sh 'rm -rf plt*' - } - }} - - stage('Collisions to equilibrium'){ steps{ - dir('Exec'){ - sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' - sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3' - sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' - sh 'python ../Scripts/tests/coll_equi_test.py' - sh 'rm -rf plt*' - } - }} - } // stages{ post { diff --git a/Scripts/collisions/compute_dE.py b/Scripts/collisions/compute_dE.py deleted file mode 100644 index 4faf5ab4..00000000 --- a/Scripts/collisions/compute_dE.py +++ /dev/null @@ -1,55 +0,0 @@ -''' -This script compute the energy bin size ( /Delta E ) for monoenergetic neutrino simulations and the antineutrino chemical potential, given: -- Number of neutrinos at equilibrium -- Volume of a cell -- Number of momentum beams isotropically distributed per cell -- Neutrinos energy bin center -- Neutrino chemical potential -- Background matter temperature -This script was used to compute the energy bin size in the test scripts and antineutrino chemical potential: coll_equi_test.py and coll_inst_test.py. -''' - -import numpy as np - -# constants -hbar = 1.05457266e-27 # erg s -h = hbar * 2 * np.pi # erg s -c = 2.99792458e10 # cm/s -hc = h * c # erg cm - -# Simulation parameters -V = 1 # Volume of a cell ( ccm ) -Ndir = 92 # Number of momentum beams isotropically distributed per cell -E = 20.0 # Neutrinos and antineutrinos energy bin center ( Mev ) -T = 7.0 # Background matter temperature ( Mev ) - -N_eq_electron_neutrino = 3.260869565e+31 # Number of electron neutrinos at equilibrium -u_electron_neutrino = 20.0 # Electron neutrino chemical potential ( Mev ) - -# Fermi-dirac distribution factor for electron neutrinos -f_eq_electron_neutrinos = 1 / ( 1 + np.exp( ( E - u_electron_neutrino ) / T ) ) # adimentional - -# We know : -# dE^3 = 3 * Neq * ( hc )^ 3 / ( dV * dOmega * feq ) -delta_E_cubic = 3 * N_eq_electron_neutrino * ( hc )**3 / ( V * ( 4 * np.pi / Ndir ) * f_eq_electron_neutrinos ) # cubic erg -# dOmega = 4 * pi / ( number directions ) - -# We know polinomial of delta E in term of delta E cubic and E ( deltaE**3 = ( E + dE / 2)**3 - ( E - dE / 2)**3 ) -coeff = [ 0.25 , 0 , 3 * E**2 , -1.0 * delta_E_cubic / ( 1.60218e-6**3 ) ] -# Solving for this polinomial -deltaE = np.roots(coeff) - -# Extracting just the real root -dE=0 -for complex_deltaE in deltaE: - if (np.imag(complex_deltaE)==0): - print(f'Delta energy bin in MeV = {np.real(complex_deltaE)}') - dE=np.real(complex_deltaE) - -# Electron neutrino flavor -N_eq_electron_antineutrino = 2.717391304e+31 # Number of electron antineutrinos at equilibrium - -# Computing electron antineutrino chemical potential -f_eq_electron_antineutrino = 3 * N_eq_electron_antineutrino * ( hc )**3 / ( V * ( 4 * np.pi / Ndir ) * delta_E_cubic ) -u_electron_antineutrino = E - T * np.log( 1 / f_eq_electron_antineutrino - 1 ) -print(f'Electron neutrino chemical potential in MeV = {u_electron_antineutrino}') \ No newline at end of file diff --git a/Scripts/data_reduction/amrex_plot_tools.py b/Scripts/data_reduction/amrex_plot_tools.py index ee717d97..94c8d094 100644 --- a/Scripts/data_reduction/amrex_plot_tools.py +++ b/Scripts/data_reduction/amrex_plot_tools.py @@ -26,14 +26,18 @@ def get_particle_keys(NF, ignore_pos=False, xp_only=False): "pupy", "pupz", "pupt", - "N00_Re", - "N01_Re", - "N01_Im", - "N11_Re", - "N00_Rebar", - "N01_Rebar", - "N01_Imbar", - "N11_Rebar"] + "N", + "L", + "f00_Re", + "f01_Re", + "f01_Im", + "f11_Re", + "Nbar", + "Lbar", + "f00_Rebar", + "f01_Rebar", + "f01_Imbar", + "f11_Rebar"] if(NF==3): real_quantities = ["pos_x", "pos_y", @@ -46,24 +50,28 @@ def get_particle_keys(NF, ignore_pos=False, xp_only=False): "pupy", "pupz", "pupt", - "N00_Re", - "N01_Re", - "N01_Im", - "N02_Re", - "N02_Im", - "N11_Re", - "N12_Re", - "N12_Im", - "N22_Re", - "N00_Rebar", - "N01_Rebar", - "N01_Imbar", - "N02_Rebar", - "N02_Imbar", - "N11_Rebar", - "N12_Rebar", - "N12_Imbar", - "N22_Rebar"] + "N", + "L", + "f00_Re", + "f01_Re", + "f01_Im", + "f02_Re", + "f02_Im", + "f11_Re", + "f12_Re", + "f12_Im", + "f22_Re", + "Nbar", + "Lbar", + "f00_Rebar", + "f01_Rebar", + "f01_Imbar", + "f02_Rebar", + "f02_Imbar", + "f11_Rebar", + "f12_Rebar", + "f12_Imbar", + "f22_Rebar"] if xp_only: real_quantities = real_quantities[:11] if ignore_pos: real_quantities = real_quantities[7:] diff --git a/Scripts/data_reduction/reduce_data.py b/Scripts/data_reduction/reduce_data.py index 3b4fbbe2..665f3866 100644 --- a/Scripts/data_reduction/reduce_data.py +++ b/Scripts/data_reduction/reduce_data.py @@ -385,9 +385,7 @@ def reduce_data(directory=".", nproc=4, do_average=True, do_fft=True, do_angular if(data_format=="Emu"): yt_descriptor = "boxlib" convert_N_to_inv_ccm = 1.0 - directories = glob.glob("plt*") - directories = [d for d in directories if ".h5" not in d] - directories = sorted(directories, key=lambda x: int(x.lstrip("plt"))) + directories = sorted(glob.glob("plt?????")) # get NF eds = emu.EmuDataset(directories[0]) diff --git a/Scripts/data_reduction/reduce_data_fft.py b/Scripts/data_reduction/reduce_data_fft.py index 15949757..7abbe3fe 100755 --- a/Scripts/data_reduction/reduce_data_fft.py +++ b/Scripts/data_reduction/reduce_data_fft.py @@ -9,9 +9,7 @@ parser.add_argument("-o", "--output", type=str, default="reduced_data_fft.h5", help="Name of the output file (default: reduced_data_fft.h5)") args = parser.parse_args() -directories = glob.glob("plt*") -directories = [d for d in directories if ".h5" not in d] -directories = sorted(directories, key=lambda x: int(x.lstrip("plt"))) +directories = sorted(glob.glob("plt?????")) t = [] diff --git a/Scripts/initial_conditions/initial_condition_tools.py b/Scripts/initial_conditions/initial_condition_tools.py index d0885bdd..a3639f9f 100644 --- a/Scripts/initial_conditions/initial_condition_tools.py +++ b/Scripts/initial_conditions/initial_condition_tools.py @@ -230,7 +230,7 @@ def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_g # get variable keys rkey, ikey = amrex.get_particle_keys(NF, ignore_pos=True) nelements = len(rkey) - + # generate the list of particle info particles = np.zeros((nparticles,nelements)) @@ -240,10 +240,22 @@ def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_g # save the total number density of neutrinos for each particle n_flavorsummed = np.sum(n_particle, axis=2) # [particle, nu/nubar] - for nu_nubar, suffix in zip(range(2), ["","bar"]): + for nu_nubar, suffix in zip(range(2), ["","bar"]): + nvarname = "N"+suffix + particles[:,rkey[nvarname]] = n_flavorsummed[:,nu_nubar] + for flavor in range(NF): - fvarname = "N"+str(flavor)+str(flavor)+"_Re"+suffix - particles[:,rkey[fvarname]] = n_particle[:,nu_nubar, flavor] + fvarname = "f"+str(flavor)+str(flavor)+"_Re"+suffix + particles[:,rkey[fvarname]] = n_particle[:,nu_nubar, flavor] / n_flavorsummed[:,nu_nubar] + particles[:,rkey[fvarname]][np.where(n_flavorsummed[:,nu_nubar]==0)] = 1./NF # ensure that trace stays equal to 1 + + # double check that the number densities are correct + particle_n = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]]) + particle_fmag = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]] * mu[:,nu_nubar, flavor]) + #print("nu/nubar,flavor =", nu_nubar, flavor) + #print("output/input ndens =",particle_n, nnu[nu_nubar,flavor]) + #print("output/input fluxfac =",particle_fmag / particle_n, fluxfac[nu_nubar,flavor]) + #print() return particles diff --git a/Scripts/initial_conditions/st0_msw_test.py b/Scripts/initial_conditions/st0_msw_test.py index 1c6767d5..2efafa1d 100644 --- a/Scripts/initial_conditions/st0_msw_test.py +++ b/Scripts/initial_conditions/st0_msw_test.py @@ -37,7 +37,10 @@ p[rkey["pupx"]] = phat[ip,0] * energy_erg p[rkey["pupy"]] = phat[ip,1] * energy_erg p[rkey["pupz"]] = phat[ip,2] * energy_erg - p[rkey["N00_Re"]] = 1 - p[rkey["N00_Rebar"]] = 1 + p[rkey["N"] ] = ndens_per_particle + p[rkey["Nbar"]] = ndens_per_particle + p[rkey["f00_Re"]] = 1 + p[rkey["f00_Rebar"]] = 1 + -write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st1_bipolar_test.py b/Scripts/initial_conditions/st1_bipolar_test.py index 4f8948c1..04a15e5f 100644 --- a/Scripts/initial_conditions/st1_bipolar_test.py +++ b/Scripts/initial_conditions/st1_bipolar_test.py @@ -34,6 +34,7 @@ rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) nelements = len(rkey) + # generate the list of particle info particles = np.zeros((nparticles,nelements)) for ip in range(len(phat)): @@ -42,7 +43,10 @@ p[rkey["pupx"]] = phat[ip,0] * energy_erg p[rkey["pupy"]] = phat[ip,1] * energy_erg p[rkey["pupz"]] = phat[ip,2] * energy_erg - p[rkey["N00_Re"]] = ndens_per_particle - p[rkey["N00_Rebar"]] = ndens_per_particle + p[rkey["N"] ] = ndens_per_particle + p[rkey["Nbar"]] = ndens_per_particle + p[rkey["f00_Re"]] = 1 + p[rkey["f00_Rebar"]] = 1 -write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st2_2beam_fast_flavor.py b/Scripts/initial_conditions/st2_2beam_fast_flavor.py index 6839e9d5..740b5de1 100644 --- a/Scripts/initial_conditions/st2_2beam_fast_flavor.py +++ b/Scripts/initial_conditions/st2_2beam_fast_flavor.py @@ -43,7 +43,10 @@ p[rkey["pupx"]] = u[0] * energy_erg p[rkey["pupy"]] = u[1] * energy_erg p[rkey["pupz"]] = u[2] * energy_erg - p[rkey["N00_Re"]] = ndens_per_particle * (1. + u[2]) - p[rkey["N00_Rebar"]] = ndens_per_particle * (1. - u[2]) + p[rkey["N"] ] = ndens_per_particle * (1. + u[2]) + p[rkey["Nbar"]] = ndens_per_particle * (1. - u[2]) + p[rkey["f00_Re"]] = 1 + p[rkey["f00_Rebar"]] = 1 + write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py b/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py index c2fd2da8..bbcdfab4 100644 --- a/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py +++ b/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py @@ -35,6 +35,7 @@ rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) nelements = len(rkey) + # generate the list of particle info particles = np.zeros((nparticles,nelements)) for ip in range(len(phat)): @@ -44,7 +45,10 @@ p[rkey["pupx"]] = u[0] * energy_erg p[rkey["pupy"]] = u[1] * energy_erg p[rkey["pupz"]] = u[2] * energy_erg - p[rkey["N00_Re"]] = ndens_per_particle * (1. + u[2]) - p[rkey["N00_Rebar"]] = ndens_per_particle * (1. - u[2]) + p[rkey["N"] ] = ndens_per_particle * (1. + u[2]) + p[rkey["Nbar"]] = ndens_per_particle * (1. - u[2]) + p[rkey["f00_Re"]] = 1 + p[rkey["f00_Rebar"]] = 1 + -write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st7_empty_particles.py b/Scripts/initial_conditions/st7_empty_particles.py deleted file mode 100644 index dca380ad..00000000 --- a/Scripts/initial_conditions/st7_empty_particles.py +++ /dev/null @@ -1,22 +0,0 @@ -import h5py -import numpy as np -import sys -import os -importpath = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(importpath) -sys.path.append(importpath+"/../data_analysis") -from initial_condition_tools import uniform_sphere, write_particles, moment_interpolate_particles, linear_interpolate -import amrex_plot_tools as amrex - -# generation parameters -# MUST MATCH THE INPUTS IN THE EMU INPUT FILE! -NF = 3 -nphi_equator = 16 -energy_erg = 50 * 1e6*amrex.eV - -nnu = np.zeros((2,NF)) -fnu = np.zeros((2,NF,3)) - -particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, linear_interpolate) # [particle, variable] - -write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st8_coll_inst_test.py b/Scripts/initial_conditions/st8_coll_inst_test.py deleted file mode 100644 index 0b925b21..00000000 --- a/Scripts/initial_conditions/st8_coll_inst_test.py +++ /dev/null @@ -1,36 +0,0 @@ -import h5py -import numpy as np -import sys -import os -importpath = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(importpath) -sys.path.append(importpath+"/../data_reduction") -from initial_condition_tools import uniform_sphere, moment_interpolate_particles, minerbo_interpolate, write_particles -import amrex_plot_tools as amrex - -# These initial conditions are intended to replicate the collisional instability outputs in "Collisional Flavor Instabilities of Supernova Neutrinos" by L. Johns [2104.11369]. -# Simulation parameters -NF = 2 -nphi_equator = 16 -nnue = 3.0e+33 # 1/ccm -nnua = 2.5e+33 # 1/ccm -nnux = 1.0e+33 # 1/ccm -fnue = np.array([0.0 , 0.0 , 0.0]) -fnua = np.array([0.0 , 0.0 , 0.0]) -fnux = np.array([0.0 , 0.0 , 0.0]) -energy_erg = 20.0 # MeV -energy_erg *= 1e6*amrex.eV # erg - -nnu = np.zeros((2,NF)) -nnu[0,0] = nnue -nnu[1,0] = nnua -nnu[:,1:] = nnux - -fnu = np.zeros((2,NF,3)) -fnu[0,0,:] = nnue * fnue -fnu[1,0,:] = nnua * fnua -fnu[:,1:,:] = nnu[:,1:,np.newaxis] * fnux[np.newaxis,np.newaxis,:] - -particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, minerbo_interpolate) # [particle, variable] - -write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 5207b5f7..ebb8d6db 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -94,22 +94,33 @@ def delete_generated_files(): #==================================# # FlavoredNeutrinoContainer.H_fill # #==================================# - vars = ["N"] + vars = ["f"] tails = ["","bar"] code = [] for t in tails: + code += ["N"+t] # number of neutrinos + code += ["L"+t] # length of isospin vector, units of number of neutrinos for v in vars: A = HermitianMatrix(args.N, v+"{}{}_{}"+t) code += A.header() - code += ["TrHN"] - code += ["Vphase"] + code += ["TrHf"] - code_lines = [code[i]+"," for i in range(len(code))] - write_code(code_lines, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.H_fill")) + code = [code[i]+"," for i in range(len(code))] + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.H_fill")) #========================================================# # FlavoredNeutrinoContainerInit.H_particle_varnames_fill # #========================================================# + vars = ["f"] + tails = ["","bar"] + code = [] + for t in tails: + code += ["N"+t] + code += ["L"+t] + for v in vars: + A = HermitianMatrix(args.N, v+"{}{}_{}"+t) + code += A.header() + code += ["TrHf"] code_string = 'attribute_names = {"time", "x", "y", "z", "pupx", "pupy", "pupz", "pupt", ' code = ['"{}"'.format(c) for c in code] code_string = code_string + ", ".join(code) + "};" @@ -167,8 +178,8 @@ def delete_generated_files(): "*p.rdata(PIdx::pupz)*p.rdata(PIdx::pupz)/p.rdata(PIdx::pupt)/p.rdata(PIdx::pupt));"]) code = [] for t in tails: - string3 = ")" - flist = HermitianMatrix(args.N, "N{}{}_{}"+t).header() + string3 = ")*p.rdata(PIdx::N"+t+")" + flist = HermitianMatrix(args.N, "f{}{}_{}"+t).header() for ivar in range(len(deposit_vars)): deplist = HermitianMatrix(args.N, deposit_vars[ivar]+"{}{}_{}"+t).header() for icomp in range(len(flist)): @@ -182,10 +193,10 @@ def delete_generated_files(): code = [] for t in tails: # diagonal averages - N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") + N = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") Nlist = N.header_diagonals(); for i in range(len(Nlist)): - code.append("TrN += "+Nlist[i]+";") + code.append("Trf += "+Nlist[i]+";") write_code(code, os.path.join(args.emu_home, "Source/generated_files", "DataReducer.cpp_fill_particles")) @@ -426,85 +437,94 @@ def sgn(t,var): line += "sqrt(2.) * PhysConst::GF * sx(i) * sy(j) * sz(k) * (inside_parentheses);" code.append(line) code.append("") - - code.append("T_pp += sx(i) * sy(j) * sz(k) * sarr(i, j, k, GIdx::T);") - code.append("Ye_pp += sx(i) * sy(j) * sz(k) * sarr(i, j, k, GIdx::Ye);") - code.append("rho_pp += sx(i) * sy(j) * sz(k) * sarr(i, j, k, GIdx::rho);") - code.append("") - write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_interpolate_from_mesh_fill")) #========================# # Evolve.cpp_dfdt_fill # #========================# - # Generate quantum kinetic equations - - # Define useful constants + # Set up Hermitian matrices A, B, C hbar = sympy.symbols("PhysConst\:\:hbar",real=True) - attenuation_to_hamiltonian = sympy.symbols("parms->attenuation_hamiltonians", real=True) - V_phase = sympy.symbols("p.rdata(PIdx\:\:Vphase)", real=True) - pi = sympy.symbols("MathConst\:\:pi", real=True) - c = sympy.symbols("PhysConst\:\:c", real=True) - - # List that will store the QKE code. code = [] - - # Looping over neutrinos(tail: no tail) and antineutrinos(tail: bar) for t in tails: + H = HermitianMatrix(args.N, "V{}{}_{}"+t) + F = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") - # Define Fermi-Dirac distribution matrix f_eq = diag( f_e , f_x ) from input parameters - f_eq = HermitianMatrix(args.N, "f_eq_{}{}_{}"+t) # Fermi-dirac distribution matrix ----> To be used in calculation of QKE in sympy format - f_eq_cpp = HermitianMatrix(args.N, "f_eq"+t+"[{}][{}]") # Fermi-dirac distribution matrix ----> Using the systaxis of line 183 of the Evolve.cpp file - f_eq.H = f_eq_cpp.H # Assigning input mean free paths to SymPy matrix - f_eq_temp_declare = ["amrex::Real {}".format(line) for line in f_eq.code()] # - code.append(f_eq_temp_declare) - - # Define Gamma matrix from input parameters : Gamma = diag( k*_e , k*_x ) / 2 . ka is the inverse mean free path for flavor a, including Pauli blocking term. * means that Pauli blocking term is already in the inverse mean free path values. - Gamma = HermitianMatrix(args.N, "Gamma_{}{}_{}"+t) # Inverse mean free path matrix. Gamma = diag( k*e , k*x ) / 2. ----> To be used in calculation of QKE in sympy format - IMFP_abs = HermitianMatrix(args.N, "IMFP_abs"+t+"[{}][{}]") # Inverse mean free path matrix IMFP_abs = diag( k*e , k*x ) ----> Using the systaxis of line 181 of the Evolve.cpp file - Gamma.H = IMFP_abs.H / 2 # Compute Gamma - Gamma_temp_declare = ["amrex::Real {}".format(line) for line in Gamma.code()] - code.append(Gamma_temp_declare) - - # Define N_eq matrix - f_eq = HermitianMatrix(args.N, "f_eq_{}{}_{}"+t) # Fermi-dirac distribution matrix f_eq = diag( fe , fx ) - N_eq = HermitianMatrix(args.N, "N_eq_{}{}_{}"+t) # Equilibrium neutrino number matrix N_eq equals the integral of f_eq, where the integral is over the phase space that the particle represents. - N_eq.H = f_eq.H * V_phase / ( 2 * pi * hbar * c )**3 - N_eq_temp_declare = ["amrex::Real {}".format(line) for line in N_eq.code()] - code.append(N_eq_temp_declare) - - # Define collision term - Gamma = HermitianMatrix(args.N, "Gamma_{}{}_{}"+t) # Inverse mean free path matrix. Gamma = diag( k*e , k*x ) / 2. ka is the inverse mean free path for flavor a, including Pauli blocking term. - N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") # Neutrino number matrix - N_eq = HermitianMatrix(args.N, "N_eq_{}{}_{}"+t) # Equilibrium neutrino number matrix N_eq equals the integral of f_eq, where the integral is over the phase space that the particle represents. - C = HermitianMatrix(args.N, "C_{}{}_{}"+t) # Collision term C = { gamma , N_eq - N }, {} means anticonmutator - C.H = Gamma.H * ( N_eq.H - N.H ) + ( N_eq.H - N.H ) * Gamma.H # Compute collision term - C_temp_declare = ["amrex::Real {}".format(line) for line in C.code()] - code.append(C_temp_declare) - - # Writing QKE - C = HermitianMatrix(args.N, "C_{}{}_{}"+t) # Collision term C = { gamma , N_eq - N }, {} means anticonmutator - H = HermitianMatrix(args.N, "V{}{}_{}"+t) # Hamiltonian - N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") # Neutrino number matrix - dNdt_temp = HermitianMatrix(args.N, "dNdt{}{}_{}"+t) # Temporary matrix for dNdt - dNdt_temp.H = C.H * c + ((H*N - N*H).times(-sympy.I/hbar)).H * attenuation_to_hamiltonian # Compute quantum kinetic equation - dNdt_temp_declare = ["amrex::Real {}".format(line) for line in dNdt_temp.code()] - code.append(dNdt_temp_declare) + # G = Temporary variables for dFdt + G = HermitianMatrix(args.N, "dfdt{}{}_{}"+t) + + # Calculate C = i * [A,B] + #Fnew.anticommutator(H,F).times(sympy.I * dt); + G.H = ((H*F - F*H).times(-sympy.I/hbar)).H + + # Write the temporary variables for dFdt + Gdeclare = ["amrex::Real {}".format(line) for line in G.code()] + code.append(Gdeclare) # Store dFdt back into the particle data for F - dNdt = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") - dNdt_empty = HermitianMatrix(args.N, "dNdt{}{}_{}"+t) - dNdt.H = dNdt_empty.H + dFdt = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") + Gempty = HermitianMatrix(args.N, "dfdt{}{}_{}"+t) + dFdt.H = Gempty.H - # Write out dNdt->N - code.append(dNdt.code()) + # Write out dFdt->F + code.append(dFdt.code()) # store Tr(H*F) for estimating numerical errors - TrHN = (H*N).trace(); - code.append(["p.rdata(PIdx::TrHN) += ("+sympy.cxxcode(sympy.simplify(TrHN))+");"]) + TrHf = (H*F).trace(); + code.append(["p.rdata(PIdx::TrHf) += p.rdata(PIdx::N"+t+") * ("+sympy.cxxcode(sympy.simplify(TrHf))+");"]) code = [line for sublist in code for line in sublist] write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill")) - + #================================================# + # FlavoredNeutrinoContainer.cpp_Renormalize_fill # + #================================================# + code = [] + for t in tails: + # make sure the trace is 1 + code.append("sumP = 0;") + f = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") + fdlist = f.header_diagonals() + flist = f.header() + for fii in fdlist: + code.append("sumP += " + fii + ";") + code.append("error = sumP-1.0;") + code.append("if( std::abs(error) > 100.*parms->maxError) amrex::Abort();") + code.append("if( std::abs(error) > parms->maxError ) {") + for fii in fdlist: + code.append(fii + " -= error/"+str(args.N)+";") + code.append("}") + code.append("") + + # make sure diagonals are positive + for fii in fdlist: + code.append("if("+fii+"<-100.*parms->maxError) amrex::Abort();") + code.append("if("+fii+"<-parms->maxError) "+fii+"=0;") + code.append("") + + # make sure the flavor vector length is what it would be with a 1 in only one diagonal + length = sympy.symbols("length",real=True) + length = f.SU_vector_magnitude() + target_length = "p.rdata(PIdx::L"+t+")" + code.append("length = "+sympy.cxxcode(sympy.simplify(length))+";") + code.append("error = length-"+str(target_length)+";") + code.append("if( std::abs(error) > 100.*parms->maxError) amrex::Abort();") + code.append("if( std::abs(error) > parms->maxError) {") + for fii in flist: + code.append(fii+" /= length/"+str(target_length)+";") + code.append("}") + code.append("") + + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.cpp_Renormalize_fill")) + # Write code to output file, using a template if one is provided + # write_code(code, "code.cpp", args.output_template) + + + #====================================================# + # FlavoredNeutrinoContainerInit.cpp_set_trace_length # + #====================================================# + code = [] + for t in tails: + f = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") + code.append("p.rdata(PIdx::L"+t+") = "+sympy.cxxcode(sympy.simplify(f.SU_vector_magnitude()))+";" ) + write_code(code, os.path.join(args.emu_home, "Source/generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length")) diff --git a/Scripts/tests/coll_equi_test.py b/Scripts/tests/coll_equi_test.py deleted file mode 100644 index 089feef5..00000000 --- a/Scripts/tests/coll_equi_test.py +++ /dev/null @@ -1,60 +0,0 @@ -import numpy as np -import argparse -import glob -import EmuReader -import sys -import os -importpath = os.path.dirname(os.path.realpath(__file__))+"/../data_reduction/" -sys.path.append(importpath) -import amrex_plot_tools as amrex - -parser = argparse.ArgumentParser() -parser.add_argument("-na", "--no_assert", action="store_true", help="If --no_assert is supplied, do not raise assertion errors if the test error > tolerance.") -args = parser.parse_args() - -# physical constants -theta12 = 33.82*np.pi/180. # radians -dm21c4 = 7.39e-5 * amrex.eV**2 # erg^2 - -tolerance = 2e-2 -NF=3 - -if __name__ == "__main__": - - rkey, ikey = amrex.get_particle_keys(NF) - - N_ee = [] - N_uu = [] - N_tt = [] - N_eebar = [] - N_uubar = [] - N_ttbar = [] - - idata, rdata = EmuReader.read_particle_data('plt01000', ptype="neutrinos") - - for i in range(len(rdata)): - p = rdata[i] - N_ee.append(p[rkey["N00_Re"]]) - N_uu.append(p[rkey["N11_Re"]]) - N_tt.append(p[rkey["N22_Re"]]) - N_eebar.append(p[rkey["N00_Rebar"]]) - N_uubar.append(p[rkey["N11_Rebar"]]) - N_ttbar.append(p[rkey["N22_Rebar"]]) - - print(f'average N_ee {np.average(N_ee)}') - print(f'average N_uu {np.average(N_uu)}') - print(f'average N_tt {np.average(N_tt)}') - print(f'average N_eebar {np.average(N_eebar)}') - print(f'average N_uubar {np.average(N_uubar)}') - print(f'average N_ttbar {np.average(N_ttbar)}') - - def myassert(condition): - if not args.no_assert: - assert(condition) - - myassert( np.all(np.isclose(N_ee, np.array(1e33), atol=1e33/100)) ) - myassert( np.all(np.isclose(N_uu, np.array(1e33), atol=1e33/100)) ) - myassert( np.all(np.isclose(N_tt, np.array(1e33), atol=1e33/100)) ) - myassert( np.all(np.isclose(N_eebar, np.array(1e33), atol=1e33/100)) ) - myassert( np.all(np.isclose(N_uubar, np.array(1e33), atol=1e33/100)) ) - myassert( np.all(np.isclose(N_ttbar, np.array(1e33), atol=1e33/100)) ) \ No newline at end of file diff --git a/Scripts/tests/coll_inst_test.py b/Scripts/tests/coll_inst_test.py deleted file mode 100644 index e5f0ad49..00000000 --- a/Scripts/tests/coll_inst_test.py +++ /dev/null @@ -1,276 +0,0 @@ -''' -This test script is used to reproduce the isotropic 2-flavor simulation in "Collisional Flavor Instabilities of Supernova Neutrinos" by L. Johns [2104.11369]. -The points of comparison are the LSA conducted in this paper (equation 14) and Julien's script that reproduces the same results (script received via private communication). -Created by Erick Urquilla. University of Tennessee Knoxville, USA. -''' - -import numpy as np -import argparse -import glob -import EmuReader -import sys -import os -importpath = os.path.dirname(os.path.realpath(__file__))+"/../data_reduction/" -sys.path.append(importpath) -import amrex_plot_tools as amrex -import numpy as np -import h5py -import glob - -parser = argparse.ArgumentParser() -parser.add_argument("-na", "--no_assert", action="store_true", help="If --no_assert is supplied, do not raise assertion errors if the test error > tolerance.") -args = parser.parse_args() - -if __name__ == "__main__": - - # Create a list of data files to read - directories = glob.glob("plt*_reduced_data.h5") - # Sort the data file names by time step number - directories = sorted(directories, key=lambda x: int(x.split("plt")[1].split("_")[0])) - - N_avg_mag = np.zeros((len(directories),2,2)) - Nbar_avg_mag = np.zeros((len(directories),2,2)) - F_avg_mag = np.zeros((len(directories),3,2,2)) - Fbar_avg_mag = np.zeros((len(directories),3,2,2)) - t = np.zeros(len(directories)) - - for i in range(len(directories)): - with h5py.File(directories[i], 'r') as hf: - N_avg_mag[i] = np.array(hf['N_avg_mag(1|ccm)'][:][0]) - Nbar_avg_mag[i] = np.array(hf['Nbar_avg_mag(1|ccm)'][:][0]) - F_avg_mag[i] = np.array(hf['F_avg_mag(1|ccm)'][:][0]) - Fbar_avg_mag[i] = np.array(hf['Fbar_avg_mag(1|ccm)'][:][0]) - t[i] = np.array(hf['t(s)'][:][0]) - - # Fit the exponential function ( y = a e ^ ( b x ) ) to the data - l1 = 50 # initial item for fit - l2 = 200 # last item for fit - coefficients = np.polyfit(t[l1:l2], np.log(N_avg_mag[:,0,1][l1:l2]), 1) - coefficients_bar = np.polyfit(t[l1:l2], np.log(Nbar_avg_mag[:,0,1][l1:l2]), 1) - a = np.exp(coefficients[1]) - b = coefficients[0] - abar = np.exp(coefficients_bar[1]) - bbar = coefficients_bar[0] - print(f'{b} ---> EMU : Im Omega') - print(f'{bbar} ---> EMU : Im Omegabar') - - # Plots - import matplotlib.pyplot as plt - - # Plots N and Nbar - plt.plot(t, N_avg_mag[:,0,0], label = r'$N_{ee}$') - plt.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$') - plt.plot(t, N_avg_mag[:,1,1], label = r'$N_{uu}$') - plt.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{N}_{ee}$') - plt.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') - plt.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') - plt.legend() - plt.xlabel(r'$t$ (s)') - plt.ylabel(r'$N$ and $\bar{N}$') - plt.savefig('N_and_Nbar.pdf') - plt.clf() - - # Plots N and F - plt.plot(t, N_avg_mag[:,0,0], label = r'$N_{ee}$') - plt.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$') - plt.plot(t, N_avg_mag[:,1,1], label = r'$N_{uu}$') - plt.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {b}') - plt.plot(t, F_avg_mag[:,0,0,1], label = r'$F^x_{eu}$') - plt.plot(t, F_avg_mag[:,1,0,1], label = r'$F^y_{eu}$') - plt.plot(t, F_avg_mag[:,2,0,1], label = r'$F^z_{eu}$') - plt.legend() - plt.xlabel(r'$t$ (s)') - plt.ylabel(r'$N$ and $\vec{F}$') - plt.yscale('log') - plt.savefig('N_and_F.pdf') - plt.clf() - - # Plots Nbar and Fbar - plt.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{N}_{ee}$') - plt.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') - plt.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') - plt.plot(t[l1:l2], Nbar_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {bbar}') - plt.plot(t, Fbar_avg_mag[:,0,0,1], label = r'$\bar{F}^x_{eu}$') - plt.plot(t, Fbar_avg_mag[:,1,0,1], label = r'$\bar{F}^y_{eu}$') - plt.plot(t, Fbar_avg_mag[:,2,0,1], label = r'$\bar{F}^z_{eu}$') - plt.legend() - plt.xlabel(r'$t$ (s)') - plt.ylabel(r'$\bar{N}$ and $\vec{\bar{F}}$') - plt.yscale('log') - plt.savefig('Nbar_and_Fbar.pdf') - plt.clf() - - ###################################################################################### - ###################################################################################### - # LSA in "Collisional flavor instabilities of supernova neutrinos", L. Johns [2104.11369] - - h = 6.6260755e-27 # erg s - hbar = h/(2.*np.pi) # erg s - c = 2.99792458e10 # cm/s - MeV = 1.60218e-6 # erg - eV = MeV/1e6 # erg - GF_GeV2 = 1.1663787e-5 # GeV^-2 - GF = GF_GeV2 / (1000*MeV)**2 * (hbar*c)**3 # erg cm^3 - - Nee = 3e33 # cm^-3 - Neebar = 2.5e33 # cm^-3 - Nxx = 1e33 # cm^-3 - - opac_rescale = 1e4 - - kappa_e = 1/(0.417*1e5)*opac_rescale # cm^-1 - kappa_ebar = 1/(4.36*1e5)*opac_rescale # cm^-1 - kappa_x = 0.*opac_rescale # cm^-1 - - # Collision rates (in s^-1) - Gamma_plus = (kappa_e+kappa_x)/2 * c - Gamma_minus = (kappa_e-kappa_x)/2 * c - Gammabar_plus = (kappa_ebar+kappa_x)/2 * c - Gammabar_minus= (kappa_ebar - kappa_x)/2 * c - - omega = 0.304*1e-5 * c # Delta m^2/2E, in s^-1 - mu = np.sqrt(2)*GF/hbar # s^-1.cm^3 - - S = Nee - Nxx + Neebar - Nxx - D = Nee - Nxx - Neebar + Nxx - - ImOmega_Lucas_LSA = ( ( Gamma_plus - Gammabar_plus ) / 2 ) * ( mu * S / np.sqrt( ( mu * D )**2 + 4 * omega * mu * S ) ) - ( Gamma_plus + Gammabar_plus ) / 2 - - print(f'{ImOmega_Lucas_LSA} ---> Im Omega and Omegabar : LSA in equation 14 of L. Johns [2104.11369]') - - ###################################################################################### - ###################################################################################### - - def myassert(condition): - if not args.no_assert: - assert(condition) - - b_lsa = ImOmega_Lucas_LSA - rel_error = np.abs( b - b_lsa ) / np.abs( ( b + b_lsa ) / 2 ) - rel_error_bar = np.abs( bbar - b_lsa ) / np.abs( ( bbar + b_lsa ) / 2 ) - rel_error_max = 0.05 - - print(f"{rel_error} ---> relative error in ImOmega : EMU") - print(f"{rel_error_bar} ---> relative error in ImOmegabar : EMU") - - myassert( rel_error < rel_error_max ) - myassert( rel_error_bar < rel_error_max ) - - ###################################################################################### - ###################################################################################### - - """ - Created on Wed Jun 5 13:11:50 2024 - Solves the isotropic QKE following "Collisional flavor instabilities of supernova neutrinos", L. Johns [2104.11369] - - @author: jfroustey - """ - - import numpy as np - import matplotlib.pyplot as plt - from scipy.integrate import solve_ivp - - h = 6.6260755e-27 # erg s - hbar = h/(2.*np.pi) # erg s - c = 2.99792458e10 # cm/s - MeV = 1.60218e-6 # erg - eV = MeV/1e6 # erg - GF_GeV2 = 1.1663787e-5 # GeV^-2 - GF = GF_GeV2 / (1000*MeV)**2 * (hbar*c)**3 # erg cm^3 - - Nee = 3e33 # cm^-3 - Neebar = 2.5e33 # cm^-3 - Nxx = 1e33 # cm^-3 - - opac_rescale = 1e4 - - kappa_e = 1/(0.417*1e5)*opac_rescale # cm^-1 - kappa_ebar = 1/(4.36*1e5)*opac_rescale # cm^-1 - kappa_x = 0.*opac_rescale # cm^-1 - - # Collision rates (in s^-1) - Gamma_plus = (kappa_e+kappa_x)/2 * c - Gamma_minus = (kappa_e-kappa_x)/2 * c - Gammabar_plus = (kappa_ebar+kappa_x)/2 * c - Gammabar_minus= (kappa_ebar - kappa_x)/2 * c - - # Vacuum term - - theta = 1e-6 - c2t = np.cos(2*theta) - s2t = np.sin(2*theta) - omega = 0.304*1e-5 * c # Delta m^2/2E, in s^-1 - - P0_AE = (Nee+Nxx)/2 - Pz_AE = (Nee-Nxx)/2 - Pbar0_AE = (Neebar+Nxx)/2 - Pbarz_AE = (Neebar-Nxx)/2 - - mu = np.sqrt(2)*GF/hbar # s^-1.cm^3 - - def QKE(t,y): - P0, Px, Py, Pz, Pbar0, Pbarx, Pbary, Pbarz = y - deriv = np.zeros(8) - - # Variation of P0, Pbar0 - deriv[0] = Gamma_plus*(P0_AE-P0) + Gamma_minus*(Pz_AE-Pz) - deriv[4] = Gammabar_plus*(Pbar0_AE-Pbar0) + Gammabar_minus*(Pbarz_AE - Pbarz) - - # Spatial parts - deriv[1] = omega*c2t*Py + mu*((Py-Pbary)*Pz - (Pz-Pbarz)*Py) - Gamma_plus*Px - deriv[2] = omega*(-s2t*Pz-c2t*Px) + mu*((Pz-Pbarz)*Px - (Px-Pbarx)*Pz) - Gamma_plus*Py - deriv[3] = omega*s2t*Py + mu*((Px-Pbarx)*Py - (Py-Pbary)*Px) + Gamma_plus*(Pz_AE-Pz) + Gamma_minus*(P0_AE-P0) - - deriv[5] = -omega*c2t*Pbary + mu*((Py-Pbary)*Pbarz - (Pz-Pbarz)*Pbary) - Gammabar_plus*Pbarx - deriv[6] = -omega*(-s2t*Pbarz - c2t*Pbarx) + mu*((Pz-Pbarz)*Pbarx - (Px-Pbarx)*Pbarz) - Gammabar_plus*Pbary - deriv[7] = -omega*s2t*Pbary + mu*((Px-Pbarx)*Pbary - (Py-Pbary)*Pbarx) + Gammabar_plus*(Pbarz_AE-Pbarz) + Gammabar_minus*(Pbar0_AE-Pbar0) - - return deriv - - time = np.linspace(0,90e-6/opac_rescale,2000) - y0 = np.array([P0_AE, 0., 0., Pz_AE, Pbar0_AE, 0., 0., Pbarz_AE]) - - myrtol, myatol = 1e-5, 1e-8 - sol = solve_ivp(QKE, (time[0],time[-1]), y0, t_eval=time, rtol=myrtol, atol=myatol) - - # PLOTS - plt.plot(time, sol.y[0,:]+sol.y[3,:], color='k', lw=2, label=r'$n_{\nu_e}$') - plt.plot(time, sol.y[4,:]+sol.y[7,:], color='k', lw=1.5, label=r'$n_{\bar{\nu}_e}$') - plt.plot(time, sol.y[0,:]-sol.y[3,:], color='k', lw=1, label=r'$n_{\nu_x}$') - - plt.plot(time,np.sqrt(sol.y[1,:]**2+sol.y[2,:]**2), lw=2, color='teal',label=r'$\nu_e - \nu_x$'+" coherence density") - plt.legend() - plt.grid(ls=':',color='C7') - plt.xlabel(r'$t \ (\mathrm{s})$') - plt.xlim(time[0],time[-1]) - plt.title(f"Opacities scaled by {opac_rescale:.1e}, with rtol={myrtol:.1e}, atol={myatol:.1e}") - plt.tight_layout() - plt.savefig(f"Johns_CFI_rescale_{opac_rescale:.0e}_rtol_{myrtol:.0e}_atol_{myatol:.0e}.pdf") - plt.close() - - ###################################################################################### - ###################################################################################### - - p1 = 150 # initial point for fit - p2 = 500 # final point for fit - N_eu_julien = np.sqrt(sol.y[1,:]**2+sol.y[2,:]**2) - coefficients = np.polyfit(time[p1:p2], np.log(N_eu_julien[p1:p2]), 1) - aj = np.exp(coefficients[1]) - bj = coefficients[0] - print(f'{bj} ---> Im Omega Julien') - rel_error_j = np.abs( bj - b_lsa ) / np.abs( ( bj + b_lsa ) / 2 ) - print(f"{rel_error_j} ---> relative erroror in Julien script") - - # Plotting Julien, EMU and Lucas LSA data - plt.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$ EMU') - plt.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega EMU = {b}', linestyle = 'dashed') - plt.plot(time,N_eu_julien, label = r'$N_{eu}$ Julien script') - plt.plot(time[p1:p2],N_eu_julien[p1:p2], label = f'Im Omega Julien = {bj}', linestyle = 'dashed') - plt.plot(time[p1:p2], 1e23*np.exp(ImOmega_Lucas_LSA*time[p1:p2]), label = f'Im Omega Lucas LSA = {ImOmega_Lucas_LSA}', linestyle = 'dashed') - plt.xlabel(r'$t \ (\mathrm{s})$') - plt.ylabel(r'$N_{eu}$') - plt.title(f"Collisional flavor instability test") - plt.yscale('log') - plt.legend() - plt.savefig('EMU_Julien_LucasLSA_Neu.pdf') - plt.close() \ No newline at end of file diff --git a/Scripts/tests/convergence.sh b/Scripts/tests/convergence.sh new file mode 100644 index 00000000..3816a379 --- /dev/null +++ b/Scripts/tests/convergence.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# echo the commands +set -x + +# All runs will use these +DIM=3 +EXEC=./main${DIM}d.gnu.TPROF.MPI.ex +MPINUM=4 + +RUNPARAMS=" +cfl_factor=-1 +nsteps=1000000 +end_time=5.0e-11" + +# Each integrator will set these +INTPARAMS="" +INTNAME="" + +# Clean up any existing output files before we start +rm -rf plt* +rm -rf single_neutrino*.png +rm -rf msw_test_*.txt + +# Define a function for a single run +do_single () { + mpiexec -n ${MPINUM} ${EXEC} inputs_msw_test ${RUNPARAMS} flavor_cfl_factor=${FCFL} ${INTPARAMS} + echo "cfl: ${FCFL}" >> msw_test_${INTNAME}.txt + python3 msw_test.py -na >> msw_test_${INTNAME}.txt + python3 plot_first_particle.py + mv single_neutrino.png single_neutrino_fcfl_${FCFL}_${INTNAME}.png + rm -rf plt* +} + +# Define a function for running convergence +do_convergence () { + FCFL=0.1 + do_single + + FCFL=0.05 + do_single + + FCFL=0.025 + do_single + + FCFL=0.0125 + do_single + + FCFL=0.00625 + do_single + + FCFL=0.003125 + do_single +} + +# Forward Euler convergence +INTPARAMS=" +integration.type=0" + +INTNAME="fe" + +do_convergence + +# Trapezoid convergence +INTPARAMS=" +integration.type=1 +integration.rk.type=2" + +INTNAME="trapz" + +do_convergence + +# SSPRK3 Convergence +INTPARAMS=" +integration.type=1 +integration.rk.type=3" + +INTNAME="ssprk3" + +do_convergence + +# RK4 Convergence +INTPARAMS=" +integration.type=1 +integration.rk.type=4" + +INTNAME="rk4" + +do_convergence diff --git a/Scripts/tests/fast_flavor_k_test.py b/Scripts/tests/fast_flavor_k_test.py index 8b661152..5703e49c 100644 --- a/Scripts/tests/fast_flavor_k_test.py +++ b/Scripts/tests/fast_flavor_k_test.py @@ -30,10 +30,10 @@ rkey, ikey = amrex.get_particle_keys(NF) t = [] - NexR = [] - NexI = [] - NexRbar = [] - NexIbar = [] + fexR = [] + fexI = [] + fexRbar = [] + fexIbar = [] pupt = [] nfiles = len(glob.glob("plt[0-9][0-9][0-9][0-9][0-9]")) @@ -43,18 +43,18 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata t.append(p[0][rkey["time"]]) - NexR.append(np.max(np.abs(p[:,rkey["N01_Re"]]))) - NexI.append(np.max(np.abs(p[:,rkey["N01_Im"]]))) - NexRbar.append(np.max(np.abs(p[:,rkey["N01_Rebar"]]))) - NexIbar.append(np.max(np.abs(p[:,rkey["N01_Imbar"]]))) + fexR.append(np.max(np.abs(p[:,rkey["f01_Re"]]))) + fexI.append(np.max(np.abs(p[:,rkey["f01_Im"]]))) + fexRbar.append(np.max(np.abs(p[:,rkey["f01_Rebar"]]))) + fexIbar.append(np.max(np.abs(p[:,rkey["f01_Imbar"]]))) pupt.append(p[0][rkey["pupt"]]) t = np.array(t) - NexR = np.array(NexR) - NexI = np.array(NexI) - NexRbar = np.array(NexRbar) - NexIbar = np.array(NexIbar) - print(NexR) + fexR = np.array(fexR) + fexI = np.array(fexI) + fexRbar = np.array(fexRbar) + fexIbar = np.array(fexIbar) + print(fexR) # The neutrino energy we set E = 50. * 1e6*amrex.eV @@ -70,27 +70,27 @@ # get growth rate from each diagonal component dt = t[i1]-t[i0] - NexRomega = np.log(np.abs(NexR[i1]/NexR[i0])) / dt - NexIomega = np.log(np.abs(NexI[i1]/NexI[i0])) / dt - NexRbaromega = np.log(np.abs(NexRbar[i1]/NexRbar[i0])) / dt - NexIbaromega = np.log(np.abs(NexIbar[i1]/NexIbar[i0])) / dt + fexRomega = np.log(np.abs(fexR[i1]/fexR[i0])) / dt + fexIomega = np.log(np.abs(fexI[i1]/fexI[i0])) / dt + fexRbaromega = np.log(np.abs(fexRbar[i1]/fexRbar[i0])) / dt + fexIbaromega = np.log(np.abs(fexIbar[i1]/fexIbar[i0])) / dt - print("growth rates:",NexRomega,NexIomega,NexRbaromega,NexIbaromega) - print("growth rates / theoretical:",NexRomega/ImOmega,NexIomega/ImOmega,NexRbaromega/ImOmega,NexIbaromega/ImOmega) + print("growth rates:",fexRomega,fexIomega,fexRbaromega,fexIbaromega) + print("growth rates / theoretical:",fexRomega/ImOmega,fexIomega/ImOmega,fexRbaromega/ImOmega,fexIbaromega/ImOmega) def myassert(condition): if not args.no_assert: assert(condition) - NexRerror = np.abs(ImOmega - NexRomega) / ImOmega - myassert( NexRerror < tolerance ) + fexRerror = np.abs(ImOmega - fexRomega) / ImOmega + myassert( fexRerror < tolerance ) - NexIerror = np.abs(ImOmega - NexIomega) / ImOmega - myassert( NexIerror < tolerance ) + fexIerror = np.abs(ImOmega - fexIomega) / ImOmega + myassert( fexIerror < tolerance ) - NexRbarerror = np.abs(ImOmega - NexRbaromega) / ImOmega - myassert( NexRbarerror < tolerance ) + fexRbarerror = np.abs(ImOmega - fexRbaromega) / ImOmega + myassert( fexRbarerror < tolerance ) - NexIbarerror = np.abs(ImOmega - NexIbaromega) / ImOmega - myassert( NexIbarerror < tolerance ) + fexIbarerror = np.abs(ImOmega - fexIbaromega) / ImOmega + myassert( fexIbarerror < tolerance ) diff --git a/Scripts/tests/fast_flavor_test.py b/Scripts/tests/fast_flavor_test.py index cd50fd75..ff9d7647 100644 --- a/Scripts/tests/fast_flavor_test.py +++ b/Scripts/tests/fast_flavor_test.py @@ -26,10 +26,10 @@ rkey, ikey = amrex.get_particle_keys(NF) t = [] - NexR = [] - NexI = [] - NexRbar = [] - NexIbar = [] + fexR = [] + fexI = [] + fexRbar = [] + fexIbar = [] pupt = [] nfiles = len(glob.glob("plt[0-9][0-9][0-9][0-9][0-9]")) @@ -39,18 +39,17 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata[0] t.append(p[rkey["time"]]) - NexR.append(p[rkey["N01_Re"]]) - NexI.append(p[rkey["N01_Im"]]) - p = rdata[1] - NexRbar.append(p[rkey["N01_Rebar"]]) - NexIbar.append(p[rkey["N01_Imbar"]]) + fexR.append(p[rkey["f01_Re"]]) + fexI.append(p[rkey["f01_Im"]]) + fexRbar.append(p[rkey["f01_Rebar"]]) + fexIbar.append(p[rkey["f01_Imbar"]]) pupt.append(p[rkey["pupt"]]) t = np.array(t) - NexR = np.array(NexR) - NexI = np.array(NexI) - NexRbar = np.array(NexRbar) - NexIbar = np.array(NexIbar) + fexR = np.array(fexR) + fexI = np.array(fexI) + fexRbar = np.array(fexRbar) + fexIbar = np.array(fexIbar) # The neutrino energy we set E = 50. * 1e6*amrex.eV @@ -62,32 +61,32 @@ # get growth rate from each diagonal component dt = t[i1]-t[i0] - NexRomega = np.log(NexR[i1]/NexR[i0]) / dt - NexIomega = np.log(NexI[i1]/NexI[i0]) / dt - NexRbaromega = np.log(NexRbar[i1]/NexRbar[i0]) / dt - NexIbaromega = np.log(NexIbar[i1]/NexIbar[i0]) / dt + fexRomega = np.log(fexR[i1]/fexR[i0]) / dt + fexIomega = np.log(fexI[i1]/fexI[i0]) / dt + fexRbaromega = np.log(fexRbar[i1]/fexRbar[i0]) / dt + fexIbaromega = np.log(fexIbar[i1]/fexIbar[i0]) / dt def myassert(condition): if not args.no_assert: assert(condition) - print("growth rates:",NexRomega,NexIomega,NexRbaromega,NexIbaromega) + print("growth rates:",fexRomega,fexIomega,fexRbaromega,fexIbaromega) print(dt,t[i0],t[i1]) - print(NexR[i1],NexR[i0]) - print(NexI[i1],NexI[i0]) - print(NexRbar[i1],NexRbar[i0]) - print(NexIbar[i1],NexIbar[i0]) + print(fexR[i1],fexR[i0]) + print(fexI[i1],fexI[i0]) + print(fexRbar[i1],fexRbar[i0]) + print(fexIbar[i1],fexIbar[i0]) - NexRerror = np.abs(ImOmega - NexRomega) / ImOmega - myassert( NexRerror < tolerance ) + fexRerror = np.abs(ImOmega - fexRomega) / ImOmega + myassert( fexRerror < tolerance ) - NexIerror = np.abs(ImOmega - NexIomega) / ImOmega - myassert( NexIerror < tolerance ) + fexIerror = np.abs(ImOmega - fexIomega) / ImOmega + myassert( fexIerror < tolerance ) - NexRbarerror = np.abs(ImOmega - NexRbaromega) / ImOmega - myassert( NexRbarerror < tolerance ) + fexRbarerror = np.abs(ImOmega - fexRbaromega) / ImOmega + myassert( fexRbarerror < tolerance ) - NexIbarerror = np.abs(ImOmega - NexIbaromega) / ImOmega - myassert( NexIbarerror < tolerance ) + fexIbarerror = np.abs(ImOmega - fexIbaromega) / ImOmega + myassert( fexIbarerror < tolerance ) diff --git a/Scripts/tests/msw_test.py b/Scripts/tests/msw_test.py index aab770cb..0c4f24ac 100644 --- a/Scripts/tests/msw_test.py +++ b/Scripts/tests/msw_test.py @@ -35,10 +35,10 @@ rkey, ikey = amrex.get_particle_keys(NF) t = [] - Nee = [] - Nxx = [] - Neebar = [] - Nxxbar = [] + fee = [] + fxx = [] + feebar = [] + fxxbar = [] pupt = [] nfiles = len(glob.glob("plt[0-9][0-9][0-9][0-9][0-9]")) @@ -48,17 +48,17 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata[0] t.append(p[rkey["time"]]) - Nee.append(p[rkey["N00_Re"]]) - Nxx.append(p[rkey["N11_Re"]]) - Neebar.append(p[rkey["N00_Rebar"]]) - Nxxbar.append(p[rkey["N11_Rebar"]]) + fee.append(p[rkey["f00_Re"]]) + fxx.append(p[rkey["f11_Re"]]) + feebar.append(p[rkey["f00_Rebar"]]) + fxxbar.append(p[rkey["f11_Rebar"]]) pupt.append(p[rkey["pupt"]]) t = np.array(t) - Nee = np.array(Nee) - Nxx = np.array(Nxx) - Neebar = np.array(Neebar) - Nxxbar = np.array(Nxxbar) + fee = np.array(fee) + fxx = np.array(fxx) + feebar = np.array(feebar) + fxxbar = np.array(fxxbar) # The neutrino energy we set #E = dm21c4 * np.sin(2.*theta12) / (8.*np.pi*hbar*clight) @@ -83,31 +83,31 @@ def myassert(condition): assert(condition) # calculate errors - Nee_analytic = Psurv(dm2_eff, sin2_eff, E) - error_ee = np.max(np.abs( Nee - Nee_analytic ) ) + fee_analytic = Psurv(dm2_eff, sin2_eff, E) + error_ee = np.max(np.abs( fee - fee_analytic ) ) print("f_ee error:", error_ee) myassert( error_ee < tolerance ) - Nxx_analytic = 1. - Psurv(dm2_eff, sin2_eff, E) - error_xx = np.max(np.abs( Nxx - Nxx_analytic ) ) + fxx_analytic = 1. - Psurv(dm2_eff, sin2_eff, E) + error_xx = np.max(np.abs( fxx - fxx_analytic ) ) print("f_xx error:", error_xx) myassert( error_xx < tolerance ) - Neebar_analytic = Psurv(dm2_effbar, sin2_effbar, E) - error_eebar = np.max(np.abs( Neebar - Neebar_analytic ) ) + feebar_analytic = Psurv(dm2_effbar, sin2_effbar, E) + error_eebar = np.max(np.abs( feebar - feebar_analytic ) ) print("f_eebar error:", error_eebar) myassert( error_eebar < tolerance ) - Nxxbar_analytic = 1. - Psurv(dm2_effbar, sin2_effbar, E) - error_xxbar = np.max(np.abs( Nxxbar - Nxxbar_analytic ) ) + fxxbar_analytic = 1. - Psurv(dm2_effbar, sin2_effbar, E) + error_xxbar = np.max(np.abs( fxxbar - fxxbar_analytic ) ) print("f_xxbar error:", error_xxbar) myassert( error_xxbar < tolerance ) - conservation_error = np.max(np.abs( (Nee+Nxx) -1. )) + conservation_error = np.max(np.abs( (fee+fxx) -1. )) print("conservation_error:", conservation_error) myassert(conservation_error < tolerance) - conservation_errorbar = np.max(np.abs( (Neebar+Nxxbar) -1. )) + conservation_errorbar = np.max(np.abs( (feebar+fxxbar) -1. )) print("conservation_errorbar:", conservation_errorbar) myassert(conservation_errorbar < tolerance) diff --git a/Scripts/tests/plot_convergence.py b/Scripts/tests/plot_convergence.py new file mode 100644 index 00000000..4be2b8c3 --- /dev/null +++ b/Scripts/tests/plot_convergence.py @@ -0,0 +1,98 @@ +import numpy as np +import matplotlib.pyplot as plt + +# Read forward Euler data +class ConvergenceData(object): + def __init__(self, filename=None): + self.data = {"cfl": [], + "f_ee error": [], + "f_xx error": [], + "f_eebar error": [], + "f_xxbar error": []} + + if filename: + self.readfrom(filename) + + def readfrom(self, filename): + f = open(filename, "r") + + while True: + entry = [f.readline().strip() for i in range(9)] + if not entry[0]: + break + for line in entry: + ls = line.split(":") + name = ls[0].strip() + value = ls[-1].strip() + for k in self.data.keys(): + if name == k: + self.data[k].append(float(value)) + + f.close() + + for k in self.data.keys(): + self.data[k] = np.array(self.data[k]) + + def get(self, key): + return self.data[key] + + def keys(self): + return self.data.keys() + + def error_keys(self): + return [k for k in self.data.keys() if k != "cfl"] + + def average_convergence(self, key): + # get the average convergence order for the keyed quantity + err = self.get(key) + cfl = self.get("cfl") + + orders = [] + for i in range(len(err)-1): + order = np.log10(err[i+1]/err[i]) / np.log10(cfl[i+1]/cfl[i]) + orders.append(order) + orders = np.array(orders) + + order_average = np.average(orders) + return order_average + + def plot_on_axis(self, axis, key, label, color): + log_cfl = np.log10(self.get("cfl")) + log_err = np.log10(self.get(key)) + axis.plot(log_cfl, log_err, label=label, marker="o", linestyle="None", color=color) + + order = self.average_convergence(key) + iMaxErr = np.argmax(log_err) + intercept = log_err[iMaxErr] - order * log_cfl[iMaxErr] + log_order_err = intercept + order * log_cfl + axis.plot(log_cfl, log_order_err, label="$O({}) = {:0.2f}$".format(label, order), marker="None", linestyle="--", color=color) + + +cdata = {} +cdata["fe"] = ConvergenceData("msw_test_fe.txt") +cdata["trapz"] = ConvergenceData("msw_test_trapz.txt") +cdata["ssprk3"] = ConvergenceData("msw_test_ssprk3.txt") +cdata["rk4"] = ConvergenceData("msw_test_rk4.txt") + +variables = cdata["fe"].error_keys() + +for v in variables: + fig, ax = plt.subplots() + + ax.set_xlabel("log10 flavor CFL") + ax.set_ylabel("log10 {}".format(v)) + + colors = ["red", "blue", "green", "magenta"] + + for k, c in zip(cdata.keys(), colors): + cd = cdata[k] + cd.plot_on_axis(ax, v, k, c) + + ax.invert_xaxis() + + ax.legend(loc=(1.05, 0.0)) + fig.tight_layout() + + plt.savefig("convergence_{}.eps".format(v.replace(" ","_"))) + plt.savefig("convergence_{}.png".format(v.replace(" ","_")), dpi=300) + plt.clf() diff --git a/Source/Constants.H b/Source/Constants.H index 21883969..1e9eeab6 100644 --- a/Source/Constants.H +++ b/Source/Constants.H @@ -18,7 +18,6 @@ namespace PhysConst static constexpr amrex::Real GF = 1.1663787e-5/*GeV^-2*//(1e9*1e9*CGSUnitsConst::eV*CGSUnitsConst::eV) * hbarc*hbarc*hbarc; //erg cm^3 static constexpr amrex::Real Mp = 1.6726219e-24; // g static constexpr amrex::Real sin2thetaW = 0.23122; - static constexpr amrex::Real kB = 1.380658e-16; // erg/K } namespace MathConst diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index 4c3c15ab..d339beb6 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -64,8 +64,8 @@ void DataReducer::InitializeFiles() #endif } file0D.createDataSet("N_offdiag_mag(1|ccm)", dataspace, create_datatype(), props); - file0D.createDataSet("sumTrN", dataspace, create_datatype(), props); - file0D.createDataSet("sumTrHN", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrf", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrHf", dataspace, create_datatype(), props); #else @@ -104,9 +104,9 @@ void DataReducer::InitializeFiles() j++; outfile << j << ":N_offdiag_mag(1|ccm)\t"; j++; - outfile << j << ":sumTrN\t"; + outfile << j << ":sumTrf\t"; j++; - outfile << j << ":sumTrHN\t"; + outfile << j << ":sumTrHf\t"; outfile << std::endl; outfile.close(); @@ -130,15 +130,15 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, amrex::ReduceOps reduce_ops; auto particleResult = amrex::ParticleReduce< ReduceData >(neutrinos, [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { - Real TrHN = p.rdata(PIdx::TrHN); - Real TrN = 0; + Real TrHf = p.rdata(PIdx::TrHf); + Real Trf = 0; #include "generated_files/DataReducer.cpp_fill_particles" - return GpuTuple{TrN,TrHN}; + return GpuTuple{Trf,TrHf}; }, reduce_ops); - Real TrN = amrex::get<0>(particleResult); - Real TrHN = amrex::get<1>(particleResult); - ParallelDescriptor::ReduceRealSum(TrN); - ParallelDescriptor::ReduceRealSum(TrHN); + Real Trf = amrex::get<0>(particleResult); + Real TrHf = amrex::get<1>(particleResult); + ParallelDescriptor::ReduceRealSum(Trf); + ParallelDescriptor::ReduceRealSum(TrHf); //=============================// // Do reductions over the grid // @@ -290,8 +290,8 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, #endif } append_0D(file0D, "N_offdiag_mag(1|ccm)", N_offdiag_mag); - append_0D(file0D, "sumTrN", TrN); - append_0D(file0D, "sumTrHN", TrHN); + append_0D(file0D, "sumTrf", Trf); + append_0D(file0D, "sumTrHf", TrHf); #else std::ofstream outfile; outfile.open(filename0D, std::ofstream::app); @@ -324,8 +324,8 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, #endif } outfile << N_offdiag_mag << "\t"; - outfile << TrN << "\t"; - outfile << TrHN << "\t"; + outfile << Trf << "\t"; + outfile << TrHf << "\t"; outfile << std::endl; outfile.close(); #endif diff --git a/Source/Evolve.H b/Source/Evolve.H index 395634e7..55f23dec 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -26,7 +26,7 @@ namespace GIdx void Initialize(); } -amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); +amrex::Real compute_dt(const amrex::Geometry& geom, const amrex::Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor, const Real max_adaptive_speedup); void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 62011ae7..f6f63d4e 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -19,19 +19,19 @@ namespace GIdx } } -Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const TestParams* parms) +Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const Real flavor_cfl_factor, const Real max_adaptive_speedup) { - AMREX_ASSERT(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0); + AMREX_ASSERT(cfl_factor > 0.0 || flavor_cfl_factor > 0.0); // translation part of timestep limit const auto dxi = geom.CellSizeArray(); Real dt_translation = 0.0; - if (parms->cfl_factor > 0.0) { - dt_translation = std::min(std::min(dxi[0],dxi[1]), dxi[2]) / PhysConst::c * parms->cfl_factor; + if (cfl_factor > 0.0) { + dt_translation = std::min(std::min(dxi[0],dxi[1]), dxi[2]) / PhysConst::c * cfl_factor; } Real dt_flavor = 0.0; - if (parms->flavor_cfl_factor > 0.0 && parms->collision_cfl_factor > 0.0) { + if (flavor_cfl_factor > 0.0) { // define the reduction operator to get the max contribution to // the potential from matter and neutrinos // compute "effective" potential (ergs) that produces characteristic timescale @@ -39,9 +39,10 @@ Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutr ReduceOps reduce_op; ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; - for (MFIter mfi(state); mfi.isValid(); ++mfi) { + for (MFIter mfi(state); mfi.isValid(); ++mfi) + { const Box& bx = mfi.fabbox(); - auto const& fab = state.array(mfi); + auto const& fab = state.array(mfi); reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { @@ -49,44 +50,25 @@ Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutr #include "generated_files/Evolve.cpp_compute_dt_fill" return {V_adaptive, V_stupid}; }); - } + } - // extract the reduced values from the combined reduced data structure - auto rv = reduce_data.value(); - Real Vmax_adaptive = amrex::get<0>(rv) + FlavoredNeutrinoContainer::Vvac_max; - Real Vmax_stupid = amrex::get<1>(rv) + FlavoredNeutrinoContainer::Vvac_max; + // extract the reduced values from the combined reduced data structure + auto rv = reduce_data.value(); + Real Vmax_adaptive = amrex::get<0>(rv) + FlavoredNeutrinoContainer::Vvac_max; + Real Vmax_stupid = amrex::get<1>(rv) + FlavoredNeutrinoContainer::Vvac_max; - // reduce across MPI ranks - ParallelDescriptor::ReduceRealMax(Vmax_adaptive); - ParallelDescriptor::ReduceRealMax(Vmax_stupid ); + // reduce across MPI ranks + ParallelDescriptor::ReduceRealMax(Vmax_adaptive); + ParallelDescriptor::ReduceRealMax(Vmax_stupid ); - // define the dt associated with each method - Real dt_flavor_adaptive = std::numeric_limits::max(); - Real dt_flavor_stupid = std::numeric_limits::max(); - Real dt_flavor_absorption = std::numeric_limits::max(); // Initialize with infinity + // define the dt associated with each method + Real dt_flavor_adaptive = PhysConst::hbar/Vmax_adaptive*flavor_cfl_factor; + Real dt_flavor_stupid = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; - if (parms->attenuation_hamiltonians != 0) { - dt_flavor_adaptive = PhysConst::hbar / Vmax_adaptive * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; - dt_flavor_stupid = PhysConst::hbar / Vmax_stupid * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; - } - - if (parms->IMFP_method == 1) { - // Use the IMFPs from the input file and find the maximum absorption IMFP - double max_IMFP_abs = std::numeric_limits::lowest(); // Initialize max to lowest possible value - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < NUM_FLAVORS; ++j) { - max_IMFP_abs = std::max(max_IMFP_abs, parms->IMFP_abs[i][j]); - } - } - // Calculate dt_flavor_absorption - dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor; - } - - // pick the appropriate timestep - dt_flavor = min(dt_flavor_stupid, dt_flavor_adaptive, dt_flavor_absorption); - if(parms->max_adaptive_speedup>1) { - dt_flavor = min(dt_flavor_stupid*parms->max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); - } + // pick the appropriate timestep + dt_flavor = dt_flavor_stupid; + if(max_adaptive_speedup>1) + dt_flavor = min(dt_flavor_stupid*max_adaptive_speedup, dt_flavor_adaptive); } Real dt = 0.0; @@ -164,11 +146,6 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M const ParticleInterpolator sy(delta_y, shape_factor_order_y); const ParticleInterpolator sz(delta_z, shape_factor_order_z); - // The following variables contains temperature, electron fraction, and density interpolated from grid quantities to particle positions - Real T_pp = 0; - Real Ye_pp = 0; - Real rho_pp = 0; - for (int k = sz.first(); k <= sz.last(); ++k) { for (int j = sy.first(); j <= sy.last(); ++j) { for (int i = sx.first(); i <= sx.last(); ++i) { @@ -177,44 +154,6 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } } - // Declare matrices to be used in quantum kinetic equation calculation - Real IMFP_abs[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino inverse mean free path matrix: diag( k_e , k_u , k_t ) - Real IMFP_absbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino inverse mean free path matrix: diag( kbar_e , kbar_u , kbar_t ) - Real f_eq[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino equilibrium Fermi-dirac distribution matrix: f_eq = diag( f_e , f_u , f_t ) - Real f_eqbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino equilibrium Fermi-dirac distribution matrix: f_eq = diag( fbar_e , fbar_u , fbar_t ) - - // Initialize matrices with zeros - for (int i=0; iIMFP_method==1){ - for (int i=0; iIMFP_abs[0][i]; // Read absorption inverse mean free path from input parameters file. - IMFP_absbar[i][i] = parms->IMFP_abs[1][i]; // Read absorption inverse mean free path from input parameters file. - - // Calculate the Fermi-Dirac distribution for neutrinos and antineutrinos. - f_eq[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - parms->munu[0][i] ) / T_pp ) ); - f_eqbar[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - parms->munu[1][i] ) / T_pp ) ); - - // Include the Pauli blocking term - if (parms->Do_Pauli_blocking == 1){ - IMFP_abs[i][i] = IMFP_abs[i][i] / ( 1 - f_eq[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). - IMFP_absbar[i][i] = IMFP_absbar[i][i] / ( 1 - f_eqbar[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). - } - } - } - else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0 or 1"); - - #include "generated_files/Evolve.cpp_dfdt_fill" - // set the dfdt values into p.rdata p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; @@ -224,6 +163,11 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M p.rdata(PIdx::pupy) = 0; p.rdata(PIdx::pupz) = 0; p.rdata(PIdx::pupt) = 0; - p.rdata(PIdx::Vphase) = 0; + p.rdata(PIdx::N) = 0; + p.rdata(PIdx::Nbar) = 0; + p.rdata(PIdx::L) = 0; + p.rdata(PIdx::Lbar) = 0; + + #include "generated_files/Evolve.cpp_dfdt_fill" }); } diff --git a/Source/FlavoredNeutrinoContainer.H b/Source/FlavoredNeutrinoContainer.H index 42e13c2a..cb6bcd68 100644 --- a/Source/FlavoredNeutrinoContainer.H +++ b/Source/FlavoredNeutrinoContainer.H @@ -89,6 +89,8 @@ public: Redistribute(lev_min, lev_max, nGrow, local); } + void Renormalize(const TestParams* parms); + amrex::Vector get_attribute_names() const { return attribute_names; diff --git a/Source/FlavoredNeutrinoContainer.cpp b/Source/FlavoredNeutrinoContainer.cpp index 6abd1907..cc15a333 100644 --- a/Source/FlavoredNeutrinoContainer.cpp +++ b/Source/FlavoredNeutrinoContainer.cpp @@ -82,4 +82,27 @@ UpdateLocationFrom(FlavoredNeutrinoContainer& Ploc) } }); } -} \ No newline at end of file +} + +void FlavoredNeutrinoContainer:: +Renormalize(const TestParams* parms) +{ + BL_PROFILE("FlavoredNeutrinoContainer::Renormalize"); + + const int lev = 0; + +#ifdef _OPENMP +#pragma omp parallel +#endif + for (FNParIter pti(*this, lev); pti.isValid(); ++pti) + { + const int np = pti.numParticles(); + ParticleType * pstruct = &(pti.GetArrayOfStructs()[0]); + + amrex::ParallelFor (np, [=] AMREX_GPU_DEVICE (int i) { + ParticleType& p = pstruct[i]; + Real sumP, length, error; + #include "generated_files/FlavoredNeutrinoContainer.cpp_Renormalize_fill" + }); + } +} diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 356fa612..6885b413 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -10,13 +10,8 @@ using namespace amrex; // Particle distribution in momentum space // //=========================================// -Gpu::ManagedVector> read_particle_data(std::string filename){ - - // This function reads the input file containing the initial conditions of the particles. - // It reads the momentum, energy, and flavor occupation matrices for neutrinos and antineutrinos. - - // This array will save the particles information - Gpu::ManagedVector> particle_data; +Gpu::ManagedVector > read_particle_data(std::string filename){ + Gpu::ManagedVector > particle_data; // open the file as a stream std::ifstream file(filename); @@ -37,11 +32,9 @@ Gpu::ManagedVector> read_particle_data(std::string if(NF_in != NUM_FLAVORS) amrex::Print() << "Error: number of flavors in particle data file does not match the number of flavors Emu was compiled for." << std::endl; AMREX_ASSERT(NF_in == NUM_FLAVORS); - // Loop over every line in the initial condition file. - // This is equivalent to looping over every particle. - // Save every particle's information in the array particle_data. while(std::getline(file, line)){ ss = std::stringstream(line); + // skip over the first four attributes (x,y,z,t) for(int i=4; i> temp_particle[i]; particle_data.push_back(temp_particle); @@ -258,14 +251,24 @@ InitParticles(const TestParams* parms) for(int i_attrib=0; i_attrib= 0); - AMREX_ASSERT(p.rdata(PIdx::N11_Re ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::N00_Rebar) >= 0); - AMREX_ASSERT(p.rdata(PIdx::N11_Rebar) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::Nbar ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::L ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::Lbar ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::f00_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::f11_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::f00_Rebar) >= 0); + AMREX_ASSERT(p.rdata(PIdx::f11_Rebar) >= 0); + Real trace = p.rdata(PIdx::f00_Re ) + p.rdata(PIdx::f11_Re ); + Real tracebar = p.rdata(PIdx::f00_Rebar) + p.rdata(PIdx::f11_Rebar); #if NUM_FLAVORS==3 - AMREX_ASSERT(p.rdata(PIdx::N22_Re ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::N22_Rebar) >= 0); + AMREX_ASSERT(p.rdata(PIdx::f22_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::f22_Rebar) >= 0); + trace += p.rdata(PIdx::f22_Re ); + tracebar += p.rdata(PIdx::f22_Rebar); #endif + AMREX_ASSERT(std::abs(trace -1)<1e-6); + AMREX_ASSERT(std::abs(tracebar-1)<1e-6); // Set particle position p.pos(0) = x; @@ -279,18 +282,8 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::time) = 0; // scale particle numbers based on number of points per cell and the cell volume - p.rdata(PIdx::N00_Re ) *= scale_fac; - p.rdata(PIdx::N11_Re ) *= scale_fac; - p.rdata(PIdx::N00_Rebar) *= scale_fac; - p.rdata(PIdx::N11_Rebar) *= scale_fac; -#if NUM_FLAVORS==3 - p.rdata(PIdx::N22_Re ) *= scale_fac; - p.rdata(PIdx::N22_Rebar) *= scale_fac; -#endif - - if(parms->IMFP_method == 1){ - p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/(3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); - } + p.rdata(PIdx::N ) *= scale_fac; + p.rdata(PIdx::Nbar) *= scale_fac; //=====================// // Apply Perturbations // @@ -299,37 +292,37 @@ InitParticles(const TestParams* parms) // random perturbations to the off-diagonals Real rand; symmetric_uniform(&rand, engine); - p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); + p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N01_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); + p.rdata(PIdx::f01_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); + p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); + p.rdata(PIdx::f01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); #if NUM_FLAVORS==3 symmetric_uniform(&rand, engine); - p.rdata(PIdx::N02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); + p.rdata(PIdx::f02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); + p.rdata(PIdx::f02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::N22_Re )); + p.rdata(PIdx::f12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::N22_Re )); + p.rdata(PIdx::f12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N22_Rebar)); + p.rdata(PIdx::f02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N22_Rebar)); + p.rdata(PIdx::f02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar)); + p.rdata(PIdx::f12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar)); + p.rdata(PIdx::f12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); #endif } if(parms->perturbation_type == 1){ // Perturb real part of e-mu component only sinusoidally in z Real nu_k = (2.*M_PI) / parms->perturbation_wavelength_cm; - p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); - p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); + p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); + p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); } if(parms->perturbation_type == 2){ // random perturbations of the diagonals @@ -349,6 +342,9 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::f22_Rebar) *= 1. + parms->perturbation_amplitude*rand; #endif } + + +#include "generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length" } // loop over direction } // loop over location diff --git a/Source/Parameters.H b/Source/Parameters.H index d8ce7ee6..4ee8121a 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -22,8 +22,8 @@ struct TestParams : public amrex::Gpu::Managed Real end_time; int write_plot_every; int write_plot_particles_every; - Real rho_in, Ye_in, kT_in; // g/ccm, 1, erg - Real cfl_factor, flavor_cfl_factor,collision_cfl_factor; + Real rho_in, Ye_in, T_in; // g/ccm, 1, MeV + Real cfl_factor, flavor_cfl_factor; Real max_adaptive_speedup; bool do_restart; std::string restart_dir; @@ -42,18 +42,7 @@ struct TestParams : public amrex::Gpu::Managed int perturbation_type; Real perturbation_wavelength_cm; Real perturbation_amplitude; - - // absorption opacities and equilibrium neutrino chemical potentials - int IMFP_method; - int Do_Pauli_blocking = 0; // If 1, it will multiply the opacities by 1 / (1 - f_eq); if 0, do nothing. - Real IMFP_abs[2][NUM_FLAVORS]; - Real IMFP_scat[2][NUM_FLAVORS]; - Real munu[2][NUM_FLAVORS]; // equilibrium electron neutrino chemical potential in erg (CGS unist) - Real delta_E; // erg (CGS unist) - - // attenuation to hamiltonians - Real attenuation_hamiltonians; - + void Initialize(){ ParmParse pp; pp.get("ncell", ncell); @@ -66,10 +55,9 @@ struct TestParams : public amrex::Gpu::Managed pp.get("end_time", end_time); pp.get("rho_g_ccm", rho_in); pp.get("Ye", Ye_in); - pp.get("T_MeV", kT_in); + pp.get("T_MeV", T_in); pp.get("cfl_factor", cfl_factor); pp.get("flavor_cfl_factor", flavor_cfl_factor); - pp.get("collision_cfl_factor", collision_cfl_factor); pp.get("max_adaptive_speedup", max_adaptive_speedup); pp.get("write_plot_every", write_plot_every); pp.get("write_plot_particles_every", write_plot_particles_every); @@ -77,9 +65,6 @@ struct TestParams : public amrex::Gpu::Managed pp.get("restart_dir", restart_dir); pp.get("maxError", maxError); - // convert to cgs - kT_in *= 1e6 * CGSUnitsConst::eV; // erg - // angular grid pp.get("particle_data_filename",particle_data_filename); @@ -89,59 +74,28 @@ struct TestParams : public amrex::Gpu::Managed if(perturbation_type == 1) pp.get("perturbation_wavelength_cm", perturbation_wavelength_cm); - // neutrino physics parameters for 2-flavor - pp.get("mass1_eV", mass1); - pp.get("mass2_eV", mass2); - pp.get("theta12_degrees", theta12); - pp.get("alpha1_degrees", alpha1); - mass1 *= CGSUnitsConst::eV/PhysConst::c2; - mass2 *= CGSUnitsConst::eV/PhysConst::c2; - theta12 *= M_PI/180.; - alpha1 *= M_PI/180.; - if(NUM_FLAVORS>=2){ - pp.get("mass3_eV", mass3); - pp.get("theta13_degrees", theta13); - pp.get("theta23_degrees", theta23); - pp.get("alpha2_degrees", alpha2); - pp.get("deltaCP_degrees", deltaCP); - mass3 *= CGSUnitsConst::eV/PhysConst::c2; - theta13 *= M_PI/180.; - theta23 *= M_PI/180.; - alpha2 *= M_PI/180.; - deltaCP *= M_PI/180.; - } - - // attenuation to hamiltonians - pp.get("attenuation_hamiltonians", attenuation_hamiltonians); + // neutrino physics parameters for 2-flavor + pp.get("mass1_eV", mass1); + pp.get("mass2_eV", mass2); + pp.get("theta12_degrees", theta12); + pp.get("alpha1_degrees", alpha1); + mass1 *= CGSUnitsConst::eV/PhysConst::c2; + mass2 *= CGSUnitsConst::eV/PhysConst::c2; + theta12 *= M_PI/180.; + alpha1 *= M_PI/180.; - // absorption opacities and equilibrium neutrino chemical potentials - pp.get("IMFP_method", IMFP_method); - pp.get("Do_Pauli_blocking", Do_Pauli_blocking); // If 1, it will multiply the opacities by 1 / (1 - f_eq); if 0, do nothing. - if(IMFP_method==0){ - // do nothing - all values will be set to zero - } - else if(IMFP_method==1){ - for(int i=0;i=2){ + pp.get("mass3_eV", mass3); + pp.get("theta13_degrees", theta13); + pp.get("theta23_degrees", theta23); + pp.get("alpha2_degrees", alpha2); + pp.get("deltaCP_degrees", deltaCP); + mass3 *= CGSUnitsConst::eV/PhysConst::c2; + theta13 *= M_PI/180.; + theta23 *= M_PI/180.; + alpha2 *= M_PI/180.; + deltaCP *= M_PI/180.; } - pp.get("delta_E", delta_E); - delta_E*= 1e6*CGSUnitsConst::eV; // erg - } - else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0(do not collisions) and 1(do collisions)"); } }; diff --git a/Source/main.cpp b/Source/main.cpp index f6b5fa50..5bbb2118 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -82,7 +82,7 @@ void evolve_flavor(const TestParams* parms) state.setVal(0.0); state.setVal(parms->rho_in,GIdx::rho,1); // g/ccm state.setVal(parms->Ye_in,GIdx::Ye,1); - state.setVal(parms->kT_in,GIdx::T,1); // erg + state.setVal(parms->T_in,GIdx::T,1); // MeV state.FillBoundary(geom.periodicity()); // initialize the grid variable names @@ -117,7 +117,7 @@ void evolve_flavor(const TestParams* parms) // Deposit particles to grid deposit_to_mesh(neutrinos_old, state, geom); - + // Write plotfile after initialization DataReducer rd; if (not parms->do_restart) { @@ -148,7 +148,6 @@ void evolve_flavor(const TestParams* parms) // B) We only Redistribute the integrator new data at the end of the timestep, not all the RHS data. // Thus, this copy clears the old RHS particles and creates particles in the RHS container corresponding // to the current particles in neutrinos. - neutrinos_rhs.copyParticles(neutrinos, true); // Step 3: Interpolate Mesh to construct the neutrino RHS in place @@ -173,6 +172,9 @@ void evolve_flavor(const TestParams* parms) // since Redistribute() applies periodic boundary conditions. neutrinos.SyncLocation(Sync::PositionToCoordinate); + // Renormalize the neutrino state + neutrinos.Renormalize(parms); + // Get which step the integrator is on const int step = integrator.get_step_number(); const Real time = integrator.get_time(); @@ -195,7 +197,7 @@ void evolve_flavor(const TestParams* parms) // Note: this won't be the same as the new-time grid data // because the last deposit_to_mesh call was at either the old time (forward Euler) // or the final RK stage, if using Runge-Kutta. - const Real dt = compute_dt(geom, state, neutrinos, parms); + const Real dt = compute_dt(geom,parms->cfl_factor,state,neutrinos,parms->flavor_cfl_factor,parms->max_adaptive_speedup); integrator.set_timestep(dt); }; @@ -204,7 +206,7 @@ void evolve_flavor(const TestParams* parms) integrator.set_post_timestep(post_timestep_fun); // Get a starting timestep - const Real starting_dt = compute_dt(geom, state, neutrinos_old, parms); + const Real starting_dt = compute_dt(geom,parms->cfl_factor,state,neutrinos_old,parms->flavor_cfl_factor, parms->max_adaptive_speedup); // Do all the science! amrex::Print() << "Starting timestepping loop... " << std::endl; diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 5413306c..c3e89d37 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -1,10 +1,6 @@ perturbation_type = 0 perturbation_amplitude = 1e-6 -# attenuation parameters to time derivative of N due to hamiltonians -attenuation_hamiltonians = 1 - -collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 @@ -82,8 +78,3 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 -################# -# opacity stuff # -################# -IMFP_method = 0 -Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. \ No newline at end of file diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 4414c0aa..e4d58c56 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -1,12 +1,8 @@ -collision_cfl_factor = 1e-3 cfl_factor = 0.5 max_adaptive_speedup = 0 flavor_cfl_factor = .5 maxError = 1e-6 -# attenuation parameters to time derivative of N due to hamiltonians -attenuation_hamiltonians = 1 - perturbation_type = 0 perturbation_amplitude = 0 @@ -82,9 +78,3 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 - -################# -# opacity stuff # -################# -IMFP_method = 0 -Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. \ No newline at end of file diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test deleted file mode 100644 index 49f5ea92..00000000 --- a/sample_inputs/inputs_coll_equi_test +++ /dev/null @@ -1,113 +0,0 @@ -perturbation_type = 0 -perturbation_amplitude = 1e-6 - -# attenuation parameters to time derivative of N due to hamiltonians -attenuation_hamiltonians = 0 - -collision_cfl_factor = 0.04 -cfl_factor = 1e10 -flavor_cfl_factor = 0.5 -max_adaptive_speedup = 0 -maxError = 1e-6 - -integration.type = 1 -integration.rk.type = 4 - -# Domain size in 3D index space -ncell = (2, 2, 2) -Lx = 20 -Ly = 20 -Lz = 20 - -# Number of particles per cell -nppc = (1, 1, 1) -particle_data_filename = "particle_input.dat" - -# Maximum size of each grid in the domain -max_grid_size = 16 - -# Number of steps to run -nsteps = 1000 - -# Simulation end time -end_time = 5.0e9 - -# Make FPE signal errors so we get a Backtrace -amrex.fpe_trap_invalid=1 - -# give background fluid conditions -rho_g_ccm = 0 -T_MeV = 10 -Ye = 1 - -# Write plotfiles -write_plot_every = 1000 - -# Write particle data in plotfiles -write_plot_particles_every = 1000 - -# checkpointing -do_restart = 0 -restart_dir = "" - -############################### -# NEUTRINO PHYSICS PARAMETERS # -############################### -# see first column of table 14.7 in http://pdg.lbl.gov/2019/reviews/rpp2019-rev-neutrino-mixing.pdf - -# mass state 1 mass in eV [NO/IO:-sqrt(7.39e-5)] -mass1_eV = 0 #-0.008596511 - -# mass state 2 mass in eV (define at 0 arbitrarily because oscillations only sensitive to deltaM^2) -mass2_eV = 0 - -# mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] -mass3_eV = 0.049487372 - -# 1-2 mixing angle in degrees [NO/IO:33.82] -theta12_degrees = 1e-6 - -# 2-3 mixing angle in degrees [NO:8.61 IO:8.65] -theta23_degrees = 8.61 - -# 1-3 mixing angle in degrees [NO:48.3 IO:48.6] -theta13_degrees = 48.3 - -# Majorana angle 1 in degrees -alpha1_degrees = 0 - -# Majorana angle 2 in degrees -alpha2_degrees = 0 - -# CP-violating phase in degrees [NO:222 IO:285] -deltaCP_degrees = 222 - -################# -# opacity stuff # -################# -IMFP_method = 1 - -Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. - -IMFP_abs0_cm = 5e-4 -IMFP_abs1_cm = 5e-4 -IMFP_abs2_cm = 5e-4 -IMFP_abs0bar_cm = 5e-4 -IMFP_abs1bar_cm = 5e-4 -IMFP_abs2bar_cm = 5e-4 - -munu0_MeV = 0 -munu1_MeV = 0 -munu2_MeV = 0 -munu0bar_MeV = 0 -munu1bar_MeV = 0 -munu2bar_MeV = 0 - -IMFP_scat0_cm = 0 -IMFP_scat1_cm = 0 -IMFP_scat2_cm = 0 -IMFP_scat0bar_cm = 0 -IMFP_scat1bar_cm = 0 -IMFP_scat2bar_cm = 0 - -delta_E = 0.8394810651882109 # Mev diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test deleted file mode 100644 index decf1ec7..00000000 --- a/sample_inputs/inputs_collisional_instability_test +++ /dev/null @@ -1,113 +0,0 @@ -perturbation_type = 0 -perturbation_amplitude = 0.0 - -# attenuation parameters to time derivative of N due to hamiltonians -attenuation_hamiltonians = 1.0 - -collision_cfl_factor = 1e-3 -cfl_factor = 0.5 -flavor_cfl_factor = 0.5 -max_adaptive_speedup = 0 -maxError = 1e-6 - -integration.type = 1 -integration.rk.type = 4 - -# Domain size in 3D index space -ncell = (1, 1, 1) -Lx = 1 # cm -Ly = 1 # cm -Lz = 1 # cm - -# Number of particles per cell -nppc = (1, 1, 1) -particle_data_filename = "particle_input.dat" - -# Maximum size of each grid in the domain -max_grid_size = 16 - -# Number of steps to run -nsteps = 100000 - -# Simulation end time -end_time = 5.0e19 - -# Make FPE signal errors so we get a Backtrace -amrex.fpe_trap_invalid=1 - -# give background fluid conditions -rho_g_ccm = 0 -T_MeV = 7.0 -Ye = 0 - -# Write plotfiles -write_plot_every = 100 - -# Write particle data in plotfiles -write_plot_particles_every = 100 - -# checkpointing -do_restart = 0 -restart_dir = "" - -############################### -# NEUTRINO PHYSICS PARAMETERS # -############################### -# see first column of table 14.7 in http://pdg.lbl.gov/2019/reviews/rpp2019-rev-neutrino-mixing.pdf - -# mass state 1 mass in eV [NO/IO:-sqrt(7.39e-5)] -mass1_eV = 0.04866 #-0.008596511 - -# mass state 2 mass in eV (define at 0 arbitrarily because oscillations only sensitive to deltaM^2) -mass2_eV = 0 - -# mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] -mass3_eV = 0 - -# 1-2 mixing angle in degrees [NO/IO:33.82] -theta12_degrees = 1e-6 - -# 2-3 mixing angle in degrees [NO:8.61 IO:8.65] -theta23_degrees = 0 - -# 1-3 mixing angle in degrees [NO:48.3 IO:48.6] -theta13_degrees = 0 - -# Majorana angle 1 in degrees -alpha1_degrees = 0 - -# Majorana angle 2 in degrees -alpha2_degrees = 0 - -# CP-violating phase in degrees [NO:222 IO:285] -deltaCP_degrees = 0 - -################# -# opacity stuff # -################# -IMFP_method = 1 - -Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. - -IMFP_abs0_cm = 2.398082e-1 -IMFP_abs1_cm = 0 -IMFP_abs2_cm = 0 -IMFP_abs0bar_cm = 2.29358e-2 -IMFP_abs1bar_cm = 0 -IMFP_abs2bar_cm = 0 - -munu0_MeV = 20.0 -munu1_MeV = 0 -munu2_MeV = 0 -munu0bar_MeV = 17.644694342915507 -munu1bar_MeV = 0 -munu2bar_MeV = 0 - -IMFP_scat0_cm = 0 -IMFP_scat1_cm = 0 -IMFP_scat2_cm = 0 -IMFP_scat0bar_cm = 0 -IMFP_scat1bar_cm = 0 -IMFP_scat2bar_cm = 0 - -delta_E = 2.272540842052914 # Mev diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index c07c6936..2406fbeb 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -1,12 +1,8 @@ -collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = .5 max_adaptive_speedup = 0 maxError = 1e-6 -# attenuation parameters to time derivative of N due to hamiltonians -attenuation_hamiltonians = 1 - perturbation_type = 0 perturbation_amplitude = 0 @@ -82,9 +78,3 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 - -################# -# opacity stuff # -################# -IMFP_method = 0 -Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index c4e877ad..dc858a8e 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -2,10 +2,6 @@ perturbation_type = 1 perturbation_amplitude = 1e-6 perturbation_wavelength_cm = 1 -# attenuation parameters to time derivative of N due to hamiltonians -attenuation_hamiltonians = 1 - -collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 @@ -83,8 +79,3 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 -################# -# opacity stuff # -################# -IMFP_method = 0 -Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. \ No newline at end of file diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index 17cd8dfd..cdd14ac7 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -1,12 +1,8 @@ -collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = 0.1 max_adaptive_speedup = 0 maxError = 1e-6 -# attenuation parameters to time derivative of N due to hamiltonians -attenuation_hamiltonians = 1 - perturbation_type = 0 perturbation_amplitude = 0 @@ -82,9 +78,3 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 - -################# -# opacity stuff # -################# -IMFP_method = 0 -Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. \ No newline at end of file From 94846e2b40e6fa6813a2b29f024154ed420689b3 Mon Sep 17 00:00:00 2001 From: Sherwood Richers <5142652+srichers@users.noreply.github.com> Date: Fri, 23 Aug 2024 09:18:12 -0400 Subject: [PATCH 159/276] Collisions (again) (#98) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Particles store the phase space volume they occupy * set evolution equations for N and L to be zero in code generation script in anticipation of changing them based on opacities. Set Vphase to not evolve in time. * whitespace * infrastructure to set IMFPs and chemical potentials in the parameter file * redefining f --> Nrho (before was f ---> Nrho) * computing volume of phase space * deleting lines that convert twice mu to erg * adding delta E as an input parameter to compute phase space volume * adding new input parameters to input files to run collitions * adding collition term in QKE * moving line that compute derivates, now it set the phase volume to zero meaning its time derivative its zero after it is used in collision term * considering the particle that carry antineutrinos in the fast flavor test * Fixing an error in the collision term * including adaptative time step for absortion and emission * creating vacuum paticles initial condition script * adding attenuation parameter to the time derivative of N_Ab due to the hamiltonians * solving mistake to compute time step due to collisions * adding collision test input file * testing if the particles converge to the expected equilibrium distribution * solving issue to copy GNUmakefile_jenkins to Exec directory * doing fast compilation in collision test * clarifying better the collision time steps, now delta t collision is collision_CFL / ( c * max_IMFP ) * deletting renomarlization function. It will not be used anymore * solving mistake in compilation comand to compile the code for 3 flavors * Setting default collision time step to infinite. It will change if the code do collisions. * setting right indentation * including attenuation parameter to compute the time steps due to potentials * computing minimum between time steps due to potentials and collision even if max_adaptive_speedup>1 is not satisfy * solve mistake in chemical potential units in coment * add coment on the energy units * delete the individual parms arguments * creating collisional flavor instability test * delete print line * now reads plt* files for time steps of more than 5 digits * now reads plt files for time steps with more than 5 digits * add function to initialize the background enviroment Ye, rho and T * insert chemical potential for neutrinos and antineutrino independently * set python script that generate initial condition for collisional instability test * set input file for collisional instability test * Create a Jenkins test for collisional flavor instability * change cuda version to 7.0 * delete std cout line * correct cuda version for jenkins * change order of commands first reduce_data_fft.py then reduce_data.py * delete function to initialize background quantities on this branch * Solve mistake on collisional instability test. Now it read the right input file * Solve mistake on collisional instability test. Now it read the right python script * add python script for collision instability test * solve problem of defining chemical potentials for antineutrinos in the input parameter file * rename f -> N for particles Beware: grid also has a Nij variable which is different * finish renaming f -> N everywhere * comment lines that assert if N and Nbar are negatives, this lines will be deleted since now N and Nbar will no longer be used * change f for N in utility script to generate initial particles conditions * modify initial condition script for msw test. now it gives N_ab instead of N and f_ab as input * add comments to the function that read the initial conditions of the particles * change msw test to include the change f_ab ---> N_ab * reading N_ab for the particle initial condition file instead of N and f_ab independenly * removing f_ab variables for N_ab when setting perturbation * delete no needed lines * repare coll_equi_test.py test to consider N_ab instead of f_ab * making bypolar test work with the changes f_ab ---> N_ab * making fast flavor test work for f_ab ---> N_ab * making fast_flavor_k_test works for f_ab ---> N_ab * delete print line * delete print lines * make attenuation parameter symbolic in code generation scripts * move f-->N in a few more places * change f to N in test scripts * change Trf to TrN * remove unused line * setting right initial conditions for collisional instability test * inteporlate T, Ye, rho from grid to particles, but return right value multiplied by 4 * solve mistake in interpolating rho, Ye and T from grid to particles posision, now the generation code script does not repeate the code 4 times * delete print lines * using the interpolated value of T for the calculation of the equlibrium distribution * collisional instability test do not include a random perturbation on the off diagonal component of the density matrix. Not it use vacuum potentials. * upload julien nsm input script * solve mistake in energy average * set right initial conditions for julien nsm simulation * Modify collision instability test The collision instaiblity test now includes the LSA in equation 14 of L. Johns [2104.11369] Julien Frousley script to solve isotropic QKE is used to compare EMU solution Plots are included all over the test but were commented * Add collisions directory to store usefull script for simulations containing collisions The script compute_dE.py is used to compute the energy bin size base on the expected equilibrium number of neutrinos * Comments added to compute_dE.py script * reordered scripts in jenkinsfile to clean up PR * added line in data reduction scripts to prevent glob from picking up generated hdf5 files * Deleting unnecessary variables in the script that generates initial conditions for the collision to equilibrium test (coll_equi_test.py). This commit reduces code size by removing redundant variables in the st7_empty_particles.py script (Scripts/tests/coll_equi_test.py), which generates a set of particles with zero neutrinos and antineutrinos. * Delete Julien's NSM input file from the initial conditions script directory. * Removed opacities and chemical potential entries from input files that do not include collision simulations. To avoid future confusion, I deleted the collision-related input options (opacities, chemical potential, and energy bin size) from input files used for tests without collisions. * Added a more accurate description of the assert error for opacity methods. The assert error now clearly indicates that the only available opacity methods are 0 (no collisions) and 1 (with collisions). * Change variable type from double to Real to keep consistency * Adding more comments to the script that computes the energy bin size for collision tests. * Delete commented lines that are under development. I deleted commented lines containing Julien's script that computes the collisional flavor instability evolution. This could potentially be included in the future to solidify the collisional instability test. * Delete Python script of a previous test that will no longer be used. * Delete Python script that generates the initial conditions of particles in Julien's NSM simulation. * Delete old script used for testing convergence. It will no longer be used. * Change N_eq to f_eq since this better represents the collision term. f_eq is the Fermi-Dirac neutrino distribution for a given particle energy. * Include option to multiply the inverse mean free paths by the Pauli blocking term. A new input parameter was created (Do_Pauli_blocking). If set to 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if set to 0, it will do nothing. * Adding a semicolon * Adding the input parameter Do_Pauli_blocking to all input files. I added the input parameter Do_Pauli_blocking to all input files to avoid errors caused by not reading this parameter during execution. If this parameter is set to 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if set to 0, it will do nothing. In the tests that don't involve collisions, it will not have any effect. * Correction in the collision term of QKE The collision term implemented in this commit is C = { gamma , N_eq - N }. Here Gamma = diag( k*_e , k*_x ) / 2 . k*_a is the inverse mean free path for flavor a, including Pauli blocking term. * means that Pauli blocking term is already in the inverse mean free path values. Equilibrium neutrino number matrix N_eq equals the integral of f_eq, where the integral is over the phase space that the particle represents. * Modify the script to compute the energy bin size and antineutrino chemical potential for the collision instability test. This change emerged from the correction made to the collision term and the necessity of obtaining the antineutrino chemical potential to ensure the test runs properly. * Setting up the collisional instability test Set the appropriate input parameters for the collisional instability test, generate plots to visualize the results, and add comments to the Python script that generates the initial conditions for this test. * convert f variables in new commits to development to N * Deleting input parameter Do_Pauli_blocking in tests that do not use it This change simplifies the input parameters file for simulations that do not run collisions. Now, the input parameter Do_Pauli_blocking only appears in the input files for collision-to-equilibrium and collisional instability tests. * Creating two separate scripts to compute the energy bin size for collision tests: Now, instead of a single script to compute the energy bin size, I have included two separate scripts—one for the collisional instability test and another for the particles-to-equilibrium test. * Setting script for particles to equilibrium test Unnecessary lines were deleted, the correct energy bin size was set in the input parameter file. Added comments to the code. * Changing input parameters for collisional instability test to speed up the test Reducing the number of time steps and the written files in the collisional instability test to improve performance in Jenkins. --------- Co-authored-by: Sherwood Richers Co-authored-by: erickurquilla1999 Co-authored-by: erickurquilla1999 --- Jenkinsfile | 23 ++ .../collisions/compute_dE_coll_equi_test.py | 47 +++ .../collisions/compute_dE_coll_inst_test.py | 55 ++++ Scripts/data_reduction/amrex_plot_tools.py | 60 ++-- Scripts/data_reduction/reduce_data.py | 4 +- Scripts/data_reduction/reduce_data_fft.py | 4 +- .../initial_condition_tools.py | 20 +- Scripts/initial_conditions/st0_msw_test.py | 9 +- .../initial_conditions/st1_bipolar_test.py | 10 +- .../st2_2beam_fast_flavor.py | 7 +- .../st3_2beam_fast_flavor_nonzerok.py | 10 +- .../initial_conditions/st7_empty_particles.py | 22 ++ .../initial_conditions/st8_coll_inst_test.py | 36 +++ Scripts/symbolic_hermitians/generate_code.py | 164 +++++------ Scripts/tests/coll_equi_test.py | 61 ++++ Scripts/tests/coll_inst_test.py | 276 ++++++++++++++++++ Scripts/tests/convergence.sh | 89 ------ Scripts/tests/fast_flavor_k_test.py | 54 ++-- Scripts/tests/fast_flavor_test.py | 59 ++-- Scripts/tests/msw_test.py | 44 +-- Scripts/tests/plot_convergence.py | 98 ------- Source/Constants.H | 1 + Source/DataReducer.cpp | 30 +- Source/Evolve.H | 2 +- Source/Evolve.cpp | 114 ++++++-- Source/FlavoredNeutrinoContainer.H | 2 - Source/FlavoredNeutrinoContainer.cpp | 25 +- Source/FlavoredNeutrinoContainerInit.cpp | 92 +++--- Source/Parameters.H | 97 ++++-- Source/main.cpp | 12 +- sample_inputs/inputs_1d_fiducial | 8 + sample_inputs/inputs_bipolar_test | 9 + sample_inputs/inputs_coll_equi_test | 113 +++++++ .../inputs_collisional_instability_test | 113 +++++++ sample_inputs/inputs_fast_flavor | 9 + sample_inputs/inputs_fast_flavor_nonzerok | 8 + sample_inputs/inputs_msw_test | 9 + 37 files changed, 1215 insertions(+), 581 deletions(-) create mode 100644 Scripts/collisions/compute_dE_coll_equi_test.py create mode 100644 Scripts/collisions/compute_dE_coll_inst_test.py create mode 100644 Scripts/initial_conditions/st7_empty_particles.py create mode 100644 Scripts/initial_conditions/st8_coll_inst_test.py create mode 100644 Scripts/tests/coll_equi_test.py create mode 100644 Scripts/tests/coll_inst_test.py delete mode 100644 Scripts/tests/convergence.sh delete mode 100644 Scripts/tests/plot_convergence.py create mode 100644 sample_inputs/inputs_coll_equi_test create mode 100644 sample_inputs/inputs_collisional_instability_test diff --git a/Jenkinsfile b/Jenkinsfile index 84c48283..be2611aa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -90,6 +90,29 @@ pipeline { } }} + stage('Collisions flavor instability'){ steps{ + dir('Exec'){ + sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' + sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' + sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' + sh 'python ../Scripts/data_reduction/reduce_data.py' + sh 'python ../Scripts/tests/coll_inst_test.py' + sh 'rm -rf plt*' + } + }} + + stage('Collisions to equilibrium'){ steps{ + dir('Exec'){ + sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' + sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3' + sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' + sh 'python ../Scripts/tests/coll_equi_test.py' + sh 'rm -rf plt*' + } + }} + } // stages{ post { diff --git a/Scripts/collisions/compute_dE_coll_equi_test.py b/Scripts/collisions/compute_dE_coll_equi_test.py new file mode 100644 index 00000000..c0743e6d --- /dev/null +++ b/Scripts/collisions/compute_dE_coll_equi_test.py @@ -0,0 +1,47 @@ +''' +This script compute the energy bin size ( /Delta E ) for monoenergetic neutrino simulations given: +- Number of neutrinos at equilibrium +- Volume of a cell +- Number of momentum beams isotropically distributed per cell +- Neutrinos energy bin center +- Neutrino chemical potential +- Background matter temperature +This script was used to compute the energy bin size in the test scripts: coll_equi_test.py. +''' + +import numpy as np + +# constants +hbar = 1.05457266e-27 # erg s +h = hbar * 2 * np.pi # erg s +c = 2.99792458e10 # cm/s +hc = h * c # erg cm + +# Simulation parameters +V = 10**3 # Volume of a cell ( ccm ) +Ndir = 92 # Number of momentum beams isotropically distributed per cell +E = 50.0 # Neutrinos and antineutrinos energy bin center ( Mev ) +T = 10.0 # Background matter temperature ( Mev ) + +N_eq_electron_neutrino = 1e33 # Number of electron neutrinos at equilibrium +u_electron_neutrino = 0.0 # Electron neutrino chemical potential ( Mev ) + +# Fermi-dirac distribution factor for electron neutrinos +f_eq_electron_neutrinos = 1 / ( 1 + np.exp( ( E - u_electron_neutrino ) / T ) ) # adimentional + +# We know : +# dE^3 = 3 * Neq * ( hc )^ 3 / ( dV * dOmega * feq ) +delta_E_cubic = 3 * N_eq_electron_neutrino * ( hc )**3 / ( V * ( 4 * np.pi / Ndir ) * f_eq_electron_neutrinos ) # cubic erg +# dOmega = 4 * pi / ( number directions ) + +# We know polinomial of delta E in term of delta E cubic and E ( deltaE**3 = ( E + dE / 2)**3 - ( E - dE / 2)**3 ) +coeff = [ 0.25 , 0 , 3 * E**2 , -1.0 * delta_E_cubic / ( 1.60218e-6**3 ) ] +# Solving for this polinomial +deltaE = np.roots(coeff) + +# Extracting just the real root +dE=0 +for complex_deltaE in deltaE: + if (np.imag(complex_deltaE)==0): + print(f'Delta energy bin in MeV = {np.real(complex_deltaE)}') + dE=np.real(complex_deltaE) \ No newline at end of file diff --git a/Scripts/collisions/compute_dE_coll_inst_test.py b/Scripts/collisions/compute_dE_coll_inst_test.py new file mode 100644 index 00000000..dec22d2c --- /dev/null +++ b/Scripts/collisions/compute_dE_coll_inst_test.py @@ -0,0 +1,55 @@ +''' +This script compute the energy bin size ( /Delta E ) for monoenergetic neutrino simulations and the antineutrino chemical potential, given: +- Number of neutrinos at equilibrium +- Volume of a cell +- Number of momentum beams isotropically distributed per cell +- Neutrinos energy bin center +- Neutrino chemical potential +- Background matter temperature +This script was used to compute the energy bin size in the test scripts and antineutrino chemical potential: coll_inst_test.py. +''' + +import numpy as np + +# constants +hbar = 1.05457266e-27 # erg s +h = hbar * 2 * np.pi # erg s +c = 2.99792458e10 # cm/s +hc = h * c # erg cm + +# Simulation parameters +V = 1 # Volume of a cell ( ccm ) +Ndir = 92 # Number of momentum beams isotropically distributed per cell +E = 20.0 # Neutrinos and antineutrinos energy bin center ( Mev ) +T = 7.0 # Background matter temperature ( Mev ) + +N_eq_electron_neutrino = 3.260869565e+31 # Number of electron neutrinos at equilibrium +u_electron_neutrino = 20.0 # Electron neutrino chemical potential ( Mev ) + +# Fermi-dirac distribution factor for electron neutrinos +f_eq_electron_neutrinos = 1 / ( 1 + np.exp( ( E - u_electron_neutrino ) / T ) ) # adimentional + +# We know : +# dE^3 = 3 * Neq * ( hc )^ 3 / ( dV * dOmega * feq ) +delta_E_cubic = 3 * N_eq_electron_neutrino * ( hc )**3 / ( V * ( 4 * np.pi / Ndir ) * f_eq_electron_neutrinos ) # cubic erg +# dOmega = 4 * pi / ( number directions ) + +# We know polinomial of delta E in term of delta E cubic and E ( deltaE**3 = ( E + dE / 2)**3 - ( E - dE / 2)**3 ) +coeff = [ 0.25 , 0 , 3 * E**2 , -1.0 * delta_E_cubic / ( 1.60218e-6**3 ) ] +# Solving for this polinomial +deltaE = np.roots(coeff) + +# Extracting just the real root +dE=0 +for complex_deltaE in deltaE: + if (np.imag(complex_deltaE)==0): + print(f'Delta energy bin in MeV = {np.real(complex_deltaE)}') + dE=np.real(complex_deltaE) + +# Electron neutrino flavor +N_eq_electron_antineutrino = 2.717391304e+31 # Number of electron antineutrinos at equilibrium + +# Computing electron antineutrino chemical potential +f_eq_electron_antineutrino = 3 * N_eq_electron_antineutrino * ( hc )**3 / ( V * ( 4 * np.pi / Ndir ) * delta_E_cubic ) +u_electron_antineutrino = E - T * np.log( 1 / f_eq_electron_antineutrino - 1 ) +print(f'Electron neutrino chemical potential in MeV = {u_electron_antineutrino}') \ No newline at end of file diff --git a/Scripts/data_reduction/amrex_plot_tools.py b/Scripts/data_reduction/amrex_plot_tools.py index 94c8d094..ee717d97 100644 --- a/Scripts/data_reduction/amrex_plot_tools.py +++ b/Scripts/data_reduction/amrex_plot_tools.py @@ -26,18 +26,14 @@ def get_particle_keys(NF, ignore_pos=False, xp_only=False): "pupy", "pupz", "pupt", - "N", - "L", - "f00_Re", - "f01_Re", - "f01_Im", - "f11_Re", - "Nbar", - "Lbar", - "f00_Rebar", - "f01_Rebar", - "f01_Imbar", - "f11_Rebar"] + "N00_Re", + "N01_Re", + "N01_Im", + "N11_Re", + "N00_Rebar", + "N01_Rebar", + "N01_Imbar", + "N11_Rebar"] if(NF==3): real_quantities = ["pos_x", "pos_y", @@ -50,28 +46,24 @@ def get_particle_keys(NF, ignore_pos=False, xp_only=False): "pupy", "pupz", "pupt", - "N", - "L", - "f00_Re", - "f01_Re", - "f01_Im", - "f02_Re", - "f02_Im", - "f11_Re", - "f12_Re", - "f12_Im", - "f22_Re", - "Nbar", - "Lbar", - "f00_Rebar", - "f01_Rebar", - "f01_Imbar", - "f02_Rebar", - "f02_Imbar", - "f11_Rebar", - "f12_Rebar", - "f12_Imbar", - "f22_Rebar"] + "N00_Re", + "N01_Re", + "N01_Im", + "N02_Re", + "N02_Im", + "N11_Re", + "N12_Re", + "N12_Im", + "N22_Re", + "N00_Rebar", + "N01_Rebar", + "N01_Imbar", + "N02_Rebar", + "N02_Imbar", + "N11_Rebar", + "N12_Rebar", + "N12_Imbar", + "N22_Rebar"] if xp_only: real_quantities = real_quantities[:11] if ignore_pos: real_quantities = real_quantities[7:] diff --git a/Scripts/data_reduction/reduce_data.py b/Scripts/data_reduction/reduce_data.py index 665f3866..3b4fbbe2 100644 --- a/Scripts/data_reduction/reduce_data.py +++ b/Scripts/data_reduction/reduce_data.py @@ -385,7 +385,9 @@ def reduce_data(directory=".", nproc=4, do_average=True, do_fft=True, do_angular if(data_format=="Emu"): yt_descriptor = "boxlib" convert_N_to_inv_ccm = 1.0 - directories = sorted(glob.glob("plt?????")) + directories = glob.glob("plt*") + directories = [d for d in directories if ".h5" not in d] + directories = sorted(directories, key=lambda x: int(x.lstrip("plt"))) # get NF eds = emu.EmuDataset(directories[0]) diff --git a/Scripts/data_reduction/reduce_data_fft.py b/Scripts/data_reduction/reduce_data_fft.py index 7abbe3fe..15949757 100755 --- a/Scripts/data_reduction/reduce_data_fft.py +++ b/Scripts/data_reduction/reduce_data_fft.py @@ -9,7 +9,9 @@ parser.add_argument("-o", "--output", type=str, default="reduced_data_fft.h5", help="Name of the output file (default: reduced_data_fft.h5)") args = parser.parse_args() -directories = sorted(glob.glob("plt?????")) +directories = glob.glob("plt*") +directories = [d for d in directories if ".h5" not in d] +directories = sorted(directories, key=lambda x: int(x.lstrip("plt"))) t = [] diff --git a/Scripts/initial_conditions/initial_condition_tools.py b/Scripts/initial_conditions/initial_condition_tools.py index a3639f9f..d0885bdd 100644 --- a/Scripts/initial_conditions/initial_condition_tools.py +++ b/Scripts/initial_conditions/initial_condition_tools.py @@ -230,7 +230,7 @@ def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_g # get variable keys rkey, ikey = amrex.get_particle_keys(NF, ignore_pos=True) nelements = len(rkey) - + # generate the list of particle info particles = np.zeros((nparticles,nelements)) @@ -240,22 +240,10 @@ def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_g # save the total number density of neutrinos for each particle n_flavorsummed = np.sum(n_particle, axis=2) # [particle, nu/nubar] - for nu_nubar, suffix in zip(range(2), ["","bar"]): - nvarname = "N"+suffix - particles[:,rkey[nvarname]] = n_flavorsummed[:,nu_nubar] - + for nu_nubar, suffix in zip(range(2), ["","bar"]): for flavor in range(NF): - fvarname = "f"+str(flavor)+str(flavor)+"_Re"+suffix - particles[:,rkey[fvarname]] = n_particle[:,nu_nubar, flavor] / n_flavorsummed[:,nu_nubar] - particles[:,rkey[fvarname]][np.where(n_flavorsummed[:,nu_nubar]==0)] = 1./NF # ensure that trace stays equal to 1 - - # double check that the number densities are correct - particle_n = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]]) - particle_fmag = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]] * mu[:,nu_nubar, flavor]) - #print("nu/nubar,flavor =", nu_nubar, flavor) - #print("output/input ndens =",particle_n, nnu[nu_nubar,flavor]) - #print("output/input fluxfac =",particle_fmag / particle_n, fluxfac[nu_nubar,flavor]) - #print() + fvarname = "N"+str(flavor)+str(flavor)+"_Re"+suffix + particles[:,rkey[fvarname]] = n_particle[:,nu_nubar, flavor] return particles diff --git a/Scripts/initial_conditions/st0_msw_test.py b/Scripts/initial_conditions/st0_msw_test.py index 2efafa1d..1c6767d5 100644 --- a/Scripts/initial_conditions/st0_msw_test.py +++ b/Scripts/initial_conditions/st0_msw_test.py @@ -37,10 +37,7 @@ p[rkey["pupx"]] = phat[ip,0] * energy_erg p[rkey["pupy"]] = phat[ip,1] * energy_erg p[rkey["pupz"]] = phat[ip,2] * energy_erg - p[rkey["N"] ] = ndens_per_particle - p[rkey["Nbar"]] = ndens_per_particle - p[rkey["f00_Re"]] = 1 - p[rkey["f00_Rebar"]] = 1 - + p[rkey["N00_Re"]] = 1 + p[rkey["N00_Rebar"]] = 1 -write_particles(np.array(particles), NF, "particle_input.dat") +write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file diff --git a/Scripts/initial_conditions/st1_bipolar_test.py b/Scripts/initial_conditions/st1_bipolar_test.py index 04a15e5f..4f8948c1 100644 --- a/Scripts/initial_conditions/st1_bipolar_test.py +++ b/Scripts/initial_conditions/st1_bipolar_test.py @@ -34,7 +34,6 @@ rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) nelements = len(rkey) - # generate the list of particle info particles = np.zeros((nparticles,nelements)) for ip in range(len(phat)): @@ -43,10 +42,7 @@ p[rkey["pupx"]] = phat[ip,0] * energy_erg p[rkey["pupy"]] = phat[ip,1] * energy_erg p[rkey["pupz"]] = phat[ip,2] * energy_erg - p[rkey["N"] ] = ndens_per_particle - p[rkey["Nbar"]] = ndens_per_particle - p[rkey["f00_Re"]] = 1 - p[rkey["f00_Rebar"]] = 1 + p[rkey["N00_Re"]] = ndens_per_particle + p[rkey["N00_Rebar"]] = ndens_per_particle - -write_particles(np.array(particles), NF, "particle_input.dat") +write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file diff --git a/Scripts/initial_conditions/st2_2beam_fast_flavor.py b/Scripts/initial_conditions/st2_2beam_fast_flavor.py index 740b5de1..6839e9d5 100644 --- a/Scripts/initial_conditions/st2_2beam_fast_flavor.py +++ b/Scripts/initial_conditions/st2_2beam_fast_flavor.py @@ -43,10 +43,7 @@ p[rkey["pupx"]] = u[0] * energy_erg p[rkey["pupy"]] = u[1] * energy_erg p[rkey["pupz"]] = u[2] * energy_erg - p[rkey["N"] ] = ndens_per_particle * (1. + u[2]) - p[rkey["Nbar"]] = ndens_per_particle * (1. - u[2]) - p[rkey["f00_Re"]] = 1 - p[rkey["f00_Rebar"]] = 1 - + p[rkey["N00_Re"]] = ndens_per_particle * (1. + u[2]) + p[rkey["N00_Rebar"]] = ndens_per_particle * (1. - u[2]) write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py b/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py index bbcdfab4..c2fd2da8 100644 --- a/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py +++ b/Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py @@ -35,7 +35,6 @@ rkey, ikey = amrex.get_particle_keys(NF,ignore_pos=True) nelements = len(rkey) - # generate the list of particle info particles = np.zeros((nparticles,nelements)) for ip in range(len(phat)): @@ -45,10 +44,7 @@ p[rkey["pupx"]] = u[0] * energy_erg p[rkey["pupy"]] = u[1] * energy_erg p[rkey["pupz"]] = u[2] * energy_erg - p[rkey["N"] ] = ndens_per_particle * (1. + u[2]) - p[rkey["Nbar"]] = ndens_per_particle * (1. - u[2]) - p[rkey["f00_Re"]] = 1 - p[rkey["f00_Rebar"]] = 1 - + p[rkey["N00_Re"]] = ndens_per_particle * (1. + u[2]) + p[rkey["N00_Rebar"]] = ndens_per_particle * (1. - u[2]) -write_particles(np.array(particles), NF, "particle_input.dat") +write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file diff --git a/Scripts/initial_conditions/st7_empty_particles.py b/Scripts/initial_conditions/st7_empty_particles.py new file mode 100644 index 00000000..dca380ad --- /dev/null +++ b/Scripts/initial_conditions/st7_empty_particles.py @@ -0,0 +1,22 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../data_analysis") +from initial_condition_tools import uniform_sphere, write_particles, moment_interpolate_particles, linear_interpolate +import amrex_plot_tools as amrex + +# generation parameters +# MUST MATCH THE INPUTS IN THE EMU INPUT FILE! +NF = 3 +nphi_equator = 16 +energy_erg = 50 * 1e6*amrex.eV + +nnu = np.zeros((2,NF)) +fnu = np.zeros((2,NF,3)) + +particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, linear_interpolate) # [particle, variable] + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/initial_conditions/st8_coll_inst_test.py b/Scripts/initial_conditions/st8_coll_inst_test.py new file mode 100644 index 00000000..0b925b21 --- /dev/null +++ b/Scripts/initial_conditions/st8_coll_inst_test.py @@ -0,0 +1,36 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../data_reduction") +from initial_condition_tools import uniform_sphere, moment_interpolate_particles, minerbo_interpolate, write_particles +import amrex_plot_tools as amrex + +# These initial conditions are intended to replicate the collisional instability outputs in "Collisional Flavor Instabilities of Supernova Neutrinos" by L. Johns [2104.11369]. +# Simulation parameters +NF = 2 +nphi_equator = 16 +nnue = 3.0e+33 # 1/ccm +nnua = 2.5e+33 # 1/ccm +nnux = 1.0e+33 # 1/ccm +fnue = np.array([0.0 , 0.0 , 0.0]) +fnua = np.array([0.0 , 0.0 , 0.0]) +fnux = np.array([0.0 , 0.0 , 0.0]) +energy_erg = 20.0 # MeV +energy_erg *= 1e6*amrex.eV # erg + +nnu = np.zeros((2,NF)) +nnu[0,0] = nnue +nnu[1,0] = nnua +nnu[:,1:] = nnux + +fnu = np.zeros((2,NF,3)) +fnu[0,0,:] = nnue * fnue +fnu[1,0,:] = nnua * fnua +fnu[:,1:,:] = nnu[:,1:,np.newaxis] * fnux[np.newaxis,np.newaxis,:] + +particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, minerbo_interpolate) # [particle, variable] + +write_particles(np.array(particles), NF, "particle_input.dat") diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index ebb8d6db..5207b5f7 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -94,33 +94,22 @@ def delete_generated_files(): #==================================# # FlavoredNeutrinoContainer.H_fill # #==================================# - vars = ["f"] + vars = ["N"] tails = ["","bar"] code = [] for t in tails: - code += ["N"+t] # number of neutrinos - code += ["L"+t] # length of isospin vector, units of number of neutrinos for v in vars: A = HermitianMatrix(args.N, v+"{}{}_{}"+t) code += A.header() - code += ["TrHf"] + code += ["TrHN"] + code += ["Vphase"] - code = [code[i]+"," for i in range(len(code))] - write_code(code, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.H_fill")) + code_lines = [code[i]+"," for i in range(len(code))] + write_code(code_lines, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.H_fill")) #========================================================# # FlavoredNeutrinoContainerInit.H_particle_varnames_fill # #========================================================# - vars = ["f"] - tails = ["","bar"] - code = [] - for t in tails: - code += ["N"+t] - code += ["L"+t] - for v in vars: - A = HermitianMatrix(args.N, v+"{}{}_{}"+t) - code += A.header() - code += ["TrHf"] code_string = 'attribute_names = {"time", "x", "y", "z", "pupx", "pupy", "pupz", "pupt", ' code = ['"{}"'.format(c) for c in code] code_string = code_string + ", ".join(code) + "};" @@ -178,8 +167,8 @@ def delete_generated_files(): "*p.rdata(PIdx::pupz)*p.rdata(PIdx::pupz)/p.rdata(PIdx::pupt)/p.rdata(PIdx::pupt));"]) code = [] for t in tails: - string3 = ")*p.rdata(PIdx::N"+t+")" - flist = HermitianMatrix(args.N, "f{}{}_{}"+t).header() + string3 = ")" + flist = HermitianMatrix(args.N, "N{}{}_{}"+t).header() for ivar in range(len(deposit_vars)): deplist = HermitianMatrix(args.N, deposit_vars[ivar]+"{}{}_{}"+t).header() for icomp in range(len(flist)): @@ -193,10 +182,10 @@ def delete_generated_files(): code = [] for t in tails: # diagonal averages - N = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") + N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") Nlist = N.header_diagonals(); for i in range(len(Nlist)): - code.append("Trf += "+Nlist[i]+";") + code.append("TrN += "+Nlist[i]+";") write_code(code, os.path.join(args.emu_home, "Source/generated_files", "DataReducer.cpp_fill_particles")) @@ -437,94 +426,85 @@ def sgn(t,var): line += "sqrt(2.) * PhysConst::GF * sx(i) * sy(j) * sz(k) * (inside_parentheses);" code.append(line) code.append("") + + code.append("T_pp += sx(i) * sy(j) * sz(k) * sarr(i, j, k, GIdx::T);") + code.append("Ye_pp += sx(i) * sy(j) * sz(k) * sarr(i, j, k, GIdx::Ye);") + code.append("rho_pp += sx(i) * sy(j) * sz(k) * sarr(i, j, k, GIdx::rho);") + code.append("") + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_interpolate_from_mesh_fill")) #========================# # Evolve.cpp_dfdt_fill # #========================# - # Set up Hermitian matrices A, B, C + # Generate quantum kinetic equations + + # Define useful constants hbar = sympy.symbols("PhysConst\:\:hbar",real=True) - code = [] - for t in tails: - H = HermitianMatrix(args.N, "V{}{}_{}"+t) - F = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") + attenuation_to_hamiltonian = sympy.symbols("parms->attenuation_hamiltonians", real=True) + V_phase = sympy.symbols("p.rdata(PIdx\:\:Vphase)", real=True) + pi = sympy.symbols("MathConst\:\:pi", real=True) + c = sympy.symbols("PhysConst\:\:c", real=True) - # G = Temporary variables for dFdt - G = HermitianMatrix(args.N, "dfdt{}{}_{}"+t) + # List that will store the QKE code. + code = [] - # Calculate C = i * [A,B] - #Fnew.anticommutator(H,F).times(sympy.I * dt); - G.H = ((H*F - F*H).times(-sympy.I/hbar)).H + # Looping over neutrinos(tail: no tail) and antineutrinos(tail: bar) + for t in tails: - # Write the temporary variables for dFdt - Gdeclare = ["amrex::Real {}".format(line) for line in G.code()] - code.append(Gdeclare) + # Define Fermi-Dirac distribution matrix f_eq = diag( f_e , f_x ) from input parameters + f_eq = HermitianMatrix(args.N, "f_eq_{}{}_{}"+t) # Fermi-dirac distribution matrix ----> To be used in calculation of QKE in sympy format + f_eq_cpp = HermitianMatrix(args.N, "f_eq"+t+"[{}][{}]") # Fermi-dirac distribution matrix ----> Using the systaxis of line 183 of the Evolve.cpp file + f_eq.H = f_eq_cpp.H # Assigning input mean free paths to SymPy matrix + f_eq_temp_declare = ["amrex::Real {}".format(line) for line in f_eq.code()] # + code.append(f_eq_temp_declare) + + # Define Gamma matrix from input parameters : Gamma = diag( k*_e , k*_x ) / 2 . ka is the inverse mean free path for flavor a, including Pauli blocking term. * means that Pauli blocking term is already in the inverse mean free path values. + Gamma = HermitianMatrix(args.N, "Gamma_{}{}_{}"+t) # Inverse mean free path matrix. Gamma = diag( k*e , k*x ) / 2. ----> To be used in calculation of QKE in sympy format + IMFP_abs = HermitianMatrix(args.N, "IMFP_abs"+t+"[{}][{}]") # Inverse mean free path matrix IMFP_abs = diag( k*e , k*x ) ----> Using the systaxis of line 181 of the Evolve.cpp file + Gamma.H = IMFP_abs.H / 2 # Compute Gamma + Gamma_temp_declare = ["amrex::Real {}".format(line) for line in Gamma.code()] + code.append(Gamma_temp_declare) + + # Define N_eq matrix + f_eq = HermitianMatrix(args.N, "f_eq_{}{}_{}"+t) # Fermi-dirac distribution matrix f_eq = diag( fe , fx ) + N_eq = HermitianMatrix(args.N, "N_eq_{}{}_{}"+t) # Equilibrium neutrino number matrix N_eq equals the integral of f_eq, where the integral is over the phase space that the particle represents. + N_eq.H = f_eq.H * V_phase / ( 2 * pi * hbar * c )**3 + N_eq_temp_declare = ["amrex::Real {}".format(line) for line in N_eq.code()] + code.append(N_eq_temp_declare) + + # Define collision term + Gamma = HermitianMatrix(args.N, "Gamma_{}{}_{}"+t) # Inverse mean free path matrix. Gamma = diag( k*e , k*x ) / 2. ka is the inverse mean free path for flavor a, including Pauli blocking term. + N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") # Neutrino number matrix + N_eq = HermitianMatrix(args.N, "N_eq_{}{}_{}"+t) # Equilibrium neutrino number matrix N_eq equals the integral of f_eq, where the integral is over the phase space that the particle represents. + C = HermitianMatrix(args.N, "C_{}{}_{}"+t) # Collision term C = { gamma , N_eq - N }, {} means anticonmutator + C.H = Gamma.H * ( N_eq.H - N.H ) + ( N_eq.H - N.H ) * Gamma.H # Compute collision term + C_temp_declare = ["amrex::Real {}".format(line) for line in C.code()] + code.append(C_temp_declare) + + # Writing QKE + C = HermitianMatrix(args.N, "C_{}{}_{}"+t) # Collision term C = { gamma , N_eq - N }, {} means anticonmutator + H = HermitianMatrix(args.N, "V{}{}_{}"+t) # Hamiltonian + N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") # Neutrino number matrix + dNdt_temp = HermitianMatrix(args.N, "dNdt{}{}_{}"+t) # Temporary matrix for dNdt + dNdt_temp.H = C.H * c + ((H*N - N*H).times(-sympy.I/hbar)).H * attenuation_to_hamiltonian # Compute quantum kinetic equation + dNdt_temp_declare = ["amrex::Real {}".format(line) for line in dNdt_temp.code()] + code.append(dNdt_temp_declare) # Store dFdt back into the particle data for F - dFdt = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") - Gempty = HermitianMatrix(args.N, "dfdt{}{}_{}"+t) - dFdt.H = Gempty.H + dNdt = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") + dNdt_empty = HermitianMatrix(args.N, "dNdt{}{}_{}"+t) + dNdt.H = dNdt_empty.H - # Write out dFdt->F - code.append(dFdt.code()) + # Write out dNdt->N + code.append(dNdt.code()) # store Tr(H*F) for estimating numerical errors - TrHf = (H*F).trace(); - code.append(["p.rdata(PIdx::TrHf) += p.rdata(PIdx::N"+t+") * ("+sympy.cxxcode(sympy.simplify(TrHf))+");"]) + TrHN = (H*N).trace(); + code.append(["p.rdata(PIdx::TrHN) += ("+sympy.cxxcode(sympy.simplify(TrHN))+");"]) code = [line for sublist in code for line in sublist] write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill")) - #================================================# - # FlavoredNeutrinoContainer.cpp_Renormalize_fill # - #================================================# - code = [] - for t in tails: - # make sure the trace is 1 - code.append("sumP = 0;") - f = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") - fdlist = f.header_diagonals() - flist = f.header() - for fii in fdlist: - code.append("sumP += " + fii + ";") - code.append("error = sumP-1.0;") - code.append("if( std::abs(error) > 100.*parms->maxError) amrex::Abort();") - code.append("if( std::abs(error) > parms->maxError ) {") - for fii in fdlist: - code.append(fii + " -= error/"+str(args.N)+";") - code.append("}") - code.append("") - - # make sure diagonals are positive - for fii in fdlist: - code.append("if("+fii+"<-100.*parms->maxError) amrex::Abort();") - code.append("if("+fii+"<-parms->maxError) "+fii+"=0;") - code.append("") - - # make sure the flavor vector length is what it would be with a 1 in only one diagonal - length = sympy.symbols("length",real=True) - length = f.SU_vector_magnitude() - target_length = "p.rdata(PIdx::L"+t+")" - code.append("length = "+sympy.cxxcode(sympy.simplify(length))+";") - code.append("error = length-"+str(target_length)+";") - code.append("if( std::abs(error) > 100.*parms->maxError) amrex::Abort();") - code.append("if( std::abs(error) > parms->maxError) {") - for fii in flist: - code.append(fii+" /= length/"+str(target_length)+";") - code.append("}") - code.append("") - - write_code(code, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.cpp_Renormalize_fill")) - # Write code to output file, using a template if one is provided - # write_code(code, "code.cpp", args.output_template) - - - #====================================================# - # FlavoredNeutrinoContainerInit.cpp_set_trace_length # - #====================================================# - code = [] - for t in tails: - f = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") - code.append("p.rdata(PIdx::L"+t+") = "+sympy.cxxcode(sympy.simplify(f.SU_vector_magnitude()))+";" ) - write_code(code, os.path.join(args.emu_home, "Source/generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length")) + diff --git a/Scripts/tests/coll_equi_test.py b/Scripts/tests/coll_equi_test.py new file mode 100644 index 00000000..b2e3145d --- /dev/null +++ b/Scripts/tests/coll_equi_test.py @@ -0,0 +1,61 @@ +''' +This script tests the equilibrium value of the collision term. +Created by Erick Urquilla, University of Tennessee, Knoxville. +''' +import numpy as np +import argparse +import glob +import EmuReader +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__))+"/../data_reduction/" +sys.path.append(importpath) +import amrex_plot_tools as amrex + +parser = argparse.ArgumentParser() +parser.add_argument("-na", "--no_assert", action="store_true", help="If --no_assert is supplied, do not raise assertion errors if the test error > tolerance.") +args = parser.parse_args() + +NF=3 + +if __name__ == "__main__": + + rkey, ikey = amrex.get_particle_keys(NF) + + N_ee = [] + N_uu = [] + N_tt = [] + N_eebar = [] + N_uubar = [] + N_ttbar = [] + + idata, rdata = EmuReader.read_particle_data('plt01000', ptype="neutrinos") + + for i in range(len(rdata)): + p = rdata[i] + N_ee.append(p[rkey["N00_Re"]]) + N_uu.append(p[rkey["N11_Re"]]) + N_tt.append(p[rkey["N22_Re"]]) + N_eebar.append(p[rkey["N00_Rebar"]]) + N_uubar.append(p[rkey["N11_Rebar"]]) + N_ttbar.append(p[rkey["N22_Rebar"]]) + + print(f'The expected theoretical value for the following quantities is 1e33.') + print(f'The numerical values obtained using EMU are:') + print(f'average N_ee {np.average(N_ee)}') + print(f'average N_uu {np.average(N_uu)}') + print(f'average N_tt {np.average(N_tt)}') + print(f'average N_eebar {np.average(N_eebar)}') + print(f'average N_uubar {np.average(N_uubar)}') + print(f'average N_ttbar {np.average(N_ttbar)}') + + def myassert(condition): + if not args.no_assert: + assert(condition) + + myassert( np.all(np.isclose(N_ee, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_uu, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_tt, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_eebar, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_uubar, np.array(1e33), atol=1e33/100)) ) + myassert( np.all(np.isclose(N_ttbar, np.array(1e33), atol=1e33/100)) ) \ No newline at end of file diff --git a/Scripts/tests/coll_inst_test.py b/Scripts/tests/coll_inst_test.py new file mode 100644 index 00000000..cc3ba35a --- /dev/null +++ b/Scripts/tests/coll_inst_test.py @@ -0,0 +1,276 @@ +''' +This test script is used to reproduce the isotropic 2-flavor simulation in "Collisional Flavor Instabilities of Supernova Neutrinos" by L. Johns [2104.11369]. +The points of comparison are the LSA conducted in this paper (equation 14) and Julien's script that reproduces the same results (script received via private communication). +Created by Erick Urquilla. University of Tennessee Knoxville, USA. +''' + +import numpy as np +import argparse +import glob +import EmuReader +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__))+"/../data_reduction/" +sys.path.append(importpath) +import amrex_plot_tools as amrex +import numpy as np +import h5py +import glob + +parser = argparse.ArgumentParser() +parser.add_argument("-na", "--no_assert", action="store_true", help="If --no_assert is supplied, do not raise assertion errors if the test error > tolerance.") +args = parser.parse_args() + +if __name__ == "__main__": + + # Create a list of data files to read + directories = glob.glob("plt*_reduced_data.h5") + # Sort the data file names by time step number + directories = sorted(directories, key=lambda x: int(x.split("plt")[1].split("_")[0])) + + N_avg_mag = np.zeros((len(directories),2,2)) + Nbar_avg_mag = np.zeros((len(directories),2,2)) + F_avg_mag = np.zeros((len(directories),3,2,2)) + Fbar_avg_mag = np.zeros((len(directories),3,2,2)) + t = np.zeros(len(directories)) + + for i in range(len(directories)): + with h5py.File(directories[i], 'r') as hf: + N_avg_mag[i] = np.array(hf['N_avg_mag(1|ccm)'][:][0]) + Nbar_avg_mag[i] = np.array(hf['Nbar_avg_mag(1|ccm)'][:][0]) + F_avg_mag[i] = np.array(hf['F_avg_mag(1|ccm)'][:][0]) + Fbar_avg_mag[i] = np.array(hf['Fbar_avg_mag(1|ccm)'][:][0]) + t[i] = np.array(hf['t(s)'][:][0]) + + # Fit the exponential function ( y = a e ^ ( b x ) ) to the data + l1 = 10 # initial item for fit + l2 = 40 # last item for fit + coefficients = np.polyfit(t[l1:l2], np.log(N_avg_mag[:,0,1][l1:l2]), 1) + coefficients_bar = np.polyfit(t[l1:l2], np.log(Nbar_avg_mag[:,0,1][l1:l2]), 1) + a = np.exp(coefficients[1]) + b = coefficients[0] + abar = np.exp(coefficients_bar[1]) + bbar = coefficients_bar[0] + print(f'{b} ---> EMU : Im Omega') + print(f'{bbar} ---> EMU : Im Omegabar') + + # Plots + import matplotlib.pyplot as plt + + # Plots N and Nbar + plt.plot(t, N_avg_mag[:,0,0], label = r'$N_{ee}$') + plt.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$') + plt.plot(t, N_avg_mag[:,1,1], label = r'$N_{uu}$') + plt.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{N}_{ee}$') + plt.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') + plt.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') + plt.legend() + plt.xlabel(r'$t$ (s)') + plt.ylabel(r'$N$ and $\bar{N}$') + plt.savefig('N_and_Nbar.pdf') + plt.clf() + + # Plots N and F + plt.plot(t, N_avg_mag[:,0,0], label = r'$N_{ee}$') + plt.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$') + plt.plot(t, N_avg_mag[:,1,1], label = r'$N_{uu}$') + plt.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {b}') + plt.plot(t, F_avg_mag[:,0,0,1], label = r'$F^x_{eu}$') + plt.plot(t, F_avg_mag[:,1,0,1], label = r'$F^y_{eu}$') + plt.plot(t, F_avg_mag[:,2,0,1], label = r'$F^z_{eu}$') + plt.legend() + plt.xlabel(r'$t$ (s)') + plt.ylabel(r'$N$ and $\vec{F}$') + plt.yscale('log') + plt.savefig('N_and_F.pdf') + plt.clf() + + # Plots Nbar and Fbar + plt.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{N}_{ee}$') + plt.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') + plt.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') + plt.plot(t[l1:l2], Nbar_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {bbar}') + plt.plot(t, Fbar_avg_mag[:,0,0,1], label = r'$\bar{F}^x_{eu}$') + plt.plot(t, Fbar_avg_mag[:,1,0,1], label = r'$\bar{F}^y_{eu}$') + plt.plot(t, Fbar_avg_mag[:,2,0,1], label = r'$\bar{F}^z_{eu}$') + plt.legend() + plt.xlabel(r'$t$ (s)') + plt.ylabel(r'$\bar{N}$ and $\vec{\bar{F}}$') + plt.yscale('log') + plt.savefig('Nbar_and_Fbar.pdf') + plt.clf() + + ###################################################################################### + ###################################################################################### + # LSA in "Collisional flavor instabilities of supernova neutrinos", L. Johns [2104.11369] + + h = 6.6260755e-27 # erg s + hbar = h/(2.*np.pi) # erg s + c = 2.99792458e10 # cm/s + MeV = 1.60218e-6 # erg + eV = MeV/1e6 # erg + GF_GeV2 = 1.1663787e-5 # GeV^-2 + GF = GF_GeV2 / (1000*MeV)**2 * (hbar*c)**3 # erg cm^3 + + Nee = 3e33 # cm^-3 + Neebar = 2.5e33 # cm^-3 + Nxx = 1e33 # cm^-3 + + opac_rescale = 1e4 + + kappa_e = 1/(0.417*1e5)*opac_rescale # cm^-1 + kappa_ebar = 1/(4.36*1e5)*opac_rescale # cm^-1 + kappa_x = 0.*opac_rescale # cm^-1 + + # Collision rates (in s^-1) + Gamma_plus = (kappa_e+kappa_x)/2 * c + Gamma_minus = (kappa_e-kappa_x)/2 * c + Gammabar_plus = (kappa_ebar+kappa_x)/2 * c + Gammabar_minus= (kappa_ebar - kappa_x)/2 * c + + omega = 0.304*1e-5 * c # Delta m^2/2E, in s^-1 + mu = np.sqrt(2)*GF/hbar # s^-1.cm^3 + + S = Nee - Nxx + Neebar - Nxx + D = Nee - Nxx - Neebar + Nxx + + ImOmega_Lucas_LSA = ( ( Gamma_plus - Gammabar_plus ) / 2 ) * ( mu * S / np.sqrt( ( mu * D )**2 + 4 * omega * mu * S ) ) - ( Gamma_plus + Gammabar_plus ) / 2 + + print(f'{ImOmega_Lucas_LSA} ---> Im Omega and Omegabar : LSA in equation 14 of L. Johns [2104.11369]') + + ###################################################################################### + ###################################################################################### + + def myassert(condition): + if not args.no_assert: + assert(condition) + + b_lsa = ImOmega_Lucas_LSA + rel_error = np.abs( b - b_lsa ) / np.abs( ( b + b_lsa ) / 2 ) + rel_error_bar = np.abs( bbar - b_lsa ) / np.abs( ( bbar + b_lsa ) / 2 ) + rel_error_max = 0.05 + + print(f"{rel_error} ---> relative error in ImOmega : EMU") + print(f"{rel_error_bar} ---> relative error in ImOmegabar : EMU") + + myassert( rel_error < rel_error_max ) + myassert( rel_error_bar < rel_error_max ) + + ###################################################################################### + ###################################################################################### + + """ + Created on Wed Jun 5 13:11:50 2024 + Solves the isotropic QKE following "Collisional flavor instabilities of supernova neutrinos", L. Johns [2104.11369] + + @author: jfroustey + """ + + import numpy as np + import matplotlib.pyplot as plt + from scipy.integrate import solve_ivp + + h = 6.6260755e-27 # erg s + hbar = h/(2.*np.pi) # erg s + c = 2.99792458e10 # cm/s + MeV = 1.60218e-6 # erg + eV = MeV/1e6 # erg + GF_GeV2 = 1.1663787e-5 # GeV^-2 + GF = GF_GeV2 / (1000*MeV)**2 * (hbar*c)**3 # erg cm^3 + + Nee = 3e33 # cm^-3 + Neebar = 2.5e33 # cm^-3 + Nxx = 1e33 # cm^-3 + + opac_rescale = 1e4 + + kappa_e = 1/(0.417*1e5)*opac_rescale # cm^-1 + kappa_ebar = 1/(4.36*1e5)*opac_rescale # cm^-1 + kappa_x = 0.*opac_rescale # cm^-1 + + # Collision rates (in s^-1) + Gamma_plus = (kappa_e+kappa_x)/2 * c + Gamma_minus = (kappa_e-kappa_x)/2 * c + Gammabar_plus = (kappa_ebar+kappa_x)/2 * c + Gammabar_minus= (kappa_ebar - kappa_x)/2 * c + + # Vacuum term + + theta = 1e-6 + c2t = np.cos(2*theta) + s2t = np.sin(2*theta) + omega = 0.304*1e-5 * c # Delta m^2/2E, in s^-1 + + P0_AE = (Nee+Nxx)/2 + Pz_AE = (Nee-Nxx)/2 + Pbar0_AE = (Neebar+Nxx)/2 + Pbarz_AE = (Neebar-Nxx)/2 + + mu = np.sqrt(2)*GF/hbar # s^-1.cm^3 + + def QKE(t,y): + P0, Px, Py, Pz, Pbar0, Pbarx, Pbary, Pbarz = y + deriv = np.zeros(8) + + # Variation of P0, Pbar0 + deriv[0] = Gamma_plus*(P0_AE-P0) + Gamma_minus*(Pz_AE-Pz) + deriv[4] = Gammabar_plus*(Pbar0_AE-Pbar0) + Gammabar_minus*(Pbarz_AE - Pbarz) + + # Spatial parts + deriv[1] = omega*c2t*Py + mu*((Py-Pbary)*Pz - (Pz-Pbarz)*Py) - Gamma_plus*Px + deriv[2] = omega*(-s2t*Pz-c2t*Px) + mu*((Pz-Pbarz)*Px - (Px-Pbarx)*Pz) - Gamma_plus*Py + deriv[3] = omega*s2t*Py + mu*((Px-Pbarx)*Py - (Py-Pbary)*Px) + Gamma_plus*(Pz_AE-Pz) + Gamma_minus*(P0_AE-P0) + + deriv[5] = -omega*c2t*Pbary + mu*((Py-Pbary)*Pbarz - (Pz-Pbarz)*Pbary) - Gammabar_plus*Pbarx + deriv[6] = -omega*(-s2t*Pbarz - c2t*Pbarx) + mu*((Pz-Pbarz)*Pbarx - (Px-Pbarx)*Pbarz) - Gammabar_plus*Pbary + deriv[7] = -omega*s2t*Pbary + mu*((Px-Pbarx)*Pbary - (Py-Pbary)*Pbarx) + Gammabar_plus*(Pbarz_AE-Pbarz) + Gammabar_minus*(Pbar0_AE-Pbar0) + + return deriv + + time = np.linspace(0,90e-6/opac_rescale,2000) + y0 = np.array([P0_AE, 0., 0., Pz_AE, Pbar0_AE, 0., 0., Pbarz_AE]) + + myrtol, myatol = 1e-5, 1e-8 + sol = solve_ivp(QKE, (time[0],time[-1]), y0, t_eval=time, rtol=myrtol, atol=myatol) + + # PLOTS + plt.plot(time, sol.y[0,:]+sol.y[3,:], color='k', lw=2, label=r'$n_{\nu_e}$') + plt.plot(time, sol.y[4,:]+sol.y[7,:], color='k', lw=1.5, label=r'$n_{\bar{\nu}_e}$') + plt.plot(time, sol.y[0,:]-sol.y[3,:], color='k', lw=1, label=r'$n_{\nu_x}$') + + plt.plot(time,np.sqrt(sol.y[1,:]**2+sol.y[2,:]**2), lw=2, color='teal',label=r'$\nu_e - \nu_x$'+" coherence density") + plt.legend() + plt.grid(ls=':',color='C7') + plt.xlabel(r'$t \ (\mathrm{s})$') + plt.xlim(time[0],time[-1]) + plt.title(f"Opacities scaled by {opac_rescale:.1e}, with rtol={myrtol:.1e}, atol={myatol:.1e}") + plt.tight_layout() + plt.savefig(f"Johns_CFI_rescale_{opac_rescale:.0e}_rtol_{myrtol:.0e}_atol_{myatol:.0e}.pdf") + plt.close() + + ###################################################################################### + ###################################################################################### + + p1 = 150 # initial point for fit + p2 = 500 # final point for fit + N_eu_julien = np.sqrt(sol.y[1,:]**2+sol.y[2,:]**2) + coefficients = np.polyfit(time[p1:p2], np.log(N_eu_julien[p1:p2]), 1) + aj = np.exp(coefficients[1]) + bj = coefficients[0] + print(f'{bj} ---> Im Omega Julien') + rel_error_j = np.abs( bj - b_lsa ) / np.abs( ( bj + b_lsa ) / 2 ) + print(f"{rel_error_j} ---> relative erroror in Julien script") + + # Plotting Julien, EMU and Lucas LSA data + plt.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$ EMU') + plt.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega EMU = {b}', linestyle = 'dashed') + plt.plot(time,N_eu_julien, label = r'$N_{eu}$ Julien script') + plt.plot(time[p1:p2],N_eu_julien[p1:p2], label = f'Im Omega Julien = {bj}', linestyle = 'dashed') + plt.plot(time[p1:p2], 1e23*np.exp(ImOmega_Lucas_LSA*time[p1:p2]), label = f'Im Omega Lucas LSA = {ImOmega_Lucas_LSA}', linestyle = 'dashed') + plt.xlabel(r'$t \ (\mathrm{s})$') + plt.ylabel(r'$N_{eu}$') + plt.title(f"Collisional flavor instability test") + plt.yscale('log') + plt.legend() + plt.savefig('EMU_Julien_LucasLSA_Neu.pdf') + plt.close() \ No newline at end of file diff --git a/Scripts/tests/convergence.sh b/Scripts/tests/convergence.sh deleted file mode 100644 index 3816a379..00000000 --- a/Scripts/tests/convergence.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -# echo the commands -set -x - -# All runs will use these -DIM=3 -EXEC=./main${DIM}d.gnu.TPROF.MPI.ex -MPINUM=4 - -RUNPARAMS=" -cfl_factor=-1 -nsteps=1000000 -end_time=5.0e-11" - -# Each integrator will set these -INTPARAMS="" -INTNAME="" - -# Clean up any existing output files before we start -rm -rf plt* -rm -rf single_neutrino*.png -rm -rf msw_test_*.txt - -# Define a function for a single run -do_single () { - mpiexec -n ${MPINUM} ${EXEC} inputs_msw_test ${RUNPARAMS} flavor_cfl_factor=${FCFL} ${INTPARAMS} - echo "cfl: ${FCFL}" >> msw_test_${INTNAME}.txt - python3 msw_test.py -na >> msw_test_${INTNAME}.txt - python3 plot_first_particle.py - mv single_neutrino.png single_neutrino_fcfl_${FCFL}_${INTNAME}.png - rm -rf plt* -} - -# Define a function for running convergence -do_convergence () { - FCFL=0.1 - do_single - - FCFL=0.05 - do_single - - FCFL=0.025 - do_single - - FCFL=0.0125 - do_single - - FCFL=0.00625 - do_single - - FCFL=0.003125 - do_single -} - -# Forward Euler convergence -INTPARAMS=" -integration.type=0" - -INTNAME="fe" - -do_convergence - -# Trapezoid convergence -INTPARAMS=" -integration.type=1 -integration.rk.type=2" - -INTNAME="trapz" - -do_convergence - -# SSPRK3 Convergence -INTPARAMS=" -integration.type=1 -integration.rk.type=3" - -INTNAME="ssprk3" - -do_convergence - -# RK4 Convergence -INTPARAMS=" -integration.type=1 -integration.rk.type=4" - -INTNAME="rk4" - -do_convergence diff --git a/Scripts/tests/fast_flavor_k_test.py b/Scripts/tests/fast_flavor_k_test.py index 5703e49c..8b661152 100644 --- a/Scripts/tests/fast_flavor_k_test.py +++ b/Scripts/tests/fast_flavor_k_test.py @@ -30,10 +30,10 @@ rkey, ikey = amrex.get_particle_keys(NF) t = [] - fexR = [] - fexI = [] - fexRbar = [] - fexIbar = [] + NexR = [] + NexI = [] + NexRbar = [] + NexIbar = [] pupt = [] nfiles = len(glob.glob("plt[0-9][0-9][0-9][0-9][0-9]")) @@ -43,18 +43,18 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata t.append(p[0][rkey["time"]]) - fexR.append(np.max(np.abs(p[:,rkey["f01_Re"]]))) - fexI.append(np.max(np.abs(p[:,rkey["f01_Im"]]))) - fexRbar.append(np.max(np.abs(p[:,rkey["f01_Rebar"]]))) - fexIbar.append(np.max(np.abs(p[:,rkey["f01_Imbar"]]))) + NexR.append(np.max(np.abs(p[:,rkey["N01_Re"]]))) + NexI.append(np.max(np.abs(p[:,rkey["N01_Im"]]))) + NexRbar.append(np.max(np.abs(p[:,rkey["N01_Rebar"]]))) + NexIbar.append(np.max(np.abs(p[:,rkey["N01_Imbar"]]))) pupt.append(p[0][rkey["pupt"]]) t = np.array(t) - fexR = np.array(fexR) - fexI = np.array(fexI) - fexRbar = np.array(fexRbar) - fexIbar = np.array(fexIbar) - print(fexR) + NexR = np.array(NexR) + NexI = np.array(NexI) + NexRbar = np.array(NexRbar) + NexIbar = np.array(NexIbar) + print(NexR) # The neutrino energy we set E = 50. * 1e6*amrex.eV @@ -70,27 +70,27 @@ # get growth rate from each diagonal component dt = t[i1]-t[i0] - fexRomega = np.log(np.abs(fexR[i1]/fexR[i0])) / dt - fexIomega = np.log(np.abs(fexI[i1]/fexI[i0])) / dt - fexRbaromega = np.log(np.abs(fexRbar[i1]/fexRbar[i0])) / dt - fexIbaromega = np.log(np.abs(fexIbar[i1]/fexIbar[i0])) / dt + NexRomega = np.log(np.abs(NexR[i1]/NexR[i0])) / dt + NexIomega = np.log(np.abs(NexI[i1]/NexI[i0])) / dt + NexRbaromega = np.log(np.abs(NexRbar[i1]/NexRbar[i0])) / dt + NexIbaromega = np.log(np.abs(NexIbar[i1]/NexIbar[i0])) / dt - print("growth rates:",fexRomega,fexIomega,fexRbaromega,fexIbaromega) - print("growth rates / theoretical:",fexRomega/ImOmega,fexIomega/ImOmega,fexRbaromega/ImOmega,fexIbaromega/ImOmega) + print("growth rates:",NexRomega,NexIomega,NexRbaromega,NexIbaromega) + print("growth rates / theoretical:",NexRomega/ImOmega,NexIomega/ImOmega,NexRbaromega/ImOmega,NexIbaromega/ImOmega) def myassert(condition): if not args.no_assert: assert(condition) - fexRerror = np.abs(ImOmega - fexRomega) / ImOmega - myassert( fexRerror < tolerance ) + NexRerror = np.abs(ImOmega - NexRomega) / ImOmega + myassert( NexRerror < tolerance ) - fexIerror = np.abs(ImOmega - fexIomega) / ImOmega - myassert( fexIerror < tolerance ) + NexIerror = np.abs(ImOmega - NexIomega) / ImOmega + myassert( NexIerror < tolerance ) - fexRbarerror = np.abs(ImOmega - fexRbaromega) / ImOmega - myassert( fexRbarerror < tolerance ) + NexRbarerror = np.abs(ImOmega - NexRbaromega) / ImOmega + myassert( NexRbarerror < tolerance ) - fexIbarerror = np.abs(ImOmega - fexIbaromega) / ImOmega - myassert( fexIbarerror < tolerance ) + NexIbarerror = np.abs(ImOmega - NexIbaromega) / ImOmega + myassert( NexIbarerror < tolerance ) diff --git a/Scripts/tests/fast_flavor_test.py b/Scripts/tests/fast_flavor_test.py index ff9d7647..cd50fd75 100644 --- a/Scripts/tests/fast_flavor_test.py +++ b/Scripts/tests/fast_flavor_test.py @@ -26,10 +26,10 @@ rkey, ikey = amrex.get_particle_keys(NF) t = [] - fexR = [] - fexI = [] - fexRbar = [] - fexIbar = [] + NexR = [] + NexI = [] + NexRbar = [] + NexIbar = [] pupt = [] nfiles = len(glob.glob("plt[0-9][0-9][0-9][0-9][0-9]")) @@ -39,17 +39,18 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata[0] t.append(p[rkey["time"]]) - fexR.append(p[rkey["f01_Re"]]) - fexI.append(p[rkey["f01_Im"]]) - fexRbar.append(p[rkey["f01_Rebar"]]) - fexIbar.append(p[rkey["f01_Imbar"]]) + NexR.append(p[rkey["N01_Re"]]) + NexI.append(p[rkey["N01_Im"]]) + p = rdata[1] + NexRbar.append(p[rkey["N01_Rebar"]]) + NexIbar.append(p[rkey["N01_Imbar"]]) pupt.append(p[rkey["pupt"]]) t = np.array(t) - fexR = np.array(fexR) - fexI = np.array(fexI) - fexRbar = np.array(fexRbar) - fexIbar = np.array(fexIbar) + NexR = np.array(NexR) + NexI = np.array(NexI) + NexRbar = np.array(NexRbar) + NexIbar = np.array(NexIbar) # The neutrino energy we set E = 50. * 1e6*amrex.eV @@ -61,32 +62,32 @@ # get growth rate from each diagonal component dt = t[i1]-t[i0] - fexRomega = np.log(fexR[i1]/fexR[i0]) / dt - fexIomega = np.log(fexI[i1]/fexI[i0]) / dt - fexRbaromega = np.log(fexRbar[i1]/fexRbar[i0]) / dt - fexIbaromega = np.log(fexIbar[i1]/fexIbar[i0]) / dt + NexRomega = np.log(NexR[i1]/NexR[i0]) / dt + NexIomega = np.log(NexI[i1]/NexI[i0]) / dt + NexRbaromega = np.log(NexRbar[i1]/NexRbar[i0]) / dt + NexIbaromega = np.log(NexIbar[i1]/NexIbar[i0]) / dt def myassert(condition): if not args.no_assert: assert(condition) - print("growth rates:",fexRomega,fexIomega,fexRbaromega,fexIbaromega) + print("growth rates:",NexRomega,NexIomega,NexRbaromega,NexIbaromega) print(dt,t[i0],t[i1]) - print(fexR[i1],fexR[i0]) - print(fexI[i1],fexI[i0]) - print(fexRbar[i1],fexRbar[i0]) - print(fexIbar[i1],fexIbar[i0]) + print(NexR[i1],NexR[i0]) + print(NexI[i1],NexI[i0]) + print(NexRbar[i1],NexRbar[i0]) + print(NexIbar[i1],NexIbar[i0]) - fexRerror = np.abs(ImOmega - fexRomega) / ImOmega - myassert( fexRerror < tolerance ) + NexRerror = np.abs(ImOmega - NexRomega) / ImOmega + myassert( NexRerror < tolerance ) - fexIerror = np.abs(ImOmega - fexIomega) / ImOmega - myassert( fexIerror < tolerance ) + NexIerror = np.abs(ImOmega - NexIomega) / ImOmega + myassert( NexIerror < tolerance ) - fexRbarerror = np.abs(ImOmega - fexRbaromega) / ImOmega - myassert( fexRbarerror < tolerance ) + NexRbarerror = np.abs(ImOmega - NexRbaromega) / ImOmega + myassert( NexRbarerror < tolerance ) - fexIbarerror = np.abs(ImOmega - fexIbaromega) / ImOmega - myassert( fexIbarerror < tolerance ) + NexIbarerror = np.abs(ImOmega - NexIbaromega) / ImOmega + myassert( NexIbarerror < tolerance ) diff --git a/Scripts/tests/msw_test.py b/Scripts/tests/msw_test.py index 0c4f24ac..aab770cb 100644 --- a/Scripts/tests/msw_test.py +++ b/Scripts/tests/msw_test.py @@ -35,10 +35,10 @@ rkey, ikey = amrex.get_particle_keys(NF) t = [] - fee = [] - fxx = [] - feebar = [] - fxxbar = [] + Nee = [] + Nxx = [] + Neebar = [] + Nxxbar = [] pupt = [] nfiles = len(glob.glob("plt[0-9][0-9][0-9][0-9][0-9]")) @@ -48,17 +48,17 @@ idata, rdata = EmuReader.read_particle_data(plotfile, ptype="neutrinos") p = rdata[0] t.append(p[rkey["time"]]) - fee.append(p[rkey["f00_Re"]]) - fxx.append(p[rkey["f11_Re"]]) - feebar.append(p[rkey["f00_Rebar"]]) - fxxbar.append(p[rkey["f11_Rebar"]]) + Nee.append(p[rkey["N00_Re"]]) + Nxx.append(p[rkey["N11_Re"]]) + Neebar.append(p[rkey["N00_Rebar"]]) + Nxxbar.append(p[rkey["N11_Rebar"]]) pupt.append(p[rkey["pupt"]]) t = np.array(t) - fee = np.array(fee) - fxx = np.array(fxx) - feebar = np.array(feebar) - fxxbar = np.array(fxxbar) + Nee = np.array(Nee) + Nxx = np.array(Nxx) + Neebar = np.array(Neebar) + Nxxbar = np.array(Nxxbar) # The neutrino energy we set #E = dm21c4 * np.sin(2.*theta12) / (8.*np.pi*hbar*clight) @@ -83,31 +83,31 @@ def myassert(condition): assert(condition) # calculate errors - fee_analytic = Psurv(dm2_eff, sin2_eff, E) - error_ee = np.max(np.abs( fee - fee_analytic ) ) + Nee_analytic = Psurv(dm2_eff, sin2_eff, E) + error_ee = np.max(np.abs( Nee - Nee_analytic ) ) print("f_ee error:", error_ee) myassert( error_ee < tolerance ) - fxx_analytic = 1. - Psurv(dm2_eff, sin2_eff, E) - error_xx = np.max(np.abs( fxx - fxx_analytic ) ) + Nxx_analytic = 1. - Psurv(dm2_eff, sin2_eff, E) + error_xx = np.max(np.abs( Nxx - Nxx_analytic ) ) print("f_xx error:", error_xx) myassert( error_xx < tolerance ) - feebar_analytic = Psurv(dm2_effbar, sin2_effbar, E) - error_eebar = np.max(np.abs( feebar - feebar_analytic ) ) + Neebar_analytic = Psurv(dm2_effbar, sin2_effbar, E) + error_eebar = np.max(np.abs( Neebar - Neebar_analytic ) ) print("f_eebar error:", error_eebar) myassert( error_eebar < tolerance ) - fxxbar_analytic = 1. - Psurv(dm2_effbar, sin2_effbar, E) - error_xxbar = np.max(np.abs( fxxbar - fxxbar_analytic ) ) + Nxxbar_analytic = 1. - Psurv(dm2_effbar, sin2_effbar, E) + error_xxbar = np.max(np.abs( Nxxbar - Nxxbar_analytic ) ) print("f_xxbar error:", error_xxbar) myassert( error_xxbar < tolerance ) - conservation_error = np.max(np.abs( (fee+fxx) -1. )) + conservation_error = np.max(np.abs( (Nee+Nxx) -1. )) print("conservation_error:", conservation_error) myassert(conservation_error < tolerance) - conservation_errorbar = np.max(np.abs( (feebar+fxxbar) -1. )) + conservation_errorbar = np.max(np.abs( (Neebar+Nxxbar) -1. )) print("conservation_errorbar:", conservation_errorbar) myassert(conservation_errorbar < tolerance) diff --git a/Scripts/tests/plot_convergence.py b/Scripts/tests/plot_convergence.py deleted file mode 100644 index 4be2b8c3..00000000 --- a/Scripts/tests/plot_convergence.py +++ /dev/null @@ -1,98 +0,0 @@ -import numpy as np -import matplotlib.pyplot as plt - -# Read forward Euler data -class ConvergenceData(object): - def __init__(self, filename=None): - self.data = {"cfl": [], - "f_ee error": [], - "f_xx error": [], - "f_eebar error": [], - "f_xxbar error": []} - - if filename: - self.readfrom(filename) - - def readfrom(self, filename): - f = open(filename, "r") - - while True: - entry = [f.readline().strip() for i in range(9)] - if not entry[0]: - break - for line in entry: - ls = line.split(":") - name = ls[0].strip() - value = ls[-1].strip() - for k in self.data.keys(): - if name == k: - self.data[k].append(float(value)) - - f.close() - - for k in self.data.keys(): - self.data[k] = np.array(self.data[k]) - - def get(self, key): - return self.data[key] - - def keys(self): - return self.data.keys() - - def error_keys(self): - return [k for k in self.data.keys() if k != "cfl"] - - def average_convergence(self, key): - # get the average convergence order for the keyed quantity - err = self.get(key) - cfl = self.get("cfl") - - orders = [] - for i in range(len(err)-1): - order = np.log10(err[i+1]/err[i]) / np.log10(cfl[i+1]/cfl[i]) - orders.append(order) - orders = np.array(orders) - - order_average = np.average(orders) - return order_average - - def plot_on_axis(self, axis, key, label, color): - log_cfl = np.log10(self.get("cfl")) - log_err = np.log10(self.get(key)) - axis.plot(log_cfl, log_err, label=label, marker="o", linestyle="None", color=color) - - order = self.average_convergence(key) - iMaxErr = np.argmax(log_err) - intercept = log_err[iMaxErr] - order * log_cfl[iMaxErr] - log_order_err = intercept + order * log_cfl - axis.plot(log_cfl, log_order_err, label="$O({}) = {:0.2f}$".format(label, order), marker="None", linestyle="--", color=color) - - -cdata = {} -cdata["fe"] = ConvergenceData("msw_test_fe.txt") -cdata["trapz"] = ConvergenceData("msw_test_trapz.txt") -cdata["ssprk3"] = ConvergenceData("msw_test_ssprk3.txt") -cdata["rk4"] = ConvergenceData("msw_test_rk4.txt") - -variables = cdata["fe"].error_keys() - -for v in variables: - fig, ax = plt.subplots() - - ax.set_xlabel("log10 flavor CFL") - ax.set_ylabel("log10 {}".format(v)) - - colors = ["red", "blue", "green", "magenta"] - - for k, c in zip(cdata.keys(), colors): - cd = cdata[k] - cd.plot_on_axis(ax, v, k, c) - - ax.invert_xaxis() - - ax.legend(loc=(1.05, 0.0)) - fig.tight_layout() - - plt.savefig("convergence_{}.eps".format(v.replace(" ","_"))) - plt.savefig("convergence_{}.png".format(v.replace(" ","_")), dpi=300) - plt.clf() diff --git a/Source/Constants.H b/Source/Constants.H index 1e9eeab6..21883969 100644 --- a/Source/Constants.H +++ b/Source/Constants.H @@ -18,6 +18,7 @@ namespace PhysConst static constexpr amrex::Real GF = 1.1663787e-5/*GeV^-2*//(1e9*1e9*CGSUnitsConst::eV*CGSUnitsConst::eV) * hbarc*hbarc*hbarc; //erg cm^3 static constexpr amrex::Real Mp = 1.6726219e-24; // g static constexpr amrex::Real sin2thetaW = 0.23122; + static constexpr amrex::Real kB = 1.380658e-16; // erg/K } namespace MathConst diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index d339beb6..4c3c15ab 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -64,8 +64,8 @@ void DataReducer::InitializeFiles() #endif } file0D.createDataSet("N_offdiag_mag(1|ccm)", dataspace, create_datatype(), props); - file0D.createDataSet("sumTrf", dataspace, create_datatype(), props); - file0D.createDataSet("sumTrHf", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrN", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrHN", dataspace, create_datatype(), props); #else @@ -104,9 +104,9 @@ void DataReducer::InitializeFiles() j++; outfile << j << ":N_offdiag_mag(1|ccm)\t"; j++; - outfile << j << ":sumTrf\t"; + outfile << j << ":sumTrN\t"; j++; - outfile << j << ":sumTrHf\t"; + outfile << j << ":sumTrHN\t"; outfile << std::endl; outfile.close(); @@ -130,15 +130,15 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, amrex::ReduceOps reduce_ops; auto particleResult = amrex::ParticleReduce< ReduceData >(neutrinos, [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { - Real TrHf = p.rdata(PIdx::TrHf); - Real Trf = 0; + Real TrHN = p.rdata(PIdx::TrHN); + Real TrN = 0; #include "generated_files/DataReducer.cpp_fill_particles" - return GpuTuple{Trf,TrHf}; + return GpuTuple{TrN,TrHN}; }, reduce_ops); - Real Trf = amrex::get<0>(particleResult); - Real TrHf = amrex::get<1>(particleResult); - ParallelDescriptor::ReduceRealSum(Trf); - ParallelDescriptor::ReduceRealSum(TrHf); + Real TrN = amrex::get<0>(particleResult); + Real TrHN = amrex::get<1>(particleResult); + ParallelDescriptor::ReduceRealSum(TrN); + ParallelDescriptor::ReduceRealSum(TrHN); //=============================// // Do reductions over the grid // @@ -290,8 +290,8 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, #endif } append_0D(file0D, "N_offdiag_mag(1|ccm)", N_offdiag_mag); - append_0D(file0D, "sumTrf", Trf); - append_0D(file0D, "sumTrHf", TrHf); + append_0D(file0D, "sumTrN", TrN); + append_0D(file0D, "sumTrHN", TrHN); #else std::ofstream outfile; outfile.open(filename0D, std::ofstream::app); @@ -324,8 +324,8 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, #endif } outfile << N_offdiag_mag << "\t"; - outfile << Trf << "\t"; - outfile << TrHf << "\t"; + outfile << TrN << "\t"; + outfile << TrHN << "\t"; outfile << std::endl; outfile.close(); #endif diff --git a/Source/Evolve.H b/Source/Evolve.H index 55f23dec..395634e7 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -26,7 +26,7 @@ namespace GIdx void Initialize(); } -amrex::Real compute_dt(const amrex::Geometry& geom, const amrex::Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const Real flavor_cfl_factor, const Real max_adaptive_speedup); +amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index f6f63d4e..62011ae7 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -19,19 +19,19 @@ namespace GIdx } } -Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const Real flavor_cfl_factor, const Real max_adaptive_speedup) +Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const TestParams* parms) { - AMREX_ASSERT(cfl_factor > 0.0 || flavor_cfl_factor > 0.0); + AMREX_ASSERT(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0); // translation part of timestep limit const auto dxi = geom.CellSizeArray(); Real dt_translation = 0.0; - if (cfl_factor > 0.0) { - dt_translation = std::min(std::min(dxi[0],dxi[1]), dxi[2]) / PhysConst::c * cfl_factor; + if (parms->cfl_factor > 0.0) { + dt_translation = std::min(std::min(dxi[0],dxi[1]), dxi[2]) / PhysConst::c * parms->cfl_factor; } Real dt_flavor = 0.0; - if (flavor_cfl_factor > 0.0) { + if (parms->flavor_cfl_factor > 0.0 && parms->collision_cfl_factor > 0.0) { // define the reduction operator to get the max contribution to // the potential from matter and neutrinos // compute "effective" potential (ergs) that produces characteristic timescale @@ -39,10 +39,9 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta ReduceOps reduce_op; ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; - for (MFIter mfi(state); mfi.isValid(); ++mfi) - { + for (MFIter mfi(state); mfi.isValid(); ++mfi) { const Box& bx = mfi.fabbox(); - auto const& fab = state.array(mfi); + auto const& fab = state.array(mfi); reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { @@ -50,25 +49,44 @@ Real compute_dt(const Geometry& geom, const Real cfl_factor, const MultiFab& sta #include "generated_files/Evolve.cpp_compute_dt_fill" return {V_adaptive, V_stupid}; }); - } + } - // extract the reduced values from the combined reduced data structure - auto rv = reduce_data.value(); - Real Vmax_adaptive = amrex::get<0>(rv) + FlavoredNeutrinoContainer::Vvac_max; - Real Vmax_stupid = amrex::get<1>(rv) + FlavoredNeutrinoContainer::Vvac_max; + // extract the reduced values from the combined reduced data structure + auto rv = reduce_data.value(); + Real Vmax_adaptive = amrex::get<0>(rv) + FlavoredNeutrinoContainer::Vvac_max; + Real Vmax_stupid = amrex::get<1>(rv) + FlavoredNeutrinoContainer::Vvac_max; - // reduce across MPI ranks - ParallelDescriptor::ReduceRealMax(Vmax_adaptive); - ParallelDescriptor::ReduceRealMax(Vmax_stupid ); + // reduce across MPI ranks + ParallelDescriptor::ReduceRealMax(Vmax_adaptive); + ParallelDescriptor::ReduceRealMax(Vmax_stupid ); - // define the dt associated with each method - Real dt_flavor_adaptive = PhysConst::hbar/Vmax_adaptive*flavor_cfl_factor; - Real dt_flavor_stupid = PhysConst::hbar/Vmax_stupid *flavor_cfl_factor; + // define the dt associated with each method + Real dt_flavor_adaptive = std::numeric_limits::max(); + Real dt_flavor_stupid = std::numeric_limits::max(); + Real dt_flavor_absorption = std::numeric_limits::max(); // Initialize with infinity - // pick the appropriate timestep - dt_flavor = dt_flavor_stupid; - if(max_adaptive_speedup>1) - dt_flavor = min(dt_flavor_stupid*max_adaptive_speedup, dt_flavor_adaptive); + if (parms->attenuation_hamiltonians != 0) { + dt_flavor_adaptive = PhysConst::hbar / Vmax_adaptive * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; + dt_flavor_stupid = PhysConst::hbar / Vmax_stupid * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; + } + + if (parms->IMFP_method == 1) { + // Use the IMFPs from the input file and find the maximum absorption IMFP + double max_IMFP_abs = std::numeric_limits::lowest(); // Initialize max to lowest possible value + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < NUM_FLAVORS; ++j) { + max_IMFP_abs = std::max(max_IMFP_abs, parms->IMFP_abs[i][j]); + } + } + // Calculate dt_flavor_absorption + dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor; + } + + // pick the appropriate timestep + dt_flavor = min(dt_flavor_stupid, dt_flavor_adaptive, dt_flavor_absorption); + if(parms->max_adaptive_speedup>1) { + dt_flavor = min(dt_flavor_stupid*parms->max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); + } } Real dt = 0.0; @@ -146,6 +164,11 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M const ParticleInterpolator sy(delta_y, shape_factor_order_y); const ParticleInterpolator sz(delta_z, shape_factor_order_z); + // The following variables contains temperature, electron fraction, and density interpolated from grid quantities to particle positions + Real T_pp = 0; + Real Ye_pp = 0; + Real rho_pp = 0; + for (int k = sz.first(); k <= sz.last(); ++k) { for (int j = sy.first(); j <= sy.last(); ++j) { for (int i = sx.first(); i <= sx.last(); ++i) { @@ -154,6 +177,44 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } } + // Declare matrices to be used in quantum kinetic equation calculation + Real IMFP_abs[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino inverse mean free path matrix: diag( k_e , k_u , k_t ) + Real IMFP_absbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino inverse mean free path matrix: diag( kbar_e , kbar_u , kbar_t ) + Real f_eq[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino equilibrium Fermi-dirac distribution matrix: f_eq = diag( f_e , f_u , f_t ) + Real f_eqbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino equilibrium Fermi-dirac distribution matrix: f_eq = diag( fbar_e , fbar_u , fbar_t ) + + // Initialize matrices with zeros + for (int i=0; iIMFP_method==1){ + for (int i=0; iIMFP_abs[0][i]; // Read absorption inverse mean free path from input parameters file. + IMFP_absbar[i][i] = parms->IMFP_abs[1][i]; // Read absorption inverse mean free path from input parameters file. + + // Calculate the Fermi-Dirac distribution for neutrinos and antineutrinos. + f_eq[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - parms->munu[0][i] ) / T_pp ) ); + f_eqbar[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - parms->munu[1][i] ) / T_pp ) ); + + // Include the Pauli blocking term + if (parms->Do_Pauli_blocking == 1){ + IMFP_abs[i][i] = IMFP_abs[i][i] / ( 1 - f_eq[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). + IMFP_absbar[i][i] = IMFP_absbar[i][i] / ( 1 - f_eqbar[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). + } + } + } + else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0 or 1"); + + #include "generated_files/Evolve.cpp_dfdt_fill" + // set the dfdt values into p.rdata p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; @@ -163,11 +224,6 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M p.rdata(PIdx::pupy) = 0; p.rdata(PIdx::pupz) = 0; p.rdata(PIdx::pupt) = 0; - p.rdata(PIdx::N) = 0; - p.rdata(PIdx::Nbar) = 0; - p.rdata(PIdx::L) = 0; - p.rdata(PIdx::Lbar) = 0; - - #include "generated_files/Evolve.cpp_dfdt_fill" + p.rdata(PIdx::Vphase) = 0; }); } diff --git a/Source/FlavoredNeutrinoContainer.H b/Source/FlavoredNeutrinoContainer.H index cb6bcd68..42e13c2a 100644 --- a/Source/FlavoredNeutrinoContainer.H +++ b/Source/FlavoredNeutrinoContainer.H @@ -89,8 +89,6 @@ public: Redistribute(lev_min, lev_max, nGrow, local); } - void Renormalize(const TestParams* parms); - amrex::Vector get_attribute_names() const { return attribute_names; diff --git a/Source/FlavoredNeutrinoContainer.cpp b/Source/FlavoredNeutrinoContainer.cpp index cc15a333..6abd1907 100644 --- a/Source/FlavoredNeutrinoContainer.cpp +++ b/Source/FlavoredNeutrinoContainer.cpp @@ -82,27 +82,4 @@ UpdateLocationFrom(FlavoredNeutrinoContainer& Ploc) } }); } -} - -void FlavoredNeutrinoContainer:: -Renormalize(const TestParams* parms) -{ - BL_PROFILE("FlavoredNeutrinoContainer::Renormalize"); - - const int lev = 0; - -#ifdef _OPENMP -#pragma omp parallel -#endif - for (FNParIter pti(*this, lev); pti.isValid(); ++pti) - { - const int np = pti.numParticles(); - ParticleType * pstruct = &(pti.GetArrayOfStructs()[0]); - - amrex::ParallelFor (np, [=] AMREX_GPU_DEVICE (int i) { - ParticleType& p = pstruct[i]; - Real sumP, length, error; - #include "generated_files/FlavoredNeutrinoContainer.cpp_Renormalize_fill" - }); - } -} +} \ No newline at end of file diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 6885b413..65d68b8f 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -10,8 +10,13 @@ using namespace amrex; // Particle distribution in momentum space // //=========================================// -Gpu::ManagedVector > read_particle_data(std::string filename){ - Gpu::ManagedVector > particle_data; +Gpu::ManagedVector> read_particle_data(std::string filename){ + + // This function reads the input file containing the initial conditions of the particles. + // It reads the momentum, energy, and flavor occupation matrices for neutrinos and antineutrinos. + + // This array will save the particles information + Gpu::ManagedVector> particle_data; // open the file as a stream std::ifstream file(filename); @@ -32,9 +37,11 @@ Gpu::ManagedVector > read_particle_data(std::strin if(NF_in != NUM_FLAVORS) amrex::Print() << "Error: number of flavors in particle data file does not match the number of flavors Emu was compiled for." << std::endl; AMREX_ASSERT(NF_in == NUM_FLAVORS); + // Loop over every line in the initial condition file. + // This is equivalent to looping over every particle. + // Save every particle's information in the array particle_data. while(std::getline(file, line)){ ss = std::stringstream(line); - // skip over the first four attributes (x,y,z,t) for(int i=4; i> temp_particle[i]; particle_data.push_back(temp_particle); @@ -251,24 +258,14 @@ InitParticles(const TestParams* parms) for(int i_attrib=0; i_attrib= 0); - AMREX_ASSERT(p.rdata(PIdx::Nbar ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::L ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::Lbar ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f00_Re ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f11_Re ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f00_Rebar) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f11_Rebar) >= 0); - Real trace = p.rdata(PIdx::f00_Re ) + p.rdata(PIdx::f11_Re ); - Real tracebar = p.rdata(PIdx::f00_Rebar) + p.rdata(PIdx::f11_Rebar); + AMREX_ASSERT(p.rdata(PIdx::N00_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N11_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N00_Rebar) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N11_Rebar) >= 0); #if NUM_FLAVORS==3 - AMREX_ASSERT(p.rdata(PIdx::f22_Re ) >= 0); - AMREX_ASSERT(p.rdata(PIdx::f22_Rebar) >= 0); - trace += p.rdata(PIdx::f22_Re ); - tracebar += p.rdata(PIdx::f22_Rebar); + AMREX_ASSERT(p.rdata(PIdx::N22_Re ) >= 0); + AMREX_ASSERT(p.rdata(PIdx::N22_Rebar) >= 0); #endif - AMREX_ASSERT(std::abs(trace -1)<1e-6); - AMREX_ASSERT(std::abs(tracebar-1)<1e-6); // Set particle position p.pos(0) = x; @@ -282,8 +279,18 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::time) = 0; // scale particle numbers based on number of points per cell and the cell volume - p.rdata(PIdx::N ) *= scale_fac; - p.rdata(PIdx::Nbar) *= scale_fac; + p.rdata(PIdx::N00_Re ) *= scale_fac; + p.rdata(PIdx::N11_Re ) *= scale_fac; + p.rdata(PIdx::N00_Rebar) *= scale_fac; + p.rdata(PIdx::N11_Rebar) *= scale_fac; +#if NUM_FLAVORS==3 + p.rdata(PIdx::N22_Re ) *= scale_fac; + p.rdata(PIdx::N22_Rebar) *= scale_fac; +#endif + + if(parms->IMFP_method == 1){ + p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/(3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); + } //=====================// // Apply Perturbations // @@ -292,59 +299,56 @@ InitParticles(const TestParams* parms) // random perturbations to the off-diagonals Real rand; symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); + p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); + p.rdata(PIdx::N01_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + p.rdata(PIdx::N01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); #if NUM_FLAVORS==3 symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::N22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Re ) - p.rdata(PIdx::f22_Re )); + p.rdata(PIdx::N12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::N22_Re )); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::f12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::f11_Rebar) - p.rdata(PIdx::f22_Rebar)); + p.rdata(PIdx::N12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar)); #endif } if(parms->perturbation_type == 1){ // Perturb real part of e-mu component only sinusoidally in z Real nu_k = (2.*M_PI) / parms->perturbation_wavelength_cm; - p.rdata(PIdx::f01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Re ) - p.rdata(PIdx::f11_Re )); - p.rdata(PIdx::f01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::f00_Rebar) - p.rdata(PIdx::f11_Rebar)); + p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); + p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); } if(parms->perturbation_type == 2){ // random perturbations of the diagonals Real rand; symmetric_uniform(&rand, engine); - p.rdata(PIdx::f00_Re) *= 1. + parms->perturbation_amplitude*rand; + p.rdata(PIdx::N00_Re) *= 1. + parms->perturbation_amplitude*rand; symmetric_uniform(&rand, engine); - p.rdata(PIdx::f00_Rebar) *= 1. + parms->perturbation_amplitude*rand; + p.rdata(PIdx::N00_Rebar) *= 1. + parms->perturbation_amplitude*rand; symmetric_uniform(&rand, engine); - p.rdata(PIdx::f11_Re) *= 1. + parms->perturbation_amplitude*rand; + p.rdata(PIdx::N11_Re) *= 1. + parms->perturbation_amplitude*rand; symmetric_uniform(&rand, engine); - p.rdata(PIdx::f11_Rebar) *= 1. + parms->perturbation_amplitude*rand; + p.rdata(PIdx::N11_Rebar) *= 1. + parms->perturbation_amplitude*rand; #if NUM_FLAVORS==3 symmetric_uniform(&rand, engine); - p.rdata(PIdx::f22_Re) *= 1. + parms->perturbation_amplitude*rand; + p.rdata(PIdx::N22_Re) *= 1. + parms->perturbation_amplitude*rand; symmetric_uniform(&rand, engine); - p.rdata(PIdx::f22_Rebar) *= 1. + parms->perturbation_amplitude*rand; + p.rdata(PIdx::N22_Rebar) *= 1. + parms->perturbation_amplitude*rand; #endif } - - -#include "generated_files/FlavoredNeutrinoContainerInit.cpp_set_trace_length" } // loop over direction } // loop over location diff --git a/Source/Parameters.H b/Source/Parameters.H index 4ee8121a..201e0cd3 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -22,8 +22,8 @@ struct TestParams : public amrex::Gpu::Managed Real end_time; int write_plot_every; int write_plot_particles_every; - Real rho_in, Ye_in, T_in; // g/ccm, 1, MeV - Real cfl_factor, flavor_cfl_factor; + Real rho_in, Ye_in, kT_in; // g/ccm, 1, erg + Real cfl_factor, flavor_cfl_factor,collision_cfl_factor; Real max_adaptive_speedup; bool do_restart; std::string restart_dir; @@ -42,7 +42,18 @@ struct TestParams : public amrex::Gpu::Managed int perturbation_type; Real perturbation_wavelength_cm; Real perturbation_amplitude; - + + // absorption opacities and equilibrium neutrino chemical potentials + int IMFP_method; + int Do_Pauli_blocking; // If 1, it will multiply the opacities by 1 / (1 - f_eq); if 0, do nothing. + Real IMFP_abs[2][NUM_FLAVORS]; + Real IMFP_scat[2][NUM_FLAVORS]; + Real munu[2][NUM_FLAVORS]; // equilibrium electron neutrino chemical potential in erg (CGS unist) + Real delta_E; // erg (CGS unist) + + // attenuation to hamiltonians + Real attenuation_hamiltonians; + void Initialize(){ ParmParse pp; pp.get("ncell", ncell); @@ -55,9 +66,10 @@ struct TestParams : public amrex::Gpu::Managed pp.get("end_time", end_time); pp.get("rho_g_ccm", rho_in); pp.get("Ye", Ye_in); - pp.get("T_MeV", T_in); + pp.get("T_MeV", kT_in); pp.get("cfl_factor", cfl_factor); pp.get("flavor_cfl_factor", flavor_cfl_factor); + pp.get("collision_cfl_factor", collision_cfl_factor); pp.get("max_adaptive_speedup", max_adaptive_speedup); pp.get("write_plot_every", write_plot_every); pp.get("write_plot_particles_every", write_plot_particles_every); @@ -65,6 +77,9 @@ struct TestParams : public amrex::Gpu::Managed pp.get("restart_dir", restart_dir); pp.get("maxError", maxError); + // convert to cgs + kT_in *= 1e6 * CGSUnitsConst::eV; // erg + // angular grid pp.get("particle_data_filename",particle_data_filename); @@ -74,28 +89,60 @@ struct TestParams : public amrex::Gpu::Managed if(perturbation_type == 1) pp.get("perturbation_wavelength_cm", perturbation_wavelength_cm); - // neutrino physics parameters for 2-flavor - pp.get("mass1_eV", mass1); - pp.get("mass2_eV", mass2); - pp.get("theta12_degrees", theta12); - pp.get("alpha1_degrees", alpha1); - mass1 *= CGSUnitsConst::eV/PhysConst::c2; - mass2 *= CGSUnitsConst::eV/PhysConst::c2; - theta12 *= M_PI/180.; - alpha1 *= M_PI/180.; - - if(NUM_FLAVORS>=2){ - pp.get("mass3_eV", mass3); - pp.get("theta13_degrees", theta13); - pp.get("theta23_degrees", theta23); - pp.get("alpha2_degrees", alpha2); - pp.get("deltaCP_degrees", deltaCP); - mass3 *= CGSUnitsConst::eV/PhysConst::c2; - theta13 *= M_PI/180.; - theta23 *= M_PI/180.; - alpha2 *= M_PI/180.; - deltaCP *= M_PI/180.; + // neutrino physics parameters for 2-flavor + pp.get("mass1_eV", mass1); + pp.get("mass2_eV", mass2); + pp.get("theta12_degrees", theta12); + pp.get("alpha1_degrees", alpha1); + mass1 *= CGSUnitsConst::eV/PhysConst::c2; + mass2 *= CGSUnitsConst::eV/PhysConst::c2; + theta12 *= M_PI/180.; + alpha1 *= M_PI/180.; + if(NUM_FLAVORS>=2){ + pp.get("mass3_eV", mass3); + pp.get("theta13_degrees", theta13); + pp.get("theta23_degrees", theta23); + pp.get("alpha2_degrees", alpha2); + pp.get("deltaCP_degrees", deltaCP); + mass3 *= CGSUnitsConst::eV/PhysConst::c2; + theta13 *= M_PI/180.; + theta23 *= M_PI/180.; + alpha2 *= M_PI/180.; + deltaCP *= M_PI/180.; + } + + // attenuation to hamiltonians + pp.get("attenuation_hamiltonians", attenuation_hamiltonians); + + // absorption opacities and equilibrium neutrino chemical potentials + pp.get("IMFP_method", IMFP_method); + if(IMFP_method==0){ + // do nothing - all values will be set to zero + } + else if(IMFP_method==1){ + for(int i=0;irho_in,GIdx::rho,1); // g/ccm state.setVal(parms->Ye_in,GIdx::Ye,1); - state.setVal(parms->T_in,GIdx::T,1); // MeV + state.setVal(parms->kT_in,GIdx::T,1); // erg state.FillBoundary(geom.periodicity()); // initialize the grid variable names @@ -117,7 +117,7 @@ void evolve_flavor(const TestParams* parms) // Deposit particles to grid deposit_to_mesh(neutrinos_old, state, geom); - + // Write plotfile after initialization DataReducer rd; if (not parms->do_restart) { @@ -148,6 +148,7 @@ void evolve_flavor(const TestParams* parms) // B) We only Redistribute the integrator new data at the end of the timestep, not all the RHS data. // Thus, this copy clears the old RHS particles and creates particles in the RHS container corresponding // to the current particles in neutrinos. + neutrinos_rhs.copyParticles(neutrinos, true); // Step 3: Interpolate Mesh to construct the neutrino RHS in place @@ -172,9 +173,6 @@ void evolve_flavor(const TestParams* parms) // since Redistribute() applies periodic boundary conditions. neutrinos.SyncLocation(Sync::PositionToCoordinate); - // Renormalize the neutrino state - neutrinos.Renormalize(parms); - // Get which step the integrator is on const int step = integrator.get_step_number(); const Real time = integrator.get_time(); @@ -197,7 +195,7 @@ void evolve_flavor(const TestParams* parms) // Note: this won't be the same as the new-time grid data // because the last deposit_to_mesh call was at either the old time (forward Euler) // or the final RK stage, if using Runge-Kutta. - const Real dt = compute_dt(geom,parms->cfl_factor,state,neutrinos,parms->flavor_cfl_factor,parms->max_adaptive_speedup); + const Real dt = compute_dt(geom, state, neutrinos, parms); integrator.set_timestep(dt); }; @@ -206,7 +204,7 @@ void evolve_flavor(const TestParams* parms) integrator.set_post_timestep(post_timestep_fun); // Get a starting timestep - const Real starting_dt = compute_dt(geom,parms->cfl_factor,state,neutrinos_old,parms->flavor_cfl_factor, parms->max_adaptive_speedup); + const Real starting_dt = compute_dt(geom, state, neutrinos_old, parms); // Do all the science! amrex::Print() << "Starting timestepping loop... " << std::endl; diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index c3e89d37..0557d522 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -1,6 +1,10 @@ perturbation_type = 0 perturbation_amplitude = 1e-6 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + +collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 @@ -78,3 +82,7 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 +################# +# opacity stuff # +################# +IMFP_method = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index e4d58c56..c36df0fb 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -1,8 +1,12 @@ +collision_cfl_factor = 1e-3 cfl_factor = 0.5 max_adaptive_speedup = 0 flavor_cfl_factor = .5 maxError = 1e-6 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + perturbation_type = 0 perturbation_amplitude = 0 @@ -78,3 +82,8 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test new file mode 100644 index 00000000..5c68dbbe --- /dev/null +++ b/sample_inputs/inputs_coll_equi_test @@ -0,0 +1,113 @@ +perturbation_type = 0 +perturbation_amplitude = 1e-6 + +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 0 + +collision_cfl_factor = 0.04 +cfl_factor = 1e10 +flavor_cfl_factor = 0.5 +max_adaptive_speedup = 0 +maxError = 1e-6 + +integration.type = 1 +integration.rk.type = 4 + +# Domain size in 3D index space +ncell = (2, 2, 2) +Lx = 20 +Ly = 20 +Lz = 20 + +# Number of particles per cell +nppc = (1, 1, 1) +particle_data_filename = "particle_input.dat" + +# Maximum size of each grid in the domain +max_grid_size = 16 + +# Number of steps to run +nsteps = 1000 + +# Simulation end time +end_time = 5.0e9 + +# Make FPE signal errors so we get a Backtrace +amrex.fpe_trap_invalid=1 + +# give background fluid conditions +rho_g_ccm = 0 +T_MeV = 10 +Ye = 1 + +# Write plotfiles +write_plot_every = 1000 + +# Write particle data in plotfiles +write_plot_particles_every = 1000 + +# checkpointing +do_restart = 0 +restart_dir = "" + +############################### +# NEUTRINO PHYSICS PARAMETERS # +############################### +# see first column of table 14.7 in http://pdg.lbl.gov/2019/reviews/rpp2019-rev-neutrino-mixing.pdf + +# mass state 1 mass in eV [NO/IO:-sqrt(7.39e-5)] +mass1_eV = 0 #-0.008596511 + +# mass state 2 mass in eV (define at 0 arbitrarily because oscillations only sensitive to deltaM^2) +mass2_eV = 0 + +# mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] +mass3_eV = 0.049487372 + +# 1-2 mixing angle in degrees [NO/IO:33.82] +theta12_degrees = 1e-6 + +# 2-3 mixing angle in degrees [NO:8.61 IO:8.65] +theta23_degrees = 8.61 + +# 1-3 mixing angle in degrees [NO:48.3 IO:48.6] +theta13_degrees = 48.3 + +# Majorana angle 1 in degrees +alpha1_degrees = 0 + +# Majorana angle 2 in degrees +alpha2_degrees = 0 + +# CP-violating phase in degrees [NO:222 IO:285] +deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 1 + +Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. + +IMFP_abs0_cm = 5e-4 +IMFP_abs1_cm = 5e-4 +IMFP_abs2_cm = 5e-4 +IMFP_abs0bar_cm = 5e-4 +IMFP_abs1bar_cm = 5e-4 +IMFP_abs2bar_cm = 5e-4 + +munu0_MeV = 0 +munu1_MeV = 0 +munu2_MeV = 0 +munu0bar_MeV = 0 +munu1bar_MeV = 0 +munu2bar_MeV = 0 + +IMFP_scat0_cm = 0 +IMFP_scat1_cm = 0 +IMFP_scat2_cm = 0 +IMFP_scat0bar_cm = 0 +IMFP_scat1bar_cm = 0 +IMFP_scat2bar_cm = 0 + +delta_E = 0.8339001570751987 # Mev diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test new file mode 100644 index 00000000..e8304064 --- /dev/null +++ b/sample_inputs/inputs_collisional_instability_test @@ -0,0 +1,113 @@ +perturbation_type = 0 +perturbation_amplitude = 0.0 + +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1.0 + +collision_cfl_factor = 1e-3 +cfl_factor = 0.5 +flavor_cfl_factor = 0.5 +max_adaptive_speedup = 0 +maxError = 1e-6 + +integration.type = 1 +integration.rk.type = 4 + +# Domain size in 3D index space +ncell = (1, 1, 1) +Lx = 1 # cm +Ly = 1 # cm +Lz = 1 # cm + +# Number of particles per cell +nppc = (1, 1, 1) +particle_data_filename = "particle_input.dat" + +# Maximum size of each grid in the domain +max_grid_size = 16 + +# Number of steps to run +nsteps = 40000 + +# Simulation end time +end_time = 5.0e19 + +# Make FPE signal errors so we get a Backtrace +amrex.fpe_trap_invalid=1 + +# give background fluid conditions +rho_g_ccm = 0 +T_MeV = 7.0 +Ye = 0 + +# Write plotfiles +write_plot_every = 500 + +# Write particle data in plotfiles +write_plot_particles_every = 500 + +# checkpointing +do_restart = 0 +restart_dir = "" + +############################### +# NEUTRINO PHYSICS PARAMETERS # +############################### +# see first column of table 14.7 in http://pdg.lbl.gov/2019/reviews/rpp2019-rev-neutrino-mixing.pdf + +# mass state 1 mass in eV [NO/IO:-sqrt(7.39e-5)] +mass1_eV = 0.04866 #-0.008596511 + +# mass state 2 mass in eV (define at 0 arbitrarily because oscillations only sensitive to deltaM^2) +mass2_eV = 0 + +# mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] +mass3_eV = 0 + +# 1-2 mixing angle in degrees [NO/IO:33.82] +theta12_degrees = 1e-6 + +# 2-3 mixing angle in degrees [NO:8.61 IO:8.65] +theta23_degrees = 0 + +# 1-3 mixing angle in degrees [NO:48.3 IO:48.6] +theta13_degrees = 0 + +# Majorana angle 1 in degrees +alpha1_degrees = 0 + +# Majorana angle 2 in degrees +alpha2_degrees = 0 + +# CP-violating phase in degrees [NO:222 IO:285] +deltaCP_degrees = 0 + +################# +# opacity stuff # +################# +IMFP_method = 1 + +Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. + +IMFP_abs0_cm = 2.398082e-1 +IMFP_abs1_cm = 0 +IMFP_abs2_cm = 0 +IMFP_abs0bar_cm = 2.29358e-2 +IMFP_abs1bar_cm = 0 +IMFP_abs2bar_cm = 0 + +munu0_MeV = 20.0 +munu1_MeV = 0 +munu2_MeV = 0 +munu0bar_MeV = 17.644694342915507 +munu1bar_MeV = 0 +munu2bar_MeV = 0 + +IMFP_scat0_cm = 0 +IMFP_scat1_cm = 0 +IMFP_scat2_cm = 0 +IMFP_scat0bar_cm = 0 +IMFP_scat1bar_cm = 0 +IMFP_scat2bar_cm = 0 + +delta_E = 2.272540842052914 # Mev diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index 2406fbeb..c67c0043 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -1,8 +1,12 @@ +collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = .5 max_adaptive_speedup = 0 maxError = 1e-6 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + perturbation_type = 0 perturbation_amplitude = 0 @@ -78,3 +82,8 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index dc858a8e..808cd76e 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -2,6 +2,10 @@ perturbation_type = 1 perturbation_amplitude = 1e-6 perturbation_wavelength_cm = 1 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + +collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 @@ -79,3 +83,7 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 +################# +# opacity stuff # +################# +IMFP_method = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index cdd14ac7..163af873 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -1,8 +1,12 @@ +collision_cfl_factor = 1e-3 cfl_factor = 0.5 flavor_cfl_factor = 0.1 max_adaptive_speedup = 0 maxError = 1e-6 +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 1 + perturbation_type = 0 perturbation_amplitude = 0 @@ -78,3 +82,8 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 0 \ No newline at end of file From f694f712fb42abfc5d5f8b7607f7caf414d55379 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 26 Aug 2024 17:04:05 -0400 Subject: [PATCH 160/276] Resolving conflict between development and Swapnil's EOS branch. The conflict occurred in the assignment of opacities into the opacity matrices. --- Source/Evolve.cpp | 142 +++++++++++++++++++--------------------------- 1 file changed, 57 insertions(+), 85 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index ad7356b1..75dc082f 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -193,45 +193,45 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } } -<<<<<<< HEAD - // determine the IMFPs and equilibrium distribution value - // create 2 x NF matrix to store absorption IMFPs - // and 2 x NF matrix to store scattering IMFPs - // and 2 x NF matrix to store equilibrium distribution values - Real IMFP_abs[2][NUM_FLAVORS]; - Real IMFP_scat[2][NUM_FLAVORS]; - Real N_eq[2][NUM_FLAVORS]; // equilibrium distribution function (dimensionless) - Real munu[2][NUM_FLAVORS]; // equilibrium chemical potential (erg) - Real att_ham = parms->attenuation_hamiltonians; - - // fill the IMFP values - if(parms->IMFP_method==0){ - // fill with all zeros - for (int i=0; i<2; ++i) { - for (int j=0; jIMFP_method==1){ - // use the IMFPs from the input file - for(int i=0; i<2; i++){ //0->neutrino or 1->antineutrino - for(int j=0; jelectron, 1->heavy(muon), 2->heavy(tau); all heavy same for current table - IMFP_abs[i][j] = parms->IMFP_abs[i][j]; - IMFP_scat[i][j] = parms->IMFP_scat[i][j]; - munu[i][j] = parms->munu[i][j]; //munu -> "mu_e" - "muhat" - } + } + + // If opacity_method is 1, the code will use the inverse mean free paths in the input parameters to compute the collision term. + if(parms->IMFP_method==1){ + for (int i=0; iIMFP_abs[0][i]; // Read absorption inverse mean free path from input parameters file. + IMFP_absbar[i][i] = parms->IMFP_abs[1][i]; // Read absorption inverse mean free path from input parameters file. + munu[i][i] = parms->munu[0][i]; // Read neutrino chemical potential from input parameters file. + munubar[i][i] = parms->munu[1][i]; // Read antineutrino chemical potential from input parameters file. + } } else if(parms->IMFP_method==2){ // use the IMFPs from NuLib table and munu from EoS table. //FIXME: Set the value of rho, temperature and Ye using interpolation from the EoS table. - double rho = 1.0e6; //g/cm^3 - double temperature = 0.6103379806197231; //0.05 //MeV - double Ye = 0.035; + Real rho = rho_pp; //g/cm^3 + Real temperature = T_pp; //0.05 //MeV + Real Ye = Ye_pp; //-------------------- Values from EoS table ------------------------------ double mue_out, muhat_out; @@ -246,12 +246,13 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M #endif const double munu_val = mue_out - muhat_out; //munu -> "mu_e" - "muhat" - - for(int i=0; i<2; i++){ - for(int j=0; jneutrino or 1->antineutrino - for(int j=1; jelectron, 1->heavy(muon), 2->heavy(tau); all heavy same for current table - IMFP_abs[i][j] = absorption_opacity; - IMFP_scat[i][j] = scattering_opacity; - } + for (int i=1; ineutrino or 1->antineutrino + // for(int j=1; jelectron, 1->heavy(muon), 2->heavy(tau); all heavy same for current table + IMFP_abs[i][i] = absorption_opacity ; // ... fix it ... + IMFP_absbar[i][i] = absorption_opacity ; // ... fix it ... + IMFP_scat[i][i] = scattering_opacity ; // ... fix it ... + IMFP_scatbar[i][i] = scattering_opacity ; // ... fix it ... + // } } //----------------------------------------------------------------------- } else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0, 1 or 2"); - // calculate the equilibrium distribution. Really munu and temperature should be interpolated from the grid. - for(int i=0; i<2; i++){ - for(int j=0; jkT_in; - N_eq[i][j] = 1. / (1. + exp(exponent)); - } - } -======= - // Declare matrices to be used in quantum kinetic equation calculation - Real IMFP_abs[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino inverse mean free path matrix: diag( k_e , k_u , k_t ) - Real IMFP_absbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino inverse mean free path matrix: diag( kbar_e , kbar_u , kbar_t ) - Real f_eq[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino equilibrium Fermi-dirac distribution matrix: f_eq = diag( f_e , f_u , f_t ) - Real f_eqbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino equilibrium Fermi-dirac distribution matrix: f_eq = diag( fbar_e , fbar_u , fbar_t ) - - // Initialize matrices with zeros - for (int i=0; iIMFP_method==1){ - for (int i=0; iIMFP_abs[0][i]; // Read absorption inverse mean free path from input parameters file. - IMFP_absbar[i][i] = parms->IMFP_abs[1][i]; // Read absorption inverse mean free path from input parameters file. + for (int i=0; ineutrino or 1->antineutrino - // Calculate the Fermi-Dirac distribution for neutrinos and antineutrinos. - f_eq[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - parms->munu[0][i] ) / T_pp ) ); - f_eqbar[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - parms->munu[1][i] ) / T_pp ) ); + // Calculate the Fermi-Dirac distribution for neutrinos and antineutrinos. + f_eq[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - munu[i][i] ) / T_pp ) ); + f_eqbar[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - munubar[i][i] ) / T_pp ) ); - // Include the Pauli blocking term - if (parms->Do_Pauli_blocking == 1){ - IMFP_abs[i][i] = IMFP_abs[i][i] / ( 1 - f_eq[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). - IMFP_absbar[i][i] = IMFP_absbar[i][i] / ( 1 - f_eqbar[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). - } + // Include the Pauli blocking term + if (parms->Do_Pauli_blocking == 1){ + IMFP_abs[i][i] = IMFP_abs[i][i] / ( 1 - f_eq[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). + IMFP_absbar[i][i] = IMFP_absbar[i][i] / ( 1 - f_eqbar[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). } + } - else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0 or 1"); ->>>>>>> 94846e2b40e6fa6813a2b29f024154ed420689b3 #include "generated_files/Evolve.cpp_dfdt_fill" From cb7512c76467066409f7ea5868da2806fa0e4dba Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 27 Aug 2024 09:10:10 -0400 Subject: [PATCH 161/276] Clean up the code and add comments --- Source/Evolve.cpp | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 75dc082f..c7276e28 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -195,9 +195,9 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M // Declare matrices to be used in quantum kinetic equation calculation Real IMFP_abs[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino inverse mean free path matrix for nucleon absortion: diag( k_e , k_u , k_t ) - Real IMFP_absbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino inverse mean free path matrix for nucleon absortion: diag( kbar_e , kbar_u , kbar_t ) - Real IMFP_scat[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino inverse mean free path matrix for nucleon scatteting - Real IMFP_scatbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino inverse mean free path matrix for nucleon scatteting + Real IMFP_absbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino inverse mean free path matrix for nucleon absortion: diag( kbar_e , kbar_u , kbar_t ) + Real IMFP_scat[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino inverse mean free path matrix for scatteting: diag( k_e , k_u , k_t ) + Real IMFP_scatbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino inverse mean free path matrix for scatteting: diag( kbar_e , kbar_u , kbar_t ) Real f_eq[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino equilibrium Fermi-dirac distribution matrix: f_eq = diag( f_e , f_u , f_t ) Real f_eqbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino equilibrium Fermi-dirac distribution matrix: f_eq = diag( fbar_e , fbar_u , fbar_t ) Real munu[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino chemical potential matrix: munu = diag ( munu_e , munu_x) @@ -226,15 +226,16 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } } + // If opacity_method is 2, the code interpolate inverse mean free paths from NuLib table and electron neutrino chemical potential from EoS table to compute the collision term. else if(parms->IMFP_method==2){ - // use the IMFPs from NuLib table and munu from EoS table. - //FIXME: Set the value of rho, temperature and Ye using interpolation from the EoS table. - Real rho = rho_pp; //g/cm^3 - Real temperature = T_pp; //0.05 //MeV - Real Ye = Ye_pp; + + // Assign temperature, electron fraction, and density at the particle's position to new variables for interpolation of chemical potentials and inverse mean free paths. + Real rho = rho_pp; // Density of background matter at this particle's position g/cm^3 + Real temperature = T_pp; // Temperature of background matter at this particle's position 0.05 //MeV + Real Ye = Ye_pp; // Electron fraction of background matter at this particle's position //-------------------- Values from EoS table ------------------------------ - double mue_out, muhat_out; + double mue_out, muhat_out; // mue_out : Electron chemical potential. muhat_out : neutron minus proton chemical potential int keyerr, anyerr; EOS_tabulated_obj.get_mue_muhat(rho, temperature, Ye, mue_out, muhat_out, keyerr, anyerr); if (anyerr) assert(0); //If there is an error in interpolation call, stop execution. @@ -244,7 +245,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M printf("(Evolve.cpp) mu_e interpolated = %f\n", mue_out); printf("(Evolve.cpp) muhat interpolated = %f\n", muhat_out); #endif - + // munu_val : electron neutrino chemical potential const double munu_val = mue_out - muhat_out; //munu -> "mu_e" - "muhat" // !!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -314,7 +315,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0, 1 or 2"); - for (int i=0; ineutrino or 1->antineutrino + for (int i=0; i Date: Tue, 27 Aug 2024 09:13:00 -0400 Subject: [PATCH 162/276] Filling the chemical potential matrix to be used in the QKE calculation with the electron neutrino and antineutrino values interpolated from the equation of state table. --- Source/Evolve.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index c7276e28..7acf3571 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -248,12 +248,8 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M // munu_val : electron neutrino chemical potential const double munu_val = mue_out - muhat_out; //munu -> "mu_e" - "muhat" - // !!!!!!!!!!!!!!!!!!!!!!!!!!!! - for (int i=0; i Date: Tue, 10 Sep 2024 22:57:36 -0400 Subject: [PATCH 163/276] Do not create particles at outer boundary when periodic BC is used --- Source/FlavoredNeutrinoContainerInit.cpp | 10 +++-- Source/main.cpp | 57 +++++++++++++++++++----- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index f88695c7..c8931358 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -391,7 +391,7 @@ CreateParticlesAtBoundary(const TestParams* parms) // determine the number of directions per location int ndirs_per_loc = particle_data.size(); - amrex::Print() << "Using " << ndirs_per_loc << " directions." << std::endl; + //amrex::Print() << "Using " << ndirs_per_loc << " directions." << std::endl; const Real scale_fac = dx[0]*dx[1]*dx[2]/nlocs_per_cell; //FIXME: We need to use outflow boundary condition, not periodic boundary condition. Put an assert(!periodic) here. @@ -415,7 +415,7 @@ CreateParticlesAtBoundary(const TestParams* parms) const auto lo = amrex::lbound(tile_box); const auto hi = amrex::ubound(tile_box); - printf("tile_box = [%d, %d, %d] x [%d, %d, %d] \n", lo.x, lo.y, lo.z, hi.x, hi.y, hi.z); + //printf("tile_box = [%d, %d, %d] x [%d, %d, %d] \n", lo.x, lo.y, lo.z, hi.x, hi.y, hi.z); Gpu::ManagedVector counts(tile_box.numPts(), 0); //PODVector > counts(n, 0) @@ -514,7 +514,7 @@ CreateParticlesAtBoundary(const TestParams* parms) auto new_size = old_size + num_to_add; particle_tile.resize(new_size); - printf("num_to_add = %d, old_size = %lu, new_size = %lu \n", num_to_add, old_size, new_size); + //printf("num_to_add = %d, old_size = %lu, new_size = %lu \n", num_to_add, old_size, new_size); //Returns the next particle ID for this processor. // Particle IDs start at 1 and are never reused. The pair, consisting of the ID and the CPU on which the particle is "born", is a globally unique identifier for a particle. @@ -618,7 +618,7 @@ CreateParticlesAtBoundary(const TestParams* parms) break; } - printf("x = %f, y = %f, z = %f \n", x, y, z); + //printf("x = %f, y = %f, z = %f \n", x, y, z); if (!create_particle_this_cell) continue; //printf("CREATE PARTRICLE AT: i = %d, j = %d, k = %d \n", i, j, k); @@ -764,6 +764,8 @@ CreateParticlesAtBoundary(const TestParams* parms) } // loop over direction } // loop over location }); // loop over grid cells + + //printf("Finished creating particles at boundary. \n"); } // loop over multifabs } // CreateParticlesAtBoundary() diff --git a/Source/main.cpp b/Source/main.cpp index ed125cdb..d08bff88 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -40,9 +40,38 @@ using namespace amrex; void evolve_flavor(const TestParams* parms) { - // Periodicity and Boundary Conditions - // Defaults to Periodic in all dimensions - Vector is_periodic(AMREX_SPACEDIM, 1); + + //The BC will be set using parameter file. + //Option 0: use periodic BC + //Option 1: create particles at boundary. + + //FIXME: FIXME: Define this in parameter file. + const int BC_type = 0; //0=periodic, 1=outer. + + int BC_type_val; + enum BC_type_enum {PERIODIC, OUTER}; + + if (BC_type == 0){ + BC_type_val = BC_type_enum::PERIODIC; //use periodic BC + } else if (BC_type == 1){ + BC_type_val = BC_type_enum::OUTER; //use outer BC + } else { + amrex::Abort("BC_type is incorrect."); + } + + int periodic_flag; + if (BC_type_val == BC_type_enum::PERIODIC){ + //1=yes, use periodic + periodic_flag = 1; + } else if (BC_type_val == BC_type_enum::OUTER){ + //2=no, do not use periodic. + periodic_flag = 0; + } else { + amrex::Abort("BC_type is incorrect."); + } + + Vector is_periodic(AMREX_SPACEDIM, periodic_flag); + Vector domain_lo_bc_types(AMREX_SPACEDIM, BCType::int_dir); Vector domain_hi_bc_types(AMREX_SPACEDIM, BCType::int_dir); @@ -75,7 +104,7 @@ void evolve_flavor(const TestParams* parms) const IntVect ngrow(1 + (1+shape_factor_order_vec)/2); for(int i=0; incell[i] >= ngrow[i]); - printf("ngrow = [%d, %d, %d] \n", ngrow[0], ngrow[1], ngrow[2]); + //printf("ngrow = [%d, %d, %d] \n", ngrow[0], ngrow[1], ngrow[2]); // We want 1 component (this is one real scalar field on the domain) const int ncomp = GIdx::ncomp; @@ -84,7 +113,7 @@ void evolve_flavor(const TestParams* parms) MultiFab state(ba, dm, ncomp, ngrow); //FIXME: FIXME: Define this in parameter file. - const int read_rho_T_Ye_from_table = 1; + const int read_rho_T_Ye_from_table = 0; // initialize with NaNs ... state.setVal(0.0); @@ -191,12 +220,14 @@ void evolve_flavor(const TestParams* parms) //FIXME: Think carefully where to call this function. //Create particles at outer boundary - neutrinos.CreateParticlesAtBoundary(parms); - neutrinos.CreateParticlesAtBoundary(parms); - neutrinos.CreateParticlesAtBoundary(parms); - neutrinos.CreateParticlesAtBoundary(parms); - neutrinos.CreateParticlesAtBoundary(parms); - neutrinos.CreateParticlesAtBoundary(parms); + if (BC_type_val == BC_type_enum::OUTER){ + neutrinos.CreateParticlesAtBoundary(parms); + neutrinos.CreateParticlesAtBoundary(parms); + neutrinos.CreateParticlesAtBoundary(parms); + neutrinos.CreateParticlesAtBoundary(parms); + neutrinos.CreateParticlesAtBoundary(parms); + neutrinos.CreateParticlesAtBoundary(parms); + } //Create particles at inner boundary //TODO: This needs to be implemented. @@ -216,7 +247,9 @@ void evolve_flavor(const TestParams* parms) const int step = integrator.get_step_number(); const Real time = integrator.get_time(); + printf("Writing reduced data to file... \n"); rd.WriteReducedData0D(geom, state, neutrinos, time, step+1); + printf("Done. \n"); run_fom += neutrinos.TotalNumberOfParticles(); @@ -234,8 +267,10 @@ void evolve_flavor(const TestParams* parms) // Note: this won't be the same as the new-time grid data // because the last deposit_to_mesh call was at either the old time (forward Euler) // or the final RK stage, if using Runge-Kutta. + printf("Setting next timestep... \n"); const Real dt = compute_dt(geom, state, neutrinos, parms); integrator.set_timestep(dt); + printf("Done. \n"); }; // Attach our RHS and post timestep hooks to the integrator From b683cc13abb52910eb14cbd85d4db07059170018 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Tue, 10 Sep 2024 23:27:39 -0400 Subject: [PATCH 164/276] Use NUM_FLAVORS = 2 instead of 3 in makefiles/GNUmakefile_jenkins_HDF5_CUDA Use of NUM_FLAVORS = 3 currently fails --- makefiles/GNUmakefile_jenkins_HDF5_CUDA | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefiles/GNUmakefile_jenkins_HDF5_CUDA b/makefiles/GNUmakefile_jenkins_HDF5_CUDA index 4919c22e..72f9e71a 100644 --- a/makefiles/GNUmakefile_jenkins_HDF5_CUDA +++ b/makefiles/GNUmakefile_jenkins_HDF5_CUDA @@ -1,4 +1,4 @@ -NUM_FLAVORS = 3 +NUM_FLAVORS = 2 SHAPE_FACTOR_ORDER = 2 NUM_MOMENTS = 3 From 6c967a8f2a6a0d650a96564a1a4ba7511e95eff7 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Thu, 12 Sep 2024 13:04:21 -0400 Subject: [PATCH 165/276] Remove ../Scripts/babysitting/avgfee_HDF5.py test from Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a4493ab0..6f40b2b0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ pipeline { sh 'make realclean; make generate; make -j' sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi_3F.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.ex ../sample_inputs/inputs_1d_fiducial' - sh 'python3 ../Scripts/babysitting/avgfee_HDF5.py' + /*sh 'python3 ../Scripts/babysitting/avgfee_HDF5.py'*/ sh 'rm -rf plt*' } }} From 9f7712684904dd2eb580d0981544829a4addc8c5 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Thu, 12 Sep 2024 15:45:06 -0400 Subject: [PATCH 166/276] use GNUmakefile_jenkins_HDF5_CUDA in Jenkinsfile Collisions flavor instability and Collisions to equilibrium test Resolve merge conflict in sample_inputs/inputs_coll_equi_test --- Jenkinsfile | 4 ++-- sample_inputs/inputs_coll_equi_test | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6f40b2b0..04fafc5b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -92,7 +92,7 @@ pipeline { stage('Collisions flavor instability'){ steps{ dir('Exec'){ - sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' + sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' @@ -104,7 +104,7 @@ pipeline { stage('Collisions to equilibrium'){ steps{ dir('Exec'){ - sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile' + sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3' sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index a2ca0bc5..5c68dbbe 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -110,8 +110,4 @@ IMFP_scat0bar_cm = 0 IMFP_scat1bar_cm = 0 IMFP_scat2bar_cm = 0 -<<<<<<< HEAD -delta_E = 0.8394810651882109 # Mev -======= delta_E = 0.8339001570751987 # Mev ->>>>>>> 94846e2b40e6fa6813a2b29f024154ed420689b3 From 9cccd62861a46824bebec308f2a668b90ff1181d Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Mon, 16 Sep 2024 16:22:34 -0400 Subject: [PATCH 167/276] Use actual pup* and dt for Vphase calculation --- Source/FlavoredNeutrinoContainer.H | 2 +- Source/FlavoredNeutrinoContainerInit.cpp | 29 ++++++++++++------------ Source/main.cpp | 14 +++++++----- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/Source/FlavoredNeutrinoContainer.H b/Source/FlavoredNeutrinoContainer.H index 7454af59..0dcb4d98 100644 --- a/Source/FlavoredNeutrinoContainer.H +++ b/Source/FlavoredNeutrinoContainer.H @@ -80,7 +80,7 @@ public: void InitParticles(const TestParams* parms); template - void CreateParticlesAtBoundary(const TestParams* parms); + void CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt); void SyncLocation(int type); diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index c8931358..216be3f4 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -368,7 +368,7 @@ InitParticles(const TestParams* parms) //==================================================================================================================// template void FlavoredNeutrinoContainer:: -CreateParticlesAtBoundary(const TestParams* parms) +CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) { BL_PROFILE("FlavoredNeutrinoContainer::CreateParticlesAtBoundary"); @@ -674,14 +674,13 @@ CreateParticlesAtBoundary(const TestParams* parms) (3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); //p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*V_momentum; - const Real dt = 0.1; //FIXME: FIXME: This is a dummy value. Set correct value from time integrator. - const Real clight = 1.0; //FIXME: FIXME: This is a dummy value. Set correct value. - const Real pupx_ = 1.0; //FIXME: FIXME: This is a dummy value. Set correct value. - const Real pupy_ = 1.0; //FIXME: FIXME: This is a dummy value. Set correct value. - const Real pupz_ = 1.0; //FIXME: FIXME: This is a dummy value. Set correct value. - const Real pupt_ = 1.0; //FIXME: FIXME: This is a dummy value. Set correct value. - printf("(WARNING) Using dummy values: dt = %f, clight = %f, pupt = %f etc.\n", dt, clight, pupt_); - + const Real dt = current_dt; + const Real clight = PhysConst::c; + const Real pupx_ = p.rdata(PIdx::pupx); //TODO: Review this. + const Real pupy_ = p.rdata(PIdx::pupy); //TODO: Review this. + const Real pupz_ = p.rdata(PIdx::pupz); //TODO: Review this. + const Real pupt_ = p.rdata(PIdx::pupt); //TODO: Review this. + switch (DIRECTION) { //Create particles in +ve x direction at lower x boundary. @@ -772,16 +771,16 @@ CreateParticlesAtBoundary(const TestParams* parms) //We need to explicitly instantiate the template function for different use cases. //DIRECTION == BoundaryParticleCreationDirection::I_PLUS (+ve x direction at lower x boundary.) -template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt); //DIRECTION == BoundaryParticleCreationDirection::I_MINUS (-ve x direction at upper x boundary.) -template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt); //DIRECTION == BoundaryParticleCreationDirection::J_PLUS (+ve y direction at lower y boundary.) -template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt); //DIRECTION == BoundaryParticleCreationDirection::J_MINUS (-ve y direction at upper y boundary.) -template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt); //DIRECTION == BoundaryParticleCreationDirection::K_PLUS (+ve z direction at lower z boundary.) -template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt); //DIRECTION == BoundaryParticleCreationDirection::K_MINUS (-ve z direction at upper z boundary.) -template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms); +template void FlavoredNeutrinoContainer::CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt); diff --git a/Source/main.cpp b/Source/main.cpp index d08bff88..96c85bba 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -218,15 +218,17 @@ void evolve_flavor(const TestParams* parms) // Use the latest-time neutrino data auto& neutrinos = neutrinos_new; + const Real current_dt = integrator.get_timestep(); //FIXME: FIXME: Pass this to neutrinos.CreateParticlesAtBoundary. + //FIXME: Think carefully where to call this function. //Create particles at outer boundary if (BC_type_val == BC_type_enum::OUTER){ - neutrinos.CreateParticlesAtBoundary(parms); - neutrinos.CreateParticlesAtBoundary(parms); - neutrinos.CreateParticlesAtBoundary(parms); - neutrinos.CreateParticlesAtBoundary(parms); - neutrinos.CreateParticlesAtBoundary(parms); - neutrinos.CreateParticlesAtBoundary(parms); + neutrinos.CreateParticlesAtBoundary(parms, current_dt); + neutrinos.CreateParticlesAtBoundary(parms, current_dt); + neutrinos.CreateParticlesAtBoundary(parms, current_dt); + neutrinos.CreateParticlesAtBoundary(parms, current_dt); + neutrinos.CreateParticlesAtBoundary(parms, current_dt); + neutrinos.CreateParticlesAtBoundary(parms, current_dt); } //Create particles at inner boundary From 6db027b2425444650e97758a71eafac27a0ae84b Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Tue, 24 Sep 2024 16:09:14 -0400 Subject: [PATCH 168/276] Fix particle creation at boundary and calculate total Vphase --- Source/DataReducer.cpp | 18 ++-- Source/Evolve.cpp | 32 +++++++ Source/FlavoredNeutrinoContainerInit.cpp | 103 +++++++++++++++-------- Source/main.cpp | 6 +- 4 files changed, 116 insertions(+), 43 deletions(-) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index b4da473c..b90799e1 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -112,6 +112,8 @@ void DataReducer::InitializeFiles() outfile << j << ":sumTrN\t"; j++; outfile << j << ":sumTrHN\t"; + j++; + outfile << j << ":Vphase\t"; outfile << std::endl; outfile.close(); @@ -132,18 +134,23 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, // Do reductions over the particles // //==================================// using PType = typename FlavoredNeutrinoContainer::ParticleType; - amrex::ReduceOps reduce_ops; - auto particleResult = amrex::ParticleReduce< ReduceData >(neutrinos, - [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { + amrex::ReduceOps reduce_ops; + auto particleResult = amrex::ParticleReduce< ReduceData >(neutrinos, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { Real TrHN = p.rdata(PIdx::TrHN); - Real TrN = 0; + Real TrN = 0; + Real Vphase = p.rdata(PIdx::Vphase); #include "generated_files/DataReducer.cpp_fill_particles" - return GpuTuple{TrN,TrHN}; + return GpuTuple{TrN,TrHN, Vphase}; }, reduce_ops); Real TrN = amrex::get<0>(particleResult); Real TrHN = amrex::get<1>(particleResult); + Real Vphase = amrex::get<2>(particleResult); ParallelDescriptor::ReduceRealSum(TrN); ParallelDescriptor::ReduceRealSum(TrHN); + ParallelDescriptor::ReduceRealSum(Vphase); + + printf("TrN=%g, TrHN=%g, Vphase=%g\n", TrN, TrHN, Vphase); //=============================// // Do reductions over the grid // @@ -331,6 +338,7 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, outfile << N_offdiag_mag << "\t"; outfile << TrN << "\t"; outfile << TrHN << "\t"; + outfile << Vphase << "\t"; outfile << std::endl; outfile.close(); #endif diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 7acf3571..31d23957 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -1,3 +1,4 @@ +#include "FlavoredNeutrinoContainer.H" #include "Evolve.H" #include "Constants.H" #include "ParticleInterpolator.H" @@ -166,6 +167,37 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M NuLib_tabulated NuLib_tabulated_obj(alltables_nulib, logrho_nulib, logtemp_nulib, yes_nulib, helperVarsReal_nulib, helperVarsInt_nulib); + + + //--------------------------- + const int lev = 0; + +#ifdef _OPENMP +#pragma omp parallel +#endif + for (FNParIter pti(neutrinos_rhs, lev); pti.isValid(); ++pti) + { + const int np = pti.numParticles(); + //printf("(Evolve.cpp) number of particles: np = %d \n", np); + FlavoredNeutrinoContainer::ParticleType* pstruct = &(pti.GetArrayOfStructs()[0]); + + amrex::ParallelFor (np, [=] AMREX_GPU_DEVICE (int i) { + FlavoredNeutrinoContainer::ParticleType& p = pstruct[i]; + + //printf("(Inside Evolve.cpp)i =%d, Vphase = %g \n", i, p.rdata(PIdx::Vphase)); + /*p.pos(0) = p.rdata(PIdx::x); + p.pos(1) = p.rdata(PIdx::y); + p.pos(2) = p.rdata(PIdx::z); + + p.rdata(PIdx::x) = p.pos(0); + p.rdata(PIdx::y) = p.pos(1); + p.rdata(PIdx::z) = p.pos(2);*/ + + }); + } + //-------------------------- + + amrex::MeshToParticle(neutrinos_rhs, state, 0, [=] AMREX_GPU_DEVICE (FlavoredNeutrinoContainer::ParticleType& p, amrex::Array4 const& sarr) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 216be3f4..adbcd81c 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -290,6 +290,7 @@ InitParticles(const TestParams* parms) if(parms->IMFP_method == 1){ p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/(3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); + //printf("(Inside FlavoredNeutrinoContainerInit.cpp) Vphase = %g \n", p.rdata(PIdx::Vphase)); } //=====================// @@ -385,7 +386,6 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) // array of direction vectors //TODO: We can use a different custom file to set particle data at boundary points. - //FIXME: The first line of particle_input.dat is just NFLAVORS (and not an array). Is it correct? Gpu::ManagedVector > particle_data = read_particle_data(parms->particle_data_filename);; auto* particle_data_p = particle_data.dataPtr(); @@ -423,9 +423,11 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) Gpu::ManagedVector offsets(tile_box.numPts()); unsigned int* poffset = offsets.dataPtr(); + + const int buffer = 0; //TODO: TODO: Set this appropriately. // Determine how many particles to add to the particle tile per cell - //This loops runs over all the particles in a given box. + //This loop runs over all the particles in a given box. //For each particle, it calculates a unique "cellid". //It then adds the pcount for that cell by adding ndirs_per_loc value to it (which is number of particles per location emitted). //From amrex documentation: Tiling is turned off if GPU is enabled so that more parallelism is exposed to GPU kernels. @@ -442,32 +444,32 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) { //Create particles in +ve x direction at lower x boundary. case BoundaryParticleCreationDirection::I_PLUS: - if (i==0) create_particle_this_cell = true; + if (i==0+buffer) create_particle_this_cell = true; break; //Create particles in -ve x direction at upper x boundary. case BoundaryParticleCreationDirection::I_MINUS: - if (i==ncellx-1) create_particle_this_cell = true; + if (i==ncellx-1-buffer) create_particle_this_cell = true; break; //Create particles in +ve y direction at lower y boundary. case BoundaryParticleCreationDirection::J_PLUS: - if (j==0) create_particle_this_cell = true; + if (j==0+buffer) create_particle_this_cell = true; break; //Create particles in -ve y direction at upper y boundary. case BoundaryParticleCreationDirection::J_MINUS: - if (j==ncelly-1) create_particle_this_cell = true; + if (j==ncelly-1-buffer) create_particle_this_cell = true; break; //Create particles in +ve z direction at lower z boundary. case BoundaryParticleCreationDirection::K_PLUS: - if (k==0 ) create_particle_this_cell = true; + if (k==0+buffer) create_particle_this_cell = true; break; //Create particles in -ve z direction at upper z boundary. case BoundaryParticleCreationDirection::K_MINUS: - if (k==ncellz-1) create_particle_this_cell = true; + if (k==ncellz-1-buffer) create_particle_this_cell = true; break; default: @@ -494,9 +496,10 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) }); // Determine total number of particles to add to the particle tile - Gpu::inclusive_scan(counts.begin(), counts.end(), offsets.begin()); //This sets the value of "offsets" + //Gpu::inclusive_scan(counts.begin(), counts.end(), offsets.begin()); //This sets the value of "offsets" + Gpu::exclusive_scan(counts.begin(), counts.end(), offsets.begin()); //This sets the value of "offsets" - int num_to_add = offsets[tile_box.numPts()-1]; + int num_to_add = offsets[tile_box.numPts()-1] + counts[tile_box.numPts()-1]; if (num_to_add == 0) continue; // this will be the particle ID for the first new particle in the tile @@ -514,7 +517,12 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) auto new_size = old_size + num_to_add; particle_tile.resize(new_size); + for (int i = 0; i< offsets.size(); i++){ + offsets[i] += old_size; + } + //printf("num_to_add = %d, old_size = %lu, new_size = %lu \n", num_to_add, old_size, new_size); + //printf("Particle ID = %lu \n", ParticleType::NextID()); //Returns the next particle ID for this processor. // Particle IDs start at 1 and are never reused. The pair, consisting of the ID and the CPU on which the particle is "born", is a globally unique identifier for a particle. @@ -566,48 +574,48 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) { //Create particles in +ve x direction at lower x boundary. case BoundaryParticleCreationDirection::I_PLUS: - if (i==0) create_particle_this_cell = true; - x = plo[0]; //VC, lower x boundary + if (i==0+buffer) create_particle_this_cell = true; + x = plo[0]+dx[0]*buffer; //VC, lower x boundary y = plo[1] + (j + r[1])*dx[1]; //CC z = plo[2] + (k + r[2])*dx[2]; //CC break; //Create particles in -ve x direction at upper x boundary. case BoundaryParticleCreationDirection::I_MINUS: - if (i==ncellx-1) create_particle_this_cell = true; - x = plo[0] + ncellx*dx[0]; //VC, upper x boundary + if (i==ncellx-1-buffer) create_particle_this_cell = true; + x = plo[0] + (ncellx-buffer)*dx[0]; //VC, upper x boundary y = plo[1] + (j + r[1])*dx[1]; //CC z = plo[2] + (k + r[2])*dx[2]; //CC break; //Create particles in +ve y direction at lower y boundary. case BoundaryParticleCreationDirection::J_PLUS: - if (j==0) create_particle_this_cell = true; - y = plo[1]; //VC, lower y boundary + if (j==0+buffer) create_particle_this_cell = true; + y = plo[1]+dx[1]*buffer; //VC, lower y boundary x = plo[0] + (i + r[0])*dx[0]; //CC z = plo[2] + (k + r[2])*dx[2]; //CC break; //Create particles in -ve y direction at upper y boundary. case BoundaryParticleCreationDirection::J_MINUS: - if (j==ncelly-1) create_particle_this_cell = true; - y = plo[1] + ncelly*dx[1]; //VC, upper y boundary + if (j==ncelly-1-buffer) create_particle_this_cell = true; + y = plo[1] + (ncelly-buffer)*dx[1]; //VC, upper y boundary x = plo[0] + (i + r[0])*dx[0]; //CC z = plo[2] + (k + r[2])*dx[2]; //CC break; //Create particles in +ve z direction at lower z boundary. case BoundaryParticleCreationDirection::K_PLUS: - if (k==0 ) create_particle_this_cell = true; - z = plo[2]; //VC, lower z boundary + if (k==0+buffer) create_particle_this_cell = true; + z = plo[2]+dx[2]*buffer; //VC, lower z boundary x = plo[0] + (i + r[0])*dx[0]; //CC y = plo[1] + (j + r[1])*dx[1]; //CC break; //Create particles in -ve z direction at upper z boundary. case BoundaryParticleCreationDirection::K_MINUS: - if (k==ncellz-1) create_particle_this_cell = true; - z = plo[2] + ncellz*dx[2]; //VC, upper z boundary + if (k==ncellz-1-buffer) create_particle_this_cell = true; + z = plo[2] + (ncellz-buffer)*dx[2]; //VC, upper z boundary x = plo[0] + (i + r[0])*dx[0]; //CC y = plo[1] + (j + r[1])*dx[1]; //CC break; @@ -625,7 +633,8 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) for(int i_direction=0; i_directionIMFP_method == 1){ - const Real V_momentum = 4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/ - (3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); + const Real V_momentum = 4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/(3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); //p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*V_momentum; const Real dt = current_dt; @@ -685,32 +693,38 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) { //Create particles in +ve x direction at lower x boundary. case BoundaryParticleCreationDirection::I_PLUS: - p.rdata(PIdx::Vphase) = dx[1]*dx[2]*clight*dt*V_momentum*pupx_/pupt_; + p.rdata(PIdx::Vphase) = dx[1]*dx[2]*clight*dt*V_momentum*std::abs(pupx_/pupt_); + //printf("(Inside FlavoredNeutrinoContainerInit.cpp:I_PLUS) Vphase = %g \n", p.rdata(PIdx::Vphase)); break; //Create particles in -ve x direction at upper x boundary. case BoundaryParticleCreationDirection::I_MINUS: - p.rdata(PIdx::Vphase) = dx[1]*dx[2]*clight*dt*V_momentum*pupx_/pupt_; + p.rdata(PIdx::Vphase) = dx[1]*dx[2]*clight*dt*V_momentum*std::abs(pupx_/pupt_); + //printf("(Inside FlavoredNeutrinoContainerInit.cpp:I_MINUS) Vphase = %g \n", p.rdata(PIdx::Vphase)); break; //Create particles in +ve y direction at lower y boundary. case BoundaryParticleCreationDirection::J_PLUS: - p.rdata(PIdx::Vphase) = dx[0]*dx[2]*clight*dt*V_momentum*pupy_/pupt_; + p.rdata(PIdx::Vphase) = dx[0]*dx[2]*clight*dt*V_momentum*std::abs(pupy_/pupt_); + //printf("(Inside FlavoredNeutrinoContainerInit.cpp:J_PLUS) Vphase = %g \n", p.rdata(PIdx::Vphase)); break; //Create particles in -ve y direction at upper y boundary. case BoundaryParticleCreationDirection::J_MINUS: - p.rdata(PIdx::Vphase) = dx[0]*dx[2]*clight*dt*V_momentum*pupy_/pupt_; + p.rdata(PIdx::Vphase) = dx[0]*dx[2]*clight*dt*V_momentum*std::abs(pupy_/pupt_); + //printf("(Inside FlavoredNeutrinoContainerInit.cpp:J_MINUS) Vphase = %g \n", p.rdata(PIdx::Vphase)); break; //Create particles in +ve z direction at lower z boundary. case BoundaryParticleCreationDirection::K_PLUS: - p.rdata(PIdx::Vphase) = dx[0]*dx[1]*clight*dt*V_momentum*pupz_/pupt_; + p.rdata(PIdx::Vphase) = dx[0]*dx[1]*clight*dt*V_momentum*std::abs(pupz_/pupt_); + //printf("(Inside FlavoredNeutrinoContainerInit.cpp:K_PLUS) Vphase = %g \n", p.rdata(PIdx::Vphase)); break; //Create particles in -ve z direction at upper z boundary. case BoundaryParticleCreationDirection::K_MINUS: - p.rdata(PIdx::Vphase) = dx[0]*dx[1]*clight*dt*V_momentum*pupz_/pupt_; + p.rdata(PIdx::Vphase) = dx[0]*dx[1]*clight*dt*V_momentum*std::abs(pupz_/pupt_); + //printf("(Inside FlavoredNeutrinoContainerInit.cpp:K_MINUS) Vphase = %g \n", p.rdata(PIdx::Vphase)); break; default: @@ -720,12 +734,13 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) } } + //TODO: Do not apply perturbation in this case. //=====================// // Apply Perturbations // //=====================// if(parms->perturbation_type == 0){ // random perturbations to the off-diagonals - Real rand; + /*Real rand; symmetric_uniform(&rand, engine); p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); symmetric_uniform(&rand, engine); @@ -733,9 +748,13 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) symmetric_uniform(&rand, engine); p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); + p.rdata(PIdx::N01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar));*/ + p.rdata(PIdx::N01_Re) = - p.rdata(PIdx::N11_Re); + p.rdata(PIdx::N01_Im) = - p.rdata(PIdx::N11_Re); + p.rdata(PIdx::N01_Rebar) = - p.rdata(PIdx::N11_Rebar); + p.rdata(PIdx::N01_Imbar) = - p.rdata(PIdx::N11_Rebar); #if NUM_FLAVORS==3 - symmetric_uniform(&rand, engine); + /*symmetric_uniform(&rand, engine); p.rdata(PIdx::N02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); symmetric_uniform(&rand, engine); p.rdata(PIdx::N02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); @@ -750,14 +769,24 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) symmetric_uniform(&rand, engine); p.rdata(PIdx::N12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar)); symmetric_uniform(&rand, engine); - p.rdata(PIdx::N12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar)); + p.rdata(PIdx::N12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar));*/ + p.rdata(PIdx::N02_Re) = - p.rdata(PIdx::N22_Re); + p.rdata(PIdx::N02_Im) = - p.rdata(PIdx::N22_Re); + p.rdata(PIdx::N12_Re) = - p.rdata(PIdx::N22_Re); + p.rdata(PIdx::N12_Im) = - p.rdata(PIdx::N22_Re); + p.rdata(PIdx::N02_Rebar) = - p.rdata(PIdx::N22_Rebar); + p.rdata(PIdx::N02_Imbar) = - p.rdata(PIdx::N22_Rebar); + p.rdata(PIdx::N12_Rebar) = - p.rdata(PIdx::N22_Rebar); + p.rdata(PIdx::N12_Imbar) = - p.rdata(PIdx::N22_Rebar); #endif } if(parms->perturbation_type == 1){ // Perturb real part of e-mu component only sinusoidally in z - Real nu_k = (2.*M_PI) / parms->perturbation_wavelength_cm; + /*Real nu_k = (2.*M_PI) / parms->perturbation_wavelength_cm; p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); - p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); + p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar));*/ + p.rdata(PIdx::N01_Re) = - p.rdata(PIdx::N11_Re); + p.rdata(PIdx::N01_Rebar) = - p.rdata(PIdx::N11_Rebar); } } // loop over direction diff --git a/Source/main.cpp b/Source/main.cpp index 96c85bba..d604231c 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -46,7 +46,7 @@ void evolve_flavor(const TestParams* parms) //Option 1: create particles at boundary. //FIXME: FIXME: Define this in parameter file. - const int BC_type = 0; //0=periodic, 1=outer. + const int BC_type = 1; //0=periodic, 1=outer. int BC_type_val; enum BC_type_enum {PERIODIC, OUTER}; @@ -74,6 +74,9 @@ void evolve_flavor(const TestParams* parms) Vector domain_lo_bc_types(AMREX_SPACEDIM, BCType::int_dir); Vector domain_hi_bc_types(AMREX_SPACEDIM, BCType::int_dir); + //Vector domain_lo_bc_types(AMREX_SPACEDIM, BCType::foextrap); + //Vector domain_hi_bc_types(AMREX_SPACEDIM, BCType::foextrap); + // Define the index space of the domain @@ -272,6 +275,7 @@ void evolve_flavor(const TestParams* parms) printf("Setting next timestep... \n"); const Real dt = compute_dt(geom, state, neutrinos, parms); integrator.set_timestep(dt); + //printf("current_dt = %g, dt = %g \n", current_dt, dt); printf("Done. \n"); }; From 0aa255931480d235bf17073953c1476b777fd7ed Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Thu, 26 Sep 2024 14:53:58 -0400 Subject: [PATCH 169/276] Set correct BC_type (should be moved to parameter file) --- Source/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/main.cpp b/Source/main.cpp index d604231c..aec06e0b 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -46,7 +46,7 @@ void evolve_flavor(const TestParams* parms) //Option 1: create particles at boundary. //FIXME: FIXME: Define this in parameter file. - const int BC_type = 1; //0=periodic, 1=outer. + const int BC_type = 0; //0=periodic, 1=outer. int BC_type_val; enum BC_type_enum {PERIODIC, OUTER}; From 08af2bd80ee46a26d15d5bf5d37d6a3c7db17c4b Mon Sep 17 00:00:00 2001 From: Swapnil Shankar <55387159+shankar-1729@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:55:14 -0400 Subject: [PATCH 170/276] Swapnil/cleanup development (#102) * Cleanup * more cleanup, enable compiling Emu with and without MPI * Just use AMREX_USE_MPI instead of creating another precompiler definition * cleanup --- Source/Evolve.cpp | 21 ++--- Source/FlavoredNeutrinoContainerInit.cpp | 108 +++++------------------ Source/NuLibTableHelpers.H | 8 -- Source/ReadEosTable.cpp | 44 +-------- Source/ReadInput_RhoTempYe.cpp | 11 +-- Source/ReadNuLibTable.cpp | 27 +----- Source/main.cpp | 4 - 7 files changed, 33 insertions(+), 190 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 31d23957..e639e959 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -168,34 +168,25 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M yes_nulib, helperVarsReal_nulib, helperVarsInt_nulib); - - //--------------------------- + +//The following commented loop can be used to print information about each particle in case debug is needed in future. +/* const int lev = 0; - #ifdef _OPENMP #pragma omp parallel #endif for (FNParIter pti(neutrinos_rhs, lev); pti.isValid(); ++pti) { const int np = pti.numParticles(); - //printf("(Evolve.cpp) number of particles: np = %d \n", np); FlavoredNeutrinoContainer::ParticleType* pstruct = &(pti.GetArrayOfStructs()[0]); amrex::ParallelFor (np, [=] AMREX_GPU_DEVICE (int i) { FlavoredNeutrinoContainer::ParticleType& p = pstruct[i]; - - //printf("(Inside Evolve.cpp)i =%d, Vphase = %g \n", i, p.rdata(PIdx::Vphase)); - /*p.pos(0) = p.rdata(PIdx::x); - p.pos(1) = p.rdata(PIdx::y); - p.pos(2) = p.rdata(PIdx::z); - - p.rdata(PIdx::x) = p.pos(0); - p.rdata(PIdx::y) = p.pos(1); - p.rdata(PIdx::z) = p.pos(2);*/ - + //printf("(Inside Evolve.cpp) Partile i = %d, Vphase = %g \n", i, p.rdata(PIdx::Vphase)); }); } - //-------------------------- +*/ + amrex::MeshToParticle(neutrinos_rhs, state, 0, diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index adbcd81c..f05e2335 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -290,7 +290,6 @@ InitParticles(const TestParams* parms) if(parms->IMFP_method == 1){ p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/(3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); - //printf("(Inside FlavoredNeutrinoContainerInit.cpp) Vphase = %g \n", p.rdata(PIdx::Vphase)); } //=====================// @@ -377,8 +376,6 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) const auto dx = Geom(lev).CellSizeArray(); const auto plo = Geom(lev).ProbLoArray(); const auto& a_bounds = Geom(lev).ProbDomain(); - - //printf("plo = [%f, %f, %f] \n", plo[0], plo[1], plo[2]); const int nlocs_per_cell = AMREX_D_TERM( parms->nppc[0], *parms->nppc[1], @@ -391,12 +388,8 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) // determine the number of directions per location int ndirs_per_loc = particle_data.size(); - //amrex::Print() << "Using " << ndirs_per_loc << " directions." << std::endl; const Real scale_fac = dx[0]*dx[1]*dx[2]/nlocs_per_cell; - //FIXME: We need to use outflow boundary condition, not periodic boundary condition. Put an assert(!periodic) here. - - // Loop over multifabs // #ifdef _OPENMP #pragma omp parallel @@ -415,8 +408,6 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) const auto lo = amrex::lbound(tile_box); const auto hi = amrex::ubound(tile_box); - //printf("tile_box = [%d, %d, %d] x [%d, %d, %d] \n", lo.x, lo.y, lo.z, hi.x, hi.y, hi.z); - Gpu::ManagedVector counts(tile_box.numPts(), 0); //PODVector > counts(n, 0) unsigned int* pcount = counts.dataPtr(); @@ -424,7 +415,8 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) Gpu::ManagedVector offsets(tile_box.numPts()); unsigned int* poffset = offsets.dataPtr(); - const int buffer = 0; //TODO: TODO: Set this appropriately. + //TODO: This can be used to emit particles not exactly at the boundary, but at n cells away from the boundary (n = buffer). + const int buffer = 0; // Determine how many particles to add to the particle tile per cell //This loop runs over all the particles in a given box. @@ -479,8 +471,7 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) } if (!create_particle_this_cell) continue; - //printf("CREATE PARTRICLE AT: i = %d, j = %d, k = %d \n", i, j, k); - + int ix = i - lo.x; int iy = j - lo.y; int iz = k - lo.z; @@ -496,7 +487,6 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) }); // Determine total number of particles to add to the particle tile - //Gpu::inclusive_scan(counts.begin(), counts.end(), offsets.begin()); //This sets the value of "offsets" Gpu::exclusive_scan(counts.begin(), counts.end(), offsets.begin()); //This sets the value of "offsets" int num_to_add = offsets[tile_box.numPts()-1] + counts[tile_box.numPts()-1]; @@ -521,9 +511,6 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) offsets[i] += old_size; } - //printf("num_to_add = %d, old_size = %lu, new_size = %lu \n", num_to_add, old_size, new_size); - //printf("Particle ID = %lu \n", ParticleType::NextID()); - //Returns the next particle ID for this processor. // Particle IDs start at 1 and are never reused. The pair, consisting of the ID and the CPU on which the particle is "born", is a globally unique identifier for a particle. //The maximum of this value across all processors must be checkpointed and then restored on restart so that we don't reuse particle IDs. @@ -559,11 +546,6 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) get_position_unit_cell(r, parms->nppc, i_loc); - //Real x = plo[0] + (i + r[0])*dx[0]; - //Real x = plo[0] for i_plus direction i.e VC, (y,z) remain same CC. - //Real x = plo[0] + (i+1)*dx[0] for i_minus direction i.e. VC, (y,z) remain same CC. - //Real y = plo[1] + (j + r[1])*dx[1]; - //Real z = plo[2] + (k + r[2])*dx[2]; Real x, y, z; bool create_particle_this_cell = false; @@ -626,14 +608,10 @@ CreateParticlesAtBoundary(const TestParams* parms, const Real current_dt) break; } - //printf("x = %f, y = %f, z = %f \n", x, y, z); - if (!create_particle_this_cell) continue; - //printf("CREATE PARTRICLE AT: i = %d, j = %d, k = %d \n", i, j, k); for(int i_direction=0; i_directionperturbation_type == 0){ - // random perturbations to the off-diagonals - /*Real rand; - symmetric_uniform(&rand, engine); - p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::N01_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar)); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::N01_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar));*/ - p.rdata(PIdx::N01_Re) = - p.rdata(PIdx::N11_Re); - p.rdata(PIdx::N01_Im) = - p.rdata(PIdx::N11_Re); - p.rdata(PIdx::N01_Rebar) = - p.rdata(PIdx::N11_Rebar); - p.rdata(PIdx::N01_Imbar) = - p.rdata(PIdx::N11_Rebar); + //Set off-diagonal terms to zero //TODO: This should be reviewed once. + p.rdata(PIdx::N01_Re) = 0.0; + p.rdata(PIdx::N01_Im) = 0.0; + p.rdata(PIdx::N01_Rebar) = 0.0; + p.rdata(PIdx::N01_Imbar) = 0.0; #if NUM_FLAVORS==3 - /*symmetric_uniform(&rand, engine); - p.rdata(PIdx::N02_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::N02_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N22_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::N12_Re) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::N22_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::N12_Im) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Re ) - p.rdata(PIdx::N22_Re )); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::N02_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N22_Rebar)); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::N02_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N22_Rebar)); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::N12_Rebar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar)); - symmetric_uniform(&rand, engine); - p.rdata(PIdx::N12_Imbar) = parms->perturbation_amplitude*rand * (p.rdata(PIdx::N11_Rebar) - p.rdata(PIdx::N22_Rebar));*/ - p.rdata(PIdx::N02_Re) = - p.rdata(PIdx::N22_Re); - p.rdata(PIdx::N02_Im) = - p.rdata(PIdx::N22_Re); - p.rdata(PIdx::N12_Re) = - p.rdata(PIdx::N22_Re); - p.rdata(PIdx::N12_Im) = - p.rdata(PIdx::N22_Re); - p.rdata(PIdx::N02_Rebar) = - p.rdata(PIdx::N22_Rebar); - p.rdata(PIdx::N02_Imbar) = - p.rdata(PIdx::N22_Rebar); - p.rdata(PIdx::N12_Rebar) = - p.rdata(PIdx::N22_Rebar); - p.rdata(PIdx::N12_Imbar) = - p.rdata(PIdx::N22_Rebar); + p.rdata(PIdx::N02_Re) = 0.0; + p.rdata(PIdx::N02_Im) = 0.0; + p.rdata(PIdx::N12_Re) = 0.0; + p.rdata(PIdx::N12_Im) = 0.0; + p.rdata(PIdx::N02_Rebar) = 0.0; + p.rdata(PIdx::N02_Imbar) = 0.0; + p.rdata(PIdx::N12_Rebar) = 0.0; + p.rdata(PIdx::N12_Imbar) = 0.0; #endif - } - if(parms->perturbation_type == 1){ - // Perturb real part of e-mu component only sinusoidally in z - /*Real nu_k = (2.*M_PI) / parms->perturbation_wavelength_cm; - p.rdata(PIdx::N01_Re) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Re ) - p.rdata(PIdx::N11_Re )); - p.rdata(PIdx::N01_Rebar) = parms->perturbation_amplitude*sin(nu_k*p.pos(2)) * (p.rdata(PIdx::N00_Rebar) - p.rdata(PIdx::N11_Rebar));*/ - p.rdata(PIdx::N01_Re) = - p.rdata(PIdx::N11_Re); - p.rdata(PIdx::N01_Rebar) = - p.rdata(PIdx::N11_Rebar); - } } // loop over direction } // loop over location }); // loop over grid cells - //printf("Finished creating particles at boundary. \n"); } // loop over multifabs } // CreateParticlesAtBoundary() diff --git a/Source/NuLibTableHelpers.H b/Source/NuLibTableHelpers.H index cc41bc1c..e40ae532 100644 --- a/Source/NuLibTableHelpers.H +++ b/Source/NuLibTableHelpers.H @@ -72,14 +72,6 @@ get_interp_spots_nulib(const double x, const double y, const double z, double &d idx[6] = NTABLES_NULIB*(ix + nrho*((iy-1) + ntemp*((iz-1) + nye*(idx_species + nspecies*idx_group)))); idx[7] = NTABLES_NULIB*((ix-1) + nrho*((iy-1) + ntemp*((iz-1) + nye*(idx_species + nspecies*idx_group)))); - //idx[0] = NTABLES_NULIB*(ix + nrho*(iy + ntemp*iz)); - //idx[1] = NTABLES_NULIB*((ix-1) + nrho*(iy + ntemp*iz)); - //idx[2] = NTABLES_NULIB*(ix + nrho*((iy-1) + ntemp*iz)); - //idx[3] = NTABLES_NULIB*(ix + nrho*(iy + ntemp*(iz-1))); - //idx[4] = NTABLES_NULIB*((ix-1) + nrho*((iy-1) + ntemp*iz)); - //idx[5] = NTABLES_NULIB*((ix-1) + nrho*(iy + ntemp*(iz-1))); - //idx[6] = NTABLES_NULIB*(ix + nrho*((iy-1) + ntemp*(iz-1))); - //idx[7] = NTABLES_NULIB*((ix-1) + nrho*((iy-1) + ntemp*(iz-1))); // set up aux vars for interpolation delx = logrho_nulib[ix] - x; diff --git a/Source/ReadEosTable.cpp b/Source/ReadEosTable.cpp index eba72b95..1db37657 100644 --- a/Source/ReadEosTable.cpp +++ b/Source/ReadEosTable.cpp @@ -7,9 +7,7 @@ #include "EosTable.H" -// mini NoMPI -#define HAVE_CAPABILITY_MPI //FIXME: This should be defined only when USE_MPI = TRUE -#ifdef HAVE_CAPABILITY_MPI +#ifdef AMREX_USE_MPI #include #define BCAST(buffer, size) MPI_Bcast(buffer, size, MPI_BYTE, my_reader_process, MPI_COMM_WORLD) #else @@ -54,17 +52,10 @@ namespace nuc_eos_private { void ReadEosTable(const std::string nuceos_table_name) { using namespace nuc_eos_private; - //std::string nuceos_table_name = "/home/sshanka/000_UTK_projects/Emu/Exec/SFHo.h5"; amrex::Print() << "(ReadEosTable.cpp) Using table: " << nuceos_table_name << std::endl; //TODO: int my_reader_process = 0; //reader_process; - /*if (my_reader_process < 0 || my_reader_process >= CCTK_nProcs(cctkGH)) - { - CCTK_VWarn(CCTK_WARN_COMPLAIN, __LINE__, __FILE__, CCTK_THORNSTRING, - "Requested IO process %d out of range. Reverting to process 0.", my_reader_process); - my_reader_process = 0; - }*/ const int read_table_on_single_process = 1; //const int doIO = !read_table_on_single_process || CCTK_MyProc(cctkGH) == my_reader_process; //TODO: @@ -187,16 +178,11 @@ void ReadEosTable(const std::string nuceos_table_name) { // free memory of temporary array myManagedArena.deallocate(alltables_temp, nrho_ * ntemp_ * nye_ * NTABLES); - // convert units, convert logs to natural log + // convert logs to natural log // The latter is great, because exp() is way faster than pow() - // pressure - //energy_shift_ = energy_shift_ * EPSGF; //Old code. - //energy_shift_ = energy_shift_; //Let's not convert units yet. - + for(int i=0;i dx = geom.CellSizeArray(); - //const auto plo = geom.ProbLoArray(); - //const auto dxi = geom.InvCellSizeArray(); - //const Real inv_cell_volume = dxi[0]*dxi[1]*dxi[2]; - - //const int shape_factor_order_x = geom.Domain().length(0) > 1 ? SHAPE_FACTOR_ORDER : 0; - //const int shape_factor_order_y = geom.Domain().length(1) > 1 ? SHAPE_FACTOR_ORDER : 0; - //const int shape_factor_order_z = geom.Domain().length(2) > 1 ? SHAPE_FACTOR_ORDER : 0; - + //always access mf comp index as (GIdx::rho - start_comp) //Example: Amrex tutorials -> ExampleCodes/MPMD/Case-2/main.cpp. @@ -35,8 +28,6 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom) amrex::Real y = (j+0.5) * dx[1]; amrex::Real z = (k+0.5) * dx[2]; - //printf("Inside MFIter: x=%f, y=%f, z=%f\n", x, y, z); - //TODO: Find the global (i, j, k) from the amrex domain. call them (ig, jg, kg). //TODO: Then get the values from GPU-array for (ig, jg, kg) and set them to corresponding MultiFabs here. mf_array(i, j, k, GIdx::rho - start_comp) = -404.0; //FIXME: diff --git a/Source/ReadNuLibTable.cpp b/Source/ReadNuLibTable.cpp index 6947ad3f..bbed8577 100644 --- a/Source/ReadNuLibTable.cpp +++ b/Source/ReadNuLibTable.cpp @@ -1,4 +1,4 @@ -#include + #include #include @@ -7,9 +7,7 @@ #include "NuLibTable.H" -// mini NoMPI -#define HAVE_CAPABILITY_MPI //FIXME: This should be defined only when USE_MPI = TRUE -#ifdef HAVE_CAPABILITY_MPI +#ifdef AMREX_USE_MPI #include #define BCAST(buffer, size) MPI_Bcast(buffer, size, MPI_BYTE, my_reader_process, MPI_COMM_WORLD) #else @@ -52,21 +50,13 @@ namespace nulib_private { int *helperVarsInt_nulib; } -//TODO: Pass the /path/to/table here in the function argument void ReadNuLibTable(const std::string nulib_table_name) { using namespace nulib_private; - //std::string nulib_table_name = "/mnt/scratch/tables/NuLib/NuLib_SFHo.h5"; amrex::Print() << "(ReadNuLibTable.cpp) Using table: " << nulib_table_name << std::endl; //TODO: int my_reader_process = 0; //reader_process; - /*if (my_reader_process < 0 || my_reader_process >= CCTK_nProcs(cctkGH)) - { - CCTK_VWarn(CCTK_WARN_COMPLAIN, __LINE__, __FILE__, CCTK_THORNSTRING, - "Requested IO process %d out of range. Reverting to process 0.", my_reader_process); - my_reader_process = 0; - }*/ const int read_table_on_single_process = 1; //const int doIO = !read_table_on_single_process || CCTK_MyProc(cctkGH) == my_reader_process; //TODO: @@ -191,9 +181,7 @@ void ReadNuLibTable(const std::string nulib_table_name) { for(int k = 0; k domain_lo_bc_types(AMREX_SPACEDIM, BCType::int_dir); Vector domain_hi_bc_types(AMREX_SPACEDIM, BCType::int_dir); - //Vector domain_lo_bc_types(AMREX_SPACEDIM, BCType::foextrap); - //Vector domain_hi_bc_types(AMREX_SPACEDIM, BCType::foextrap); // Define the index space of the domain @@ -107,8 +105,6 @@ void evolve_flavor(const TestParams* parms) const IntVect ngrow(1 + (1+shape_factor_order_vec)/2); for(int i=0; incell[i] >= ngrow[i]); - //printf("ngrow = [%d, %d, %d] \n", ngrow[0], ngrow[1], ngrow[2]); - // We want 1 component (this is one real scalar field on the domain) const int ncomp = GIdx::ncomp; From d6f2f5a993fb981f56151ab241828e5afcf74758 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 13 Sep 2024 16:27:54 -0400 Subject: [PATCH 171/276] Create a file that contains the function which will read background quantities, rho, Ye, and T, from an HDF5 file to an EMU grid --- Source/ReadHDF5RhoYeT.H | 15 ++++ Source/ReadHDF5RhoYeT.cpp | 178 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 Source/ReadHDF5RhoYeT.H create mode 100644 Source/ReadHDF5RhoYeT.cpp diff --git a/Source/ReadHDF5RhoYeT.H b/Source/ReadHDF5RhoYeT.H new file mode 100644 index 00000000..613e94c4 --- /dev/null +++ b/Source/ReadHDF5RhoYeT.H @@ -0,0 +1,15 @@ +#ifndef READ_HDF5_RHO_YE_T_H +#define READ_HDF5_RHO_YE_T_H + +void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T); + +namespace background_input_rho_T_Ye { + +// table data + extern double *rho_array_input; + extern double *T_array_input; + extern double *Ye_array_input; + +} + +#endif // READ_HDF5_RHO_YE_T_H diff --git a/Source/ReadHDF5RhoYeT.cpp b/Source/ReadHDF5RhoYeT.cpp new file mode 100644 index 00000000..462fbd32 --- /dev/null +++ b/Source/ReadHDF5RhoYeT.cpp @@ -0,0 +1,178 @@ +#include + +#include + +#define H5_USE_16_API 1 +#include "hdf5.h" + +#include "ReadHDF5RhoYeT.H" + +// mini NoMPI +#define HAVE_CAPABILITY_MPI //FIXME: This should be defined only when USE_MPI = TRUE +#ifdef HAVE_CAPABILITY_MPI +#include +#define BCAST(buffer, size) MPI_Bcast(buffer, size, MPI_BYTE, my_reader_process, MPI_COMM_WORLD) +#else +#define BCAST(buffer, size) do { /* do nothing */ } while(0) +#endif + +// Catch HDF5 errors +#define HDF5_ERROR(fn_call) \ + if(doIO) { \ + int _error_code = fn_call; \ + if (_error_code < 0) { \ + AMREX_ASSERT_WITH_MESSAGE(false, "HDF5 call failed"); \ + } \ + } + +using namespace amrex; + +static int file_is_readable(std::string filename); +static int file_is_readable(std::string filename) +{ + FILE* fp = NULL; + fp = fopen(filename.c_str(), "r"); + if(fp != NULL) + { + fclose(fp); + return 1; + } + return 0; +} + +namespace background_input_rho_T_Ye { + + double *rho_array_input; + double *T_array_input; + double *Ye_array_input; + +} + +//TODO: Pass the /path/to/table here in the function argument +void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ + using namespace background_input_rho_T_Ye; + + //std::string nuceos_table_name = "/home/sshanka/000_UTK_projects/Emu/Exec/SFHo.h5"; + amrex::Print() << "(ReadHDF5RhoYeT.cpp) Using hdf5: " << hdf5_background_rho_Ye_T << std::endl; + + //TODO: + int my_reader_process = 0; //reader_process; + /*if (my_reader_process < 0 || my_reader_process >= CCTK_nProcs(cctkGH)) + { + CCTK_VWarn(CCTK_WARN_COMPLAIN, __LINE__, __FILE__, CCTK_THORNSTRING, + "Requested IO process %d out of range. Reverting to process 0.", my_reader_process); + my_reader_process = 0; + }*/ + + const int read_table_on_single_process = 1; + //const int doIO = !read_table_on_single_process || CCTK_MyProc(cctkGH) == my_reader_process; //TODO: + const int doIO = 1; + + hid_t file; + if (doIO && !file_is_readable(hdf5_background_rho_Ye_T)) { + AMREX_ASSERT_WITH_MESSAGE(false, "Could not read hdf5_background_rho_Ye_T"); + } + + HDF5_ERROR(file = H5Fopen(hdf5_background_rho_Ye_T.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); + +// Use these two defines to easily read in a lot of variables in the same way +// The first reads in one variable of a given type completely +#define READ_BCAST_EOS_HDF5(NAME,VAR,TYPE,MEM,NELEMS) \ + do { \ + hid_t dataset; \ + HDF5_ERROR(dataset = H5Dopen(file, NAME)); \ + HDF5_ERROR(H5Dread(dataset, TYPE, MEM, H5S_ALL, H5P_DEFAULT, VAR)); \ + if (read_table_on_single_process) \ + BCAST (VAR, sizeof(*(VAR))*(NELEMS)); \ + HDF5_ERROR(H5Dclose(dataset)); \ + } while (0) +// The second reads a given variable into a hyperslab of the alltables_temp array +#define READ_BCAST_EOSTABLE_HDF5(NAME,OFF,DIMS) \ + do { \ + READ_BCAST_EOS_HDF5(NAME,&allbackgroundYeTrhos_temp[(OFF)*(DIMS)[1]],H5T_NATIVE_DOUBLE,H5S_ALL,(DIMS)[1]); \ + } while (0) + + int ncellx_; + int ncelly_; + int ncellz_; + // Read size of tables + READ_BCAST_EOS_HDF5("ncellsx", &ncellx_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ncellsy", &ncelly_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ncellsz", &ncellz_, H5T_NATIVE_INT, H5S_ALL, 1); + + printf("(ReadHDF5RhoYeT.cpp) ncellx_ = %d, ncelly_ = %d, ncellz_ = %d\n", ncellx_, ncelly_, ncellz_); + + //Allocate managed memory arena on unified memory + ManagedArenaAllocator myManagedArena; + ManagedArenaAllocator myManagedArena_Int; // REMOVE IT IF NOT NEEDED ERICK + + // Allocate memory for tables + double *allbackgroundYeTrhos_temp; + if (!(allbackgroundYeTrhos_temp = myManagedArena.allocate(ncellx_ * ncelly_ * ncellz_ * 3 ) )) { + printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); + assert(0); + } + // Allocate memory for tables + if (!(rho_array_input = myManagedArena.allocate(ncellx_ * ncelly_ * ncellz_) )) { + printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); + assert(0); + } + if (!(T_array_input = myManagedArena.allocate(ncellx_ * ncelly_ * ncellz_) )) { + printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); + assert(0); + } + if (!(Ye_array_input = myManagedArena.allocate(ncellx_ * ncelly_ * ncellz_) )) { + printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); + assert(0); + } + + // Prepare HDF5 to read hyperslabs into alltables_temp + hsize_t table_dims[2] = {3, (hsize_t)ncellx_ * ncelly_ * ncellz_}; + hsize_t var3[2] = { 1, (hsize_t)ncellx_ * ncelly_ * ncellz_}; // DELETE IF NOT NEEDED ERICK + hid_t mem3 = H5Screate_simple(2, table_dims, NULL); + + // Read alltables_temp + READ_BCAST_EOSTABLE_HDF5("rho_g|ccm", 0, table_dims); + READ_BCAST_EOSTABLE_HDF5("T_Mev", 1, table_dims); + READ_BCAST_EOSTABLE_HDF5("Ye", 2, table_dims); + + HDF5_ERROR(H5Sclose(mem3)); + HDF5_ERROR(H5Fclose(file)); + + for( int k = 0 ; k < ncellx_ ; k++ ){ + for( int j = 0 ; j < ncelly_ ; j++ ){ + for(int i = 0 ; i < ncellz_ ; i++ ) { + int index_old = i + ncellz_*(j + ncelly_*(k + ncellx_)); + int index_new = 0 + i + ncellz_*(j + ncelly_*k); + rho_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; + } + } + } + + for( int k = 0 ; k < ncellx_ ; k++ ){ + for( int j = 0 ; j < ncelly_ ; j++ ){ + for(int i = 0 ; i < ncellz_ ; i++ ) { + int index_old = i + ncellz_*(j + ncelly_*(k + ncellx_*2)); + int index_new = 0 + i + ncellz_*(j + ncelly_*k); + T_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; + } + } + } + + for( int k = 0 ; k < ncellx_ ; k++ ){ + for( int j = 0 ; j < ncelly_ ; j++ ){ + for(int i = 0 ; i < ncellz_ ; i++ ) { + int index_old = i + ncellz_*(j + ncelly_*(k + ncellx_*3)); + int index_new = 0 + i + ncellz_*(j + ncelly_*k); + Ye_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; + } + } + } + + // free memory of temporary array + myManagedArena.deallocate(allbackgroundYeTrhos_temp, ncellz_ * ncelly_ * ncellz_ * 3); + +} // ReadEOSTable + + + From 89622c0e10e47f50a760534e9841bbe5896be54b Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 13 Sep 2024 16:30:01 -0400 Subject: [PATCH 172/276] Call the function that reads the HDF5 file containing rho, Ye, and T from the function that sets these quantities in the EMU multifab mesh. --- Source/ReadInput_RhoTempYe.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp index 60e00ecc..16124244 100644 --- a/Source/ReadInput_RhoTempYe.cpp +++ b/Source/ReadInput_RhoTempYe.cpp @@ -2,7 +2,7 @@ #include "Constants.H" #include - +#include "ReadHDF5RhoYeT.H" void set_rho_T_Ye(MultiFab& state, const Geometry& geom) { @@ -16,6 +16,9 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom) //always access mf comp index as (GIdx::rho - start_comp) //Example: Amrex tutorials -> ExampleCodes/MPMD/Case-2/main.cpp. + const std::string hdf5_background_rho_Ye_T_name = "rho_Ye_T.hdf5" + ReadInputRhoYeT(hdf5_background_rho_Ye_T_name) + for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ const amrex::Box& bx = mfi.validbox(); const amrex::Array4& mf_array = rho_T_ye_state.array(mfi); From a2623d13e7e664f5adb695980835dedc9ad29e50 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 13 Sep 2024 17:17:12 -0400 Subject: [PATCH 173/276] Solving missing semicolon --- Source/ReadInput_RhoTempYe.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp index 16124244..18020fda 100644 --- a/Source/ReadInput_RhoTempYe.cpp +++ b/Source/ReadInput_RhoTempYe.cpp @@ -1,8 +1,8 @@ #include "Evolve.H" #include "Constants.H" - -#include #include "ReadHDF5RhoYeT.H" +#include "ReadInput_RhoTempYe.H" +#include void set_rho_T_Ye(MultiFab& state, const Geometry& geom) { @@ -16,8 +16,8 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom) //always access mf comp index as (GIdx::rho - start_comp) //Example: Amrex tutorials -> ExampleCodes/MPMD/Case-2/main.cpp. - const std::string hdf5_background_rho_Ye_T_name = "rho_Ye_T.hdf5" - ReadInputRhoYeT(hdf5_background_rho_Ye_T_name) + const std::string hdf5_background_rho_Ye_T_name = "rho_Ye_T.hdf5"; + ReadInputRhoYeT(hdf5_background_rho_Ye_T_name); for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ const amrex::Box& bx = mfi.validbox(); From d4e185fee1a5bb74827865bc8c24120c37c7d344 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 13 Sep 2024 17:41:04 -0400 Subject: [PATCH 174/276] Update the Make.package file to compile the file that contains the function that reads rho, Ye, and T. --- Source/Make.package | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Source/Make.package b/Source/Make.package index 4941a84f..63f5b4e4 100644 --- a/Source/Make.package +++ b/Source/Make.package @@ -7,6 +7,7 @@ CEXE_sources += FlavoredNeutrinoContainer.cpp CEXE_sources += ReadEosTable.cpp CEXE_sources += ReadNuLibTable.cpp CEXE_sources += ReadInput_RhoTempYe.cpp +CEXE_sources += ReadHDF5RhoYeT.cpp CEXE_headers += Evolve.H CEXE_headers += FlavoredNeutrinoContainer.H @@ -24,3 +25,4 @@ CEXE_headers += NuLibTable.H CEXE_headers += NuLibTableFunctions.H CEXE_headers += NuLibTableHelpers.H CEXE_headers += ReadInput_RhoTempYe.H +CEXE_headers += ReadHdf5RhoYeT.H From 060734b02854cfb843447eda2ccde8b63e724267 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 13 Sep 2024 17:52:16 -0400 Subject: [PATCH 175/276] Send input parameters to the function that sets rho, Y, and T to the EMU mesh. --- Source/ReadInput_RhoTempYe.H | 2 +- Source/ReadInput_RhoTempYe.cpp | 2 +- Source/main.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/ReadInput_RhoTempYe.H b/Source/ReadInput_RhoTempYe.H index 25414e10..b9dd6113 100644 --- a/Source/ReadInput_RhoTempYe.H +++ b/Source/ReadInput_RhoTempYe.H @@ -8,7 +8,7 @@ #include #include -void set_rho_T_Ye(MultiFab& state, const Geometry& geom); +void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms); #endif diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp index 18020fda..5182304d 100644 --- a/Source/ReadInput_RhoTempYe.cpp +++ b/Source/ReadInput_RhoTempYe.cpp @@ -4,7 +4,7 @@ #include "ReadInput_RhoTempYe.H" #include -void set_rho_T_Ye(MultiFab& state, const Geometry& geom) +void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms) { // Create an alias of the MultiFab so set_rho_T_Ye only sets rho, T and Ye. int start_comp = GIdx::rho; diff --git a/Source/main.cpp b/Source/main.cpp index 89648fd3..b973be39 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -120,7 +120,7 @@ void evolve_flavor(const TestParams* parms) //If reading from table, call function "set_rho_T_Ye". //Else set rho, T and Ye to constant value throughout the grid using values from parameter file. if (read_rho_T_Ye_from_table){ - set_rho_T_Ye(state, geom); + set_rho_T_Ye(state, geom, parms); } else { state.setVal(parms->rho_in,GIdx::rho,1); // g/ccm state.setVal(parms->Ye_in,GIdx::Ye,1); From cda1a453a207e092a096a4d7a7850cea4e4bf3f6 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 19 Sep 2024 09:16:43 -0400 Subject: [PATCH 176/276] Reading the input number of cells and domain size from the background quantities HDF5 file. This is to compare if the value read by the function is correct. --- Source/ReadHDF5RhoYeT.H | 10 +++++ Source/ReadHDF5RhoYeT.cpp | 74 ++++++++++++++++++++-------------- Source/ReadInput_RhoTempYe.cpp | 4 ++ 3 files changed, 57 insertions(+), 31 deletions(-) diff --git a/Source/ReadHDF5RhoYeT.H b/Source/ReadHDF5RhoYeT.H index 613e94c4..cd2253bb 100644 --- a/Source/ReadHDF5RhoYeT.H +++ b/Source/ReadHDF5RhoYeT.H @@ -6,6 +6,16 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T); namespace background_input_rho_T_Ye { // table data + extern int *n_cell_x; + extern int *n_cell_y; + extern int *n_cell_z; + extern double *x_min; + extern double *x_max; + extern double *y_min; + extern double *y_max; + extern double *z_min; + extern double *z_max; + extern double *rho_array_input; extern double *rho_array_input; extern double *T_array_input; extern double *Ye_array_input; diff --git a/Source/ReadHDF5RhoYeT.cpp b/Source/ReadHDF5RhoYeT.cpp index 462fbd32..8908b22c 100644 --- a/Source/ReadHDF5RhoYeT.cpp +++ b/Source/ReadHDF5RhoYeT.cpp @@ -42,6 +42,15 @@ static int file_is_readable(std::string filename) namespace background_input_rho_T_Ye { + int *n_cell_x; + int *n_cell_y; + int *n_cell_z; + double *x_min; + double *x_max; + double *y_min; + double *y_max; + double *z_min; + double *z_max; double *rho_array_input; double *T_array_input; double *Ye_array_input; @@ -91,16 +100,19 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ do { \ READ_BCAST_EOS_HDF5(NAME,&allbackgroundYeTrhos_temp[(OFF)*(DIMS)[1]],H5T_NATIVE_DOUBLE,H5S_ALL,(DIMS)[1]); \ } while (0) - - int ncellx_; - int ncelly_; - int ncellz_; - // Read size of tables - READ_BCAST_EOS_HDF5("ncellsx", &ncellx_, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("ncellsy", &ncelly_, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("ncellsz", &ncellz_, H5T_NATIVE_INT, H5S_ALL, 1); - printf("(ReadHDF5RhoYeT.cpp) ncellx_ = %d, ncelly_ = %d, ncellz_ = %d\n", ncellx_, ncelly_, ncellz_); + // Read size of tables + READ_BCAST_EOS_HDF5("ncellsx", n_cell_x, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ncellsy", n_cell_y, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ncellsz", n_cell_z, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("xmin_cm", x_min, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ymin_cm", y_min, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("zmin_cm", z_min, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("xmax_cm", x_max, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ymax_cm", y_max, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("zmax_cm", z_max, H5T_NATIVE_INT, H5S_ALL, 1); + + printf("(ReadHDF5RhoYeT.cpp) n_cell_x = %d, n_cell_y = %d, n_cell_z = %d\n", n_cell_x, n_cell_y, n_cell_z); //Allocate managed memory arena on unified memory ManagedArenaAllocator myManagedArena; @@ -108,27 +120,27 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ // Allocate memory for tables double *allbackgroundYeTrhos_temp; - if (!(allbackgroundYeTrhos_temp = myManagedArena.allocate(ncellx_ * ncelly_ * ncellz_ * 3 ) )) { + if (!(allbackgroundYeTrhos_temp = myManagedArena.allocate(n_cell_x * n_cell_y * n_cell_z * 3 ) )) { printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); assert(0); } // Allocate memory for tables - if (!(rho_array_input = myManagedArena.allocate(ncellx_ * ncelly_ * ncellz_) )) { + if (!(rho_array_input = myManagedArena.allocate(n_cell_x * n_cell_y * n_cell_z) )) { printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); assert(0); } - if (!(T_array_input = myManagedArena.allocate(ncellx_ * ncelly_ * ncellz_) )) { + if (!(T_array_input = myManagedArena.allocate(n_cell_x * n_cell_y * n_cell_z) )) { printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); assert(0); } - if (!(Ye_array_input = myManagedArena.allocate(ncellx_ * ncelly_ * ncellz_) )) { + if (!(Ye_array_input = myManagedArena.allocate(n_cell_x * n_cell_y * n_cell_z) )) { printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); assert(0); } // Prepare HDF5 to read hyperslabs into alltables_temp - hsize_t table_dims[2] = {3, (hsize_t)ncellx_ * ncelly_ * ncellz_}; - hsize_t var3[2] = { 1, (hsize_t)ncellx_ * ncelly_ * ncellz_}; // DELETE IF NOT NEEDED ERICK + hsize_t table_dims[2] = {3, (hsize_t)n_cell_x * n_cell_y * n_cell_z}; + hsize_t var3[2] = { 1, (hsize_t)n_cell_x * n_cell_y * n_cell_z}; // DELETE IF NOT NEEDED ERICK hid_t mem3 = H5Screate_simple(2, table_dims, NULL); // Read alltables_temp @@ -139,38 +151,38 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ HDF5_ERROR(H5Sclose(mem3)); HDF5_ERROR(H5Fclose(file)); - for( int k = 0 ; k < ncellx_ ; k++ ){ - for( int j = 0 ; j < ncelly_ ; j++ ){ - for(int i = 0 ; i < ncellz_ ; i++ ) { - int index_old = i + ncellz_*(j + ncelly_*(k + ncellx_)); - int index_new = 0 + i + ncellz_*(j + ncelly_*k); + for( int k = 0 ; k < n_cell_x ; k++ ){ + for( int j = 0 ; j < n_cell_y ; j++ ){ + for(int i = 0 ; i < n_cell_z ; i++ ) { + int index_old = i + n_cell_z*(j + n_cell_y*(k + n_cell_x)); + int index_new = 0 + i + n_cell_z*(j + n_cell_y*k); rho_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; } } } - for( int k = 0 ; k < ncellx_ ; k++ ){ - for( int j = 0 ; j < ncelly_ ; j++ ){ - for(int i = 0 ; i < ncellz_ ; i++ ) { - int index_old = i + ncellz_*(j + ncelly_*(k + ncellx_*2)); - int index_new = 0 + i + ncellz_*(j + ncelly_*k); + for( int k = 0 ; k < n_cell_x ; k++ ){ + for( int j = 0 ; j < n_cell_y ; j++ ){ + for(int i = 0 ; i < n_cell_z ; i++ ) { + int index_old = i + n_cell_z*(j + n_cell_y*(k + n_cell_x*2)); + int index_new = 0 + i + n_cell_z*(j + n_cell_y*k); T_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; } } } - for( int k = 0 ; k < ncellx_ ; k++ ){ - for( int j = 0 ; j < ncelly_ ; j++ ){ - for(int i = 0 ; i < ncellz_ ; i++ ) { - int index_old = i + ncellz_*(j + ncelly_*(k + ncellx_*3)); - int index_new = 0 + i + ncellz_*(j + ncelly_*k); + for( int k = 0 ; k < n_cell_x ; k++ ){ + for( int j = 0 ; j < n_cell_y ; j++ ){ + for(int i = 0 ; i < n_cell_z ; i++ ) { + int index_old = i + n_cell_z*(j + n_cell_y*(k + n_cell_x*3)); + int index_new = 0 + i + n_cell_z*(j + n_cell_y*k); Ye_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; } } } // free memory of temporary array - myManagedArena.deallocate(allbackgroundYeTrhos_temp, ncellz_ * ncelly_ * ncellz_ * 3); + myManagedArena.deallocate(allbackgroundYeTrhos_temp, n_cell_z * n_cell_y * n_cell_z * 3); } // ReadEOSTable diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp index 5182304d..71f3ec61 100644 --- a/Source/ReadInput_RhoTempYe.cpp +++ b/Source/ReadInput_RhoTempYe.cpp @@ -16,9 +16,13 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms //always access mf comp index as (GIdx::rho - start_comp) //Example: Amrex tutorials -> ExampleCodes/MPMD/Case-2/main.cpp. + using namespace background_input_rho_T_Ye; const std::string hdf5_background_rho_Ye_T_name = "rho_Ye_T.hdf5"; ReadInputRhoYeT(hdf5_background_rho_Ye_T_name); + amrex::Print() << "n_cell_x" << n_cell_x << std::endl; + amrex::Print() << "parms->ncell[0]" << parms->ncell[0] << std::endl; + for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ const amrex::Box& bx = mfi.validbox(); const amrex::Array4& mf_array = rho_T_ye_state.array(mfi); From 968d1eb968b83811dbfdd8315f066f67437d9687 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 19 Sep 2024 09:37:35 -0400 Subject: [PATCH 177/276] Resolve the issue when using the pointer that contains the number of cells in the x, y, and z directions --- Source/ReadHDF5RhoYeT.cpp | 46 +++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/Source/ReadHDF5RhoYeT.cpp b/Source/ReadHDF5RhoYeT.cpp index 8908b22c..7a79ab69 100644 --- a/Source/ReadHDF5RhoYeT.cpp +++ b/Source/ReadHDF5RhoYeT.cpp @@ -112,35 +112,33 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ READ_BCAST_EOS_HDF5("ymax_cm", y_max, H5T_NATIVE_INT, H5S_ALL, 1); READ_BCAST_EOS_HDF5("zmax_cm", z_max, H5T_NATIVE_INT, H5S_ALL, 1); - printf("(ReadHDF5RhoYeT.cpp) n_cell_x = %d, n_cell_y = %d, n_cell_z = %d\n", n_cell_x, n_cell_y, n_cell_z); + printf("(ReadHDF5RhoYeT.cpp) n_cell_x = %d, n_cell_y = %d, n_cell_z = %d\n", *n_cell_x, *n_cell_y, *n_cell_z); //Allocate managed memory arena on unified memory ManagedArenaAllocator myManagedArena; - ManagedArenaAllocator myManagedArena_Int; // REMOVE IT IF NOT NEEDED ERICK // Allocate memory for tables double *allbackgroundYeTrhos_temp; - if (!(allbackgroundYeTrhos_temp = myManagedArena.allocate(n_cell_x * n_cell_y * n_cell_z * 3 ) )) { + if (!(allbackgroundYeTrhos_temp = myManagedArena.allocate(*n_cell_x * *n_cell_y * *n_cell_z * 3 ) )) { printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); assert(0); } // Allocate memory for tables - if (!(rho_array_input = myManagedArena.allocate(n_cell_x * n_cell_y * n_cell_z) )) { + if (!(rho_array_input = myManagedArena.allocate(*n_cell_x * *n_cell_y * *n_cell_z) )) { printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); assert(0); } - if (!(T_array_input = myManagedArena.allocate(n_cell_x * n_cell_y * n_cell_z) )) { + if (!(T_array_input = myManagedArena.allocate(*n_cell_x * *n_cell_y * *n_cell_z) )) { printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); assert(0); } - if (!(Ye_array_input = myManagedArena.allocate(n_cell_x * n_cell_y * n_cell_z) )) { + if (!(Ye_array_input = myManagedArena.allocate(*n_cell_x * *n_cell_y * *n_cell_z) )) { printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); assert(0); } // Prepare HDF5 to read hyperslabs into alltables_temp - hsize_t table_dims[2] = {3, (hsize_t)n_cell_x * n_cell_y * n_cell_z}; - hsize_t var3[2] = { 1, (hsize_t)n_cell_x * n_cell_y * n_cell_z}; // DELETE IF NOT NEEDED ERICK + hsize_t table_dims[2] = {3, (hsize_t)*n_cell_x * *n_cell_y * *n_cell_z}; hid_t mem3 = H5Screate_simple(2, table_dims, NULL); // Read alltables_temp @@ -151,38 +149,38 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ HDF5_ERROR(H5Sclose(mem3)); HDF5_ERROR(H5Fclose(file)); - for( int k = 0 ; k < n_cell_x ; k++ ){ - for( int j = 0 ; j < n_cell_y ; j++ ){ - for(int i = 0 ; i < n_cell_z ; i++ ) { - int index_old = i + n_cell_z*(j + n_cell_y*(k + n_cell_x)); - int index_new = 0 + i + n_cell_z*(j + n_cell_y*k); + for( int k = 0 ; k < *n_cell_x ; k++ ){ + for( int j = 0 ; j < *n_cell_y ; j++ ){ + for(int i = 0 ; i < *n_cell_z ; i++ ) { + int index_old = i + *n_cell_z*(j + *n_cell_y*(k + *n_cell_x )); + int index_new = 0 + i + *n_cell_z*(j + *n_cell_y*k); rho_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; } } } - for( int k = 0 ; k < n_cell_x ; k++ ){ - for( int j = 0 ; j < n_cell_y ; j++ ){ - for(int i = 0 ; i < n_cell_z ; i++ ) { - int index_old = i + n_cell_z*(j + n_cell_y*(k + n_cell_x*2)); - int index_new = 0 + i + n_cell_z*(j + n_cell_y*k); + for( int k = 0 ; k < *n_cell_x ; k++ ){ + for( int j = 0 ; j < *n_cell_y ; j++ ){ + for(int i = 0 ; i < *n_cell_z ; i++ ) { + int index_old = i + *n_cell_z*(j + *n_cell_y*(k + *n_cell_x *2)); + int index_new = 0 + i + *n_cell_z*(j + *n_cell_y*k); T_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; } } } - for( int k = 0 ; k < n_cell_x ; k++ ){ - for( int j = 0 ; j < n_cell_y ; j++ ){ - for(int i = 0 ; i < n_cell_z ; i++ ) { - int index_old = i + n_cell_z*(j + n_cell_y*(k + n_cell_x*3)); - int index_new = 0 + i + n_cell_z*(j + n_cell_y*k); + for( int k = 0 ; k < *n_cell_x ; k++ ){ + for( int j = 0 ; j < *n_cell_y ; j++ ){ + for(int i = 0 ; i < *n_cell_z ; i++ ) { + int index_old = i + *n_cell_z*(j + *n_cell_y*(k + *n_cell_x *3)); + int index_new = 0 + i + *n_cell_z*(j + *n_cell_y*k); Ye_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; } } } // free memory of temporary array - myManagedArena.deallocate(allbackgroundYeTrhos_temp, n_cell_z * n_cell_y * n_cell_z * 3); + myManagedArena.deallocate(allbackgroundYeTrhos_temp, *n_cell_x * *n_cell_y * *n_cell_z * 3); } // ReadEOSTable From 40abf6bf986de7f55634fabadd1932b7032079dd Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 19 Sep 2024 20:54:26 -0400 Subject: [PATCH 178/276] Setting rho, Ye, and T in the AMReX mesh MultiFab. The indices have a problem and still need to be tested. --- Source/ReadHDF5RhoYeT.H | 12 ++++- Source/ReadHDF5RhoYeT.cpp | 99 ++++++++++++++++++++-------------- Source/ReadInput_RhoTempYe.cpp | 26 ++++++--- 3 files changed, 87 insertions(+), 50 deletions(-) diff --git a/Source/ReadHDF5RhoYeT.H b/Source/ReadHDF5RhoYeT.H index cd2253bb..9b7b6204 100644 --- a/Source/ReadHDF5RhoYeT.H +++ b/Source/ReadHDF5RhoYeT.H @@ -1,6 +1,17 @@ #ifndef READ_HDF5_RHO_YE_T_H #define READ_HDF5_RHO_YE_T_H +struct rhoYeT_input_struct { + + double *rho_input, *Ye_input, *T_input; + + //constructor for Tabulated EOS + AMREX_GPU_DEVICE AMREX_GPU_HOST rhoYeT_input_struct() = default;//Need to keep it + AMREX_GPU_DEVICE AMREX_GPU_HOST rhoYeT_input_struct(double *rho_input, double *Ye_input, double *T_input): + rho_input(rho_input), Ye_input(Ye_input), T_input(T_input) {} + +}; //struct EOS_tabulated + void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T); namespace background_input_rho_T_Ye { @@ -16,7 +27,6 @@ namespace background_input_rho_T_Ye { extern double *z_min; extern double *z_max; extern double *rho_array_input; - extern double *rho_array_input; extern double *T_array_input; extern double *Ye_array_input; diff --git a/Source/ReadHDF5RhoYeT.cpp b/Source/ReadHDF5RhoYeT.cpp index 7a79ab69..77da1bfd 100644 --- a/Source/ReadHDF5RhoYeT.cpp +++ b/Source/ReadHDF5RhoYeT.cpp @@ -83,7 +83,7 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ } HDF5_ERROR(file = H5Fopen(hdf5_background_rho_Ye_T.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); - + // Use these two defines to easily read in a lot of variables in the same way // The first reads in one variable of a given type completely #define READ_BCAST_EOS_HDF5(NAME,VAR,TYPE,MEM,NELEMS) \ @@ -100,45 +100,66 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ do { \ READ_BCAST_EOS_HDF5(NAME,&allbackgroundYeTrhos_temp[(OFF)*(DIMS)[1]],H5T_NATIVE_DOUBLE,H5S_ALL,(DIMS)[1]); \ } while (0) - + + int ncellx_; + int ncelly_; + int ncellz_; + double xmin_; + double xmax_; + double ymin_; + double ymax_; + double zmin_; + double zmax_; + // Read size of tables - READ_BCAST_EOS_HDF5("ncellsx", n_cell_x, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("ncellsy", n_cell_y, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("ncellsz", n_cell_z, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("xmin_cm", x_min, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("ymin_cm", y_min, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("zmin_cm", z_min, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("xmax_cm", x_max, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("ymax_cm", y_max, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("zmax_cm", z_max, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ncellsx", &ncellx_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ncellsy", &ncelly_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ncellsz", &ncellz_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("xmin_cm", &xmin_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ymin_cm", &ymin_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("zmin_cm", &zmin_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("xmax_cm", &xmax_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ymax_cm", &ymax_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("zmax_cm", &zmax_, H5T_NATIVE_INT, H5S_ALL, 1); + + printf("(ReadHDF5RhoYeT.cpp) ncellx_ = %d, ncelly_ = %d, ncellz_ = %d\n", ncellx_, ncelly_, ncellz_); + + n_cell_x = &ncellx_; + n_cell_y = &ncelly_; + n_cell_z = &ncellz_; + x_min = &xmin_; + x_max = &xmax_; + y_min = &ymin_; + y_max = &ymax_; + z_min = &zmin_; + z_max = &zmax_; - printf("(ReadHDF5RhoYeT.cpp) n_cell_x = %d, n_cell_y = %d, n_cell_z = %d\n", *n_cell_x, *n_cell_y, *n_cell_z); - //Allocate managed memory arena on unified memory ManagedArenaAllocator myManagedArena; + ManagedArenaAllocator myManagedArena_int; // Allocate memory for tables double *allbackgroundYeTrhos_temp; - if (!(allbackgroundYeTrhos_temp = myManagedArena.allocate(*n_cell_x * *n_cell_y * *n_cell_z * 3 ) )) { + if (!(allbackgroundYeTrhos_temp = myManagedArena.allocate(ncellx_ * ncelly_ * ncellz_ * 3 ) )) { printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); assert(0); } // Allocate memory for tables - if (!(rho_array_input = myManagedArena.allocate(*n_cell_x * *n_cell_y * *n_cell_z) )) { + if (!(rho_array_input = myManagedArena.allocate(ncellx_ * ncelly_ * ncellz_) )) { printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); assert(0); } - if (!(T_array_input = myManagedArena.allocate(*n_cell_x * *n_cell_y * *n_cell_z) )) { + if (!(T_array_input = myManagedArena.allocate(ncellx_ * ncelly_ * ncellz_) )) { printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); assert(0); } - if (!(Ye_array_input = myManagedArena.allocate(*n_cell_x * *n_cell_y * *n_cell_z) )) { + if (!(Ye_array_input = myManagedArena.allocate(ncellx_ * ncelly_ * ncellz_) )) { printf("(ReadEosTable.cpp) Cannot allocate memory for EOS table"); assert(0); } - + // Prepare HDF5 to read hyperslabs into alltables_temp - hsize_t table_dims[2] = {3, (hsize_t)*n_cell_x * *n_cell_y * *n_cell_z}; + hsize_t table_dims[2] = {3, (hsize_t)ncellx_ * ncelly_ * ncellz_}; hid_t mem3 = H5Screate_simple(2, table_dims, NULL); // Read alltables_temp @@ -146,43 +167,39 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ READ_BCAST_EOSTABLE_HDF5("T_Mev", 1, table_dims); READ_BCAST_EOSTABLE_HDF5("Ye", 2, table_dims); - HDF5_ERROR(H5Sclose(mem3)); HDF5_ERROR(H5Fclose(file)); - for( int k = 0 ; k < *n_cell_x ; k++ ){ - for( int j = 0 ; j < *n_cell_y ; j++ ){ - for(int i = 0 ; i < *n_cell_z ; i++ ) { - int index_old = i + *n_cell_z*(j + *n_cell_y*(k + *n_cell_x )); - int index_new = 0 + i + *n_cell_z*(j + *n_cell_y*k); + for( int k = 0 ; k < ncellx_ ; k++ ){ + for( int j = 0 ; j < ncelly_ ; j++ ){ + for(int i = 0 ; i < ncellz_ ; i++ ) { + int index_old = i + ncellz_*(j + ncelly_*(k + ncellx_ )); + int index_new = 0 + i + ncellz_*(j + ncelly_*k); rho_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; } } } - for( int k = 0 ; k < *n_cell_x ; k++ ){ - for( int j = 0 ; j < *n_cell_y ; j++ ){ - for(int i = 0 ; i < *n_cell_z ; i++ ) { - int index_old = i + *n_cell_z*(j + *n_cell_y*(k + *n_cell_x *2)); - int index_new = 0 + i + *n_cell_z*(j + *n_cell_y*k); + for( int k = 0 ; k < ncellx_ ; k++ ){ + for( int j = 0 ; j < ncelly_ ; j++ ){ + for(int i = 0 ; i < ncellz_ ; i++ ) { + int index_old = i + ncellz_*(j + ncelly_*(k + ncellx_ *2)); + int index_new = 0 + i + ncellz_*(j + ncelly_*k); T_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; } } } - for( int k = 0 ; k < *n_cell_x ; k++ ){ - for( int j = 0 ; j < *n_cell_y ; j++ ){ - for(int i = 0 ; i < *n_cell_z ; i++ ) { - int index_old = i + *n_cell_z*(j + *n_cell_y*(k + *n_cell_x *3)); - int index_new = 0 + i + *n_cell_z*(j + *n_cell_y*k); + for( int k = 0 ; k < ncellx_ ; k++ ){ + for( int j = 0 ; j < ncelly_ ; j++ ){ + for(int i = 0 ; i < ncellz_ ; i++ ) { + int index_old = i + ncellz_*(j + ncelly_*(k + ncellx_ *3)); + int index_new = 0 + i + ncellz_*(j + ncelly_*k); Ye_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; } } } - + // free memory of temporary array - myManagedArena.deallocate(allbackgroundYeTrhos_temp, *n_cell_x * *n_cell_y * *n_cell_z * 3); + myManagedArena.deallocate(allbackgroundYeTrhos_temp, ncellx_ * ncelly_ * ncellz_ * 3); -} // ReadEOSTable - - - +} // ReadEOSTable \ No newline at end of file diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp index 71f3ec61..b0633f30 100644 --- a/Source/ReadInput_RhoTempYe.cpp +++ b/Source/ReadInput_RhoTempYe.cpp @@ -16,12 +16,15 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms //always access mf comp index as (GIdx::rho - start_comp) //Example: Amrex tutorials -> ExampleCodes/MPMD/Case-2/main.cpp. - using namespace background_input_rho_T_Ye; const std::string hdf5_background_rho_Ye_T_name = "rho_Ye_T.hdf5"; ReadInputRhoYeT(hdf5_background_rho_Ye_T_name); - amrex::Print() << "n_cell_x" << n_cell_x << std::endl; - amrex::Print() << "parms->ncell[0]" << parms->ncell[0] << std::endl; + using namespace background_input_rho_T_Ye; + int ncell_x = *n_cell_x; + int ncell_y = *n_cell_y; + int ncell_z = *n_cell_z; + + rhoYeT_input_struct rhoYeT_input_obj(rho_array_input, Ye_array_input, T_array_input); for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ const amrex::Box& bx = mfi.validbox(); @@ -34,12 +37,19 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms amrex::Real x = (i+0.5) * dx[0]; amrex::Real y = (j+0.5) * dx[1]; amrex::Real z = (k+0.5) * dx[2]; + + int ig = i; // FIXME: Modify this based on how you calculate global indices + int jg = j; // FIXME: Modify this based on how you calculate global indices + int kg = k; // FIXME: Modify this based on how you calculate global indices + + // Compute the 1D index from 3D coordinates in the linearized array + int idx = ig + ncell_x * (jg + ncell_y * kg); + + // Set the values from the input arrays + mf_array(i, j, k, GIdx::rho - start_comp) = rhoYeT_input_obj.rho_input[idx]; // Assuming you have a rho_array_input + mf_array(i, j, k, GIdx::T - start_comp) = rhoYeT_input_obj.T_input[idx]; // Assuming you have a T_array_input + mf_array(i, j, k, GIdx::Ye - start_comp) = rhoYeT_input_obj.Ye_input[idx]; // Using Ye_array_input - //TODO: Find the global (i, j, k) from the amrex domain. call them (ig, jg, kg). - //TODO: Then get the values from GPU-array for (ig, jg, kg) and set them to corresponding MultiFabs here. - mf_array(i, j, k, GIdx::rho - start_comp) = -404.0; //FIXME: - mf_array(i, j, k, GIdx::T - start_comp) = -404.0; //FIXME: - mf_array(i, j, k, GIdx::Ye - start_comp) = -404.0; //FIXME: }); } From 6658b828cdc5634236b50b234bcf0b403662ee16 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 20 Sep 2024 16:39:06 -0400 Subject: [PATCH 179/276] Assigning the correct values of densities, temperature, and electron fraction to the mesh MultiFab. --- Source/ReadHDF5RhoYeT.cpp | 33 ++++----------------------------- Source/ReadInput_RhoTempYe.cpp | 30 ++++++++++-------------------- 2 files changed, 14 insertions(+), 49 deletions(-) diff --git a/Source/ReadHDF5RhoYeT.cpp b/Source/ReadHDF5RhoYeT.cpp index 77da1bfd..80f5dfe5 100644 --- a/Source/ReadHDF5RhoYeT.cpp +++ b/Source/ReadHDF5RhoYeT.cpp @@ -136,7 +136,6 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ //Allocate managed memory arena on unified memory ManagedArenaAllocator myManagedArena; - ManagedArenaAllocator myManagedArena_int; // Allocate memory for tables double *allbackgroundYeTrhos_temp; @@ -169,36 +168,12 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ HDF5_ERROR(H5Fclose(file)); - for( int k = 0 ; k < ncellx_ ; k++ ){ - for( int j = 0 ; j < ncelly_ ; j++ ){ - for(int i = 0 ; i < ncellz_ ; i++ ) { - int index_old = i + ncellz_*(j + ncelly_*(k + ncellx_ )); - int index_new = 0 + i + ncellz_*(j + ncelly_*k); - rho_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; - } - } + for(int i = 0 ; i < ncellx_ * ncelly_ * ncellz_ ; i++ ) { + rho_array_input[i] = allbackgroundYeTrhos_temp[ i + 0 * ncellx_ * ncelly_ * ncellz_ ]; + T_array_input [i] = allbackgroundYeTrhos_temp[ i + 1 * ncellx_ * ncelly_ * ncellz_ ]; + Ye_array_input [i] = allbackgroundYeTrhos_temp[ i + 2 * ncellx_ * ncelly_ * ncellz_ ]; } - for( int k = 0 ; k < ncellx_ ; k++ ){ - for( int j = 0 ; j < ncelly_ ; j++ ){ - for(int i = 0 ; i < ncellz_ ; i++ ) { - int index_old = i + ncellz_*(j + ncelly_*(k + ncellx_ *2)); - int index_new = 0 + i + ncellz_*(j + ncelly_*k); - T_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; - } - } - } - - for( int k = 0 ; k < ncellx_ ; k++ ){ - for( int j = 0 ; j < ncelly_ ; j++ ){ - for(int i = 0 ; i < ncellz_ ; i++ ) { - int index_old = i + ncellz_*(j + ncelly_*(k + ncellx_ *3)); - int index_new = 0 + i + ncellz_*(j + ncelly_*k); - Ye_array_input[index_new] = allbackgroundYeTrhos_temp[index_old]; - } - } - } - // free memory of temporary array myManagedArena.deallocate(allbackgroundYeTrhos_temp, ncellx_ * ncelly_ * ncellz_ * 3); diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp index b0633f30..5fe029a8 100644 --- a/Source/ReadInput_RhoTempYe.cpp +++ b/Source/ReadInput_RhoTempYe.cpp @@ -12,15 +12,11 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms MultiFab rho_T_ye_state(state, amrex::make_alias, start_comp, num_comps); amrex::GpuArray dx = geom.CellSizeArray(); - - //always access mf comp index as (GIdx::rho - start_comp) - //Example: Amrex tutorials -> ExampleCodes/MPMD/Case-2/main.cpp. const std::string hdf5_background_rho_Ye_T_name = "rho_Ye_T.hdf5"; ReadInputRhoYeT(hdf5_background_rho_Ye_T_name); using namespace background_input_rho_T_Ye; - int ncell_x = *n_cell_x; int ncell_y = *n_cell_y; int ncell_z = *n_cell_z; @@ -31,25 +27,19 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms const amrex::Array4& mf_array = rho_T_ye_state.array(mfi); amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k){ - - //x, y and z are the coordinates. - //This is not really needed. Cook up some assert statement to make sure we are at the same (x, y, z) that the table is vlaue is referring to. - amrex::Real x = (i+0.5) * dx[0]; - amrex::Real y = (j+0.5) * dx[1]; - amrex::Real z = (k+0.5) * dx[2]; - int ig = i; // FIXME: Modify this based on how you calculate global indices - int jg = j; // FIXME: Modify this based on how you calculate global indices - int kg = k; // FIXME: Modify this based on how you calculate global indices - + int ig = i; + int jg = j; + int kg = k; + // Compute the 1D index from 3D coordinates in the linearized array - int idx = ig + ncell_x * (jg + ncell_y * kg); - - // Set the values from the input arrays - mf_array(i, j, k, GIdx::rho - start_comp) = rhoYeT_input_obj.rho_input[idx]; // Assuming you have a rho_array_input - mf_array(i, j, k, GIdx::T - start_comp) = rhoYeT_input_obj.T_input[idx]; // Assuming you have a T_array_input - mf_array(i, j, k, GIdx::Ye - start_comp) = rhoYeT_input_obj.Ye_input[idx]; // Using Ye_array_input + int idx = kg + ncell_z * (jg + ncell_y * ig); + // Set the values from the input arrays + mf_array(i, j, k, GIdx::rho - start_comp) = rhoYeT_input_obj.rho_input[idx]; + mf_array(i, j, k, GIdx::T - start_comp) = rhoYeT_input_obj.T_input[idx]; + mf_array(i, j, k, GIdx::Ye - start_comp) = rhoYeT_input_obj.Ye_input[idx]; + }); } From 53b986ff0fe0bf5e78b0e357cb9d7f80a1535cc8 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 20 Sep 2024 17:44:05 -0400 Subject: [PATCH 180/276] Stop the code if the number of cells and domain size in the HDF5 background file do not agree with the number of cells and domain size in the parameter file. --- Source/ReadInput_RhoTempYe.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp index 5fe029a8..9bdf9e15 100644 --- a/Source/ReadInput_RhoTempYe.cpp +++ b/Source/ReadInput_RhoTempYe.cpp @@ -17,9 +17,32 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms ReadInputRhoYeT(hdf5_background_rho_Ye_T_name); using namespace background_input_rho_T_Ye; + int ncell_x = *n_cell_x; int ncell_y = *n_cell_y; int ncell_z = *n_cell_z; + double xmin_ = *x_min; + double xmax_ = *x_max; + double ymin_ = *y_min; + double ymax_ = *y_max; + double zmin_ = *z_min; + double zmax_ = *z_max; + double lx = xmax_ - xmin_; + double ly = ymax_ - ymin_; + double lz = zmax_ - zmin_; + amrex::Print() << "ncell_x = " << ncell_x << std::endl; + amrex::Print() << "parms->ncell[0] = " << parms->ncell[0] << std::endl; + + if (ncell_x != parms->ncell[0] || ncell_y != parms->ncell[1] || ncell_z != parms->ncell[2]) { + amrex::Print() << "The number of cells in the background data file does not match the parameter file" << std::endl; + abort(); + } + + if (lx != parms->Lx || ly != parms->Ly || lz != parms->Lz ) { + amrex::Print() << "The simulation domain in the background data file does not match the parameter file" << std::endl; + abort(); + } + rhoYeT_input_struct rhoYeT_input_obj(rho_array_input, Ye_array_input, T_array_input); for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ From 4a97daee4e115b589a9725d78371c653e59a7c2f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 23 Sep 2024 08:26:33 -0400 Subject: [PATCH 181/276] Creating a new parameter to control when the code reads the background quantities from an HDF5 file. --- Source/Parameters.H | 6 ++++++ Source/main.cpp | 5 +---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index 3e909954..ecf58fa9 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -54,6 +54,9 @@ struct TestParams : public amrex::Gpu::Managed // attenuation to hamiltonians Real attenuation_hamiltonians; + // Background matter rho, Ye, T flag + int read_rho_T_Ye_from_table; + //HDF5 table names (with full path) for EoS and NuLib tables std::string nuceos_table_name; std::string nulib_table_name; @@ -118,6 +121,9 @@ struct TestParams : public amrex::Gpu::Managed // attenuation to hamiltonians pp.get("attenuation_hamiltonians", attenuation_hamiltonians); + // Background matter rho, Ye, T flag + pp.get("read_rho_T_Ye_from_table", read_rho_T_Ye_from_table); + // absorption opacities and equilibrium neutrino chemical potentials pp.get("IMFP_method", IMFP_method); if(IMFP_method==0){ diff --git a/Source/main.cpp b/Source/main.cpp index b973be39..204fd372 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -111,15 +111,12 @@ void evolve_flavor(const TestParams* parms) // Create a MultiFab to hold our grid state data and initialize to 0.0 MultiFab state(ba, dm, ncomp, ngrow); - //FIXME: FIXME: Define this in parameter file. - const int read_rho_T_Ye_from_table = 0; - // initialize with NaNs ... state.setVal(0.0); //If reading from table, call function "set_rho_T_Ye". //Else set rho, T and Ye to constant value throughout the grid using values from parameter file. - if (read_rho_T_Ye_from_table){ + if (parms->read_rho_T_Ye_from_table){ set_rho_T_Ye(state, geom, parms); } else { state.setVal(parms->rho_in,GIdx::rho,1); // g/ccm From df87a069c13581b742accfbea9dc6d99e63040a3 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 23 Sep 2024 08:29:44 -0400 Subject: [PATCH 182/276] Adding the new parameter to control when the code reads the background quantities rho, Ye and T to the input files. --- sample_inputs/inputs_1d_fiducial | 5 +++++ sample_inputs/inputs_bipolar_test | 5 +++++ sample_inputs/inputs_coll_equi_test | 5 +++++ sample_inputs/inputs_collisional_instability_test | 5 +++++ sample_inputs/inputs_fast_flavor | 5 +++++ sample_inputs/inputs_fast_flavor_nonzerok | 5 +++++ sample_inputs/inputs_msw_test | 5 +++++ 7 files changed, 35 insertions(+) diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index e15347ca..333c9dd4 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -86,3 +86,8 @@ deltaCP_degrees = 222 # opacity stuff # ################# IMFP_method = 0 + +################# +# Background Ye, T and rho +################# +read_rho_T_Ye_from_table = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 9f12b3e1..c20f028c 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -87,3 +87,8 @@ deltaCP_degrees = 222 # opacity stuff # ################# IMFP_method = 0 + +################# +# Background Ye, T and rho +################# +read_rho_T_Ye_from_table = 0 diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index 5c68dbbe..a1c6f37c 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -111,3 +111,8 @@ IMFP_scat1bar_cm = 0 IMFP_scat2bar_cm = 0 delta_E = 0.8339001570751987 # Mev + +################# +# Background Ye, T and rho +################# +read_rho_T_Ye_from_table = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test index e8304064..089e9646 100644 --- a/sample_inputs/inputs_collisional_instability_test +++ b/sample_inputs/inputs_collisional_instability_test @@ -111,3 +111,8 @@ IMFP_scat1bar_cm = 0 IMFP_scat2bar_cm = 0 delta_E = 2.272540842052914 # Mev + +################# +# Background Ye, T and rho +################# +read_rho_T_Ye_from_table = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index 1c27ba8f..567cf2ff 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -87,3 +87,8 @@ deltaCP_degrees = 222 # opacity stuff # ################# IMFP_method = 0 + +################# +# Background Ye, T and rho +################# +read_rho_T_Ye_from_table = 0 diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 863fe0a4..32ed3dbb 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -87,3 +87,8 @@ deltaCP_degrees = 222 # opacity stuff # ################# IMFP_method = 0 + +################# +# Background Ye, T and rho +################# +read_rho_T_Ye_from_table = 0 diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index 80a3bbea..1387680a 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -87,3 +87,8 @@ deltaCP_degrees = 222 # opacity stuff # ################# IMFP_method = 0 + +################# +# Background Ye, T and rho +################# +read_rho_T_Ye_from_table = 0 From 205d92c6a621b8d5ec871e4b1667b6bc6c7b2bae Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 23 Sep 2024 10:22:05 -0400 Subject: [PATCH 183/276] Create a Python script to generate vacuum particles with different energy bins centers. --- .../st9_empty_particles_multi_energy.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 Scripts/initial_conditions/st9_empty_particles_multi_energy.py diff --git a/Scripts/initial_conditions/st9_empty_particles_multi_energy.py b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py new file mode 100644 index 00000000..6bb43b1d --- /dev/null +++ b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py @@ -0,0 +1,37 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../data_analysis") +from initial_condition_tools import uniform_sphere, write_particles, moment_interpolate_particles, linear_interpolate +import amrex_plot_tools as amrex + +# generation parameters +# MUST MATCH THE INPUTS IN THE EMU INPUT FILE! +nphi_equator = 16 + +# Energy bin centers in Mev -> Multiply to the conversion factor to ergs +# [numbers in brakets are in MeV] +energies = np.array([ 10 , 20 , 30 , 40 , 50 ]) * 1e6*amrex.eV # Energy in Erg + +nnu = np.zeros((2,NF)) +fnu = np.zeros((2,NF,3)) + +# Preallocate a NumPy array for efficiency +n_energies = len(energies) +n_particles, n_variables = moment_interpolate_particles(nphi_equator, nnu, fnu, energies[0], uniform_sphere, linear_interpolate).shape + +# Initialize a NumPy array to store all particles +particles = np.empty((n_energies, n_particles, n_variables)) + +# Fill the particles array using a loop, replacing append +for i, energy_bin in enumerate(energies): + particles[i] = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_bin, uniform_sphere, linear_interpolate) + +# Reshape the particles array +particles = particles.reshape(n_energies * n_particles, n_variables) + +# Write particles initial condition file +write_particles(np.array(particles), NF, "particle_input.dat") From 0cebd634cd7cb670e550a22eab2bd40414c08a84 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 23 Sep 2024 10:54:41 -0400 Subject: [PATCH 184/276] Now create empty particles at the energy bin center of the Nulib table. --- .../st9_empty_particles_multi_energy.py | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/Scripts/initial_conditions/st9_empty_particles_multi_energy.py b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py index 6bb43b1d..703cf9f7 100644 --- a/Scripts/initial_conditions/st9_empty_particles_multi_energy.py +++ b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py @@ -1,3 +1,7 @@ +''' +Created by Erick Urquilla, Department of Physics and Astronomy, University of Tennessee, Knoxville. +This script is used to create empty particles at the energy bin center of the Nulib table. +''' import h5py import numpy as np import sys @@ -10,24 +14,32 @@ # generation parameters # MUST MATCH THE INPUTS IN THE EMU INPUT FILE! -nphi_equator = 16 - -# Energy bin centers in Mev -> Multiply to the conversion factor to ergs -# [numbers in brakets are in MeV] -energies = np.array([ 10 , 20 , 30 , 40 , 50 ]) * 1e6*amrex.eV # Energy in Erg - +nphi_equator = 16 # number of direction in equator +NF = 3 # number of flavors + +# Energy bin centers extracted from NuLib table +energies_center_Mev = [1, 3, 5.23824, 8.00974, 11.4415, 15.6909, 20.9527, 27.4681, 35.5357, 45.5254, 57.8951, 73.2117, 92.1775, 115.662, 144.741, 180.748, 225.334, 280.542] # Energy in Mev +# Energy bin bottom extracted from NuLib table +energies_bottom_Mev = [0, 2, 4, 6.47649, 9.54299, 13.3401, 18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250] +# Energy bin top extracted from NuLib table +energies_top_Mev = [2, 4, 6.47649, 9.54299, 13.3401, 18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250, 311.085] +# Energies in ergs +energies_center_erg = np.array(energies_center_Mev) * 1e6*amrex.eV # Energy in ergs + +# Set zero number density nnu = np.zeros((2,NF)) +# Set zero number density flux fnu = np.zeros((2,NF,3)) # Preallocate a NumPy array for efficiency -n_energies = len(energies) -n_particles, n_variables = moment_interpolate_particles(nphi_equator, nnu, fnu, energies[0], uniform_sphere, linear_interpolate).shape +n_energies = len(energies_center_erg) +n_particles, n_variables = moment_interpolate_particles(nphi_equator, nnu, fnu, energies_center_erg[0], uniform_sphere, linear_interpolate).shape # Initialize a NumPy array to store all particles particles = np.empty((n_energies, n_particles, n_variables)) # Fill the particles array using a loop, replacing append -for i, energy_bin in enumerate(energies): +for i, energy_bin in enumerate(energies_center_erg): particles[i] = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_bin, uniform_sphere, linear_interpolate) # Reshape the particles array From 5b31222b0b7e6738e6cf385609a422ed1a5c1d94 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 23 Sep 2024 15:08:09 -0400 Subject: [PATCH 185/276] Deleting unnecessary printing files. --- Source/ReadInput_RhoTempYe.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp index 9bdf9e15..0de1202b 100644 --- a/Source/ReadInput_RhoTempYe.cpp +++ b/Source/ReadInput_RhoTempYe.cpp @@ -29,9 +29,6 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms double lx = xmax_ - xmin_; double ly = ymax_ - ymin_; double lz = zmax_ - zmin_; - - amrex::Print() << "ncell_x = " << ncell_x << std::endl; - amrex::Print() << "parms->ncell[0] = " << parms->ncell[0] << std::endl; if (ncell_x != parms->ncell[0] || ncell_y != parms->ncell[1] || ncell_z != parms->ncell[2]) { amrex::Print() << "The number of cells in the background data file does not match the parameter file" << std::endl; From cf915b6d104bf4c1bba081aaf9d9ad8c78cd0539 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 23 Sep 2024 15:09:27 -0400 Subject: [PATCH 186/276] Solve issue with data type when reading doubles from an HDF5 file --- Source/ReadHDF5RhoYeT.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Source/ReadHDF5RhoYeT.cpp b/Source/ReadHDF5RhoYeT.cpp index 80f5dfe5..ed05ebc4 100644 --- a/Source/ReadHDF5RhoYeT.cpp +++ b/Source/ReadHDF5RhoYeT.cpp @@ -115,14 +115,16 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ READ_BCAST_EOS_HDF5("ncellsx", &ncellx_, H5T_NATIVE_INT, H5S_ALL, 1); READ_BCAST_EOS_HDF5("ncellsy", &ncelly_, H5T_NATIVE_INT, H5S_ALL, 1); READ_BCAST_EOS_HDF5("ncellsz", &ncellz_, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("xmin_cm", &xmin_, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("ymin_cm", &ymin_, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("zmin_cm", &zmin_, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("xmax_cm", &xmax_, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("ymax_cm", &ymax_, H5T_NATIVE_INT, H5S_ALL, 1); - READ_BCAST_EOS_HDF5("zmax_cm", &zmax_, H5T_NATIVE_INT, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("xmin_cm", &xmin_, H5T_NATIVE_DOUBLE, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ymin_cm", &ymin_, H5T_NATIVE_DOUBLE, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("zmin_cm", &zmin_, H5T_NATIVE_DOUBLE, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("xmax_cm", &xmax_, H5T_NATIVE_DOUBLE, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("ymax_cm", &ymax_, H5T_NATIVE_DOUBLE, H5S_ALL, 1); + READ_BCAST_EOS_HDF5("zmax_cm", &zmax_, H5T_NATIVE_DOUBLE, H5S_ALL, 1); printf("(ReadHDF5RhoYeT.cpp) ncellx_ = %d, ncelly_ = %d, ncellz_ = %d\n", ncellx_, ncelly_, ncellz_); + printf("(ReadHDF5RhoYeT.cpp) xmin_ = %f, ymin_ = %f, zmin_ = %f\n", xmin_, ymin_, zmin_); + printf("(ReadHDF5RhoYeT.cpp) xmax_ = %f, ymax_ = %f, zmax_ = %f\n", xmax_, ymax_, zmax_); n_cell_x = &ncellx_; n_cell_y = &ncelly_; @@ -133,7 +135,7 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ y_max = &ymax_; z_min = &zmin_; z_max = &zmax_; - + //Allocate managed memory arena on unified memory ManagedArenaAllocator myManagedArena; From 97495ee56518cd1b6bb46d1e2307a79c569b240a Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 23 Sep 2024 17:13:06 -0400 Subject: [PATCH 187/276] Create an initial condition input parameter for the Fermi-Dirac test. --- sample_inputs/inputs_fermi_dirac_test | 121 ++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 sample_inputs/inputs_fermi_dirac_test diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test new file mode 100644 index 00000000..43a309da --- /dev/null +++ b/sample_inputs/inputs_fermi_dirac_test @@ -0,0 +1,121 @@ +perturbation_type = 0 +perturbation_amplitude = 1e-6 + +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 0 + +collision_cfl_factor = 0.005 +cfl_factor = 0.005 +flavor_cfl_factor = 0.005 +max_adaptive_speedup = 0 +maxError = 1e-6 + +integration.type = 1 +integration.rk.type = 4 + +# Domain size in 3D index space +ncell = (2, 2, 2) +Lx = 2.0 +Ly = 2.0 +Lz = 2.0 + +# Number of particles per cell +nppc = (1, 1, 1) +particle_data_filename = "particle_input.dat" + +# Maximum size of each grid in the domain +max_grid_size = 16 + +# Number of steps to run +nsteps = 1 + +# Simulation end time +end_time = 5.0e9 + +# Make FPE signal errors so we get a Backtrace +amrex.fpe_trap_invalid=1 + +# give background fluid conditions +rho_g_ccm = 0 +T_MeV = 10 +Ye = 1 + +# Write plotfiles +write_plot_every = 1 + +# Write particle data in plotfiles +write_plot_particles_every = 1 + +# checkpointing +do_restart = 0 +restart_dir = "" + +############################### +# NEUTRINO PHYSICS PARAMETERS # +############################### +# see first column of table 14.7 in http://pdg.lbl.gov/2019/reviews/rpp2019-rev-neutrino-mixing.pdf + +# mass state 1 mass in eV [NO/IO:-sqrt(7.39e-5)] +mass1_eV = 0 #-0.008596511 + +# mass state 2 mass in eV (define at 0 arbitrarily because oscillations only sensitive to deltaM^2) +mass2_eV = 0 + +# mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] +mass3_eV = 0.049487372 + +# 1-2 mixing angle in degrees [NO/IO:33.82] +theta12_degrees = 1e-6 + +# 2-3 mixing angle in degrees [NO:8.61 IO:8.65] +theta23_degrees = 8.61 + +# 1-3 mixing angle in degrees [NO:48.3 IO:48.6] +theta13_degrees = 48.3 + +# Majorana angle 1 in degrees +alpha1_degrees = 0 + +# Majorana angle 2 in degrees +alpha2_degrees = 0 + +# CP-violating phase in degrees [NO:222 IO:285] +deltaCP_degrees = 222 + +################# +# opacity stuff # +################# +IMFP_method = 2 + +Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. + +IMFP_abs0_cm = 5e-4 +IMFP_abs1_cm = 5e-4 +IMFP_abs2_cm = 5e-4 +IMFP_abs0bar_cm = 5e-4 +IMFP_abs1bar_cm = 5e-4 +IMFP_abs2bar_cm = 5e-4 + +munu0_MeV = 0 +munu1_MeV = 0 +munu2_MeV = 0 +munu0bar_MeV = 0 +munu1bar_MeV = 0 +munu2bar_MeV = 0 + +IMFP_scat0_cm = 0 +IMFP_scat1_cm = 0 +IMFP_scat2_cm = 0 +IMFP_scat0bar_cm = 0 +IMFP_scat1bar_cm = 0 +IMFP_scat2bar_cm = 0 + +delta_E = 0.8339001570751987 # Mev + +################# +# Background Ye, T and rho +################# +read_rho_T_Ye_from_table = 1 + +nulib_table_name = "/mnt/scratch/tables/NuLib/NuLib_SFHo.h5" +nuceos_table_name = "LS220.h5" From 9352d16d7f95ca0cb29cce59f8102c1782d1b7c2 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 23 Sep 2024 19:02:00 -0400 Subject: [PATCH 188/276] Compute the delta phase space in the input file of the particles. --- Scripts/data_reduction/amrex_plot_tools.py | 8 +++-- .../st9_empty_particles_multi_energy.py | 34 +++++++++++++------ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/Scripts/data_reduction/amrex_plot_tools.py b/Scripts/data_reduction/amrex_plot_tools.py index ee717d97..77c679b5 100644 --- a/Scripts/data_reduction/amrex_plot_tools.py +++ b/Scripts/data_reduction/amrex_plot_tools.py @@ -33,7 +33,9 @@ def get_particle_keys(NF, ignore_pos=False, xp_only=False): "N00_Rebar", "N01_Rebar", "N01_Imbar", - "N11_Rebar"] + "N11_Rebar", + "TrHN", + "Vphase"] if(NF==3): real_quantities = ["pos_x", "pos_y", @@ -63,7 +65,9 @@ def get_particle_keys(NF, ignore_pos=False, xp_only=False): "N11_Rebar", "N12_Rebar", "N12_Imbar", - "N22_Rebar"] + "N22_Rebar" + "TrHN", + "Vphase"] if xp_only: real_quantities = real_quantities[:11] if ignore_pos: real_quantities = real_quantities[7:] diff --git a/Scripts/initial_conditions/st9_empty_particles_multi_energy.py b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py index 703cf9f7..0025378f 100644 --- a/Scripts/initial_conditions/st9_empty_particles_multi_energy.py +++ b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py @@ -23,27 +23,41 @@ energies_bottom_Mev = [0, 2, 4, 6.47649, 9.54299, 13.3401, 18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250] # Energy bin top extracted from NuLib table energies_top_Mev = [2, 4, 6.47649, 9.54299, 13.3401, 18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250, 311.085] + # Energies in ergs energies_center_erg = np.array(energies_center_Mev) * 1e6*amrex.eV # Energy in ergs +energies_bottom_erg = np.array(energies_bottom_Mev) * 1e6*amrex.eV # Energy in ergs +energies_top_erg = np.array(energies_top_Mev ) * 1e6*amrex.eV # Energy in ergs -# Set zero number density -nnu = np.zeros((2,NF)) -# Set zero number density flux -fnu = np.zeros((2,NF,3)) - -# Preallocate a NumPy array for efficiency +# Gen the number of energy bins n_energies = len(energies_center_erg) -n_particles, n_variables = moment_interpolate_particles(nphi_equator, nnu, fnu, energies_center_erg[0], uniform_sphere, linear_interpolate).shape + +# get variable keys +rkey, ikey = amrex.get_particle_keys(NF, ignore_pos=True) + +# Gen the number of variables that describe each particle +n_variables = len(rkey) + +# Get the momentum distribution of the particles +phat = uniform_sphere(nphi_equator) + +# Gen the number of directions +n_directions = len(phat) + +# Gen the number of particles +n_particles = n_energies * n_directions # Initialize a NumPy array to store all particles -particles = np.empty((n_energies, n_particles, n_variables)) +particles = np.empty((n_energies, n_directions, n_variables)) # Fill the particles array using a loop, replacing append for i, energy_bin in enumerate(energies_center_erg): - particles[i] = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_bin, uniform_sphere, linear_interpolate) + particles[i , : , rkey["pupx"] : rkey["pupz"]+1 ] = energy_bin * phat + particles[i , : , rkey["pupt"] ] = energy_bin + particles[i , : , rkey["Vphase"] ] = ( 4.0 * np.pi / n_directions ) * ( ( energies_top_erg[i] ** 3 - energies_bottom_erg[i] ** 3 ) / 3.0 ) # Reshape the particles array -particles = particles.reshape(n_energies * n_particles, n_variables) +particles = particles.reshape(n_energies * n_directions, n_variables) # Write particles initial condition file write_particles(np.array(particles), NF, "particle_input.dat") From 7a3992d6c0c03ec39ce7ad76a063022d7a21a771 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 24 Sep 2024 08:51:37 -0400 Subject: [PATCH 189/276] Remove unneeded semicolon. --- Source/FlavoredNeutrinoContainerInit.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index f05e2335..c73f7dd0 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -122,7 +122,7 @@ InitParticles(const TestParams* parms) *parms->nppc[2]); // array of direction vectors - Gpu::ManagedVector > particle_data = read_particle_data(parms->particle_data_filename);; + Gpu::ManagedVector > particle_data = read_particle_data(parms->particle_data_filename); auto* particle_data_p = particle_data.dataPtr(); // determine the number of directions per location From 1ed493bb0b5a4831161f28f900458d60b778dbad Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 16:38:47 -0400 Subject: [PATCH 190/276] Redefining the phase space volume calculation based on precomputed quantities in the input script. By definition the phase space volume is Vphase = dx^3 * dOmega * dE^3 / 3 From initial conditions, Vphase gets dOmega * dE^3 / 3 In the new lines I multiply this value by the cell volume dx[0] * dx[1] * dx[2] Divide by the number of particle emission points inside the cell --- Source/FlavoredNeutrinoContainerInit.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index c73f7dd0..a95a534b 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -287,10 +287,12 @@ InitParticles(const TestParams* parms) p.rdata(PIdx::N22_Re ) *= scale_fac; p.rdata(PIdx::N22_Rebar) *= scale_fac; #endif - - if(parms->IMFP_method == 1){ - p.rdata(PIdx::Vphase) = dx[0]*dx[1]*dx[2]*4*MathConst::pi*(pow(p.rdata(PIdx::pupt)+parms->delta_E/2,3)-pow(p.rdata(PIdx::pupt)-parms->delta_E/2,3))/(3*ndirs_per_loc*parms->nppc[0]*parms->nppc[1]*parms->nppc[2]); - } + + // Set phase space volume Vphase = dx^3 * dOmega * dE^3 / 3 + // From initial conditions, Vphase gets dOmega * dE^3 / 3 + // Here we multiply this value by the cell volume dx[0] * dx[1] * dx[2] + // Divide by the number of particle emission points inside the cell + p.rdata(PIdx::Vphase) *= dx[0]*dx[1]*dx[2] / nlocs_per_cell ; //=====================// // Apply Perturbations // From d18c46ae98a9eebeef6fa6c016a7efea8ee7bc26 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 24 Sep 2024 09:31:31 -0400 Subject: [PATCH 191/276] Generate comments in Doxygen style for a function that reads the initial conditions of the particles. --- Source/FlavoredNeutrinoContainerInit.cpp | 26 ++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index a95a534b..7eee3c79 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -9,12 +9,30 @@ using namespace amrex; //=========================================// // Particle distribution in momentum space // //=========================================// - +/** + * @brief Reads the input file containing the initial conditions of the particles. + * + * This function reads the particle data from a file generated using the input Python scripts + * in the directory "Scripts/initial_conditions/---.py" and stores the momentum, energy, and flavor + * occupation matrices for neutrinos and antineutrinos. The file is expected to follow a specific + * format, with the number of flavors in the first line, followed by particle data in each + * subsequent line in the following order (2-flavor case): E*phatx, E*phaty, E*phatz, E, + * N00_Re, N01_Re, N01_Im, N11_Re, N00_Rebar, N01_Rebar, N01_Imbar, N11_Rebar, TrHN, Vphase. + * This can be generalized to the 3-flavor case. + * + * @param filename The name of the input file containing the particle data. + * + * @return A managed vector of GpuArray containing the particle information. + * Each GpuArray stores particle attributes: E*phatx, E*phaty, E*phatz, E, + * N00_Re, N01_Re, N01_Im, N11_Re, N00_Rebar, N01_Rebar, N01_Imbar, N11_Rebar, + * TrHN, Vphase. + * + * @note The file should contain the number of neutrino flavors in the first line, + * which must match the value that Emu was compiled with. If the number of flavors + * does not match, the function will print an error message and terminate execution. + */ Gpu::ManagedVector> read_particle_data(std::string filename){ - // This function reads the input file containing the initial conditions of the particles. - // It reads the momentum, energy, and flavor occupation matrices for neutrinos and antineutrinos. - // This array will save the particles information Gpu::ManagedVector> particle_data; From 0a7dfc396ea4f6375183064c10bf5039924dd4a1 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 24 Sep 2024 08:45:35 -0400 Subject: [PATCH 192/276] Using np.zeros when generating the initial condition for multi-energy particles. --- Scripts/initial_conditions/st9_empty_particles_multi_energy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/initial_conditions/st9_empty_particles_multi_energy.py b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py index 0025378f..d94e2355 100644 --- a/Scripts/initial_conditions/st9_empty_particles_multi_energy.py +++ b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py @@ -48,7 +48,7 @@ n_particles = n_energies * n_directions # Initialize a NumPy array to store all particles -particles = np.empty((n_energies, n_directions, n_variables)) +particles = np.zeros((n_energies, n_directions, n_variables)) # Fill the particles array using a loop, replacing append for i, energy_bin in enumerate(energies_center_erg): From d7516aa49591dae04423d0db9db4f9b59905511e Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 24 Sep 2024 09:47:06 -0400 Subject: [PATCH 193/276] Resolve issue with the colon. --- Scripts/data_reduction/amrex_plot_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/data_reduction/amrex_plot_tools.py b/Scripts/data_reduction/amrex_plot_tools.py index 77c679b5..97e7a65a 100644 --- a/Scripts/data_reduction/amrex_plot_tools.py +++ b/Scripts/data_reduction/amrex_plot_tools.py @@ -65,7 +65,7 @@ def get_particle_keys(NF, ignore_pos=False, xp_only=False): "N11_Rebar", "N12_Rebar", "N12_Imbar", - "N22_Rebar" + "N22_Rebar", "TrHN", "Vphase"] From 60b976216b0a6b677ea0ee307ff601f3fbb5a3a0 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 24 Sep 2024 09:55:23 -0400 Subject: [PATCH 194/276] Cleaning the Python script that generates vacuum multi-energy particles. --- .../st9_empty_particles_multi_energy.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Scripts/initial_conditions/st9_empty_particles_multi_energy.py b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py index d94e2355..4ec43d26 100644 --- a/Scripts/initial_conditions/st9_empty_particles_multi_energy.py +++ b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py @@ -2,14 +2,13 @@ Created by Erick Urquilla, Department of Physics and Astronomy, University of Tennessee, Knoxville. This script is used to create empty particles at the energy bin center of the Nulib table. ''' -import h5py import numpy as np import sys import os importpath = os.path.dirname(os.path.realpath(__file__)) sys.path.append(importpath) sys.path.append(importpath+"/../data_analysis") -from initial_condition_tools import uniform_sphere, write_particles, moment_interpolate_particles, linear_interpolate +from initial_condition_tools import uniform_sphere, write_particles import amrex_plot_tools as amrex # generation parameters @@ -29,22 +28,22 @@ energies_bottom_erg = np.array(energies_bottom_Mev) * 1e6*amrex.eV # Energy in ergs energies_top_erg = np.array(energies_top_Mev ) * 1e6*amrex.eV # Energy in ergs -# Gen the number of energy bins +# Generate the number of energy bins n_energies = len(energies_center_erg) -# get variable keys +# Get variable keys rkey, ikey = amrex.get_particle_keys(NF, ignore_pos=True) -# Gen the number of variables that describe each particle +# Generate the number of variables that describe each particle n_variables = len(rkey) # Get the momentum distribution of the particles phat = uniform_sphere(nphi_equator) -# Gen the number of directions +# Generate the number of directions n_directions = len(phat) -# Gen the number of particles +# Generate the number of particles n_particles = n_energies * n_directions # Initialize a NumPy array to store all particles From 2704addb8ebeefc5d3f0338369ca90e63f149501 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 24 Sep 2024 10:54:21 -0400 Subject: [PATCH 195/276] Modify the Python script that generates the initial conditions for the collisional instability test to compute the phase space volume. Warning: This modification is not passing the test. --- .../initial_conditions/st8_coll_inst_test.py | 95 +++++++++++++------ 1 file changed, 68 insertions(+), 27 deletions(-) diff --git a/Scripts/initial_conditions/st8_coll_inst_test.py b/Scripts/initial_conditions/st8_coll_inst_test.py index 0b925b21..5a580a33 100644 --- a/Scripts/initial_conditions/st8_coll_inst_test.py +++ b/Scripts/initial_conditions/st8_coll_inst_test.py @@ -1,4 +1,9 @@ -import h5py +''' +Created by Erick Urquilla, Department of Physics and Astronomy, University of Tennessee, Knoxville. +This script is used to create the particle initial conditions that attempt to replicate the +simulation in the paper Collisional Flavor Instabilities of Supernova Neutrinos by L. Johns +[2104.11369]. +''' import numpy as np import sys import os @@ -8,29 +13,65 @@ from initial_condition_tools import uniform_sphere, moment_interpolate_particles, minerbo_interpolate, write_particles import amrex_plot_tools as amrex -# These initial conditions are intended to replicate the collisional instability outputs in "Collisional Flavor Instabilities of Supernova Neutrinos" by L. Johns [2104.11369]. -# Simulation parameters -NF = 2 -nphi_equator = 16 -nnue = 3.0e+33 # 1/ccm -nnua = 2.5e+33 # 1/ccm -nnux = 1.0e+33 # 1/ccm -fnue = np.array([0.0 , 0.0 , 0.0]) -fnua = np.array([0.0 , 0.0 , 0.0]) -fnux = np.array([0.0 , 0.0 , 0.0]) -energy_erg = 20.0 # MeV -energy_erg *= 1e6*amrex.eV # erg - -nnu = np.zeros((2,NF)) -nnu[0,0] = nnue -nnu[1,0] = nnua -nnu[:,1:] = nnux - -fnu = np.zeros((2,NF,3)) -fnu[0,0,:] = nnue * fnue -fnu[1,0,:] = nnua * fnua -fnu[:,1:,:] = nnu[:,1:,np.newaxis] * fnux[np.newaxis,np.newaxis,:] - -particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, minerbo_interpolate) # [particle, variable] - -write_particles(np.array(particles), NF, "particle_input.dat") +# These initial conditions are intended to replicate the collisional instability outputs in +# "Collisional Flavor Instabilities of Supernova Neutrinos" by L. Johns [2104.11369]. + +NF = 2 # Number of flavors +nphi_equator = 16 # number of direction in equator ---> theta = pi/2 + +nu_e = 3.0e+33 # 1/ccm +nu_x = 1.0e+33 # 1/ccm +nu_ebar = 2.5e+33 # 1/ccm +nu_xbar = 1.0e+33 # 1/ccm + +# Energy bin size +energy_bin_size_MeV = 2.272540842052914 # Energy in Mev + +# Energy bin centers extracted from NuLib table +energies_center_Mev = [20.0] # Energy in Mev +# Energy bin bottom extracted from NuLib table +energies_bottom_Mev = [20.0-energy_bin_size_MeV/2.0] +# Energy bin top extracted from NuLib table +energies_top_Mev = [20.0+energy_bin_size_MeV/2.0] + +# Energies in ergs +energies_center_erg = np.array(energies_center_Mev) * 1e6*amrex.eV # Energy in ergs +energies_bottom_erg = np.array(energies_bottom_Mev) * 1e6*amrex.eV # Energy in ergs +energies_top_erg = np.array(energies_top_Mev ) * 1e6*amrex.eV # Energy in ergs + +# Generate the number of energy bins +n_energies = len(energies_center_erg) + +# Get variable keys +rkey, ikey = amrex.get_particle_keys(NF, ignore_pos=True) + +# Generate the number of variables that describe each particle +n_variables = len(rkey) + +# Get the momentum distribution of the particles +phat = uniform_sphere(nphi_equator) + +# Generate the number of directions +n_directions = len(phat) + +# Generate the number of particles +n_particles = n_energies * n_directions + +# Initialize a NumPy array to store all particles +particles = np.zeros((n_energies, n_directions, n_variables)) + +# Fill the particles array using a loop, replacing append +for i, energy_bin in enumerate(energies_center_erg): + particles[i , : , rkey["pupx"] : rkey["pupz"]+1 ] = energy_bin * phat + particles[i , : , rkey["pupt"] ] = energy_bin + particles[i , : , rkey["Vphase"] ] = ( 4.0 * np.pi / n_directions ) * ( ( energies_top_erg[i] ** 3 - energies_bottom_erg[i] ** 3 ) / 3.0 ) + particles[i , : , rkey["N00_Re"] ] = nu_e + particles[i , : , rkey["N11_Re"] ] = nu_x + particles[i , : , rkey["N00_Rebar"] ] = nu_ebar + particles[i , : , rkey["N11_Rebar"] ] = nu_xbar + +# Reshape the particles array +particles = particles.reshape(n_energies * n_directions, n_variables) + +# Write particles initial condition file +write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file From 11a35b1f97aae3e79090333ff57a27d00b0a59a1 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 24 Sep 2024 11:19:31 -0400 Subject: [PATCH 196/276] Simplifying the calculation of the phase space volume. The collisional instability test is still failing. --- Source/FlavoredNeutrinoContainerInit.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 7eee3c79..15362e1d 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -310,7 +310,7 @@ InitParticles(const TestParams* parms) // From initial conditions, Vphase gets dOmega * dE^3 / 3 // Here we multiply this value by the cell volume dx[0] * dx[1] * dx[2] // Divide by the number of particle emission points inside the cell - p.rdata(PIdx::Vphase) *= dx[0]*dx[1]*dx[2] / nlocs_per_cell ; + p.rdata(PIdx::Vphase) *= scale_fac ; //=====================// // Apply Perturbations // From 53d4460c91eedceb5b6bd06aa01a4eef6a50749b Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 24 Sep 2024 14:58:16 -0400 Subject: [PATCH 197/276] Solve issue with initial condition not reproducing the right neutrino number density in collisional stability test --- .../initial_conditions/st8_coll_inst_test.py | 63 +++++++++---------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/Scripts/initial_conditions/st8_coll_inst_test.py b/Scripts/initial_conditions/st8_coll_inst_test.py index 5a580a33..665b44c1 100644 --- a/Scripts/initial_conditions/st8_coll_inst_test.py +++ b/Scripts/initial_conditions/st8_coll_inst_test.py @@ -19,59 +19,54 @@ NF = 2 # Number of flavors nphi_equator = 16 # number of direction in equator ---> theta = pi/2 -nu_e = 3.0e+33 # 1/ccm -nu_x = 1.0e+33 # 1/ccm -nu_ebar = 2.5e+33 # 1/ccm -nu_xbar = 1.0e+33 # 1/ccm +# Neutrino number densities +nnue = 3.0e+33 # 1/ccm +nnua = 2.5e+33 # 1/ccm +nnux = 1.0e+33 # 1/ccm + +# Neutrino flux factors +fnue = np.array([0.0 , 0.0 , 0.0]) +fnua = np.array([0.0 , 0.0 , 0.0]) +fnux = np.array([0.0 , 0.0 , 0.0]) # Energy bin size energy_bin_size_MeV = 2.272540842052914 # Energy in Mev # Energy bin centers extracted from NuLib table -energies_center_Mev = [20.0] # Energy in Mev +energies_center_Mev = 20.0 # Energy in Mev # Energy bin bottom extracted from NuLib table -energies_bottom_Mev = [20.0-energy_bin_size_MeV/2.0] +energies_bottom_Mev = 20.0-energy_bin_size_MeV/2.0 # Energy bin top extracted from NuLib table -energies_top_Mev = [20.0+energy_bin_size_MeV/2.0] +energies_top_Mev = 20.0+energy_bin_size_MeV/2.0 # Energies in ergs energies_center_erg = np.array(energies_center_Mev) * 1e6*amrex.eV # Energy in ergs energies_bottom_erg = np.array(energies_bottom_Mev) * 1e6*amrex.eV # Energy in ergs energies_top_erg = np.array(energies_top_Mev ) * 1e6*amrex.eV # Energy in ergs -# Generate the number of energy bins -n_energies = len(energies_center_erg) - -# Get variable keys -rkey, ikey = amrex.get_particle_keys(NF, ignore_pos=True) +# Matrix to save the neutrino number densities +nnu = np.zeros((2,NF)) +nnu[0,0] = nnue +nnu[1,0] = nnua +nnu[:,1:] = nnux -# Generate the number of variables that describe each particle -n_variables = len(rkey) +# Matrix to save the neutrino number densities fluxes +fnu = np.zeros((2,NF,3)) +fnu[0,0,:] = nnue * fnue +fnu[1,0,:] = nnua * fnua +fnu[:,1:,:] = nnu[:,1:,np.newaxis] * fnux[np.newaxis,np.newaxis,:] -# Get the momentum distribution of the particles -phat = uniform_sphere(nphi_equator) +# Generate particles +particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energies_center_erg, uniform_sphere, minerbo_interpolate) # [particle, variable] # Generate the number of directions -n_directions = len(phat) - -# Generate the number of particles -n_particles = n_energies * n_directions - -# Initialize a NumPy array to store all particles -particles = np.zeros((n_energies, n_directions, n_variables)) +n_directions = len(particles) -# Fill the particles array using a loop, replacing append -for i, energy_bin in enumerate(energies_center_erg): - particles[i , : , rkey["pupx"] : rkey["pupz"]+1 ] = energy_bin * phat - particles[i , : , rkey["pupt"] ] = energy_bin - particles[i , : , rkey["Vphase"] ] = ( 4.0 * np.pi / n_directions ) * ( ( energies_top_erg[i] ** 3 - energies_bottom_erg[i] ** 3 ) / 3.0 ) - particles[i , : , rkey["N00_Re"] ] = nu_e - particles[i , : , rkey["N11_Re"] ] = nu_x - particles[i , : , rkey["N00_Rebar"] ] = nu_ebar - particles[i , : , rkey["N11_Rebar"] ] = nu_xbar +# Compute the phase space volume dOmega * dE^3 / 3 +Vphase = ( 4.0 * np.pi / n_directions ) * ( ( energies_top_erg ** 3 - energies_bottom_erg ** 3 ) / 3.0 ) -# Reshape the particles array -particles = particles.reshape(n_energies * n_directions, n_variables) +# Save V_phase in the last column of the particle array +particles[:,-1] = Vphase # Write particles initial condition file write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file From 6c7a71706f50ef1c834b6b6ae56cc508c10720e4 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 24 Sep 2024 15:13:02 -0400 Subject: [PATCH 198/276] Update the initial condition script for monoenergetic vacuum particles in the collision-to-equilibrium test. --- .../initial_conditions/st7_empty_particles.py | 74 ++++++++++++++++--- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/Scripts/initial_conditions/st7_empty_particles.py b/Scripts/initial_conditions/st7_empty_particles.py index dca380ad..5fa0b749 100644 --- a/Scripts/initial_conditions/st7_empty_particles.py +++ b/Scripts/initial_conditions/st7_empty_particles.py @@ -1,22 +1,72 @@ -import h5py +''' +Created by Erick Urquilla, Department of Physics and Astronomy, University of Tennessee, Knoxville. +This script is used to create the empty monoenergetic particles +''' import numpy as np import sys import os importpath = os.path.dirname(os.path.realpath(__file__)) sys.path.append(importpath) -sys.path.append(importpath+"/../data_analysis") -from initial_condition_tools import uniform_sphere, write_particles, moment_interpolate_particles, linear_interpolate +sys.path.append(importpath+"/../data_reduction") +from initial_condition_tools import uniform_sphere, write_particles import amrex_plot_tools as amrex -# generation parameters -# MUST MATCH THE INPUTS IN THE EMU INPUT FILE! -NF = 3 -nphi_equator = 16 -energy_erg = 50 * 1e6*amrex.eV +NF = 3 # Number of flavors +nphi_equator = 16 # number of direction in equator ---> theta = pi/2 -nnu = np.zeros((2,NF)) -fnu = np.zeros((2,NF,3)) +nu_e = 0.0 # 1/ccm +nu_x = 0.0 # 1/ccm +nu_ebar = 0.0 # 1/ccm +nu_xbar = 0.0 # 1/ccm -particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, linear_interpolate) # [particle, variable] +# Energy bin size +energy_bin_size_MeV = 0.8339001570751987 # Energy in Mev -write_particles(np.array(particles), NF, "particle_input.dat") +# Energy bin centers extracted from NuLib table +energies_center_Mev = [50.0] # Energy in Mev +# Energy bin bottom extracted from NuLib table +energies_bottom_Mev = [50.0-energy_bin_size_MeV/2.0] +# Energy bin top extracted from NuLib table +energies_top_Mev = [50.0+energy_bin_size_MeV/2.0] + +# Energies in ergs +energies_center_erg = np.array(energies_center_Mev) * 1e6*amrex.eV # Energy in ergs +energies_bottom_erg = np.array(energies_bottom_Mev) * 1e6*amrex.eV # Energy in ergs +energies_top_erg = np.array(energies_top_Mev ) * 1e6*amrex.eV # Energy in ergs + +# Generate the number of energy bins +n_energies = len(energies_center_erg) + +# Get variable keys +rkey, ikey = amrex.get_particle_keys(NF, ignore_pos=True) + +# Generate the number of variables that describe each particle +n_variables = len(rkey) + +# Get the momentum distribution of the particles +phat = uniform_sphere(nphi_equator) + +# Generate the number of directions +n_directions = len(phat) + +# Generate the number of particles +n_particles = n_energies * n_directions + +# Initialize a NumPy array to store all particles +particles = np.zeros((n_energies, n_directions, n_variables)) + +# Fill the particles array using a loop, replacing append +for i, energy_bin in enumerate(energies_center_erg): + particles[i , : , rkey["pupx"] : rkey["pupz"]+1 ] = energy_bin * phat + particles[i , : , rkey["pupt"] ] = energy_bin + particles[i , : , rkey["Vphase"] ] = ( 4.0 * np.pi / n_directions ) * ( ( energies_top_erg[i] ** 3 - energies_bottom_erg[i] ** 3 ) / 3.0 ) + particles[i , : , rkey["N00_Re"] ] = nu_e + particles[i , : , rkey["N11_Re"] ] = nu_x + particles[i , : , rkey["N00_Rebar"] ] = nu_ebar + particles[i , : , rkey["N11_Rebar"] ] = nu_xbar + +# Reshape the particles array +particles = particles.reshape(n_energies * n_directions, n_variables) + +# Write particles initial condition file +write_particles(np.array(particles), NF, "particle_input.dat") \ No newline at end of file From 436bad665e831ef3872e270bab83889e573202e4 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 25 Sep 2024 14:16:42 -0400 Subject: [PATCH 199/276] Convert the units of the background data from MeV to ergs when setting it in the MultiFab. --- Source/ReadInput_RhoTempYe.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp index 0de1202b..133a09f3 100644 --- a/Source/ReadInput_RhoTempYe.cpp +++ b/Source/ReadInput_RhoTempYe.cpp @@ -56,8 +56,8 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms int idx = kg + ncell_z * (jg + ncell_y * ig); // Set the values from the input arrays - mf_array(i, j, k, GIdx::rho - start_comp) = rhoYeT_input_obj.rho_input[idx]; - mf_array(i, j, k, GIdx::T - start_comp) = rhoYeT_input_obj.T_input[idx]; + mf_array(i, j, k, GIdx::rho - start_comp) = rhoYeT_input_obj.rho_input[idx]; // g/ccm + mf_array(i, j, k, GIdx::T - start_comp) = rhoYeT_input_obj.T_input[idx]*1e6*CGSUnitsConst::eV; //erg mf_array(i, j, k, GIdx::Ye - start_comp) = rhoYeT_input_obj.Ye_input[idx]; }); From e248f00a0692da4a76ea1d66e4b6b762cee9553c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 25 Sep 2024 14:18:34 -0400 Subject: [PATCH 200/276] Add comments and convert the temperature from ergs to MeV when interpolating the chemical potentials and neutrino opacities. --- Source/Evolve.cpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index e639e959..0af4d22e 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -204,9 +204,9 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M const ParticleInterpolator sz(delta_z, shape_factor_order_z); // The following variables contains temperature, electron fraction, and density interpolated from grid quantities to particle positions - Real T_pp = 0; + Real T_pp = 0; // erg Real Ye_pp = 0; - Real rho_pp = 0; + Real rho_pp = 0; // g/ccm for (int k = sz.first(); k <= sz.last(); ++k) { for (int j = sy.first(); j <= sy.last(); ++j) { @@ -242,10 +242,10 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M if(parms->IMFP_method==1){ for (int i=0; iIMFP_abs[0][i]; // Read absorption inverse mean free path from input parameters file. - IMFP_absbar[i][i] = parms->IMFP_abs[1][i]; // Read absorption inverse mean free path from input parameters file. - munu[i][i] = parms->munu[0][i]; // Read neutrino chemical potential from input parameters file. - munubar[i][i] = parms->munu[1][i]; // Read antineutrino chemical potential from input parameters file. + IMFP_abs[i][i] = parms->IMFP_abs[0][i]; // 1/cm : Read absorption inverse mean free path from input parameters file. + IMFP_absbar[i][i] = parms->IMFP_abs[1][i]; // 1/cm : Read absorption inverse mean free path from input parameters file. + munu[i][i] = parms->munu[0][i]; // ergs : Read neutrino chemical potential from input parameters file. + munubar[i][i] = parms->munu[1][i]; // ergs : Read antineutrino chemical potential from input parameters file. } } @@ -254,7 +254,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M // Assign temperature, electron fraction, and density at the particle's position to new variables for interpolation of chemical potentials and inverse mean free paths. Real rho = rho_pp; // Density of background matter at this particle's position g/cm^3 - Real temperature = T_pp; // Temperature of background matter at this particle's position 0.05 //MeV + Real temperature = T_pp / (1e6*CGSUnitsConst::eV); // Temperature of background matter at this particle's position 0.05 //MeV Real Ye = Ye_pp; // Electron fraction of background matter at this particle's position //-------------------- Values from EoS table ------------------------------ @@ -269,10 +269,10 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M printf("(Evolve.cpp) muhat interpolated = %f\n", muhat_out); #endif // munu_val : electron neutrino chemical potential - const double munu_val = mue_out - muhat_out; //munu -> "mu_e" - "muhat" - - munu[0][0] = munu_val; // Save neutrino chemical potential from EOS table in chemical potential matrix - munubar[0][0] = -1.0 * munu_val; // Save antineutrino chemical potential from EOS table in chemical potential matrix + const double munu_val = ( mue_out - muhat_out ) * 1e6*CGSUnitsConst::eV ; //munu -> "mu_e" - "muhat" + + munu[0][0] = munu_val; // erg : Save neutrino chemical potential from EOS table in chemical potential matrix + munubar[0][0] = -1.0 * munu_val; // erg : Save antineutrino chemical potential from EOS table in chemical potential matrix //--------------------- Values from NuLib table --------------------------- double *helperVarsReal_nulib = NuLib_tabulated_obj.get_helperVarsReal_nulib(); @@ -330,7 +330,6 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M // } } //----------------------------------------------------------------------- - } else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0, 1 or 2"); From 52fd68bf3dc68bcfc8a2e824ea80f1fad4335aca Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 25 Sep 2024 14:23:22 -0400 Subject: [PATCH 201/276] Create a Python script for the Fermi-Dirac test. --- Scripts/tests/fermi_dirac_test.py | 297 ++++++++++++++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 Scripts/tests/fermi_dirac_test.py diff --git a/Scripts/tests/fermi_dirac_test.py b/Scripts/tests/fermi_dirac_test.py new file mode 100644 index 00000000..7285c254 --- /dev/null +++ b/Scripts/tests/fermi_dirac_test.py @@ -0,0 +1,297 @@ +import numpy as np +import h5py +import glob +import matplotlib.pyplot as plt + +# physical constants +clight = 2.99792458e10 # cm/s +hbar = 1.05457266e-27 # erg s +h = 2.0*np.pi*hbar # erg s +eV = 1.60218e-12 # erg + +# Reading plt* directories +directories = np.array( glob.glob("plt*.h5") ) +# Extract directories with "old" in their names +mask = np.char.find(directories, "old") == -1 +directories = directories[mask] + +# Sort the data file names by time step number +directories = sorted(directories, key=lambda x: int(x.split("plt")[1].split(".")[0])) + +# Energy bin centers extracted from NuLib table +energies_center_Mev = np.array([1, 3, 5.23824, 8.00974, 11.4415, 15.6909, 20.9527, 27.4681, 35.5357, 45.5254, 57.8951, 73.2117, 92.1775, 115.662, 144.741, 180.748, 225.334, 280.542]) # Energy in Mev +# Energy bin bottom extracted from NuLib table +energies_bottom_Mev = np.array([0, 2, 4, 6.47649, 9.54299, 13.3401, 18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250]) +# Energy bin top extracted from NuLib table +energies_top_Mev = np.array([2, 4, 6.47649, 9.54299, 13.3401, 18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250, 311.085]) + +# Energies in ergs +energies_center_erg = np.array(energies_center_Mev) * 1e6 * eV # Energy in ergs +energies_bottom_erg = np.array(energies_bottom_Mev) * 1e6 * eV # Energy in ergs +energies_top_erg = np.array(energies_top_Mev ) * 1e6 * eV # Energy in ergs + +# Simulation parameters +Temperature_Mev = 5.02464 # Background matter temperature ( Mev ) +u_ee_MeV = -1.640496 # Electron neutrino chemical potential ( Mev ) +u_eebar_MeV = +1.640496 # Electron anti-neutrino chemical potential ( Mev ) +u_uu_MeV = 0.0 # Muon neutrino chemical potential ( Mev ) +u_uubar_MeV = 0.0 # Muon anti-neutrino chemical potential ( Mev ) +u_tt_MeV = 0.0 # Tauon neutrino chemical potential ( Mev ) +u_ttbar_MeV = 0.0 # Tauon anti-neutrino chemical potential ( Mev ) + +# Fermi-dirac distribution factor for electron neutrinos +f_eq_ee = 1 / ( 1 + np.exp( ( energies_center_Mev - u_ee_MeV ) / Temperature_Mev ) ) # adimentional +f_eq_eebar = 1 / ( 1 + np.exp( ( energies_center_Mev - u_eebar_MeV ) / Temperature_Mev ) ) # adimentional +f_eq_uu = 1 / ( 1 + np.exp( ( energies_center_Mev - u_uu_MeV ) / Temperature_Mev ) ) # adimentional +f_eq_uubar = 1 / ( 1 + np.exp( ( energies_center_Mev - u_uubar_MeV ) / Temperature_Mev ) ) # adimentional +f_eq_tt = 1 / ( 1 + np.exp( ( energies_center_Mev - u_tt_MeV ) / Temperature_Mev ) ) # adimentional +f_eq_ttbar = 1 / ( 1 + np.exp( ( energies_center_Mev - u_ttbar_MeV ) / Temperature_Mev ) ) # adimentional + +fee_last = np.zeros(len(energies_center_erg)) +fuu_last = np.zeros(len(energies_center_erg)) +ftt_last = np.zeros(len(energies_center_erg)) +feebar_last = np.zeros(len(energies_center_erg)) +fuubar_last = np.zeros(len(energies_center_erg)) +fttbar_last = np.zeros(len(energies_center_erg)) + +# Reading last plt* file generated +with h5py.File(directories[-1], 'r') as hf: + + N00_Re = np.array(hf['N00_Re']) # number of particles + N11_Re = np.array(hf['N11_Re']) # number of particles + N22_Re = np.array(hf['N22_Re']) # number of particles + N00_Rebar = np.array(hf['N00_Rebar']) # number of particles + N11_Rebar = np.array(hf['N11_Rebar']) # number of particles + N22_Rebar = np.array(hf['N22_Rebar']) # number of particles + + E = np.array(hf['pupt']) # ergs + t = np.array(hf['time']) # seconds + Vphase = np.array(hf['Vphase']) # seconds + + N_ee_last = np.zeros(len(energies_center_erg)) + N_uu_last = np.zeros(len(energies_center_erg)) + N_tt_last = np.zeros(len(energies_center_erg)) + N_ee_bar_last = np.zeros(len(energies_center_erg)) + N_uu_bar_last = np.zeros(len(energies_center_erg)) + N_tt_bar_last = np.zeros(len(energies_center_erg)) + Total_Vphase = np.zeros(len(energies_center_erg)) + + for i, energy_erg in enumerate(energies_center_erg): + mask = E == energy_erg + N_ee_last[i] = np.sum(N00_Re[mask]) + N_uu_last[i] = np.sum(N11_Re[mask]) + N_tt_last[i] = np.sum(N22_Re[mask]) + N_ee_bar_last[i] = np.sum(N00_Rebar[mask]) + N_uu_bar_last[i] = np.sum(N11_Rebar[mask]) + N_tt_bar_last[i] = np.sum(N22_Rebar[mask]) + Total_Vphase[i] = np.sum(Vphase[mask]) + + fee_last = ( h * clight )**3 * N_ee_last / Total_Vphase + fuu_last = ( h * clight )**3 * N_uu_last / Total_Vphase + ftt_last = ( h * clight )**3 * N_tt_last / Total_Vphase + feebar_last = ( h * clight )**3 * N_ee_bar_last / Total_Vphase + fuubar_last = ( h * clight )**3 * N_uu_bar_last / Total_Vphase + fttbar_last = ( h * clight )**3 * N_tt_bar_last / Total_Vphase + +error_fee = np.abs( ( fee_last - f_eq_ee ) / f_eq_ee ) +error_fuu = np.abs( ( fuu_last - f_eq_uu ) / f_eq_uu ) +error_ftt = np.abs( ( ftt_last - f_eq_tt ) / f_eq_tt ) +error_feebar = np.abs( ( feebar_last - f_eq_eebar ) / f_eq_eebar ) +error_fuubar = np.abs( ( fuubar_last - f_eq_uubar ) / f_eq_uubar ) +error_fttbar = np.abs( ( fttbar_last - f_eq_ttbar ) / f_eq_ttbar ) + +max_error_fee = np.max(error_fee) +max_error_fuu = np.max(error_fuu) +max_error_ftt = np.max(error_ftt) +max_error_feebar = np.max(error_feebar) +max_error_fuubar = np.max(error_fuubar) +max_error_fttbar = np.max(error_fttbar) + +print(f'max_error_fee = {max_error_fee}') +print(f'max_error_fuu = {max_error_fuu}') +print(f'max_error_ftt = {max_error_ftt}') +print(f'max_error_feebar = {max_error_feebar}') +print(f'max_error_fuubar = {max_error_fuubar}') +print(f'max_error_fttbar = {max_error_fttbar}') + +assert(np.all(max_error_fee<0.01)) +assert(np.all(max_error_fuu<0.01)) +assert(np.all(max_error_ftt<0.01)) +assert(np.all(max_error_feebar<0.01)) +assert(np.all(max_error_fuubar<0.01)) +assert(np.all(max_error_fttbar<0.01)) + +############################################################ +############################################################ +# PLOT SETTINGS +import matplotlib as mpl +from matplotlib.ticker import AutoLocator, AutoMinorLocator, LogLocator + +# Font settings +mpl.rcParams['font.size'] = 22 +mpl.rcParams['font.family'] = 'serif' +mpl.rc('text', usetex=True) + +# Tick settings +mpl.rcParams['xtick.major.size'] = 7 +mpl.rcParams['xtick.major.width'] = 2 +mpl.rcParams['xtick.major.pad'] = 8 +mpl.rcParams['xtick.minor.size'] = 4 +mpl.rcParams['xtick.minor.width'] = 2 +mpl.rcParams['ytick.major.size'] = 7 +mpl.rcParams['ytick.major.width'] = 2 +mpl.rcParams['ytick.minor.size'] = 4 +mpl.rcParams['ytick.minor.width'] = 2 + +# Axis linewidth +mpl.rcParams['axes.linewidth'] = 2 + +# Tick direction and enabling ticks on all sides +mpl.rcParams['xtick.direction'] = 'in' +mpl.rcParams['ytick.direction'] = 'in' +mpl.rcParams['xtick.top'] = True +mpl.rcParams['ytick.right'] = True + +# Function to apply custom tick locators and other settings to an Axes object +def apply_custom_settings(ax, leg, log_scale_y=False): + + if log_scale_y: + # Use LogLocator for the y-axis if it's in log scale + ax.set_yscale('log') + ax.yaxis.set_major_locator(LogLocator(base=10.0)) + ax.yaxis.set_minor_locator(LogLocator(base=10.0, subs='auto', numticks=100)) + else: + # Use AutoLocator for regular scales + ax.yaxis.set_major_locator(AutoLocator()) + ax.yaxis.set_minor_locator(AutoMinorLocator()) + + # Apply the AutoLocator for the x-axis + ax.xaxis.set_major_locator(AutoLocator()) + ax.xaxis.set_minor_locator(AutoMinorLocator()) + + # Legend settings + leg.get_frame().set_edgecolor('w') + leg.get_frame().set_linewidth(0.0) +############################################################ +############################################################ + +Volume = 2.0e4**3 # ccm +Tot_Vphase = Volume * ( 4.0 * np.pi ) * ( ( energies_top_erg ** 3 - energies_bottom_erg ** 3 ) / 3.0 ) + +N_eq_ee = ( 1.0 / ( h * clight )**3 ) * Tot_Vphase * f_eq_ee +N_eq_uu = ( 1.0 / ( h * clight )**3 ) * Tot_Vphase * f_eq_uu +N_eq_eebar = ( 1.0 / ( h * clight )**3 ) * Tot_Vphase * f_eq_eebar +N_eq_uubar = ( 1.0 / ( h * clight )**3 ) * Tot_Vphase * f_eq_uubar + +figfee, axfee = plt.subplots() +figfeebar, axfeebar = plt.subplots() +figfuu, axfuu = plt.subplots() +figfuubar, axfuubar = plt.subplots() + +figNee, axNee = plt.subplots() +figNeebar, axNeebar = plt.subplots() +figNuu, axNuu = plt.subplots() +figNuubar, axNuubar = plt.subplots() + +# Looping over all directories +for i in range(len(directories)): + with h5py.File(directories[i], 'r') as hf: + + N00_Re = np.array(hf['N00_Re']) # number of particles + N11_Re = np.array(hf['N11_Re']) # number of particles + N00_Rebar = np.array(hf['N00_Rebar']) # number of particles + N11_Rebar = np.array(hf['N11_Rebar']) # number of particles + E = np.array(hf['pupt']) # ergs + t = np.array(hf['time']) # seconds + Vphase = np.array(hf['Vphase']) # seconds + + Nee = np.zeros(len(energies_center_erg)) + Neebar = np.zeros(len(energies_center_erg)) + Nuu = np.zeros(len(energies_center_erg)) + Nuubar = np.zeros(len(energies_center_erg)) + Total_Vphase = np.zeros(len(energies_center_erg)) + + for i, energy_erg in enumerate(energies_center_erg): + mask = E == energy_erg + Nee[i] = np.sum(N00_Re[mask]) + Neebar[i] = np.sum(N00_Rebar[mask]) + Nuu[i] = np.sum(N11_Re[mask]) + Nuubar[i] = np.sum(N11_Rebar[mask]) + Total_Vphase[i] = np.sum(Vphase[mask]) + + fee = ( h * clight )**3 * Nee / Total_Vphase + feebar = ( h * clight )**3 * Neebar / Total_Vphase + fuu = ( h * clight )**3 * Nuu / Total_Vphase + fuubar = ( h * clight )**3 * Nuubar / Total_Vphase + + # Plot the data + axfee.plot(energies_center_erg, fee, label=f't = {t[0]:.1e} s') + axfeebar.plot(energies_center_erg, feebar, label=f't = {t[0]:.1e} s') + axfuu.plot(energies_center_erg, fuu, label=f't = {t[0]:.1e} s') + axfuubar.plot(energies_center_erg, fuubar, label=f't = {t[0]:.1e} s') + + axNee.plot(energies_center_erg, Nee, label=f't = {t[0]:.1e} s') + axNeebar.plot(energies_center_erg, Neebar, label=f't = {t[0]:.1e} s') + axNuu.plot(energies_center_erg, Nuu, label=f't = {t[0]:.1e} s') + axNuubar.plot(energies_center_erg, Nuubar, label=f't = {t[0]:.1e} s') + +axfee.plot(energies_center_erg, f_eq_ee, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') +axfeebar.plot(energies_center_erg, f_eq_eebar, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') +axfuu.plot(energies_center_erg, f_eq_uu, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') +axfuubar.plot(energies_center_erg, f_eq_uubar, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') + +axNee.plot(energies_center_erg, N_eq_ee, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') +axNeebar.plot(energies_center_erg, N_eq_eebar, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') +axNuu.plot(energies_center_erg, N_eq_uu, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') +axNuubar.plot(energies_center_erg, N_eq_uubar, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') + +# Add title and labels +axfee.set_xlabel(r'$E$ (erg)') +axfeebar.set_xlabel(r'$E$ (erg)') +axfuu.set_xlabel(r'$E$ (erg)') +axfuubar.set_xlabel(r'$E$ (erg)') +axNee.set_xlabel(r'$E$ (erg)') +axNeebar.set_xlabel(r'$E$ (erg)') +axNuu.set_xlabel(r'$E$ (erg)') +axNuubar.set_xlabel(r'$E$ (erg)') + +axfee.set_ylabel(r'$f_{{e}}^{{eq}}$') +axfeebar.set_ylabel(r'$\bar{f}_{{e}}^{{eq}}$') +axfuu.set_ylabel(r'$f_{{u}}^{{eq}}$') +axfuubar.set_ylabel(r'$\bar{f}_{{u}}^{{eq}}$') + +axNee.set_ylabel(r'$N_{{e}}$') +axNeebar.set_ylabel(r'$\bar{N}_{{e}}$') +axNuu.set_ylabel(r'$N_{{u}}$') +axNuubar.set_ylabel(r'$\bar{N}_{{u}}$') + +# Add a legend +legfee = axfee.legend(framealpha=0.0, ncol=1, fontsize=14) +apply_custom_settings(axfee, legfee) +legfeebar = axfeebar.legend(framealpha=0.0, ncol=1, fontsize=14) +apply_custom_settings(axfeebar, legfeebar) +legfuu = axfuu.legend(framealpha=0.0, ncol=1, fontsize=14) +apply_custom_settings(axfuu, legfuu) +legfuubar = axfuubar.legend(framealpha=0.0, ncol=1, fontsize=14) +apply_custom_settings(axfuubar, legfuubar) + +legNee = axNee.legend(framealpha=0.0, ncol=1, fontsize=14) +apply_custom_settings(axNee, legNee) +legNeebar = axNeebar.legend(framealpha=0.0, ncol=1, fontsize=14) +apply_custom_settings(axNeebar, legNeebar) +legNuu = axNuu.legend(framealpha=0.0, ncol=1, fontsize=14) +apply_custom_settings(axNuu, legNuu) +legNuubar = axNuubar.legend(framealpha=0.0, ncol=1, fontsize=14) +apply_custom_settings(axNuubar, legNuubar) + +# Save the plot as a PDF file +figfee.savefig('figfee.pdf',bbox_inches='tight') +figfeebar.savefig('figfeebar.pdf',bbox_inches='tight') +figfuu.savefig('figfuu.pdf',bbox_inches='tight') +figfuubar.savefig('figfuubar.pdf',bbox_inches='tight') +figNee.savefig('figNee.pdf',bbox_inches='tight') +figNeebar.savefig('figNeebar.pdf',bbox_inches='tight') +figNuu.savefig('figNuu.pdf',bbox_inches='tight') +figNuubar.savefig('figNuubar.pdf',bbox_inches='tight') + +plt.clf() \ No newline at end of file From f0582b0977e98ddb3cc243eb17035f452bdc7405 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 25 Sep 2024 14:24:22 -0400 Subject: [PATCH 202/276] Update the input parameters for the Fermi-Dirac test. --- sample_inputs/inputs_fermi_dirac_test | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index 43a309da..8b7695eb 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -4,9 +4,9 @@ perturbation_amplitude = 1e-6 # attenuation parameters to time derivative of N due to hamiltonians attenuation_hamiltonians = 0 -collision_cfl_factor = 0.005 -cfl_factor = 0.005 -flavor_cfl_factor = 0.005 +collision_cfl_factor = 100 +cfl_factor = 100 +flavor_cfl_factor = 100 max_adaptive_speedup = 0 maxError = 1e-6 @@ -14,10 +14,10 @@ integration.type = 1 integration.rk.type = 4 # Domain size in 3D index space -ncell = (2, 2, 2) -Lx = 2.0 -Ly = 2.0 -Lz = 2.0 +ncell = (2,2,2) +Lx = 2.0e4 # cm +Ly = 2.0e4 # cm +Lz = 2.0e4 # cm # Number of particles per cell nppc = (1, 1, 1) @@ -27,7 +27,7 @@ particle_data_filename = "particle_input.dat" max_grid_size = 16 # Number of steps to run -nsteps = 1 +nsteps = 2000 # Simulation end time end_time = 5.0e9 @@ -41,10 +41,10 @@ T_MeV = 10 Ye = 1 # Write plotfiles -write_plot_every = 1 +write_plot_every = 2000 # Write particle data in plotfiles -write_plot_particles_every = 1 +write_plot_particles_every = 2000 # checkpointing do_restart = 0 From 5a752febda1f3957e7c73cd9c3bf282b70c22a2e Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 25 Sep 2024 14:44:57 -0400 Subject: [PATCH 203/276] Create a python script to generate a constant background matter field. This hdf5 file is read into the EMU mesh. --- ...sm_constant_background_rho_Ye_T__writer.py | 47 +++++++++++++++++++ Scripts/collisions/nsm_grid_generator.py | 42 +++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 Scripts/collisions/nsm_constant_background_rho_Ye_T__writer.py create mode 100644 Scripts/collisions/nsm_grid_generator.py diff --git a/Scripts/collisions/nsm_constant_background_rho_Ye_T__writer.py b/Scripts/collisions/nsm_constant_background_rho_Ye_T__writer.py new file mode 100644 index 00000000..8a336ed9 --- /dev/null +++ b/Scripts/collisions/nsm_constant_background_rho_Ye_T__writer.py @@ -0,0 +1,47 @@ +''' +Created by Erick Urquilla, Department of Physics and Astronomy, University of Tennessee, Knoxville. +This script is used to generate an HDF5 file that contains constant background matter information: rho (density), Yâ‚‘ (electron fraction), and T (temperature). +The HDF5 file generated by this script will be used as input for background matter quantities in EMU. +''' + +import numpy as np +import h5py +import nsm_grid_generator + +# EMU grid parameters +ncellsx = 2 # scalar, number of cells in x-direction +ncellsy = 2 # scalar, number of cells in y-direction +ncellsz = 2 # scalar, number of cells in z-direction +xmin = 0.0 #cm +xmax = 2.0e4 #cm +ymin = 0.0 #cm +ymax = 2.0e4 #cm +zmin = 0.0 #cm +zmax = 2.0e4 #cm + +# Create EMU mesh +centers, mesh = nsm_grid_generator.create_grid([ncellsx, ncellsy, ncellsz], [[xmin, xmax], [ymin, ymax], [zmin, zmax]]) # cm + +rho_cons = 1802418929.1457505 # g/ccm +T_cons = 5.0246353 # MeV +Ye_cons = 0.46005958 # n_electron - n_positron / n_barions + +# Create arrays to store the interpolated values of T, rho, and Ye. +rho = np.full( ( ncellsx, ncellsy, ncellsz ), rho_cons ) # array of size (ncellsx, ncellsy, ncellsz) +T = np.full( ( ncellsx, ncellsy, ncellsz ), T_cons ) # array of size (ncellsx, ncellsy, ncellsz) +Ye = np.full( ( ncellsx, ncellsy, ncellsz ), Ye_cons ) # array of size (ncellsx, ncellsy, ncellsz) + +# Write hdf5 file with all the data +with h5py.File('rho_Ye_T.hdf5', 'w') as hdf: + hdf.create_dataset("ncellsx", data=ncellsx) + hdf.create_dataset("ncellsy", data=ncellsy) + hdf.create_dataset("ncellsz", data=ncellsz) + hdf.create_dataset("xmin_cm", data=xmin) + hdf.create_dataset("xmax_cm", data=xmax) + hdf.create_dataset("ymin_cm", data=ymin) + hdf.create_dataset("ymax_cm", data=ymax) + hdf.create_dataset("zmin_cm", data=zmin) + hdf.create_dataset("zmax_cm", data=zmax) + hdf.create_dataset("rho_g|ccm", data=rho) + hdf.create_dataset("T_Mev", data=T) + hdf.create_dataset("Ye", data=Ye) \ No newline at end of file diff --git a/Scripts/collisions/nsm_grid_generator.py b/Scripts/collisions/nsm_grid_generator.py new file mode 100644 index 00000000..40c88028 --- /dev/null +++ b/Scripts/collisions/nsm_grid_generator.py @@ -0,0 +1,42 @@ +''' +Created by Erick Urquilla, Department of Physics and Astronomy, University of Tennessee, Knoxville. +This script is used to Creates a 3D cartesian grid. +''' + +import numpy as np + +def create_grid(cell_numbers, dimensions): + """ + Creates a 3D grid of points and corresponding mesh for the given cell numbers and dimensions. + + Parameters: + - cell_numbers: A list or tuple of integers specifying the number of cells along each dimension (x, y, z). + - dimensions: A list or tuple of tuples, where each inner tuple contains the minimum and maximum value for that dimension (e.g., [(xmin, xmax), (ymin, ymax), (zmin, zmax)]). + + Returns: + - centers: A 2D array where each row is the (x, y, z) coordinate of a grid point center. + - mesh: A 3D array containing the mesh grid of coordinates for each dimension. + """ + + # Create arrays of face positions along each dimension + # 'faces' will contain arrays that specify the boundaries of cells in each dimension + faces = [np.linspace(d[0], d[1], n+1) for d, n in zip(dimensions, cell_numbers)] + + # Calculate the center positions of the cells in each dimension + # 'centers' will contain arrays of the center points between the cell boundaries (faces) + centers = [0.5 * (f[1:] + f[:-1]) for f in faces] + + # Create a 3D mesh grid of center points using the calculated centers for each dimension + # 'indexing="ij"' is used to maintain matrix indexing (row-major order) + X, Y, Z = np.meshgrid(*centers, indexing='ij') + + # Flatten the 3D mesh grid to obtain a 2D array of (x, y, z) coordinates for each grid point + # Each row in 'centers' corresponds to the coordinates of a single grid point + centers = np.vstack((X.flatten(), Y.flatten(), Z.flatten())).transpose() + + # Combine the 3D mesh grid into a single array 'mesh' for easy access + # 'mesh' will be a 3D array containing the X, Y, and Z coordinates of the grid + mesh = np.stack((X, Y, Z), axis=-1) + + # Return the grid point centers and the mesh grid + return centers, mesh \ No newline at end of file From 3e42aeb8953719f73c10654a1ee1d07b99809de5 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 25 Sep 2024 14:49:26 -0400 Subject: [PATCH 204/276] Update the input parameters for the Fermi-Dirac test. --- sample_inputs/inputs_fermi_dirac_test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index 8b7695eb..fc27fbee 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -117,5 +117,5 @@ delta_E = 0.8339001570751987 # Mev ################# read_rho_T_Ye_from_table = 1 -nulib_table_name = "/mnt/scratch/tables/NuLib/NuLib_SFHo.h5" -nuceos_table_name = "LS220.h5" +nulib_table_name = "../Scripts/collisions/NuLib_SFHo.h5" +nuceos_table_name = "../Scripts/collisions/LS220.h5" From 5f3f3c22e432d9c1e827eab331bf9ee010ad54f4 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 11:25:17 -0400 Subject: [PATCH 205/276] Read the background matter HDF5 file path from the input parameters. --- Source/Parameters.H | 7 ++++++- Source/ReadInput_RhoTempYe.cpp | 3 +-- sample_inputs/inputs_fermi_dirac_test | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index ecf58fa9..1866006b 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -60,6 +60,7 @@ struct TestParams : public amrex::Gpu::Managed //HDF5 table names (with full path) for EoS and NuLib tables std::string nuceos_table_name; std::string nulib_table_name; + std::string background_rho_Ye_T_table_name; void Initialize(){ ParmParse pp; @@ -123,7 +124,10 @@ struct TestParams : public amrex::Gpu::Managed // Background matter rho, Ye, T flag pp.get("read_rho_T_Ye_from_table", read_rho_T_Ye_from_table); - + if(read_rho_T_Ye_from_table==1){ + pp.get("background_rho_Ye_T_table_name", background_rho_Ye_T_table_name); + } + // absorption opacities and equilibrium neutrino chemical potentials pp.get("IMFP_method", IMFP_method); if(IMFP_method==0){ @@ -158,6 +162,7 @@ struct TestParams : public amrex::Gpu::Managed } else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0(do not collisions) and 1(do collisions)"); } + }; #endif diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp index 133a09f3..aca0f93d 100644 --- a/Source/ReadInput_RhoTempYe.cpp +++ b/Source/ReadInput_RhoTempYe.cpp @@ -13,8 +13,7 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms amrex::GpuArray dx = geom.CellSizeArray(); - const std::string hdf5_background_rho_Ye_T_name = "rho_Ye_T.hdf5"; - ReadInputRhoYeT(hdf5_background_rho_Ye_T_name); + ReadInputRhoYeT(parms->background_rho_Ye_T_table_name); using namespace background_input_rho_T_Ye; int ncell_x = *n_cell_x; diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index fc27fbee..9daf9417 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -119,3 +119,4 @@ read_rho_T_Ye_from_table = 1 nulib_table_name = "../Scripts/collisions/NuLib_SFHo.h5" nuceos_table_name = "../Scripts/collisions/LS220.h5" +background_rho_Ye_T_table_name = "./rho_Ye_T.hdf5" From 60d58588760228019b05789cb062a05ed3b2887f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 11:43:25 -0400 Subject: [PATCH 206/276] Add a script to write all particle information into HDF5 files. --- Scripts/collisions/writeparticleinfohdf5.py | 133 ++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100755 Scripts/collisions/writeparticleinfohdf5.py diff --git a/Scripts/collisions/writeparticleinfohdf5.py b/Scripts/collisions/writeparticleinfohdf5.py new file mode 100755 index 00000000..1a0f0187 --- /dev/null +++ b/Scripts/collisions/writeparticleinfohdf5.py @@ -0,0 +1,133 @@ +########################################################## +#This script write all the particle information in the plt* directories into hdf5 format files +########################################################## + +import os +os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+'/data_reduction') +import numpy as np +import matplotlib.pyplot as plt +import yt +import glob +import multiprocessing as mp +import h5py +import amrex_plot_tools as amrex +import emu_yt_module as emu +from multiprocessing import Pool +import scipy.special + +########## +# INPUTS # +########## +nproc = 1 + +dirh5 = sorted(glob.glob("*.h5")) +dirh5 = [dirh5[i].split('.')[0] for i in range(len(dirh5))] +dirall = sorted(glob.glob("plt*/neutrinos")) +dirall = [dirall[i].split('/')[0] for i in range(len(dirall))] # remove "neutrinos" + +directories=[] + +for dir1 in dirall: + thereis=0 + for dir2 in dirh5: + if dir1==dir2: + thereis=1 + break + if thereis==0: + directories.append(dir1) + +# get NF +eds = emu.EmuDataset(directories[0]) +NF = eds.get_num_flavors() +if NF==2: + rkey, ikey = amrex.get_particle_keys(NF) + labels=['pos_x','pos_y','pos_z', 'time', 'x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N11_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N11_Rebar', 'TrHN', 'Vphase'] +if NF==3: + rkey, ikey = amrex.get_particle_keys(NF) + labels=['pos_x','pos_y','pos_z','time','x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N02_Re', 'N02_Im', 'N11_Re', 'N12_Re', 'N12_Im' ,'N22_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N02_Rebar', 'N02_Imbar', 'N11_Rebar', 'N12_Rebar' ,'N12_Imbar', 'N22_Rebar', 'TrHN', 'Vphase'] + +class GridData(object): + def __init__(self, ad): + x = ad['index','x'].d + y = ad['index','y'].d + z = ad['index','z'].d + dx = ad['index','dx'].d + dy = ad['index','dy'].d + dz = ad['index','dz'].d + self.ad = ad + self.dx = dx[0] + self.dy = dy[0] + self.dz = dz[0] + self.xmin = np.min(x-dx/2.) + self.ymin = np.min(y-dy/2.) + self.zmin = np.min(z-dz/2.) + self.xmax = np.max(x+dx/2.) + self.ymax = np.max(y+dy/2.) + self.zmax = np.max(z+dz/2.) + self.nx = int((self.xmax - self.xmin) / self.dx + 0.5) + self.ny = int((self.ymax - self.ymin) / self.dy + 0.5) + self.nz = int((self.zmax - self.zmin) / self.dz + 0.5) + print(self.nx, self.ny, self.nz) + + def get_particle_cell_ids(self,rdata): + # get coordinates + x = rdata[:,rkey["x"]] + y = rdata[:,rkey["y"]] + z = rdata[:,rkey["z"]] + ix = (x/self.dx).astype(int) + iy = (y/self.dy).astype(int) + iz = (z/self.dz).astype(int) + + # HACK - get this grid's bounds using particle locations + ix -= np.min(ix) + iy -= np.min(iy) + iz -= np.min(iz) + nx = np.max(ix)+1 + ny = np.max(iy)+1 + nz = np.max(iz)+1 + idlist = (iz + nz*iy + nz*ny*ix).astype(int) + + return idlist + +def writehdf5files(dire): + + eds = emu.EmuDataset(dire) + t = eds.ds.current_time + ad = eds.ds.all_data() + + header = amrex.AMReXParticleHeader(dire+"/neutrinos/Header") + grid_data = GridData(ad) + nlevels = len(header.grids) + assert nlevels==1 + level = 0 + ngrids = len(header.grids[level]) + + # creating the file to save the particle data + hf = h5py.File(str(dire)+".h5", 'w') + + for label in labels: + hf.create_dataset(label,data=[],maxshape=(None,),chunks=True) + + # loop over all cells within each grid + for gridID in range(ngrids): + + # read particle data on a single grid + idata, rdata = amrex.read_particle_data(dire, ptype="neutrinos", level_gridID=(level,gridID)) + + # writing the particle data + for label in labels: + hf[label].resize((hf[label].shape[0] + rdata[:,rkey[label]].shape[0]), axis=0) + hf[label][-rdata[:,rkey[label]].shape[0]:] = rdata[:,rkey[label]] + + hf.close() + + return dire + +# run the write hdf5 files function in parallel +if __name__ == '__main__': + pool = Pool(nproc) + finalresult=pool.map(writehdf5files,directories) + for i in finalresult: print("completed ---> "+i) + From 87b6903a409f0f47d8af82711e0d8e6336b529a2 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 11:47:31 -0400 Subject: [PATCH 207/276] Add Fermi-Dirac test to Jenkinsfile. --- Jenkinsfile | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 04fafc5b..0c9367ac 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -98,7 +98,7 @@ pipeline { sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' sh 'python ../Scripts/data_reduction/reduce_data.py' sh 'python ../Scripts/tests/coll_inst_test.py' - sh 'rm -rf plt*' + sh 'rm -rf plt* *pdf' } }} @@ -109,7 +109,18 @@ pipeline { sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' sh 'python ../Scripts/tests/coll_equi_test.py' - sh 'rm -rf plt*' + sh 'rm -rf plt* *pdf' + } + }} + + stage('Fermi-Dirac test'){ steps{ + dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st9_empty_particles_multi_energy.py' + sh 'python ../Scripts/collisions/nsm_constant_background_rho_Ye_T__writer.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fermi_dirac_test' + sh 'python ../Scripts/collisions/writeparticleinfohdf5.py' + sh '../Scripts/tests/fermi_dirac_test.py' + sh 'rm -rf plt* *pdf' } }} From fb99e191d646280883ad185500df89289892e989 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 12:06:21 -0400 Subject: [PATCH 208/276] Updating the path in Ganon where the EOS and NuLib tables will be located, and the file names in the input parameters file in the Fermi-Dirac parameter file. --- Jenkinsfile | 6 ++---- sample_inputs/inputs_fermi_dirac_test | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0c9367ac..dc921aa9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ pipeline { triggers { pollSCM('') } // Run tests whenever a new commit is detected. - agent { dockerfile {args '--gpus all -v /mnt/scratch/tables:/tables:ro'}} // Use the Dockerfile defined in the root Flash-X directory + agent { dockerfile {args '--gpus all -v /mnt/scratch/EOS:/EOS:ro /mnt/scratch/NuLib:/NuLib:ro'}} // Use the Dockerfile defined in the root Flash-X directory environment { // Get rid of Read -1, expected , errno =1 error // See https://github.com/open-mpi/ompi/issues/4948 @@ -22,8 +22,6 @@ pipeline { } }} - - //=======// // Tests // //=======// @@ -120,7 +118,7 @@ pipeline { sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fermi_dirac_test' sh 'python ../Scripts/collisions/writeparticleinfohdf5.py' sh '../Scripts/tests/fermi_dirac_test.py' - sh 'rm -rf plt* *pdf' + sh 'rm -rf plt* *pdf rho_Ye_T.hdf5' } }} diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index 9daf9417..4f355f47 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -117,6 +117,6 @@ delta_E = 0.8339001570751987 # Mev ################# read_rho_T_Ye_from_table = 1 -nulib_table_name = "../Scripts/collisions/NuLib_SFHo.h5" -nuceos_table_name = "../Scripts/collisions/LS220.h5" +nulib_table_name = "/NuLib/NuLib_SFHo.h5" +nuceos_table_name = "/EOS/LS220.h5" background_rho_Ye_T_table_name = "./rho_Ye_T.hdf5" From 69b6bccabaec4e3b52ee6bad545f37ad449cdc47 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 14:26:27 -0400 Subject: [PATCH 209/276] Solving the issue with the Fermi-Dirac test path to the EOS and NuLib tables. --- Jenkinsfile | 2 +- sample_inputs/inputs_fermi_dirac_test | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dc921aa9..7a509170 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ pipeline { triggers { pollSCM('') } // Run tests whenever a new commit is detected. - agent { dockerfile {args '--gpus all -v /mnt/scratch/EOS:/EOS:ro /mnt/scratch/NuLib:/NuLib:ro'}} // Use the Dockerfile defined in the root Flash-X directory + agent { dockerfile {args '--gpus all -v /mnt/scratch/tables:/tables:ro'}} // Use the Dockerfile defined in the root Flash-X directory environment { // Get rid of Read -1, expected , errno =1 error // See https://github.com/open-mpi/ompi/issues/4948 diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index 4f355f47..f6621ba9 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -117,6 +117,6 @@ delta_E = 0.8339001570751987 # Mev ################# read_rho_T_Ye_from_table = 1 -nulib_table_name = "/NuLib/NuLib_SFHo.h5" -nuceos_table_name = "/EOS/LS220.h5" +nulib_table_name = "/tables/NuLib/NuLib_SFHo.h5" +nuceos_table_name = "/tables/EOS/LS220_234r_136t_50y_analmu_20091212_SVNr26.h5" background_rho_Ye_T_table_name = "./rho_Ye_T.hdf5" From aade9d1ec7395b963d8e300ba28c15c18a5cd23a Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 14:48:33 -0400 Subject: [PATCH 210/276] Trying to fix Jenkins problem when reading tables --- Jenkinsfile | 159 +++++++++++++------------- sample_inputs/inputs_fermi_dirac_test | 4 +- 2 files changed, 82 insertions(+), 81 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7a509170..8463cf00 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,7 @@ pipeline { triggers { pollSCM('') } // Run tests whenever a new commit is detected. - agent { dockerfile {args '--gpus all -v /mnt/scratch/tables:/tables:ro'}} // Use the Dockerfile defined in the root Flash-X directory + agent { dockerfile {args '--gpus all -v /mnt/scratch/tables/EOS:/EOS:ro'}} // Use the Dockerfile defined in the root Flash-X directory + agent { dockerfile {args '--gpus all -v /mnt/scratch/tables/NuLib:/NuLib:ro'}} // Use the Dockerfile defined in the root Flash-X directory environment { // Get rid of Read -1, expected , errno =1 error // See https://github.com/open-mpi/ompi/issues/4948 @@ -25,91 +26,91 @@ pipeline { //=======// // Tests // //=======// - stage('MSW'){ steps{ - dir('Exec'){ - sh 'python ../Scripts/initial_conditions/st0_msw_test.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_msw_test' - sh 'python ../Scripts/tests/msw_test.py' - sh 'rm -rf plt*' - } - }} + // stage('MSW'){ steps{ + // dir('Exec'){ + // sh 'python ../Scripts/initial_conditions/st0_msw_test.py' + // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_msw_test' + // sh 'python ../Scripts/tests/msw_test.py' + // sh 'rm -rf plt*' + // } + // }} - stage('Bipolar'){ steps{ - dir('Exec'){ - sh 'python ../Scripts/initial_conditions/st1_bipolar_test.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_bipolar_test' - sh 'rm -rf plt*' - } - }} + // stage('Bipolar'){ steps{ + // dir('Exec'){ + // sh 'python ../Scripts/initial_conditions/st1_bipolar_test.py' + // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_bipolar_test' + // sh 'rm -rf plt*' + // } + // }} - stage('Fast Flavor'){ steps{ - dir('Exec'){ - sh 'python ../Scripts/initial_conditions/st2_2beam_fast_flavor.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor' - sh 'python ../Scripts/tests/fast_flavor_test.py' - sh 'rm -rf plt*' - } - }} + // stage('Fast Flavor'){ steps{ + // dir('Exec'){ + // sh 'python ../Scripts/initial_conditions/st2_2beam_fast_flavor.py' + // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor' + // sh 'python ../Scripts/tests/fast_flavor_test.py' + // sh 'rm -rf plt*' + // } + // }} - stage('Fast Flavor k'){ steps{ - dir('Exec'){ - sh 'python ../Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor_nonzerok' - sh 'python ../Scripts/tests/fast_flavor_k_test.py' - sh 'rm -rf plt*' - } - }} + // stage('Fast Flavor k'){ steps{ + // dir('Exec'){ + // sh 'python ../Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py' + // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor_nonzerok' + // sh 'python ../Scripts/tests/fast_flavor_k_test.py' + // sh 'rm -rf plt*' + // } + // }} - stage('Fiducial 2F GPU Binary'){ steps{ - dir('Exec'){ - sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_1d_fiducial' - sh 'python ../Scripts/data_reduction/reduce_data_fft.py' - sh 'python ../Scripts/data_reduction/reduce_data.py' - sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data.h5' - sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data_fft_power.h5' - sh 'python ../Scripts/babysitting/avgfee.py' - sh 'python ../Scripts/babysitting/power_spectrum.py' - sh 'python ../Scripts/data_reduction/convertToHDF5.py' - sh 'gnuplot ../Scripts/babysitting/avgfee_gnuplot.plt' - archiveArtifacts artifacts: '*.pdf' - sh 'rm -rf plt*' - } - }} + // stage('Fiducial 2F GPU Binary'){ steps{ + // dir('Exec'){ + // sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py' + // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_1d_fiducial' + // sh 'python ../Scripts/data_reduction/reduce_data_fft.py' + // sh 'python ../Scripts/data_reduction/reduce_data.py' + // sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data.h5' + // sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data_fft_power.h5' + // sh 'python ../Scripts/babysitting/avgfee.py' + // sh 'python ../Scripts/babysitting/power_spectrum.py' + // sh 'python ../Scripts/data_reduction/convertToHDF5.py' + // sh 'gnuplot ../Scripts/babysitting/avgfee_gnuplot.plt' + // archiveArtifacts artifacts: '*.pdf' + // sh 'rm -rf plt*' + // } + // }} - stage('Fiducial 3F CPU HDF5'){ steps{ - dir('Exec'){ - sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5 GNUmakefile' - sh 'make realclean; make generate; make -j' - sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi_3F.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.ex ../sample_inputs/inputs_1d_fiducial' - /*sh 'python3 ../Scripts/babysitting/avgfee_HDF5.py'*/ - sh 'rm -rf plt*' - } - }} + // stage('Fiducial 3F CPU HDF5'){ steps{ + // dir('Exec'){ + // sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5 GNUmakefile' + // sh 'make realclean; make generate; make -j' + // sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi_3F.py' + // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.ex ../sample_inputs/inputs_1d_fiducial' + // /*sh 'python3 ../Scripts/babysitting/avgfee_HDF5.py'*/ + // sh 'rm -rf plt*' + // } + // }} - stage('Collisions flavor instability'){ steps{ - dir('Exec'){ - sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' - sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' - sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' - sh 'python ../Scripts/data_reduction/reduce_data.py' - sh 'python ../Scripts/tests/coll_inst_test.py' - sh 'rm -rf plt* *pdf' - } - }} + // stage('Collisions flavor instability'){ steps{ + // dir('Exec'){ + // sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' + // sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' + // sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' + // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' + // sh 'python ../Scripts/data_reduction/reduce_data.py' + // sh 'python ../Scripts/tests/coll_inst_test.py' + // sh 'rm -rf plt* *pdf' + // } + // }} - stage('Collisions to equilibrium'){ steps{ - dir('Exec'){ - sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' - sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3' - sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' - sh 'python ../Scripts/tests/coll_equi_test.py' - sh 'rm -rf plt* *pdf' - } - }} + // stage('Collisions to equilibrium'){ steps{ + // dir('Exec'){ + // sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' + // sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3' + // sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' + // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' + // sh 'python ../Scripts/tests/coll_equi_test.py' + // sh 'rm -rf plt* *pdf' + // } + // }} stage('Fermi-Dirac test'){ steps{ dir('Exec'){ diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index f6621ba9..e7848155 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -117,6 +117,6 @@ delta_E = 0.8339001570751987 # Mev ################# read_rho_T_Ye_from_table = 1 -nulib_table_name = "/tables/NuLib/NuLib_SFHo.h5" -nuceos_table_name = "/tables/EOS/LS220_234r_136t_50y_analmu_20091212_SVNr26.h5" +nulib_table_name = "/NuLib/NuLib_SFHo.h5" +nuceos_table_name = "/EOS/LS220_234r_136t_50y_analmu_20091212_SVNr26.h5" background_rho_Ye_T_table_name = "./rho_Ye_T.hdf5" From 91e5a54ed33b3932cbb6e76a20afe9e948b04e8d Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 14:58:22 -0400 Subject: [PATCH 211/276] Do just 2 test in Jenskins --- Jenkinsfile | 98 ++++++----------------------------------------------- 1 file changed, 10 insertions(+), 88 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8463cf00..cf6cd820 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,94 +23,16 @@ pipeline { } }} - //=======// - // Tests // - //=======// - // stage('MSW'){ steps{ - // dir('Exec'){ - // sh 'python ../Scripts/initial_conditions/st0_msw_test.py' - // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_msw_test' - // sh 'python ../Scripts/tests/msw_test.py' - // sh 'rm -rf plt*' - // } - // }} - - // stage('Bipolar'){ steps{ - // dir('Exec'){ - // sh 'python ../Scripts/initial_conditions/st1_bipolar_test.py' - // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_bipolar_test' - // sh 'rm -rf plt*' - // } - // }} - - // stage('Fast Flavor'){ steps{ - // dir('Exec'){ - // sh 'python ../Scripts/initial_conditions/st2_2beam_fast_flavor.py' - // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor' - // sh 'python ../Scripts/tests/fast_flavor_test.py' - // sh 'rm -rf plt*' - // } - // }} - - // stage('Fast Flavor k'){ steps{ - // dir('Exec'){ - // sh 'python ../Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py' - // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor_nonzerok' - // sh 'python ../Scripts/tests/fast_flavor_k_test.py' - // sh 'rm -rf plt*' - // } - // }} - - // stage('Fiducial 2F GPU Binary'){ steps{ - // dir('Exec'){ - // sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py' - // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_1d_fiducial' - // sh 'python ../Scripts/data_reduction/reduce_data_fft.py' - // sh 'python ../Scripts/data_reduction/reduce_data.py' - // sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data.h5' - // sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data_fft_power.h5' - // sh 'python ../Scripts/babysitting/avgfee.py' - // sh 'python ../Scripts/babysitting/power_spectrum.py' - // sh 'python ../Scripts/data_reduction/convertToHDF5.py' - // sh 'gnuplot ../Scripts/babysitting/avgfee_gnuplot.plt' - // archiveArtifacts artifacts: '*.pdf' - // sh 'rm -rf plt*' - // } - // }} - - // stage('Fiducial 3F CPU HDF5'){ steps{ - // dir('Exec'){ - // sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5 GNUmakefile' - // sh 'make realclean; make generate; make -j' - // sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi_3F.py' - // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.ex ../sample_inputs/inputs_1d_fiducial' - // /*sh 'python3 ../Scripts/babysitting/avgfee_HDF5.py'*/ - // sh 'rm -rf plt*' - // } - // }} - - // stage('Collisions flavor instability'){ steps{ - // dir('Exec'){ - // sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' - // sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' - // sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' - // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' - // sh 'python ../Scripts/data_reduction/reduce_data.py' - // sh 'python ../Scripts/tests/coll_inst_test.py' - // sh 'rm -rf plt* *pdf' - // } - // }} - - // stage('Collisions to equilibrium'){ steps{ - // dir('Exec'){ - // sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' - // sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3' - // sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' - // sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' - // sh 'python ../Scripts/tests/coll_equi_test.py' - // sh 'rm -rf plt* *pdf' - // } - // }} + stage('Collisions to equilibrium'){ steps{ + dir('Exec'){ + sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' + sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3' + sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' + sh 'python ../Scripts/tests/coll_equi_test.py' + sh 'rm -rf plt* *pdf' + } + }} stage('Fermi-Dirac test'){ steps{ dir('Exec'){ From 12bdf0d8bb9440e9b6ee48078483d74a100677b6 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 15:01:36 -0400 Subject: [PATCH 212/276] Trying to fix Jenkins problem when reading tables --- Jenkinsfile | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cf6cd820..be0a5c97 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,6 @@ pipeline { triggers { pollSCM('') } // Run tests whenever a new commit is detected. - agent { dockerfile {args '--gpus all -v /mnt/scratch/tables/EOS:/EOS:ro'}} // Use the Dockerfile defined in the root Flash-X directory - agent { dockerfile {args '--gpus all -v /mnt/scratch/tables/NuLib:/NuLib:ro'}} // Use the Dockerfile defined in the root Flash-X directory + agent { dockerfile {args '--gpus all -v /mnt/scratch/tables/EOS:/EOS:ro /mnt/scratch/tables/NuLib:/NuLib:ro'}} // Use the Dockerfile defined in the root Flash-X directory environment { // Get rid of Read -1, expected , errno =1 error // See https://github.com/open-mpi/ompi/issues/4948 @@ -9,18 +8,16 @@ pipeline { } stages { - //=============================// - // Set up submodules and amrex // - //=============================// - stage('Prerequisites'){ steps{ - sh 'mpicc -v' - sh 'nvidia-smi' - sh 'nvcc -V' - sh 'git submodule update --init' - sh 'cp makefiles/GNUmakefile_jenkins_HDF5_CUDA Exec/GNUmakefile' - dir('Exec'){ - sh 'make generate; make -j' - } + stage('Collisions flavor instability'){ steps{ + dir('Exec'){ + sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' + sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' + sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' + sh 'python ../Scripts/data_reduction/reduce_data.py' + sh 'python ../Scripts/tests/coll_inst_test.py' + sh 'rm -rf plt* *pdf' + } }} stage('Collisions to equilibrium'){ steps{ From 49e4e909e23874e156bcebbba1f58360cae6700b Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 15:06:59 -0400 Subject: [PATCH 213/276] Trying to fix Jenkins problem when reading tables --- Jenkinsfile | 90 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index be0a5c97..0ad1e4f3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,51 +8,67 @@ pipeline { } stages { - stage('Collisions flavor instability'){ steps{ - dir('Exec'){ - sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' - sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' - sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' - sh 'python ../Scripts/data_reduction/reduce_data.py' - sh 'python ../Scripts/tests/coll_inst_test.py' - sh 'rm -rf plt* *pdf' + //=============================// + // Set up submodules and amrex // + //=============================// + stage('Prerequisites'){ steps{ + sh 'mpicc -v' + sh 'nvidia-smi' + sh 'nvcc -V' + sh 'git submodule update --init' + sh 'cp makefiles/GNUmakefile_jenkins_HDF5_CUDA Exec/GNUmakefile' + dir('Exec'){ + sh 'make generate; make -j' + } + } } - }} - stage('Collisions to equilibrium'){ steps{ - dir('Exec'){ - sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' - sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3' - sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' - sh 'python ../Scripts/tests/coll_equi_test.py' - sh 'rm -rf plt* *pdf' - } - }} + stage('Collisions flavor instability'){ steps{ + dir('Exec'){ + sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' + sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' + sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' + sh 'python ../Scripts/data_reduction/reduce_data.py' + sh 'python ../Scripts/tests/coll_inst_test.py' + sh 'rm -rf plt* *pdf' + } + }} - stage('Fermi-Dirac test'){ steps{ - dir('Exec'){ - sh 'python ../Scripts/initial_conditions/st9_empty_particles_multi_energy.py' - sh 'python ../Scripts/collisions/nsm_constant_background_rho_Ye_T__writer.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fermi_dirac_test' - sh 'python ../Scripts/collisions/writeparticleinfohdf5.py' - sh '../Scripts/tests/fermi_dirac_test.py' - sh 'rm -rf plt* *pdf rho_Ye_T.hdf5' - } - }} + stage('Collisions to equilibrium'){ steps{ + dir('Exec'){ + sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' + sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3' + sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' + sh 'python ../Scripts/tests/coll_equi_test.py' + sh 'rm -rf plt* *pdf' + } + }} + + stage('Fermi-Dirac test'){ steps{ + dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st9_empty_particles_multi_energy.py' + sh 'python ../Scripts/collisions/nsm_constant_background_rho_Ye_T__writer.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fermi_dirac_test' + sh 'python ../Scripts/collisions/writeparticleinfohdf5.py' + sh '../Scripts/tests/fermi_dirac_test.py' + sh 'rm -rf plt* *pdf rho_Ye_T.hdf5' + } + }} } // stages{ post { always { - cleanWs( - cleanWhenNotBuilt: true, - deleteDirs: true, - disableDeferredWipeout: false, - notFailBuild: true, - patterns: [[pattern: 'submodules', type: 'EXCLUDE']] ) // allow submodules to be cached - } + cleanWs( + cleanWhenNotBuilt: true, + deleteDirs: true, + disableDeferredWipeout: false, + notFailBuild: true, + patterns: [[pattern: 'submodules', type: 'EXCLUDE']] + ) // allow submodules to be cached + } } } // pipeline{ From 16ff43ac3d762dbe673633ff0882c88fccbeb9f0 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 15:22:11 -0400 Subject: [PATCH 214/276] Trying to fix Jenkins problem when reading tables --- Jenkinsfile | 25 +------------------------ sample_inputs/inputs_fermi_dirac_test | 4 ++-- 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0ad1e4f3..36640264 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ pipeline { triggers { pollSCM('') } // Run tests whenever a new commit is detected. - agent { dockerfile {args '--gpus all -v /mnt/scratch/tables/EOS:/EOS:ro /mnt/scratch/tables/NuLib:/NuLib:ro'}} // Use the Dockerfile defined in the root Flash-X directory + agent { dockerfile {args '--gpus all -v /mnt/scratch/tables:/tables:ro'}} // Use the Dockerfile defined in the root Flash-X directory environment { // Get rid of Read -1, expected , errno =1 error // See https://github.com/open-mpi/ompi/issues/4948 @@ -23,29 +23,6 @@ pipeline { } } - stage('Collisions flavor instability'){ steps{ - dir('Exec'){ - sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' - sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' - sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' - sh 'python ../Scripts/data_reduction/reduce_data.py' - sh 'python ../Scripts/tests/coll_inst_test.py' - sh 'rm -rf plt* *pdf' - } - }} - - stage('Collisions to equilibrium'){ steps{ - dir('Exec'){ - sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' - sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3' - sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' - sh 'python ../Scripts/tests/coll_equi_test.py' - sh 'rm -rf plt* *pdf' - } - }} - stage('Fermi-Dirac test'){ steps{ dir('Exec'){ sh 'python ../Scripts/initial_conditions/st9_empty_particles_multi_energy.py' diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index e7848155..601aeee3 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -117,6 +117,6 @@ delta_E = 0.8339001570751987 # Mev ################# read_rho_T_Ye_from_table = 1 -nulib_table_name = "/NuLib/NuLib_SFHo.h5" -nuceos_table_name = "/EOS/LS220_234r_136t_50y_analmu_20091212_SVNr26.h5" +nulib_table_name = "/mnt/scratch/tables/NuLib/NuLib_SFHo.h5" +nuceos_table_name = "/mnt/scratch/tables/EOS/LS220_234r_136t_50y_analmu_20091212_SVNr26.h5" background_rho_Ye_T_table_name = "./rho_Ye_T.hdf5" From 305f55f03bc0772abbc734554dbb077583437374 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 15:34:00 -0400 Subject: [PATCH 215/276] Trying to fix Jenkins problem when reading tables --- Jenkinsfile | 2 +- sample_inputs/inputs_fermi_dirac_test | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 36640264..0317dca6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ pipeline { triggers { pollSCM('') } // Run tests whenever a new commit is detected. - agent { dockerfile {args '--gpus all -v /mnt/scratch/tables:/tables:ro'}} // Use the Dockerfile defined in the root Flash-X directory + agent { dockerfile {args '--gpus all -v /mnt/scratch/EOS:/EOS:ro -v /mnt/scratch/NuLib:/NuLib:ro'}} // Use the Dockerfile defined in the root Flash-X directory environment { // Get rid of Read -1, expected , errno =1 error // See https://github.com/open-mpi/ompi/issues/4948 diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index 601aeee3..e7848155 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -117,6 +117,6 @@ delta_E = 0.8339001570751987 # Mev ################# read_rho_T_Ye_from_table = 1 -nulib_table_name = "/mnt/scratch/tables/NuLib/NuLib_SFHo.h5" -nuceos_table_name = "/mnt/scratch/tables/EOS/LS220_234r_136t_50y_analmu_20091212_SVNr26.h5" +nulib_table_name = "/NuLib/NuLib_SFHo.h5" +nuceos_table_name = "/EOS/LS220_234r_136t_50y_analmu_20091212_SVNr26.h5" background_rho_Ye_T_table_name = "./rho_Ye_T.hdf5" From 5711c938f1fa5efe5f094d453e7bdd3fd37757b0 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 15:54:48 -0400 Subject: [PATCH 216/276] Adding all tests to Jenkins. --- Jenkinsfile | 113 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 104 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0317dca6..10b45324 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,16 +23,111 @@ pipeline { } } + stage('MSW'){ steps{ + dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st0_msw_test.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_msw_test' + sh 'python ../Scripts/tests/msw_test.py' + sh 'rm -rf plt*' + } + } + } + + stage('Bipolar'){ steps{ + dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st1_bipolar_test.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_bipolar_test' + sh 'rm -rf plt*' + } + } + } + + stage('Fast Flavor'){ steps{ + dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st2_2beam_fast_flavor.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor' + sh 'python ../Scripts/tests/fast_flavor_test.py' + sh 'rm -rf plt*' + } + } + } + + stage('Fast Flavor k'){ steps{ + dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor_nonzerok' + sh 'python ../Scripts/tests/fast_flavor_k_test.py' + sh 'rm -rf plt*' + } + } + } + + stage('Fiducial 2F GPU Binary'){ steps{ + dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_1d_fiducial' + sh 'python ../Scripts/data_reduction/reduce_data_fft.py' + sh 'python ../Scripts/data_reduction/reduce_data.py' + sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data.h5' + sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data_fft_power.h5' + sh 'python ../Scripts/babysitting/avgfee.py' + sh 'python ../Scripts/babysitting/power_spectrum.py' + sh 'python ../Scripts/data_reduction/convertToHDF5.py' + sh 'gnuplot ../Scripts/babysitting/avgfee_gnuplot.plt' + archiveArtifacts artifacts: '*.pdf' + sh 'rm -rf plt*' + } + } + } + + stage('Fiducial 3F CPU HDF5'){ steps{ + dir('Exec'){ + sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5 GNUmakefile' + sh 'make realclean; make generate; make -j' + sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi_3F.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.ex ../sample_inputs/inputs_1d_fiducial' + /*sh 'python3 ../Scripts/babysitting/avgfee_HDF5.py'*/ + sh 'rm -rf plt*' + } + } + } + + stage('Collisions flavor instability'){ steps{ + dir('Exec'){ + sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' + sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2' + sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test' + sh 'python ../Scripts/data_reduction/reduce_data.py' + sh 'python ../Scripts/tests/coll_inst_test.py' + sh 'rm -rf plt* *pdf' + } + } + } + + stage('Collisions to equilibrium'){ steps{ + dir('Exec'){ + sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' + sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3' + sh 'python ../Scripts/initial_conditions/st7_empty_particles.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test' + sh 'python ../Scripts/tests/coll_equi_test.py' + sh 'rm -rf plt* *pdf' + } + } + } + stage('Fermi-Dirac test'){ steps{ - dir('Exec'){ - sh 'python ../Scripts/initial_conditions/st9_empty_particles_multi_energy.py' - sh 'python ../Scripts/collisions/nsm_constant_background_rho_Ye_T__writer.py' - sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fermi_dirac_test' - sh 'python ../Scripts/collisions/writeparticleinfohdf5.py' - sh '../Scripts/tests/fermi_dirac_test.py' - sh 'rm -rf plt* *pdf rho_Ye_T.hdf5' - } - }} + dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st9_empty_particles_multi_energy.py' + sh 'python ../Scripts/collisions/nsm_constant_background_rho_Ye_T__writer.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fermi_dirac_test' + sh 'python ../Scripts/collisions/writeparticleinfohdf5.py' + sh 'python ../Scripts/tests/fermi_dirac_test.py' + sh 'rm -rf plt* *pdf rho_Ye_T.hdf5' + } + } + } } // stages{ From e075f8446508800bbb845fa346729937e864d3b4 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 16:21:41 -0400 Subject: [PATCH 217/276] Solving the issue of plots using LaTeX text settings that do not run in Jenkins. --- Scripts/tests/fermi_dirac_test.py | 54 +++++++++++++++---------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/Scripts/tests/fermi_dirac_test.py b/Scripts/tests/fermi_dirac_test.py index 7285c254..a1a77252 100644 --- a/Scripts/tests/fermi_dirac_test.py +++ b/Scripts/tests/fermi_dirac_test.py @@ -130,7 +130,7 @@ # Font settings mpl.rcParams['font.size'] = 22 mpl.rcParams['font.family'] = 'serif' -mpl.rc('text', usetex=True) +# mpl.rc('text', usetex=True) # Tick settings mpl.rcParams['xtick.major.size'] = 7 @@ -235,35 +235,35 @@ def apply_custom_settings(ax, leg, log_scale_y=False): axNuu.plot(energies_center_erg, Nuu, label=f't = {t[0]:.1e} s') axNuubar.plot(energies_center_erg, Nuubar, label=f't = {t[0]:.1e} s') -axfee.plot(energies_center_erg, f_eq_ee, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') -axfeebar.plot(energies_center_erg, f_eq_eebar, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') -axfuu.plot(energies_center_erg, f_eq_uu, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') -axfuubar.plot(energies_center_erg, f_eq_uubar, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') +axfee.plot(energies_center_erg, f_eq_ee, label='Fermi-Dirac T=5.02 MeV',linestyle='dotted',color = 'black') +axfeebar.plot(energies_center_erg, f_eq_eebar, label='Fermi-Dirac T=5.02 MeV',linestyle='dotted',color = 'black') +axfuu.plot(energies_center_erg, f_eq_uu, label='Fermi-Dirac T=5.02 MeV',linestyle='dotted',color = 'black') +axfuubar.plot(energies_center_erg, f_eq_uubar, label='Fermi-Dirac T=5.02 MeV',linestyle='dotted',color = 'black') -axNee.plot(energies_center_erg, N_eq_ee, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') -axNeebar.plot(energies_center_erg, N_eq_eebar, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') -axNuu.plot(energies_center_erg, N_eq_uu, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') -axNuubar.plot(energies_center_erg, N_eq_uubar, label='Fermi-Dirac $T=5.02$ MeV',linestyle='dotted',color = 'black') +axNee.plot(energies_center_erg, N_eq_ee, label='Fermi-Dirac T=5.02 MeV',linestyle='dotted',color = 'black') +axNeebar.plot(energies_center_erg, N_eq_eebar, label='Fermi-Dirac T=5.02 MeV',linestyle='dotted',color = 'black') +axNuu.plot(energies_center_erg, N_eq_uu, label='Fermi-Dirac T=5.02 MeV',linestyle='dotted',color = 'black') +axNuubar.plot(energies_center_erg, N_eq_uubar, label='Fermi-Dirac T=5.02 MeV',linestyle='dotted',color = 'black') # Add title and labels -axfee.set_xlabel(r'$E$ (erg)') -axfeebar.set_xlabel(r'$E$ (erg)') -axfuu.set_xlabel(r'$E$ (erg)') -axfuubar.set_xlabel(r'$E$ (erg)') -axNee.set_xlabel(r'$E$ (erg)') -axNeebar.set_xlabel(r'$E$ (erg)') -axNuu.set_xlabel(r'$E$ (erg)') -axNuubar.set_xlabel(r'$E$ (erg)') - -axfee.set_ylabel(r'$f_{{e}}^{{eq}}$') -axfeebar.set_ylabel(r'$\bar{f}_{{e}}^{{eq}}$') -axfuu.set_ylabel(r'$f_{{u}}^{{eq}}$') -axfuubar.set_ylabel(r'$\bar{f}_{{u}}^{{eq}}$') - -axNee.set_ylabel(r'$N_{{e}}$') -axNeebar.set_ylabel(r'$\bar{N}_{{e}}$') -axNuu.set_ylabel(r'$N_{{u}}$') -axNuubar.set_ylabel(r'$\bar{N}_{{u}}$') +axfee.set_xlabel(r'E (erg)') +axfeebar.set_xlabel(r'E (erg)') +axfuu.set_xlabel(r'E (erg)') +axfuubar.set_xlabel(r'E (erg)') +axNee.set_xlabel(r'E (erg)') +axNeebar.set_xlabel(r'E (erg)') +axNuu.set_xlabel(r'E (erg)') +axNuubar.set_xlabel(r'E (erg)') + +axfee.set_ylabel(r'f_{{e}}^{{eq}}') +axfeebar.set_ylabel(r'\bar{f}_{{e}}^{{eq}}') +axfuu.set_ylabel(r'f_{{u}}^{{eq}}') +axfuubar.set_ylabel(r'\bar{f}_{{u}}^{{eq}}') + +axNee.set_ylabel(r'N_{{e}}') +axNeebar.set_ylabel(r'\bar{N}_{{e}}') +axNuu.set_ylabel(r'N_{{u}}') +axNuubar.set_ylabel(r'\bar{N}_{{u}}') # Add a legend legfee = axfee.legend(framealpha=0.0, ncol=1, fontsize=14) From bda55b1a6f40808056b638fece4bf2fa1a2ddfd1 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 26 Sep 2024 17:02:44 -0400 Subject: [PATCH 218/276] Creating an input for the boundary condition type. --- Source/Parameters.H | 6 +++++- Source/main.cpp | 3 +-- sample_inputs/inputs_1d_fiducial | 4 +++- sample_inputs/inputs_bipolar_test | 2 ++ sample_inputs/inputs_coll_equi_test | 4 +++- sample_inputs/inputs_collisional_instability_test | 4 +++- sample_inputs/inputs_fast_flavor | 2 ++ sample_inputs/inputs_fast_flavor_nonzerok | 2 ++ sample_inputs/inputs_fermi_dirac_test | 2 ++ sample_inputs/inputs_msw_test | 2 ++ 10 files changed, 25 insertions(+), 6 deletions(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index 1866006b..03733cc5 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -61,6 +61,7 @@ struct TestParams : public amrex::Gpu::Managed std::string nuceos_table_name; std::string nulib_table_name; std::string background_rho_Ye_T_table_name; + int boundary_condition_type; void Initialize(){ ParmParse pp; @@ -121,6 +122,9 @@ struct TestParams : public amrex::Gpu::Managed // attenuation to hamiltonians pp.get("attenuation_hamiltonians", attenuation_hamiltonians); + + // Boundary condition type + pp.get("boundary_condition_type", boundary_condition_type); // Background matter rho, Ye, T flag pp.get("read_rho_T_Ye_from_table", read_rho_T_Ye_from_table); @@ -152,7 +156,7 @@ struct TestParams : public amrex::Gpu::Managed munu[1][i] *= 1e6*CGSUnitsConst::eV; } - pp.get("Do_Pauli_blocking", Do_Pauli_blocking); // If 1, it will multiply the opacities by 1 / (1 - f_eq); if 0, do nothing. + pp.get("Do_Pauli_blocking", Do_Pauli_blocking); // If 1, it will multiply the opacities by 1 / (1 - f_eq); if 0, do nothing. pp.get("delta_E", delta_E); delta_E*= 1e6*CGSUnitsConst::eV; // erg } else if (IMFP_method==2){ diff --git a/Source/main.cpp b/Source/main.cpp index 204fd372..0b99484e 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -45,8 +45,7 @@ void evolve_flavor(const TestParams* parms) //Option 0: use periodic BC //Option 1: create particles at boundary. - //FIXME: FIXME: Define this in parameter file. - const int BC_type = 0; //0=periodic, 1=outer. + const int BC_type = parm->boundary_condition_type; //0=periodic, 1=outer. int BC_type_val; enum BC_type_enum {PERIODIC, OUTER}; diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 333c9dd4..6f04c802 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -90,4 +90,6 @@ IMFP_method = 0 ################# # Background Ye, T and rho ################# -read_rho_T_Ye_from_table = 0 \ No newline at end of file +read_rho_T_Ye_from_table = 0 + +boundary_condition_type = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index c20f028c..07386d4f 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -92,3 +92,5 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 + +boundary_condition_type = 0 diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index a1c6f37c..b0c9aaa1 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -115,4 +115,6 @@ delta_E = 0.8339001570751987 # Mev ################# # Background Ye, T and rho ################# -read_rho_T_Ye_from_table = 0 \ No newline at end of file +read_rho_T_Ye_from_table = 0 + +boundary_condition_type = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test index 089e9646..99f97ebb 100644 --- a/sample_inputs/inputs_collisional_instability_test +++ b/sample_inputs/inputs_collisional_instability_test @@ -115,4 +115,6 @@ delta_E = 2.272540842052914 # Mev ################# # Background Ye, T and rho ################# -read_rho_T_Ye_from_table = 0 \ No newline at end of file +read_rho_T_Ye_from_table = 0 + +boundary_condition_type = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index 567cf2ff..449d86e6 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -92,3 +92,5 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 + +boundary_condition_type = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 32ed3dbb..c04d4a73 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -92,3 +92,5 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 + +boundary_condition_type = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index e7848155..974bafce 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -117,6 +117,8 @@ delta_E = 0.8339001570751987 # Mev ################# read_rho_T_Ye_from_table = 1 +boundary_condition_type = 0 + nulib_table_name = "/NuLib/NuLib_SFHo.h5" nuceos_table_name = "/EOS/LS220_234r_136t_50y_analmu_20091212_SVNr26.h5" background_rho_Ye_T_table_name = "./rho_Ye_T.hdf5" diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index 1387680a..69be3203 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -92,3 +92,5 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 + +boundary_condition_type = 0 From 7998bdf3dfee14bbd6b82292298130cd95346130 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Thu, 26 Sep 2024 20:53:47 -0400 Subject: [PATCH 219/276] use proper variable name --- Source/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/main.cpp b/Source/main.cpp index 0b99484e..3616c917 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -45,7 +45,7 @@ void evolve_flavor(const TestParams* parms) //Option 0: use periodic BC //Option 1: create particles at boundary. - const int BC_type = parm->boundary_condition_type; //0=periodic, 1=outer. + const int BC_type = parms->boundary_condition_type; //0=periodic, 1=outer. int BC_type_val; enum BC_type_enum {PERIODIC, OUTER}; From 7620b014e402c11a0c97da8cc6032829ea7a1037 Mon Sep 17 00:00:00 2001 From: Erick Urquilla Date: Tue, 1 Oct 2024 07:38:35 -0700 Subject: [PATCH 220/276] Include an exception for Jupyter Notebook files in the ".gitignore" file. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 6a98046a..4376b0ba 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ *.out *.app +# Jupiter files +*.ipynb_checkpoints + # Other Scripts/symbolic_hermitians/*.pyc Scripts/symbolic_hermitians/__pycache__ From 68162821fdb0877263acc9dc6754d36ee158feba Mon Sep 17 00:00:00 2001 From: Erick Urquilla Date: Tue, 1 Oct 2024 07:42:08 -0700 Subject: [PATCH 221/276] Include a GNUmakefile for Perlmutter compilation. --- makefiles/GNUmakefile_perlmutter_HDF5_CUDA | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 makefiles/GNUmakefile_perlmutter_HDF5_CUDA diff --git a/makefiles/GNUmakefile_perlmutter_HDF5_CUDA b/makefiles/GNUmakefile_perlmutter_HDF5_CUDA new file mode 100644 index 00000000..93c7005a --- /dev/null +++ b/makefiles/GNUmakefile_perlmutter_HDF5_CUDA @@ -0,0 +1,20 @@ +NUM_FLAVORS = 2 +SHAPE_FACTOR_ORDER = 2 +NUM_MOMENTS = 3 + +COMP = gnu + +DEBUG = FALSE + +USE_MPI = TRUE +USE_OMP = FALSE +USE_ACC = FALSE +USE_CUDA = TRUE +AMREX_CUDA_ARCH=80 + +USE_HDF5=TRUE +HDF5_HOME=/opt/cray/pe/hdf5-parallel/1.12.2.9/ + +EMU_HOME = .. +AMREX_HOME = ../submodules/amrex +include ../Make.Emu From 44a23228fbe8b01d7f852d2cca5faf85d8bbda26 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 8 Oct 2024 09:53:31 -0400 Subject: [PATCH 222/276] Making the ReadHDF5RhoYeT.cpp file compile without MPI and cleaning up some comments. --- Source/ReadHDF5RhoYeT.cpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Source/ReadHDF5RhoYeT.cpp b/Source/ReadHDF5RhoYeT.cpp index ed05ebc4..566357fa 100644 --- a/Source/ReadHDF5RhoYeT.cpp +++ b/Source/ReadHDF5RhoYeT.cpp @@ -7,9 +7,7 @@ #include "ReadHDF5RhoYeT.H" -// mini NoMPI -#define HAVE_CAPABILITY_MPI //FIXME: This should be defined only when USE_MPI = TRUE -#ifdef HAVE_CAPABILITY_MPI +#ifdef AMREX_USE_MPI #include #define BCAST(buffer, size) MPI_Bcast(buffer, size, MPI_BYTE, my_reader_process, MPI_COMM_WORLD) #else @@ -64,14 +62,7 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ //std::string nuceos_table_name = "/home/sshanka/000_UTK_projects/Emu/Exec/SFHo.h5"; amrex::Print() << "(ReadHDF5RhoYeT.cpp) Using hdf5: " << hdf5_background_rho_Ye_T << std::endl; - //TODO: - int my_reader_process = 0; //reader_process; - /*if (my_reader_process < 0 || my_reader_process >= CCTK_nProcs(cctkGH)) - { - CCTK_VWarn(CCTK_WARN_COMPLAIN, __LINE__, __FILE__, CCTK_THORNSTRING, - "Requested IO process %d out of range. Reverting to process 0.", my_reader_process); - my_reader_process = 0; - }*/ + int my_reader_process = 0; const int read_table_on_single_process = 1; //const int doIO = !read_table_on_single_process || CCTK_MyProc(cctkGH) == my_reader_process; //TODO: From 5c63391801f8c916893d867a5b2020be20fc3023 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 8 Oct 2024 14:03:00 -0400 Subject: [PATCH 223/276] Update default makefile to include HDF5 files in the compilation --- makefiles/GNUmakefile_default | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/makefiles/GNUmakefile_default b/makefiles/GNUmakefile_default index 7f96bca7..48f6bae7 100644 --- a/makefiles/GNUmakefile_default +++ b/makefiles/GNUmakefile_default @@ -10,7 +10,9 @@ USE_MPI = FALSE USE_OMP = FALSE USE_ACC = FALSE USE_CUDA = FALSE -USE_HDF5 = FALSE + +USE_HDF5 = TRUE +HDF5_HOME= /path/to/hdf5 EMU_HOME = .. AMREX_HOME = ../submodules/amrex From 6cfd38b2c4b5e8fb1e7df8e920a6a65a0b4ba8c5 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 8 Oct 2024 14:07:48 -0400 Subject: [PATCH 224/276] Delete comment --- Source/ReadHDF5RhoYeT.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/Source/ReadHDF5RhoYeT.cpp b/Source/ReadHDF5RhoYeT.cpp index 566357fa..727bb44c 100644 --- a/Source/ReadHDF5RhoYeT.cpp +++ b/Source/ReadHDF5RhoYeT.cpp @@ -63,9 +63,7 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ amrex::Print() << "(ReadHDF5RhoYeT.cpp) Using hdf5: " << hdf5_background_rho_Ye_T << std::endl; int my_reader_process = 0; - const int read_table_on_single_process = 1; - //const int doIO = !read_table_on_single_process || CCTK_MyProc(cctkGH) == my_reader_process; //TODO: const int doIO = 1; hid_t file; From 87c886150f6400fa81eae7d56c262b9b27c53947 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 14 Oct 2024 14:09:30 -0400 Subject: [PATCH 225/276] Setting rho, Ye, and T in the AMReX mesh MultiFab. The indices have a problem and still need to be tested. --- Source/ReadHDF5RhoYeT.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Source/ReadHDF5RhoYeT.cpp b/Source/ReadHDF5RhoYeT.cpp index 727bb44c..db04848e 100644 --- a/Source/ReadHDF5RhoYeT.cpp +++ b/Source/ReadHDF5RhoYeT.cpp @@ -125,8 +125,19 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ z_min = &zmin_; z_max = &zmax_; + n_cell_x = &ncellx_; + n_cell_y = &ncelly_; + n_cell_z = &ncellz_; + x_min = &xmin_; + x_max = &xmax_; + y_min = &ymin_; + y_max = &ymax_; + z_min = &zmin_; + z_max = &zmax_; + //Allocate managed memory arena on unified memory ManagedArenaAllocator myManagedArena; + ManagedArenaAllocator myManagedArena_int; // Allocate memory for tables double *allbackgroundYeTrhos_temp; @@ -164,7 +175,7 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ T_array_input [i] = allbackgroundYeTrhos_temp[ i + 1 * ncellx_ * ncelly_ * ncellz_ ]; Ye_array_input [i] = allbackgroundYeTrhos_temp[ i + 2 * ncellx_ * ncelly_ * ncellz_ ]; } - + // free memory of temporary array myManagedArena.deallocate(allbackgroundYeTrhos_temp, ncellx_ * ncelly_ * ncellz_ * 3); From 46c9ef533c6f07296f67dea6b58989801bac3461 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 20 Sep 2024 16:39:06 -0400 Subject: [PATCH 226/276] Assigning the correct values of densities, temperature, and electron fraction to the mesh MultiFab. --- Source/ReadHDF5RhoYeT.cpp | 1 - Source/ReadInput_RhoTempYe.cpp | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Source/ReadHDF5RhoYeT.cpp b/Source/ReadHDF5RhoYeT.cpp index db04848e..88b4d556 100644 --- a/Source/ReadHDF5RhoYeT.cpp +++ b/Source/ReadHDF5RhoYeT.cpp @@ -137,7 +137,6 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ //Allocate managed memory arena on unified memory ManagedArenaAllocator myManagedArena; - ManagedArenaAllocator myManagedArena_int; // Allocate memory for tables double *allbackgroundYeTrhos_temp; diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp index aca0f93d..77eba4f1 100644 --- a/Source/ReadInput_RhoTempYe.cpp +++ b/Source/ReadInput_RhoTempYe.cpp @@ -16,7 +16,6 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms ReadInputRhoYeT(parms->background_rho_Ye_T_table_name); using namespace background_input_rho_T_Ye; - int ncell_x = *n_cell_x; int ncell_y = *n_cell_y; int ncell_z = *n_cell_z; double xmin_ = *x_min; @@ -55,8 +54,13 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms int idx = kg + ncell_z * (jg + ncell_y * ig); // Set the values from the input arrays +<<<<<<< HEAD mf_array(i, j, k, GIdx::rho - start_comp) = rhoYeT_input_obj.rho_input[idx]; // g/ccm mf_array(i, j, k, GIdx::T - start_comp) = rhoYeT_input_obj.T_input[idx]*1e6*CGSUnitsConst::eV; //erg +======= + mf_array(i, j, k, GIdx::rho - start_comp) = rhoYeT_input_obj.rho_input[idx]; + mf_array(i, j, k, GIdx::T - start_comp) = rhoYeT_input_obj.T_input[idx]; +>>>>>>> 1eb2623 (Assigning the correct values of densities, temperature, and electron fraction to the mesh MultiFab.) mf_array(i, j, k, GIdx::Ye - start_comp) = rhoYeT_input_obj.Ye_input[idx]; }); From 66359c900a6d2461ae34cfa374008e5f6b4ca251 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 1 Oct 2024 17:17:17 -0400 Subject: [PATCH 227/276] Organizing and structuring the parameter file to avoid reading variables that are not needed in different situations. --- Source/Parameters.H | 178 +++++++++++++++++++++++++------------------- 1 file changed, 101 insertions(+), 77 deletions(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index 03733cc5..ee55d4d1 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -54,6 +54,13 @@ struct TestParams : public amrex::Gpu::Managed // attenuation to hamiltonians Real attenuation_hamiltonians; + // Black hole properties + int do_nsm; #cm + double bh_radius; #cm + double bh_center_x; #cm + double bh_center_y; #cm + double bh_center_z; #cm + // Background matter rho, Ye, T flag int read_rho_T_Ye_from_table; @@ -64,6 +71,7 @@ struct TestParams : public amrex::Gpu::Managed int boundary_condition_type; void Initialize(){ + ParmParse pp; pp.get("ncell", ncell); pp.get("Lx", Lx); @@ -73,9 +81,6 @@ struct TestParams : public amrex::Gpu::Managed pp.get("max_grid_size", max_grid_size); pp.get("nsteps", nsteps); pp.get("end_time", end_time); - pp.get("rho_g_ccm", rho_in); - pp.get("Ye", Ye_in); - pp.get("T_MeV", kT_in); pp.get("cfl_factor", cfl_factor); pp.get("flavor_cfl_factor", flavor_cfl_factor); pp.get("collision_cfl_factor", collision_cfl_factor); @@ -86,84 +91,103 @@ struct TestParams : public amrex::Gpu::Managed pp.get("restart_dir", restart_dir); pp.get("maxError", maxError); - // convert to cgs - kT_in *= 1e6 * CGSUnitsConst::eV; // erg - - // angular grid - pp.get("particle_data_filename",particle_data_filename); - - // perturbation parameters - pp.get("perturbation_amplitude", perturbation_amplitude); - pp.get("perturbation_type", perturbation_type); - if(perturbation_type == 1) - pp.get("perturbation_wavelength_cm", perturbation_wavelength_cm); - - // neutrino physics parameters for 2-flavor - pp.get("mass1_eV", mass1); - pp.get("mass2_eV", mass2); - pp.get("theta12_degrees", theta12); - pp.get("alpha1_degrees", alpha1); - mass1 *= CGSUnitsConst::eV/PhysConst::c2; - mass2 *= CGSUnitsConst::eV/PhysConst::c2; - theta12 *= M_PI/180.; - alpha1 *= M_PI/180.; - if(NUM_FLAVORS>=2){ - pp.get("mass3_eV", mass3); - pp.get("theta13_degrees", theta13); - pp.get("theta23_degrees", theta23); - pp.get("alpha2_degrees", alpha2); - pp.get("deltaCP_degrees", deltaCP); - mass3 *= CGSUnitsConst::eV/PhysConst::c2; - theta13 *= M_PI/180.; - theta23 *= M_PI/180.; - alpha2 *= M_PI/180.; - deltaCP *= M_PI/180.; - } + // angular grid + pp.get("particle_data_filename",particle_data_filename); + + // perturbation parameters + pp.get("perturbation_amplitude", perturbation_amplitude); + pp.get("perturbation_type", perturbation_type); + if(perturbation_type == 1) + pp.get("perturbation_wavelength_cm", perturbation_wavelength_cm); + + // neutrino physics parameters for 2-flavor + pp.get("mass1_eV", mass1); + pp.get("mass2_eV", mass2); + pp.get("theta12_degrees", theta12); + pp.get("alpha1_degrees", alpha1); + mass1 *= CGSUnitsConst::eV/PhysConst::c2; + mass2 *= CGSUnitsConst::eV/PhysConst::c2; + theta12 *= M_PI/180.; + alpha1 *= M_PI/180.; + if(NUM_FLAVORS>=2){ + pp.get("mass3_eV", mass3); + pp.get("theta13_degrees", theta13); + pp.get("theta23_degrees", theta23); + pp.get("alpha2_degrees", alpha2); + pp.get("deltaCP_degrees", deltaCP); + mass3 *= CGSUnitsConst::eV/PhysConst::c2; + theta13 *= M_PI/180.; + theta23 *= M_PI/180.; + alpha2 *= M_PI/180.; + deltaCP *= M_PI/180.; + } - // attenuation to hamiltonians - pp.get("attenuation_hamiltonians", attenuation_hamiltonians); - - // Boundary condition type - pp.get("boundary_condition_type", boundary_condition_type); + // attenuation to hamiltonians + pp.get("attenuation_hamiltonians", attenuation_hamiltonians); + + // Boundary condition type + pp.get("boundary_condition_type", boundary_condition_type); - // Background matter rho, Ye, T flag - pp.get("read_rho_T_Ye_from_table", read_rho_T_Ye_from_table); - if(read_rho_T_Ye_from_table==1){ - pp.get("background_rho_Ye_T_table_name", background_rho_Ye_T_table_name); - } - - // absorption opacities and equilibrium neutrino chemical potentials - pp.get("IMFP_method", IMFP_method); - if(IMFP_method==0){ - // do nothing - all values will be set to zero - } - else if(IMFP_method==1){ - for(int i=0;i Date: Tue, 1 Oct 2024 17:17:31 -0400 Subject: [PATCH 228/276] Update Fermi-Dirac test parameter file --- sample_inputs/inputs_fermi_dirac_test | 38 +++++---------------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index 974bafce..7ec41fdc 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -35,11 +35,6 @@ end_time = 5.0e9 # Make FPE signal errors so we get a Backtrace amrex.fpe_trap_invalid=1 -# give background fluid conditions -rho_g_ccm = 0 -T_MeV = 10 -Ye = 1 - # Write plotfiles write_plot_every = 2000 @@ -85,40 +80,19 @@ deltaCP_degrees = 222 ################# # opacity stuff # ################# -IMFP_method = 2 +boundary_condition_type = 0 +################# +# opacity stuff # +################# +IMFP_method = 2 Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. -IMFP_abs0_cm = 5e-4 -IMFP_abs1_cm = 5e-4 -IMFP_abs2_cm = 5e-4 -IMFP_abs0bar_cm = 5e-4 -IMFP_abs1bar_cm = 5e-4 -IMFP_abs2bar_cm = 5e-4 - -munu0_MeV = 0 -munu1_MeV = 0 -munu2_MeV = 0 -munu0bar_MeV = 0 -munu1bar_MeV = 0 -munu2bar_MeV = 0 - -IMFP_scat0_cm = 0 -IMFP_scat1_cm = 0 -IMFP_scat2_cm = 0 -IMFP_scat0bar_cm = 0 -IMFP_scat1bar_cm = 0 -IMFP_scat2bar_cm = 0 - -delta_E = 0.8339001570751987 # Mev - ################# # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 1 - -boundary_condition_type = 0 - +do_nsm = 0 nulib_table_name = "/NuLib/NuLib_SFHo.h5" nuceos_table_name = "/EOS/LS220_234r_136t_50y_analmu_20091212_SVNr26.h5" background_rho_Ye_T_table_name = "./rho_Ye_T.hdf5" From 64b35b27e3541b69462eb12fd535ea5880dc5795 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 1 Oct 2024 17:21:22 -0400 Subject: [PATCH 229/276] Add an if statement to avoid evolving particles that are inside the black hole when simulating an NSM. --- Source/Evolve.cpp | 81 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 11 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 0af4d22e..7739fa3d 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -193,6 +193,64 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M [=] AMREX_GPU_DEVICE (FlavoredNeutrinoContainer::ParticleType& p, amrex::Array4 const& sarr) { + + // Adding an if statement to avoid computing quantities of particles inside the black hole. + if(parms->IMFP_method==2){ + if(parms->do_nsm==1 ){ + + // Compute particle distance from black hole center + double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); #cm + + // Set time derivatives to zero if particles is inside the BH + if ( particle_distance_from_bh_center < parms->bh_radius ) { + + p.rdata(PIdx::time) = 1.0; // neutrinos move at one second per second! + + // set the dx/dt values + p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; + p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; + p.rdata(PIdx::z) = p.rdata(PIdx::pupz) / p.rdata(PIdx::pupt) * PhysConst::c; + // set the d(pE)/dt values + p.rdata(PIdx::pupx) = 0; + p.rdata(PIdx::pupy) = 0; + p.rdata(PIdx::pupz) = 0; + // set the dE/dt values + p.rdata(PIdx::pupt) = 0; + // set the dVphase/dt values + p.rdata(PIdx::Vphase) = 0; + + if ( NUM_FLAVORS == 2 ){ + + // set the dN/dt and dNbar/dt values + p.rdata(PIdx::N00_Re) = 0.0; + p.rdata(PIdx::N01_Re) = 0.0; + p.rdata(PIdx::N01_Im) = 0.0; + p.rdata(PIdx::N11_Re) = 0.0; + p.rdata(PIdx::N00_Rebar) = 0.0; + p.rdata(PIdx::N01_Rebar) = 0.0; + p.rdata(PIdx::N01_Imbar) = 0.0; + p.rdata(PIdx::N11_Rebar) = 0.0; + + } else if ( NUM_FLAVORS == 3 ) { + + // set the dN/dt and dNbar/dt values + p.rdata(PIdx::N02_Re) = 0.0; + p.rdata(PIdx::N02_Im) = 0.0; + p.rdata(PIdx::N12_Re) = 0.0; + p.rdata(PIdx::N12_Im) = 0.0; + p.rdata(PIdx::N22_Re) = 0.0; + p.rdata(PIdx::N02_Rebar) = 0.0; + p.rdata(PIdx::N02_Imbar) = 0.0; + p.rdata(PIdx::N12_Rebar) = 0.0; + p.rdata(PIdx::N12_Imbar) = 0.0; + p.rdata(PIdx::N22_Rebar) = 0.0; + + } + return; + } + } + } + #include "generated_files/Evolve.cpp_Vvac_fill" const amrex::Real delta_x = (p.pos(0) - plo[0]) * dxi[0]; @@ -251,7 +309,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } // If opacity_method is 2, the code interpolate inverse mean free paths from NuLib table and electron neutrino chemical potential from EoS table to compute the collision term. else if(parms->IMFP_method==2){ - + // Assign temperature, electron fraction, and density at the particle's position to new variables for interpolation of chemical potentials and inverse mean free paths. Real rho = rho_pp; // Density of background matter at this particle's position g/cm^3 Real temperature = T_pp / (1e6*CGSUnitsConst::eV); // Temperature of background matter at this particle's position 0.05 //MeV @@ -333,20 +391,21 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0, 1 or 2"); - for (int i=0; iIMFP_method==1 || parms->IMFP_method==2){ - // Calculate the Fermi-Dirac distribution for neutrinos and antineutrinos. - f_eq[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - munu[i][i] ) / T_pp ) ); - f_eqbar[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - munubar[i][i] ) / T_pp ) ); + for (int i=0; iDo_Pauli_blocking == 1){ - IMFP_abs[i][i] = IMFP_abs[i][i] / ( 1 - f_eq[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). - IMFP_absbar[i][i] = IMFP_absbar[i][i] / ( 1 - f_eqbar[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). - } + // Calculate the Fermi-Dirac distribution for neutrinos and antineutrinos. + f_eq[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - munu[i][i] ) / T_pp ) ); + f_eqbar[i][i] = 1. / ( 1. + exp( ( p.rdata( PIdx::pupt ) - munubar[i][i] ) / T_pp ) ); + // Include the Pauli blocking term + if (parms->Do_Pauli_blocking == 1){ + IMFP_abs[i][i] = IMFP_abs[i][i] / ( 1 - f_eq[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). + IMFP_absbar[i][i] = IMFP_absbar[i][i] / ( 1 - f_eqbar[i][i] ) ; // Multiply the absortion inverse mean free path by the Pauli blocking term 1 / (1 - f_eq). + } + } } - // Compute the time derivative of \( N_{ab} \) using the Quantum Kinetic Equations (QKE). #include "generated_files/Evolve.cpp_dfdt_fill" From 964ed4d3a65b1054e49faf0fedae96d11631da70 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 1 Oct 2024 17:37:09 -0400 Subject: [PATCH 230/276] Solve syntax issue. --- Source/Evolve.cpp | 2 +- Source/Parameters.H | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 7739fa3d..c9f2538c 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -199,7 +199,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M if(parms->do_nsm==1 ){ // Compute particle distance from black hole center - double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); #cm + double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm // Set time derivatives to zero if particles is inside the BH if ( particle_distance_from_bh_center < parms->bh_radius ) { diff --git a/Source/Parameters.H b/Source/Parameters.H index ee55d4d1..6a456709 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -55,11 +55,11 @@ struct TestParams : public amrex::Gpu::Managed Real attenuation_hamiltonians; // Black hole properties - int do_nsm; #cm - double bh_radius; #cm - double bh_center_x; #cm - double bh_center_y; #cm - double bh_center_z; #cm + int do_nsm; //cm + double bh_radius; //cm + double bh_center_x; //cm + double bh_center_y; //cm + double bh_center_z; //cm // Background matter rho, Ye, T flag int read_rho_T_Ye_from_table; From 8f7f7bcff1442aadd6eec4466eaae7b52b02668c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 1 Oct 2024 18:00:02 -0400 Subject: [PATCH 231/276] Writing derivatives of N and Nbar in symbolic Hermitian format. --- Scripts/symbolic_hermitians/generate_code.py | 21 ++++++++++++- Source/Evolve.cpp | 31 ++------------------ 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 5207b5f7..ec96ce36 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -507,4 +507,23 @@ def sgn(t,var): code = [line for sublist in code for line in sublist] write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill")) - + #========================# + # Evolve.cpp_dfdt_fill_zeros # + #========================# + + # List that will store the QKE code. + code = [] + + # Looping over neutrinos(tail: no tail) and antineutrinos(tail: bar) + for t in tails: + + # Store dFdt back into the particle data for F + dNdt = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") + dNdt_empty = HermitianMatrix(args.N, "0.0") + dNdt.H = dNdt_empty.H + + # Write out dNdt->N + code.append(dNdt.code()) + + code = [line for sublist in code for line in sublist] + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill_zeros")) \ No newline at end of file diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index c9f2538c..9607f94a 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -218,34 +218,9 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M p.rdata(PIdx::pupt) = 0; // set the dVphase/dt values p.rdata(PIdx::Vphase) = 0; - - if ( NUM_FLAVORS == 2 ){ - - // set the dN/dt and dNbar/dt values - p.rdata(PIdx::N00_Re) = 0.0; - p.rdata(PIdx::N01_Re) = 0.0; - p.rdata(PIdx::N01_Im) = 0.0; - p.rdata(PIdx::N11_Re) = 0.0; - p.rdata(PIdx::N00_Rebar) = 0.0; - p.rdata(PIdx::N01_Rebar) = 0.0; - p.rdata(PIdx::N01_Imbar) = 0.0; - p.rdata(PIdx::N11_Rebar) = 0.0; - - } else if ( NUM_FLAVORS == 3 ) { - - // set the dN/dt and dNbar/dt values - p.rdata(PIdx::N02_Re) = 0.0; - p.rdata(PIdx::N02_Im) = 0.0; - p.rdata(PIdx::N12_Re) = 0.0; - p.rdata(PIdx::N12_Im) = 0.0; - p.rdata(PIdx::N22_Re) = 0.0; - p.rdata(PIdx::N02_Rebar) = 0.0; - p.rdata(PIdx::N02_Imbar) = 0.0; - p.rdata(PIdx::N12_Rebar) = 0.0; - p.rdata(PIdx::N12_Imbar) = 0.0; - p.rdata(PIdx::N22_Rebar) = 0.0; - - } + + #include "generated_files/Evolve.cpp_dfdt_fill_zeros" + return; } } From 1043aae6d489a9a41d1a126faedbd586328a11c4 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 1 Oct 2024 18:58:32 -0400 Subject: [PATCH 232/276] "Creating a function to set particles to N=0 and Nbar=0 in the boundary cells." --- Source/Evolve.H | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Source/Evolve.H b/Source/Evolve.H index 395634e7..8abd4cc8 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -32,4 +32,6 @@ void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const amrex::MultiFab& state, const amrex::Geometry& geom, const TestParams* parms); +void particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const amrex::MultiFab& state, const amrex::Geometry& geom, const TestParams* parms); + #endif From 876adfc2371c9453bb9e66e7fb436324befbe57d Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 1 Oct 2024 18:59:36 -0400 Subject: [PATCH 233/276] Creating a flag for boundary conditions that set particles to N=0 and Nbar=0 in the boundary cells. --- Source/Parameters.H | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index 6a456709..1949e1d4 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -55,7 +55,8 @@ struct TestParams : public amrex::Gpu::Managed Real attenuation_hamiltonians; // Black hole properties - int do_nsm; //cm + int do_nsm; + int do_periodic_empty_bc; double bh_radius; //cm double bh_center_x; //cm double bh_center_y; //cm @@ -181,6 +182,7 @@ struct TestParams : public amrex::Gpu::Managed pp.get("do_nsm", do_nsm); if ( do_nsm == 1 ){ + pp.get("do_periodic_empty_bc", do_periodic_empty_bc); pp.get("bh_radius", bh_radius); pp.get("bh_center_x", bh_center_x); pp.get("bh_center_y", bh_center_y); From 165d2a84976b35c27fcbedb2fdee18f05e50f973 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 1 Oct 2024 19:00:44 -0400 Subject: [PATCH 234/276] Add function to set particles to N=0 and Nbar=0 in the boundary cells in Evole.cpp file. --- Source/Evolve.cpp | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 9607f94a..e0a84bf4 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -223,6 +223,20 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M return; } + + if ( parms->do_periodic_empty_bc == 1 ){ + + // Set time derivatives to zero if particles is in the boundary cells + if (p.rdata(PIdx::x) < parms->Lx / parms->ncell[0] || + p.rdata(PIdx::x) > parms->Lx - parms->Lx / parms->ncell[0] || + p.rdata(PIdx::y) < parms->Ly / parms->ncell[1] || + p.rdata(PIdx::y) > parms->Ly - parms->Ly / parms->ncell[1] || + p.rdata(PIdx::z) < parms->Lz / parms->ncell[2] || + p.rdata(PIdx::z) > parms->Lz - parms->Lz / parms->ncell[2] ) { + + #include "generated_files/Evolve.cpp_dfdt_fill_zeros" + + } } } @@ -396,3 +410,38 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M p.rdata(PIdx::Vphase) = 0; }); } + + +void particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const MultiFab& state, const Geometry& geom, const TestParams* parms) +{ + + const int lev = 0; +#ifdef _OPENMP +#pragma omp parallel +#endif + for (FNParIter pti(neutrinos, lev); pti.isValid(); ++pti) + { + const int np = pti.numParticles(); + FlavoredNeutrinoContainer::ParticleType* pstruct = &(pti.GetArrayOfStructs()[0]); + + amrex::ParallelFor (np, [=] AMREX_GPU_DEVICE (int i) { + FlavoredNeutrinoContainer::ParticleType& p = pstruct[i]; + + // Compute particle distance from black hole center + double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm + + // Set time derivatives to zero if particles is inside the BH or the boundary cells + if ( particle_distance_from_bh_center < parms->bh_radius || + p.rdata(PIdx::x) < parms->Lx / parms->ncell[0] || + p.rdata(PIdx::x) > parms->Lx - parms->Lx / parms->ncell[0] || + p.rdata(PIdx::y) < parms->Ly / parms->ncell[1] || + p.rdata(PIdx::y) > parms->Ly - parms->Ly / parms->ncell[1] || + p.rdata(PIdx::z) < parms->Lz / parms->ncell[2] || + p.rdata(PIdx::z) > parms->Lz - parms->Lz / parms->ncell[2] ) { + + #include "generated_files/Evolve.cpp_dfdt_fill_zeros" + + } + }); + } +} From 9075f001950e3083a16e39083f2a1ba09529899f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 1 Oct 2024 19:01:39 -0400 Subject: [PATCH 235/276] Add the code in main to do periodic boundary conditions but initialize particles with N=0 and Nbar=0 at the boundary --- Source/main.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Source/main.cpp b/Source/main.cpp index 3616c917..f93562e6 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -213,6 +213,15 @@ void evolve_flavor(const TestParams* parms) // Use the latest-time neutrino data auto& neutrinos = neutrinos_new; + // Do periodic boundary conditions but initialize particles with N=0 and Nbar=0 at the boundary + if ( parms->IMFP_method == 2 ){ + if ( parms->do_nsm == 1 ){ + if ( parms->do_periodic_empty_bc == 1 ){ + particles_at_boundary_cells(neutrinos, state, geom, parms); + } + } + } + const Real current_dt = integrator.get_timestep(); //FIXME: FIXME: Pass this to neutrinos.CreateParticlesAtBoundary. //FIXME: Think carefully where to call this function. From e69d99c6e91250fb7b310457c9f60eec76764995 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 1 Oct 2024 19:11:46 -0400 Subject: [PATCH 236/276] Solving bracket issue. --- Source/Evolve.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index e0a84bf4..41b8dfa7 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -237,6 +237,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M #include "generated_files/Evolve.cpp_dfdt_fill_zeros" } + } } } From 88264b5caf7879b31a24496d13065d4941daaba7 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 1 Oct 2024 19:46:08 -0400 Subject: [PATCH 237/276] Adding return when particles are in boundary cells --- Source/Evolve.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 41b8dfa7..aab28349 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -236,6 +236,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M #include "generated_files/Evolve.cpp_dfdt_fill_zeros" + return; } } } From 291b037778a2d598d5e49559373000d513ea868e Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 2 Oct 2024 08:25:46 -0400 Subject: [PATCH 238/276] Resolve the issue in the plots in the Fermi-Dirac test script. --- Scripts/tests/fermi_dirac_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Scripts/tests/fermi_dirac_test.py b/Scripts/tests/fermi_dirac_test.py index a1a77252..ad7a5206 100644 --- a/Scripts/tests/fermi_dirac_test.py +++ b/Scripts/tests/fermi_dirac_test.py @@ -211,13 +211,13 @@ def apply_custom_settings(ax, leg, log_scale_y=False): Nuubar = np.zeros(len(energies_center_erg)) Total_Vphase = np.zeros(len(energies_center_erg)) - for i, energy_erg in enumerate(energies_center_erg): + for j, energy_erg in enumerate(energies_center_erg): mask = E == energy_erg - Nee[i] = np.sum(N00_Re[mask]) - Neebar[i] = np.sum(N00_Rebar[mask]) - Nuu[i] = np.sum(N11_Re[mask]) - Nuubar[i] = np.sum(N11_Rebar[mask]) - Total_Vphase[i] = np.sum(Vphase[mask]) + Nee[j] = np.sum(N00_Re[mask]) + Neebar[j] = np.sum(N00_Rebar[mask]) + Nuu[j] = np.sum(N11_Re[mask]) + Nuubar[j] = np.sum(N11_Rebar[mask]) + Total_Vphase[j] = np.sum(Vphase[mask]) fee = ( h * clight )**3 * Nee / Total_Vphase feebar = ( h * clight )**3 * Neebar / Total_Vphase From d578339498f4c63ddfdfccda567a12f7ba6fc13e Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 14 Oct 2024 14:17:18 -0400 Subject: [PATCH 239/276] Resolve rebase conflict --- Source/ReadInput_RhoTempYe.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp index 77eba4f1..fe1ece3e 100644 --- a/Source/ReadInput_RhoTempYe.cpp +++ b/Source/ReadInput_RhoTempYe.cpp @@ -54,13 +54,8 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms int idx = kg + ncell_z * (jg + ncell_y * ig); // Set the values from the input arrays -<<<<<<< HEAD mf_array(i, j, k, GIdx::rho - start_comp) = rhoYeT_input_obj.rho_input[idx]; // g/ccm mf_array(i, j, k, GIdx::T - start_comp) = rhoYeT_input_obj.T_input[idx]*1e6*CGSUnitsConst::eV; //erg -======= - mf_array(i, j, k, GIdx::rho - start_comp) = rhoYeT_input_obj.rho_input[idx]; - mf_array(i, j, k, GIdx::T - start_comp) = rhoYeT_input_obj.T_input[idx]; ->>>>>>> 1eb2623 (Assigning the correct values of densities, temperature, and electron fraction to the mesh MultiFab.) mf_array(i, j, k, GIdx::Ye - start_comp) = rhoYeT_input_obj.Ye_input[idx]; }); From d0737dd652be7bbefe7b596553cd3e745e4d7ad3 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 14 Oct 2024 14:21:57 -0400 Subject: [PATCH 240/276] Including the declaration of the number of cells in the x direction in the function to set the background values of rho, Ye, and T --- Source/ReadInput_RhoTempYe.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/ReadInput_RhoTempYe.cpp b/Source/ReadInput_RhoTempYe.cpp index fe1ece3e..aca0f93d 100644 --- a/Source/ReadInput_RhoTempYe.cpp +++ b/Source/ReadInput_RhoTempYe.cpp @@ -16,6 +16,7 @@ void set_rho_T_Ye(MultiFab& state, const Geometry& geom, const TestParams* parms ReadInputRhoYeT(parms->background_rho_Ye_T_table_name); using namespace background_input_rho_T_Ye; + int ncell_x = *n_cell_x; int ncell_y = *n_cell_y; int ncell_z = *n_cell_z; double xmin_ = *x_min; From fea8609eabc4c71242254f74d145b712749cbb74 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 14 Oct 2024 14:30:44 -0400 Subject: [PATCH 241/276] Delete repeted lines --- Source/ReadHDF5RhoYeT.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Source/ReadHDF5RhoYeT.cpp b/Source/ReadHDF5RhoYeT.cpp index 88b4d556..8042f97f 100644 --- a/Source/ReadHDF5RhoYeT.cpp +++ b/Source/ReadHDF5RhoYeT.cpp @@ -125,16 +125,6 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ z_min = &zmin_; z_max = &zmax_; - n_cell_x = &ncellx_; - n_cell_y = &ncelly_; - n_cell_z = &ncellz_; - x_min = &xmin_; - x_max = &xmax_; - y_min = &ymin_; - y_max = &ymax_; - z_min = &zmin_; - z_max = &zmax_; - //Allocate managed memory arena on unified memory ManagedArenaAllocator myManagedArena; From bd01302d10635f6ba42a7a54af544680df65c5de Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 14 Oct 2024 14:43:24 -0400 Subject: [PATCH 242/276] cleaning up the code --- Source/Evolve.cpp | 2 +- Source/ReadHDF5RhoYeT.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index aab28349..b68daa07 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -300,7 +300,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } // If opacity_method is 2, the code interpolate inverse mean free paths from NuLib table and electron neutrino chemical potential from EoS table to compute the collision term. else if(parms->IMFP_method==2){ - + // Assign temperature, electron fraction, and density at the particle's position to new variables for interpolation of chemical potentials and inverse mean free paths. Real rho = rho_pp; // Density of background matter at this particle's position g/cm^3 Real temperature = T_pp / (1e6*CGSUnitsConst::eV); // Temperature of background matter at this particle's position 0.05 //MeV diff --git a/Source/ReadHDF5RhoYeT.cpp b/Source/ReadHDF5RhoYeT.cpp index 8042f97f..727bb44c 100644 --- a/Source/ReadHDF5RhoYeT.cpp +++ b/Source/ReadHDF5RhoYeT.cpp @@ -164,7 +164,7 @@ void ReadInputRhoYeT(const std::string hdf5_background_rho_Ye_T){ T_array_input [i] = allbackgroundYeTrhos_temp[ i + 1 * ncellx_ * ncelly_ * ncellz_ ]; Ye_array_input [i] = allbackgroundYeTrhos_temp[ i + 2 * ncellx_ * ncelly_ * ncellz_ ]; } - + // free memory of temporary array myManagedArena.deallocate(allbackgroundYeTrhos_temp, ncellx_ * ncelly_ * ncellz_ * 3); From 1ff374b0c263609acdc981b0e1ea0a7d35873f3c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 15 Oct 2024 08:41:51 -0400 Subject: [PATCH 243/276] Add indentation in if statement in parameter file --- Source/Parameters.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index 1949e1d4..9616eda1 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -99,7 +99,7 @@ struct TestParams : public amrex::Gpu::Managed pp.get("perturbation_amplitude", perturbation_amplitude); pp.get("perturbation_type", perturbation_type); if(perturbation_type == 1) - pp.get("perturbation_wavelength_cm", perturbation_wavelength_cm); + pp.get("perturbation_wavelength_cm", perturbation_wavelength_cm); // neutrino physics parameters for 2-flavor pp.get("mass1_eV", mass1); From 6e7e9af7e65c5bfb04f6f42af7a80acc83fdfe4b Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 15 Oct 2024 08:43:38 -0400 Subject: [PATCH 244/276] Delete repeated comment --- sample_inputs/inputs_fermi_dirac_test | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index 7ec41fdc..71eb0da3 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -77,16 +77,12 @@ alpha2_degrees = 0 # CP-violating phase in degrees [NO:222 IO:285] deltaCP_degrees = 222 -################# -# opacity stuff # -################# -boundary_condition_type = 0 - ################# # opacity stuff # ################# IMFP_method = 2 Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. +boundary_condition_type = 0 ################# # Background Ye, T and rho From 767cfe5a12945e9862b580b9c45a6c0a31c7124d Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 15 Oct 2024 10:04:16 -0400 Subject: [PATCH 245/276] Allowing IMFP method = 1 do periodic boundary initialization --- Source/Evolve.cpp | 27 +++++++++++++++++---------- Source/main.cpp | 8 +++----- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index b68daa07..9846250f 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -195,7 +195,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M { // Adding an if statement to avoid computing quantities of particles inside the black hole. - if(parms->IMFP_method==2){ + if( parms->IMFP_method==2 || parms->IMFP_method==1 ){ if(parms->do_nsm==1 ){ // Compute particle distance from black hole center @@ -429,21 +429,28 @@ void particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const Mul amrex::ParallelFor (np, [=] AMREX_GPU_DEVICE (int i) { FlavoredNeutrinoContainer::ParticleType& p = pstruct[i]; + if(parms->do_nsm==1 ){ + // Compute particle distance from black hole center double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm // Set time derivatives to zero if particles is inside the BH or the boundary cells - if ( particle_distance_from_bh_center < parms->bh_radius || - p.rdata(PIdx::x) < parms->Lx / parms->ncell[0] || - p.rdata(PIdx::x) > parms->Lx - parms->Lx / parms->ncell[0] || - p.rdata(PIdx::y) < parms->Ly / parms->ncell[1] || - p.rdata(PIdx::y) > parms->Ly - parms->Ly / parms->ncell[1] || - p.rdata(PIdx::z) < parms->Lz / parms->ncell[2] || - p.rdata(PIdx::z) > parms->Lz - parms->Lz / parms->ncell[2] ) { - + if ( particle_distance_from_bh_center < parms->bh_radius ) { #include "generated_files/Evolve.cpp_dfdt_fill_zeros" - } + + } + // Set time derivatives to zero if particles is inside the BH or the boundary cells + if (p.rdata(PIdx::x) < parms->Lx / parms->ncell[0] || + p.rdata(PIdx::x) > parms->Lx - parms->Lx / parms->ncell[0] || + p.rdata(PIdx::y) < parms->Ly / parms->ncell[1] || + p.rdata(PIdx::y) > parms->Ly - parms->Ly / parms->ncell[1] || + p.rdata(PIdx::z) < parms->Lz / parms->ncell[2] || + p.rdata(PIdx::z) > parms->Lz - parms->Lz / parms->ncell[2] ) { + + #include "generated_files/Evolve.cpp_dfdt_fill_zeros" + + } }); } } diff --git a/Source/main.cpp b/Source/main.cpp index f93562e6..2d0fe0d2 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -214,11 +214,9 @@ void evolve_flavor(const TestParams* parms) auto& neutrinos = neutrinos_new; // Do periodic boundary conditions but initialize particles with N=0 and Nbar=0 at the boundary - if ( parms->IMFP_method == 2 ){ - if ( parms->do_nsm == 1 ){ - if ( parms->do_periodic_empty_bc == 1 ){ - particles_at_boundary_cells(neutrinos, state, geom, parms); - } + if ( parms->IMFP_method == 1 || parms->IMFP_method == 2 ){ + if ( parms->do_periodic_empty_bc == 1 ){ + particles_at_boundary_cells(neutrinos, state, geom, parms); } } From b02a642b3924565c26a8c9fa24d2b1a4a7c743bf Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 15 Oct 2024 10:11:19 -0400 Subject: [PATCH 246/276] Allow periodic vacuum initialization of boundary condition without do_nsm flag --- Source/Parameters.H | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index 9616eda1..66c8cf82 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -147,6 +147,8 @@ struct TestParams : public amrex::Gpu::Managed if(IMFP_method==1 || IMFP_method==2 ){ pp.get("Do_Pauli_blocking", Do_Pauli_blocking); // If 1, it will multiply the opacities by 1 / (1 - f_eq); if 0, do nothing. + pp.get("do_periodic_empty_bc", do_periodic_empty_bc); + } if(IMFP_method==0) { @@ -182,7 +184,6 @@ struct TestParams : public amrex::Gpu::Managed pp.get("do_nsm", do_nsm); if ( do_nsm == 1 ){ - pp.get("do_periodic_empty_bc", do_periodic_empty_bc); pp.get("bh_radius", bh_radius); pp.get("bh_center_x", bh_center_x); pp.get("bh_center_y", bh_center_y); From 61aae6750f3e53668acadc2a65f4d97bc5e314fc Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 15 Oct 2024 11:43:27 -0400 Subject: [PATCH 247/276] Adding flag to compute do_periodic_empty_bc = 0 to avoid error --- sample_inputs/inputs_coll_equi_test | 4 ++-- sample_inputs/inputs_collisional_instability_test | 4 ++-- sample_inputs/inputs_fermi_dirac_test | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index b0c9aaa1..6960a04a 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -116,5 +116,5 @@ delta_E = 0.8339001570751987 # Mev # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 - -boundary_condition_type = 0 \ No newline at end of file +boundary_condition_type = 0 +do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test index 99f97ebb..5f3e5eed 100644 --- a/sample_inputs/inputs_collisional_instability_test +++ b/sample_inputs/inputs_collisional_instability_test @@ -116,5 +116,5 @@ delta_E = 2.272540842052914 # Mev # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 - -boundary_condition_type = 0 \ No newline at end of file +boundary_condition_type = 0 +do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index 71eb0da3..31953d87 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -89,6 +89,7 @@ boundary_condition_type = 0 ################# read_rho_T_Ye_from_table = 1 do_nsm = 0 +do_periodic_empty_bc = 0 nulib_table_name = "/NuLib/NuLib_SFHo.h5" nuceos_table_name = "/EOS/LS220_234r_136t_50y_analmu_20091212_SVNr26.h5" background_rho_Ye_T_table_name = "./rho_Ye_T.hdf5" From 5af2620ad4ce86550ec1002c4c20fc7e68861e0b Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 15 Oct 2024 11:49:41 -0400 Subject: [PATCH 248/276] Solve error in if statement that avoids computation of quantities for particles in black hole and boundary cells --- Source/Evolve.cpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 9846250f..eabd3a9b 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -223,21 +223,20 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M return; } + } + if ( parms->do_periodic_empty_bc == 1 ){ - if ( parms->do_periodic_empty_bc == 1 ){ - - // Set time derivatives to zero if particles is in the boundary cells - if (p.rdata(PIdx::x) < parms->Lx / parms->ncell[0] || - p.rdata(PIdx::x) > parms->Lx - parms->Lx / parms->ncell[0] || - p.rdata(PIdx::y) < parms->Ly / parms->ncell[1] || - p.rdata(PIdx::y) > parms->Ly - parms->Ly / parms->ncell[1] || - p.rdata(PIdx::z) < parms->Lz / parms->ncell[2] || - p.rdata(PIdx::z) > parms->Lz - parms->Lz / parms->ncell[2] ) { + // Set time derivatives to zero if particles is in the boundary cells + if (p.rdata(PIdx::x) < parms->Lx / parms->ncell[0] || + p.rdata(PIdx::x) > parms->Lx - parms->Lx / parms->ncell[0] || + p.rdata(PIdx::y) < parms->Ly / parms->ncell[1] || + p.rdata(PIdx::y) > parms->Ly - parms->Ly / parms->ncell[1] || + p.rdata(PIdx::z) < parms->Lz / parms->ncell[2] || + p.rdata(PIdx::z) > parms->Lz - parms->Lz / parms->ncell[2] ) { - #include "generated_files/Evolve.cpp_dfdt_fill_zeros" + #include "generated_files/Evolve.cpp_dfdt_fill_zeros" - return; - } + return; } } } From 364ef992776a3f6e35559a1096de5f4b3bb76591 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 15 Oct 2024 17:28:15 -0400 Subject: [PATCH 249/276] Add boundary condition test for periodic empty functions --- Scripts/tests/bc_empty_init_test.py | 254 ++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 Scripts/tests/bc_empty_init_test.py diff --git a/Scripts/tests/bc_empty_init_test.py b/Scripts/tests/bc_empty_init_test.py new file mode 100644 index 00000000..c3581281 --- /dev/null +++ b/Scripts/tests/bc_empty_init_test.py @@ -0,0 +1,254 @@ +import numpy as np +import h5py +import glob +import matplotlib.pyplot as plt +import argparse + +# Define myassert function +parser = argparse.ArgumentParser() +parser.add_argument("-na", "--no_assert", action="store_true", help="If --no_assert is supplied, do not raise assertion errors if the test error > tolerance.") +args = parser.parse_args() +def myassert(condition): + if not args.no_assert: + assert(condition) + +# physical constants +clight = 2.99792458e10 # cm/s +hbar = 1.05457266e-27 # erg s +h = 2.0*np.pi*hbar # erg s +eV = 1.60218e-12 # erg + +# Reading plt* directories +directories = np.array( glob.glob("plt*.h5") ) +# Extract directories with "old" in their names +mask = np.char.find(directories, "old") == -1 +directories = directories[mask] + +# Sort the data file names by time step number +directories = sorted(directories, key=lambda x: int(x.split("plt")[1].split(".")[0])) + +############################################################ +############################################################ +# PLOT SETTINGS +import matplotlib as mpl +from matplotlib.ticker import AutoLocator, AutoMinorLocator, LogLocator + +# Font settings +mpl.rcParams['font.size'] = 22 +mpl.rcParams['font.family'] = 'serif' +# mpl.rc('text', usetex=True) + +# Tick settings +mpl.rcParams['xtick.major.width'] = 2 +mpl.rcParams['xtick.major.pad'] = 8 +mpl.rcParams['xtick.minor.size'] = 4 +mpl.rcParams['xtick.minor.width'] = 2 +mpl.rcParams['ytick.major.size'] = 7 +mpl.rcParams['ytick.major.width'] = 2 +mpl.rcParams['ytick.minor.size'] = 4 +mpl.rcParams['ytick.minor.width'] = 2 + +# Axis linewidth +mpl.rcParams['axes.linewidth'] = 2 + +# Tick direction and enabling ticks on all sides +mpl.rcParams['xtick.direction'] = 'in' +mpl.rcParams['ytick.direction'] = 'in' +mpl.rcParams['xtick.top'] = True +mpl.rcParams['ytick.right'] = True + +# Function to apply custom tick locators and other settings to an Axes object +def apply_custom_settings(ax, leg, log_scale_y=False): + + if log_scale_y: + # Use LogLocator for the y-axis if it's in log scale + ax.set_yscale('log') + ax.yaxis.set_major_locator(LogLocator(base=10.0)) + ax.yaxis.set_minor_locator(LogLocator(base=10.0, subs='auto', numticks=100)) + else: + # Use AutoLocator for regular scales + ax.yaxis.set_major_locator(AutoLocator()) + ax.yaxis.set_minor_locator(AutoMinorLocator()) + + # Apply the AutoLocator for the x-axis + ax.xaxis.set_major_locator(AutoLocator()) + ax.xaxis.set_minor_locator(AutoMinorLocator()) + + # Legend settings + leg.get_frame().set_edgecolor('w') + leg.get_frame().set_linewidth(0.0) +############################################################ +############################################################ + +# Domain size in 3D index space +ncell = (3, 3, 3) +Lx = 3 # cm +Ly = 3 # cm +Lz = 3 # cm + +# Contains mesh faces coordinates +cell_x_faces = np.linspace(0, Lx, ncell[0] + 1) +cell_y_faces = np.linspace(0, Ly, ncell[1] + 1) +cell_z_faces = np.linspace(0, Lz, ncell[2] + 1) + +all_files_ee_ocupation_in_each_cell = np.zeros((len(directories), *ncell)) # number of particles units +all_files_eebar_ocupation_in_each_cell = np.zeros((len(directories), *ncell)) # number of particles units +all_files_uu_ocupation_in_each_cell = np.zeros((len(directories), *ncell)) # number of particles units +all_files_uubar_ocupation_in_each_cell = np.zeros((len(directories), *ncell)) # number of particles units +time = np.zeros(len(directories)) # seconds + +# Looping over all directories +for i in range(len(directories)): + + # Print directory name + print(f'{directories[i]}') + + # Open file + with h5py.File(directories[i], 'r') as hf: + + N00_Re = np.array(hf['N00_Re']) # number of particles + N11_Re = np.array(hf['N11_Re']) # number of particles + N00_Rebar = np.array(hf['N00_Rebar']) # number of particles + N11_Rebar = np.array(hf['N11_Rebar']) # number of particles + E = np.array(hf['pupt']) # ergs + t = np.array(hf['time']) # seconds + Vphase = np.array(hf['Vphase']) # seconds + pos_x = np.array(hf['pos_x']) # cm + pos_y = np.array(hf['pos_y']) # cm + pos_z = np.array(hf['pos_z']) # cm + + # Append time + if i != 0: + time[i] = t[np.nonzero(t)[0][0]] + + # Shape n_particle x 3 array + # First index runs over the particles + # Second index is the cell index in x direction + # Thrid index is the cell index in y direction + # Fourth index is the cell index in z direction + particle_cell = np.zeros((len(N00_Re),3)) + + # Find index of cell in x direction + for j in range(ncell[0]): + mask = ( (pos_x > cell_x_faces[j]) & (pos_x < cell_x_faces[j+1]) ) + particle_cell[:,0][mask] = j + + # Find index of cell in y direction + for j in range(ncell[1]): + mask = ( (pos_y > cell_y_faces[j]) & (pos_y < cell_y_faces[j+1]) ) + particle_cell[:,1][mask] = j + + # Find index of cell in z direction + for j in range(ncell[2]): + mask = ( (pos_z > cell_z_faces[j]) & (pos_z < cell_z_faces[j+1]) ) + particle_cell[:,2][mask] = j + + # Initialize arrays to store occupation numbers for each cell + ee_ocupation_in_each_cell = np.zeros(ncell) + eebar_ocupation_in_each_cell = np.zeros(ncell) + uu_ocupation_in_each_cell = np.zeros(ncell) + uubar_ocupation_in_each_cell = np.zeros(ncell) + + # Loop over all cells in the x, y, and z directions + for j in range(ncell[0]): + for k in range(ncell[1]): + for l in range(ncell[2]): + # Create a mask to identify particles in the current cell (j, k, l) + mask = ( (particle_cell[:,0] == j) & (particle_cell[:,1] == k) & (particle_cell[:,2] == l) ) + + # Print the number of particles of type N00 in the current cell + print(f'Cell ({j},{k},{l}) : N00_Re = {np.sum(N00_Re[mask])}') + print(f'Cell ({j},{k},{l}) : N00_Rebar = {np.sum(N00_Rebar[mask])}') + print(f'Cell ({j},{k},{l}) : N11_Re = {np.sum(N11_Re[mask])}') + print(f'Cell ({j},{k},{l}) : N11_Rebar = {np.sum(N11_Rebar[mask])}') + + # Sum the number of particles of each type in the current cell and store in the respective arrays + ee_ocupation_in_each_cell[j,k,l] = np.sum(N00_Re[mask]) + eebar_ocupation_in_each_cell[j,k,l] = np.sum(N00_Rebar[mask]) + uu_ocupation_in_each_cell[j,k,l] = np.sum(N11_Re[mask]) + uubar_ocupation_in_each_cell[j,k,l] = np.sum(N11_Rebar[mask]) + + # Store the occupation numbers for the current file in the all_files arrays + all_files_ee_ocupation_in_each_cell[i] = ee_ocupation_in_each_cell + all_files_eebar_ocupation_in_each_cell[i] = eebar_ocupation_in_each_cell + all_files_uu_ocupation_in_each_cell[i] = uu_ocupation_in_each_cell + all_files_uubar_ocupation_in_each_cell[i] = uubar_ocupation_in_each_cell + +N00_Re_theory = 3.0e+33 +N00_Rebar_theory = 2.5e+33 +N11_Re_theory = 1.0e+33 +N11_Rebar_theory = 1.0e+33 + +rel_error_max = 0.05 + +for i in range(ncell[0]): + for j in range(ncell[0]): + for k in range(ncell[0]): + if i==1 and j==1 and k==1: + + rel_error_ee = np.abs( all_files_ee_ocupation_in_each_cell[-1,i,j,k] - N00_Re_theory ) / N00_Re_theory + rel_error_eebar = np.abs( all_files_eebar_ocupation_in_each_cell[-1,i,j,k] - N00_Rebar_theory ) / N00_Rebar_theory + rel_error_uu = np.abs( all_files_uu_ocupation_in_each_cell[-1,i,j,k] - N11_Re_theory ) / N11_Re_theory + rel_error_uubar = np.abs( all_files_uubar_ocupation_in_each_cell[-1,i,j,k] - N11_Rebar_theory ) / N11_Rebar_theory + + print(f"{rel_error_ee} ---> relative error in ee : Cell ({j},{k},{l})") + print(f"{rel_error_eebar} ---> relative error in eebar : Cell ({j},{k},{l})") + # print(f"{rel_error_uu} ---> relative error in uu") + # print(f"{rel_error_uubar} ---> relative error in uubar") + + myassert( rel_error_ee < rel_error_max ) + myassert( rel_error_eebar < rel_error_max ) + # myassert( rel_error_uu < rel_error_max ) + # myassert( rel_error_uubar < rel_error_max ) + + else: + + print(f'Cell ({j},{k},{l})') + rel_error_ee = np.abs( all_files_ee_ocupation_in_each_cell[-1,i,j,k] ) + rel_error_eebar = np.abs( all_files_eebar_ocupation_in_each_cell[-1,i,j,k] ) + rel_error_uu = np.abs( all_files_uu_ocupation_in_each_cell[-1,i,j,k] ) + rel_error_uubar = np.abs( all_files_uubar_ocupation_in_each_cell[-1,i,j,k] ) + + print(f"{rel_error_ee} ---> relative error in ee : Cell ({j},{k},{l})") + print(f"{rel_error_eebar} ---> relative error in eebar : Cell ({j},{k},{l})") + # print(f"{rel_error_uu} ---> relative error in uu") + # print(f"{rel_error_uubar} ---> relative error in uubar") + + myassert( rel_error_ee < rel_error_max ) + myassert( rel_error_eebar < rel_error_max ) + # myassert( rel_error_uu < rel_error_max ) + # myassert( rel_error_uubar < rel_error_max ) + +print(f'all_files_ee_ocupation_in_each_cell[:,i,j,k] = {all_files_ee_ocupation_in_each_cell[:,i,j,k]}') +print(f'time = {time}') + +fig1, ax1 = plt.subplots() +for i in range(ncell[0]): + for j in range(ncell[1]): + for k in range(ncell[2]): + if i==1 and j==1 and k==1: + ax1.plot(time, all_files_ee_ocupation_in_each_cell[:,i,j,k], label=f'Cell ({i},{j},{k})', linestyle='--', color='black') + else: + ax1.plot(time, all_files_ee_ocupation_in_each_cell[:,i,j,k], label=f'Cell ({i},{j},{k})') +ax1.set_xlabel('time ($s$)') +ax1.set_ylabel('$N_{ee}$') +leg1 = ax1.legend(framealpha=0.0, ncol=3, fontsize=10) +apply_custom_settings(ax1, leg1, False) +plt.tight_layout() +fig1.savefig('electron_occupation.pdf', bbox_inches='tight') + +fig2, ax2 = plt.subplots() +for i in range(ncell[0]): + for j in range(ncell[1]): + for k in range(ncell[2]): + if i==1 and j==1 and k==1: + ax2.plot(time, all_files_eebar_ocupation_in_each_cell[:,i,j,k], label=f'Cell ({i},{j},{k})',linestyle='--', color='black') + else: + ax2.plot(time, all_files_eebar_ocupation_in_each_cell[:,i,j,k], label=f'Cell ({i},{j},{k})') + +ax2.set_xlabel('time ($s$)') +ax2.set_ylabel('$\\bar{N}_{ee}$') +leg2 = ax2.legend(framealpha=0.0, ncol=3, fontsize=10) +apply_custom_settings(ax2, leg2, False) +plt.tight_layout() +fig2.savefig('electron_occupation_bar.pdf', bbox_inches='tight') From af90ac76f9c99162ea3ed80de531325205e46d45 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 15 Oct 2024 17:28:54 -0400 Subject: [PATCH 250/276] Create initial condition for empty boundary condition test --- sample_inputs/inputs_bc_periodic_init | 120 ++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 sample_inputs/inputs_bc_periodic_init diff --git a/sample_inputs/inputs_bc_periodic_init b/sample_inputs/inputs_bc_periodic_init new file mode 100644 index 00000000..b2483775 --- /dev/null +++ b/sample_inputs/inputs_bc_periodic_init @@ -0,0 +1,120 @@ +perturbation_type = 0 +perturbation_amplitude = 0.0 + +# attenuation parameters to time derivative of N due to hamiltonians +attenuation_hamiltonians = 0 + +collision_cfl_factor = 1e-3 +cfl_factor = 0.5 +flavor_cfl_factor = 0.5 +max_adaptive_speedup = 0 +maxError = 1e-6 + +integration.type = 1 +integration.rk.type = 4 + +# Domain size in 3D index space +ncell = (3, 3, 3) +Lx = 3 # cm +Ly = 3 # cm +Lz = 3 # cm + +# Number of particles per cell +nppc = (1, 1, 1) +particle_data_filename = "particle_input.dat" + +# Maximum size of each grid in the domain +max_grid_size = 16 + +# Number of steps to run +nsteps = 1000 + +# Simulation end time +end_time = 5.0e19 + +# Make FPE signal errors so we get a Backtrace +amrex.fpe_trap_invalid=1 + +# give background fluid conditions +rho_g_ccm = 0 +T_MeV = 7.0 +Ye = 0 + +# Write plotfiles +write_plot_every = 100 + +# Write particle data in plotfiles +write_plot_particles_every = 100 + +# checkpointing +do_restart = 0 +restart_dir = "" + +############################### +# NEUTRINO PHYSICS PARAMETERS # +############################### +# see first column of table 14.7 in http://pdg.lbl.gov/2019/reviews/rpp2019-rev-neutrino-mixing.pdf + +# mass state 1 mass in eV [NO/IO:-sqrt(7.39e-5)] +mass1_eV = 0.04866 #-0.008596511 + +# mass state 2 mass in eV (define at 0 arbitrarily because oscillations only sensitive to deltaM^2) +mass2_eV = 0 + +# mass state 3 mass in eV [NO:sqrt(2.449e-3) IO:-sqrt(2.509e-3)] +mass3_eV = 0 + +# 1-2 mixing angle in degrees [NO/IO:33.82] +theta12_degrees = 1e-6 + +# 2-3 mixing angle in degrees [NO:8.61 IO:8.65] +theta23_degrees = 0 + +# 1-3 mixing angle in degrees [NO:48.3 IO:48.6] +theta13_degrees = 0 + +# Majorana angle 1 in degrees +alpha1_degrees = 0 + +# Majorana angle 2 in degrees +alpha2_degrees = 0 + +# CP-violating phase in degrees [NO:222 IO:285] +deltaCP_degrees = 0 + +################# +# opacity stuff # +################# +IMFP_method = 1 + +Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. + +IMFP_abs0_cm = 2.398082e1 +IMFP_abs1_cm = 0 +IMFP_abs2_cm = 0 +IMFP_abs0bar_cm = 2.29358e0 +IMFP_abs1bar_cm = 0 +IMFP_abs2bar_cm = 0 + +munu0_MeV = 20.0 +munu1_MeV = 0 +munu2_MeV = 0 +munu0bar_MeV = 17.644694342915507 +munu1bar_MeV = 0 +munu2bar_MeV = 0 + +IMFP_scat0_cm = 0 +IMFP_scat1_cm = 0 +IMFP_scat2_cm = 0 +IMFP_scat0bar_cm = 0 +IMFP_scat1bar_cm = 0 +IMFP_scat2bar_cm = 0 + +delta_E = 2.272540842052914 # Mev + +################# +# Background Ye, T and rho +################# +read_rho_T_Ye_from_table = 0 +boundary_condition_type = 0 +do_periodic_empty_bc = 1 \ No newline at end of file From 516f6b3e8f5ac8be712eece725a54f7fbd0e7990 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 16 Oct 2024 10:30:16 -0400 Subject: [PATCH 251/276] Add periodic empty boundary conditions to Jenkins --- Jenkinsfile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 10b45324..9f872bc2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -104,7 +104,15 @@ pipeline { } } } - + stage('BC periodic empty'){ steps{ + dir('Exec'){ + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_bc_periodic_init' + sh 'python ../Scripts/collisions/writeparticleinfohdf5.py' + sh 'python ../Scripts/tests/bc_empty_init_test.py' + sh 'rm -rf plt* *pdf' + } + } + } stage('Collisions to equilibrium'){ steps{ dir('Exec'){ sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile' From f3c7489ae3316f73d32ff4815bd81c4d114a8a47 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 16 Oct 2024 10:38:10 -0400 Subject: [PATCH 252/276] Cleaning up and writing comments --- Scripts/symbolic_hermitians/generate_code.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index ec96ce36..2e223e95 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -511,18 +511,18 @@ def sgn(t,var): # Evolve.cpp_dfdt_fill_zeros # #========================# - # List that will store the QKE code. + # List that will store the code for setting the derivative of the matrices N and Nbar to zero. code = [] # Looping over neutrinos(tail: no tail) and antineutrinos(tail: bar) for t in tails: - # Store dFdt back into the particle data for F - dNdt = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") - dNdt_empty = HermitianMatrix(args.N, "0.0") - dNdt.H = dNdt_empty.H + # Store dN/dt and dNbar/dt set to zero + dNdt = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") # Derivative of the neutrino number matrix + zero_matrix = HermitianMatrix(args.N, "0.0") # Zero matrix + dNdt.H = zero_matrix.H # Set the derivative of the neutrino number matrix to zero - # Write out dNdt->N + # Write out dN/dt = 0 code.append(dNdt.code()) code = [line for sublist in code for line in sublist] From 716a4fad41447c602adf3955d1b9425ad25261db Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 16 Oct 2024 12:53:38 -0400 Subject: [PATCH 253/276] Cleaning up and adding comments to BC empty particle test --- Scripts/tests/bc_empty_init_test.py | 62 +++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/Scripts/tests/bc_empty_init_test.py b/Scripts/tests/bc_empty_init_test.py index c3581281..8fdd2c53 100644 --- a/Scripts/tests/bc_empty_init_test.py +++ b/Scripts/tests/bc_empty_init_test.py @@ -1,3 +1,10 @@ +''' +This test script is used to check if the periodic empty boundary conditions are correctly implemented in the EMU code. +The periodic empty boundary conditions are implemented in the following way: +The particles in the boundary cells should be autamatically set to zero. +Created by Erick Urquilla. University of Tennessee Knoxville, USA. +''' + import numpy as np import h5py import glob @@ -122,10 +129,10 @@ def apply_custom_settings(ax, leg, log_scale_y=False): time[i] = t[np.nonzero(t)[0][0]] # Shape n_particle x 3 array - # First index runs over the particles - # Second index is the cell index in x direction - # Thrid index is the cell index in y direction - # Fourth index is the cell index in z direction + # The first index runs over the particles + # The second index is the cell index in the x direction + # The third index is the cell index in the y direction + # The fourth index is the cell index in the z direction particle_cell = np.zeros((len(N00_Re),3)) # Find index of cell in x direction @@ -174,6 +181,7 @@ def apply_custom_settings(ax, leg, log_scale_y=False): all_files_uu_ocupation_in_each_cell[i] = uu_ocupation_in_each_cell all_files_uubar_ocupation_in_each_cell[i] = uubar_ocupation_in_each_cell +# Theoretical values for the number of particles N00_Re_theory = 3.0e+33 N00_Rebar_theory = 2.5e+33 N11_Re_theory = 1.0e+33 @@ -186,10 +194,10 @@ def apply_custom_settings(ax, leg, log_scale_y=False): for k in range(ncell[0]): if i==1 and j==1 and k==1: - rel_error_ee = np.abs( all_files_ee_ocupation_in_each_cell[-1,i,j,k] - N00_Re_theory ) / N00_Re_theory - rel_error_eebar = np.abs( all_files_eebar_ocupation_in_each_cell[-1,i,j,k] - N00_Rebar_theory ) / N00_Rebar_theory - rel_error_uu = np.abs( all_files_uu_ocupation_in_each_cell[-1,i,j,k] - N11_Re_theory ) / N11_Re_theory - rel_error_uubar = np.abs( all_files_uubar_ocupation_in_each_cell[-1,i,j,k] - N11_Rebar_theory ) / N11_Rebar_theory + rel_error_ee = np.abs( all_files_ee_ocupation_in_each_cell[-1,i,j,k] - N00_Re_theory ) / N00_Re_theory # Calculate relative error for ee occupation number + rel_error_eebar = np.abs( all_files_eebar_ocupation_in_each_cell[-1,i,j,k] - N00_Rebar_theory ) / N00_Rebar_theory # Calculate relative error for eebar occupation number + rel_error_uu = np.abs( all_files_uu_ocupation_in_each_cell[-1,i,j,k] - N11_Re_theory ) / N11_Re_theory # Calculate relative error for uu occupation number + rel_error_uubar = np.abs( all_files_uubar_ocupation_in_each_cell[-1,i,j,k] - N11_Rebar_theory ) / N11_Rebar_theory # Calculate relative error for uubar occupation number print(f"{rel_error_ee} ---> relative error in ee : Cell ({j},{k},{l})") print(f"{rel_error_eebar} ---> relative error in eebar : Cell ({j},{k},{l})") @@ -219,36 +227,56 @@ def apply_custom_settings(ax, leg, log_scale_y=False): # myassert( rel_error_uu < rel_error_max ) # myassert( rel_error_uubar < rel_error_max ) -print(f'all_files_ee_ocupation_in_each_cell[:,i,j,k] = {all_files_ee_ocupation_in_each_cell[:,i,j,k]}') -print(f'time = {time}') - +# Create a figure and axis for plotting electron occupation numbers fig1, ax1 = plt.subplots() + +# Loop over all cells in the x, y, and z directions for i in range(ncell[0]): for j in range(ncell[1]): for k in range(ncell[2]): - if i==1 and j==1 and k==1: - ax1.plot(time, all_files_ee_ocupation_in_each_cell[:,i,j,k], label=f'Cell ({i},{j},{k})', linestyle='--', color='black') + # Plot the electron occupation number for the central cell with a different style + if i == 1 and j == 1 and k == 1: + ax1.plot(time, all_files_ee_ocupation_in_each_cell[:, i, j, k], label=f'Cell ({i},{j},{k})', linestyle='--', color='black') else: - ax1.plot(time, all_files_ee_ocupation_in_each_cell[:,i,j,k], label=f'Cell ({i},{j},{k})') + ax1.plot(time, all_files_ee_ocupation_in_each_cell[:, i, j, k], label=f'Cell ({i},{j},{k})') + +# Set the x and y axis labels ax1.set_xlabel('time ($s$)') ax1.set_ylabel('$N_{ee}$') + +# Add a legend to the plot leg1 = ax1.legend(framealpha=0.0, ncol=3, fontsize=10) + +# Apply custom settings to the plot apply_custom_settings(ax1, leg1, False) + +# Adjust layout and save the figure plt.tight_layout() fig1.savefig('electron_occupation.pdf', bbox_inches='tight') +# Create a figure and axis for plotting electron occupation numbers (bar) fig2, ax2 = plt.subplots() + +# Loop over all cells in the x, y, and z directions for i in range(ncell[0]): for j in range(ncell[1]): for k in range(ncell[2]): - if i==1 and j==1 and k==1: - ax2.plot(time, all_files_eebar_ocupation_in_each_cell[:,i,j,k], label=f'Cell ({i},{j},{k})',linestyle='--', color='black') + # Plot the electron occupation number (bar) for the central cell with a different style + if i == 1 and j == 1 and k == 1: + ax2.plot(time, all_files_eebar_ocupation_in_each_cell[:, i, j, k], label=f'Cell ({i},{j},{k})', linestyle='--', color='black') else: - ax2.plot(time, all_files_eebar_ocupation_in_each_cell[:,i,j,k], label=f'Cell ({i},{j},{k})') + ax2.plot(time, all_files_eebar_ocupation_in_each_cell[:, i, j, k], label=f'Cell ({i},{j},{k})') +# Set the x and y axis labels ax2.set_xlabel('time ($s$)') ax2.set_ylabel('$\\bar{N}_{ee}$') + +# Add a legend to the plot leg2 = ax2.legend(framealpha=0.0, ncol=3, fontsize=10) + +# Apply custom settings to the plot apply_custom_settings(ax2, leg2, False) + +# Adjust layout and save the figure plt.tight_layout() fig2.savefig('electron_occupation_bar.pdf', bbox_inches='tight') From 6e682432968f4ad2071f6d2cbaafff78b036f3e6 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 16 Oct 2024 13:44:08 -0400 Subject: [PATCH 254/276] Delete if statement to check if IMFP method is one or two and just check if do NSM is activated or empty periodic BC are activated --- Source/Evolve.cpp | 98 +++++++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 41 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index eabd3a9b..68abd095 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -194,50 +194,66 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M amrex::Array4 const& sarr) { - // Adding an if statement to avoid computing quantities of particles inside the black hole. - if( parms->IMFP_method==2 || parms->IMFP_method==1 ){ - if(parms->do_nsm==1 ){ - - // Compute particle distance from black hole center - double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm - - // Set time derivatives to zero if particles is inside the BH - if ( particle_distance_from_bh_center < parms->bh_radius ) { - - p.rdata(PIdx::time) = 1.0; // neutrinos move at one second per second! - - // set the dx/dt values - p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; - p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; - p.rdata(PIdx::z) = p.rdata(PIdx::pupz) / p.rdata(PIdx::pupt) * PhysConst::c; - // set the d(pE)/dt values - p.rdata(PIdx::pupx) = 0; - p.rdata(PIdx::pupy) = 0; - p.rdata(PIdx::pupz) = 0; - // set the dE/dt values - p.rdata(PIdx::pupt) = 0; - // set the dVphase/dt values - p.rdata(PIdx::Vphase) = 0; - - #include "generated_files/Evolve.cpp_dfdt_fill_zeros" - - return; - } + // If statement to avoid computing quantities of particles inside the black hole. + if(parms->do_nsm==1 ){ + + // Compute particle distance from black hole center + double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm + + // Set time derivatives to zero if particles is inside the BH + if ( particle_distance_from_bh_center < parms->bh_radius ) { + + p.rdata(PIdx::time) = 1.0; // neutrinos move at one second per second! + + // set the dx/dt values + p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; + p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; + p.rdata(PIdx::z) = p.rdata(PIdx::pupz) / p.rdata(PIdx::pupt) * PhysConst::c; + // set the d(pE)/dt values + p.rdata(PIdx::pupx) = 0; + p.rdata(PIdx::pupy) = 0; + p.rdata(PIdx::pupz) = 0; + // set the dE/dt values + p.rdata(PIdx::pupt) = 0; + // set the dVphase/dt values + p.rdata(PIdx::Vphase) = 0; + + // Set the dN/dt and dNbar/dt values to zero + #include "generated_files/Evolve.cpp_dfdt_fill_zeros" + + return; } - if ( parms->do_periodic_empty_bc == 1 ){ - - // Set time derivatives to zero if particles is in the boundary cells - if (p.rdata(PIdx::x) < parms->Lx / parms->ncell[0] || - p.rdata(PIdx::x) > parms->Lx - parms->Lx / parms->ncell[0] || - p.rdata(PIdx::y) < parms->Ly / parms->ncell[1] || - p.rdata(PIdx::y) > parms->Ly - parms->Ly / parms->ncell[1] || - p.rdata(PIdx::z) < parms->Lz / parms->ncell[2] || - p.rdata(PIdx::z) > parms->Lz - parms->Lz / parms->ncell[2] ) { + } + // Periodic empty boundary conditions. + // Set time derivatives to zero if particles are in the boundary cells + // Check if periodic empty boundary conditions are enabled + if (parms->do_periodic_empty_bc == 1) { - #include "generated_files/Evolve.cpp_dfdt_fill_zeros" + // Check if the particle is in the boundary cells + if (p.rdata(PIdx::x) < parms->Lx / parms->ncell[0] || + p.rdata(PIdx::x) > parms->Lx - parms->Lx / parms->ncell[0] || + p.rdata(PIdx::y) < parms->Ly / parms->ncell[1] || + p.rdata(PIdx::y) > parms->Ly - parms->Ly / parms->ncell[1] || + p.rdata(PIdx::z) < parms->Lz / parms->ncell[2] || + p.rdata(PIdx::z) > parms->Lz - parms->Lz / parms->ncell[2] ) { + + // set the dx/dt values + p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; + p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; + p.rdata(PIdx::z) = p.rdata(PIdx::pupz) / p.rdata(PIdx::pupt) * PhysConst::c; + // set the d(pE)/dt values + p.rdata(PIdx::pupx) = 0; + p.rdata(PIdx::pupy) = 0; + p.rdata(PIdx::pupz) = 0; + // set the dE/dt values + p.rdata(PIdx::pupt) = 0; + // set the dVphase/dt values + p.rdata(PIdx::Vphase) = 0; + + // Set the dN/dt and dNbar/dt values to zero + #include "generated_files/Evolve.cpp_dfdt_fill_zeros" - return; - } + return; } } From fcfce5e6284e399169e656370e57b0f91857819f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 16 Oct 2024 14:17:06 -0400 Subject: [PATCH 255/276] Solving possible issue with time step --- Source/Evolve.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 68abd095..b92c336e 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -209,6 +209,8 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; p.rdata(PIdx::z) = p.rdata(PIdx::pupz) / p.rdata(PIdx::pupt) * PhysConst::c; + // set the dt/dt = 1. Neutrinos move at one second per second + p.rdata(PIdx::time) = 1.0; // set the d(pE)/dt values p.rdata(PIdx::pupx) = 0; p.rdata(PIdx::pupy) = 0; @@ -241,6 +243,8 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; p.rdata(PIdx::z) = p.rdata(PIdx::pupz) / p.rdata(PIdx::pupt) * PhysConst::c; + // set the dt/dt = 1. Neutrinos move at one second per second + p.rdata(PIdx::time) = 1.0; // set the d(pE)/dt values p.rdata(PIdx::pupx) = 0; p.rdata(PIdx::pupy) = 0; @@ -397,6 +401,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0, 1 or 2"); + // Compute equilibrium distribution functions and include Pauli blocking term if requested if(parms->IMFP_method==1 || parms->IMFP_method==2){ for (int i=0; i Date: Wed, 16 Oct 2024 14:20:19 -0400 Subject: [PATCH 256/276] Solving issue with time step problem --- Scripts/tests/bc_empty_init_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Scripts/tests/bc_empty_init_test.py b/Scripts/tests/bc_empty_init_test.py index 8fdd2c53..c1bd2efa 100644 --- a/Scripts/tests/bc_empty_init_test.py +++ b/Scripts/tests/bc_empty_init_test.py @@ -125,8 +125,7 @@ def apply_custom_settings(ax, leg, log_scale_y=False): pos_z = np.array(hf['pos_z']) # cm # Append time - if i != 0: - time[i] = t[np.nonzero(t)[0][0]] + time[i] = t[0] # Shape n_particle x 3 array # The first index runs over the particles From 71ddce9b420e100f5b1089e141600807401aad92 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 16 Oct 2024 14:34:20 -0400 Subject: [PATCH 257/276] Add comments and clean up particles_at_boundary_cells function --- Source/Evolve.H | 17 ++++++++++++++++- Source/Evolve.cpp | 26 ++++++++++++++++++++------ Source/main.cpp | 2 +- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/Source/Evolve.H b/Source/Evolve.H index 8abd4cc8..9d24b336 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -32,6 +32,21 @@ void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const amrex::MultiFab& state, const amrex::Geometry& geom, const TestParams* parms); -void particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const amrex::MultiFab& state, const amrex::Geometry& geom, const TestParams* parms); +/** + * @brief Sets the time derivatives to zero for particles inside the black hole or boundary cells. + * + * This function iterates over all particles in the `FlavoredNeutrinoContainer` and sets their time derivatives to zero if they are inside the black hole or within the boundary cells of the simulation domain. + * + * @param neutrinos Reference to the container holding the flavored neutrinos. + * @param parms Pointer to the structure containing test parameters, including black hole properties and domain dimensions. + * + * The function performs the following steps: + * - Iterates over all particles in the container. + * - Computes the distance of each particle from the black hole center. + * - Sets the time derivatives to zero if the particle is inside the black hole radius. + * - Sets the time derivatives to zero if the particle is within the boundary cells of the simulation domain. + * + */ +void particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); #endif diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index b92c336e..1db25b54 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -439,13 +439,25 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M } -void particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const MultiFab& state, const Geometry& geom, const TestParams* parms) +/** + * @brief Sets the time derivatives to zero for particles inside the black hole or boundary cells. + * + * This function iterates over all particles in the `FlavoredNeutrinoContainer` and sets their time derivatives to zero if they are inside the black hole or within the boundary cells of the simulation domain. + * + * @param neutrinos Reference to the container holding the flavored neutrinos. + * @param parms Pointer to the structure containing test parameters, including black hole properties and domain dimensions. + * + * The function performs the following steps: + * - Iterates over all particles in the container. + * - Computes the distance of each particle from the black hole center. + * - Sets the time derivatives to zero if the particle is inside the black hole radius. + * - Sets the time derivatives to zero if the particle is within the boundary cells of the simulation domain. + * + */ +void particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const TestParams* parms) { const int lev = 0; -#ifdef _OPENMP -#pragma omp parallel -#endif for (FNParIter pti(neutrinos, lev); pti.isValid(); ++pti) { const int np = pti.numParticles(); @@ -454,18 +466,20 @@ void particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const Mul amrex::ParallelFor (np, [=] AMREX_GPU_DEVICE (int i) { FlavoredNeutrinoContainer::ParticleType& p = pstruct[i]; + // Check if the simulation involves a neutron star merger (NSM) if(parms->do_nsm==1 ){ // Compute particle distance from black hole center double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm - // Set time derivatives to zero if particles is inside the BH or the boundary cells + // Set time derivatives to zero if particles are inside the black hole if ( particle_distance_from_bh_center < parms->bh_radius ) { #include "generated_files/Evolve.cpp_dfdt_fill_zeros" } } - // Set time derivatives to zero if particles is inside the BH or the boundary cells + + // Set time derivatives to zero if particles are within the boundary cells if (p.rdata(PIdx::x) < parms->Lx / parms->ncell[0] || p.rdata(PIdx::x) > parms->Lx - parms->Lx / parms->ncell[0] || p.rdata(PIdx::y) < parms->Ly / parms->ncell[1] || diff --git a/Source/main.cpp b/Source/main.cpp index 2d0fe0d2..6dc4bb3a 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -216,7 +216,7 @@ void evolve_flavor(const TestParams* parms) // Do periodic boundary conditions but initialize particles with N=0 and Nbar=0 at the boundary if ( parms->IMFP_method == 1 || parms->IMFP_method == 2 ){ if ( parms->do_periodic_empty_bc == 1 ){ - particles_at_boundary_cells(neutrinos, state, geom, parms); + particles_at_boundary_cells(neutrinos, parms); } } From cfc154544428c9297bc3b3e66ded29b7393eabe1 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 16 Oct 2024 14:36:40 -0400 Subject: [PATCH 258/276] Change name of function particles_at_boundary_cells to empty_particles_at_boundary_cells --- Source/Evolve.H | 2 +- Source/Evolve.cpp | 2 +- Source/main.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/Evolve.H b/Source/Evolve.H index 9d24b336..baf53b0d 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -47,6 +47,6 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const a * - Sets the time derivatives to zero if the particle is within the boundary cells of the simulation domain. * */ -void particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); +void empty_particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); #endif diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 1db25b54..4d4b5030 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -454,7 +454,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M * - Sets the time derivatives to zero if the particle is within the boundary cells of the simulation domain. * */ -void particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const TestParams* parms) +void empty_particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const TestParams* parms) { const int lev = 0; diff --git a/Source/main.cpp b/Source/main.cpp index 6dc4bb3a..a4e08fcc 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -216,7 +216,7 @@ void evolve_flavor(const TestParams* parms) // Do periodic boundary conditions but initialize particles with N=0 and Nbar=0 at the boundary if ( parms->IMFP_method == 1 || parms->IMFP_method == 2 ){ if ( parms->do_periodic_empty_bc == 1 ){ - particles_at_boundary_cells(neutrinos, parms); + empty_particles_at_boundary_cells(neutrinos, parms); } } From ada8a2933b515617e1afd84136c30d77a1b81b69 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 16 Oct 2024 14:39:21 -0400 Subject: [PATCH 259/276] Add if statement to check if IMFP method is equal to 2 before the if statement for do NSM --- Source/Evolve.cpp | 57 +++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 4d4b5030..744e2158 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -195,37 +195,40 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M { // If statement to avoid computing quantities of particles inside the black hole. - if(parms->do_nsm==1 ){ - - // Compute particle distance from black hole center - double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm - - // Set time derivatives to zero if particles is inside the BH - if ( particle_distance_from_bh_center < parms->bh_radius ) { - - p.rdata(PIdx::time) = 1.0; // neutrinos move at one second per second! + if( parms->IMFP_method==2 ){ + if( parms->do_nsm==1 ){ + + // Compute particle distance from black hole center + double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm - // set the dx/dt values - p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; - p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; - p.rdata(PIdx::z) = p.rdata(PIdx::pupz) / p.rdata(PIdx::pupt) * PhysConst::c; - // set the dt/dt = 1. Neutrinos move at one second per second - p.rdata(PIdx::time) = 1.0; - // set the d(pE)/dt values - p.rdata(PIdx::pupx) = 0; - p.rdata(PIdx::pupy) = 0; - p.rdata(PIdx::pupz) = 0; - // set the dE/dt values - p.rdata(PIdx::pupt) = 0; - // set the dVphase/dt values - p.rdata(PIdx::Vphase) = 0; + // Set time derivatives to zero if particles is inside the BH + if ( particle_distance_from_bh_center < parms->bh_radius ) { - // Set the dN/dt and dNbar/dt values to zero - #include "generated_files/Evolve.cpp_dfdt_fill_zeros" - - return; + p.rdata(PIdx::time) = 1.0; // neutrinos move at one second per second! + + // set the dx/dt values + p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; + p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; + p.rdata(PIdx::z) = p.rdata(PIdx::pupz) / p.rdata(PIdx::pupt) * PhysConst::c; + // set the dt/dt = 1. Neutrinos move at one second per second + p.rdata(PIdx::time) = 1.0; + // set the d(pE)/dt values + p.rdata(PIdx::pupx) = 0; + p.rdata(PIdx::pupy) = 0; + p.rdata(PIdx::pupz) = 0; + // set the dE/dt values + p.rdata(PIdx::pupt) = 0; + // set the dVphase/dt values + p.rdata(PIdx::Vphase) = 0; + + // Set the dN/dt and dNbar/dt values to zero + #include "generated_files/Evolve.cpp_dfdt_fill_zeros" + + return; + } } } + // Periodic empty boundary conditions. // Set time derivatives to zero if particles are in the boundary cells // Check if periodic empty boundary conditions are enabled From e9219e4a039ba9494d4ff8828c910859d766fe08 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 10:34:50 -0400 Subject: [PATCH 260/276] Solve issue in periodic empty boundary condition test Before, particles did not travel the entire domain, and for this reason, the test was not valid. Now, the particles travel 10 cm, which is 3 times the simulation domain. --- Scripts/tests/bc_empty_init_test.py | 17 ++++++++++------- sample_inputs/inputs_bc_periodic_init | 16 ++++++++-------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/Scripts/tests/bc_empty_init_test.py b/Scripts/tests/bc_empty_init_test.py index c1bd2efa..72048937 100644 --- a/Scripts/tests/bc_empty_init_test.py +++ b/Scripts/tests/bc_empty_init_test.py @@ -188,9 +188,12 @@ def apply_custom_settings(ax, leg, log_scale_y=False): rel_error_max = 0.05 +# Print the distance traveled by the particles +print(f'Distance traveled by particles = {time[-1]*clight} cm') + for i in range(ncell[0]): - for j in range(ncell[0]): - for k in range(ncell[0]): + for j in range(ncell[1]): + for k in range(ncell[2]): if i==1 and j==1 and k==1: rel_error_ee = np.abs( all_files_ee_ocupation_in_each_cell[-1,i,j,k] - N00_Re_theory ) / N00_Re_theory # Calculate relative error for ee occupation number @@ -198,8 +201,8 @@ def apply_custom_settings(ax, leg, log_scale_y=False): rel_error_uu = np.abs( all_files_uu_ocupation_in_each_cell[-1,i,j,k] - N11_Re_theory ) / N11_Re_theory # Calculate relative error for uu occupation number rel_error_uubar = np.abs( all_files_uubar_ocupation_in_each_cell[-1,i,j,k] - N11_Rebar_theory ) / N11_Rebar_theory # Calculate relative error for uubar occupation number - print(f"{rel_error_ee} ---> relative error in ee : Cell ({j},{k},{l})") - print(f"{rel_error_eebar} ---> relative error in eebar : Cell ({j},{k},{l})") + # print(f"{rel_error_ee} ---> relative error in ee : Cell ({i},{j},{k})") + # print(f"{rel_error_eebar} ---> relative error in eebar : Cell ({i},{j},{k})") # print(f"{rel_error_uu} ---> relative error in uu") # print(f"{rel_error_uubar} ---> relative error in uubar") @@ -210,14 +213,14 @@ def apply_custom_settings(ax, leg, log_scale_y=False): else: - print(f'Cell ({j},{k},{l})') + # print(f'Cell ({j},{k},{l})') rel_error_ee = np.abs( all_files_ee_ocupation_in_each_cell[-1,i,j,k] ) rel_error_eebar = np.abs( all_files_eebar_ocupation_in_each_cell[-1,i,j,k] ) rel_error_uu = np.abs( all_files_uu_ocupation_in_each_cell[-1,i,j,k] ) rel_error_uubar = np.abs( all_files_uubar_ocupation_in_each_cell[-1,i,j,k] ) - print(f"{rel_error_ee} ---> relative error in ee : Cell ({j},{k},{l})") - print(f"{rel_error_eebar} ---> relative error in eebar : Cell ({j},{k},{l})") + # print(f"{rel_error_ee} ---> relative error in ee : Cell ({i},{j},{k})") + # print(f"{rel_error_eebar} ---> relative error in eebar : Cell ({i},{j},{k})") # print(f"{rel_error_uu} ---> relative error in uu") # print(f"{rel_error_uubar} ---> relative error in uubar") diff --git a/sample_inputs/inputs_bc_periodic_init b/sample_inputs/inputs_bc_periodic_init index b2483775..aef00757 100644 --- a/sample_inputs/inputs_bc_periodic_init +++ b/sample_inputs/inputs_bc_periodic_init @@ -4,9 +4,9 @@ perturbation_amplitude = 0.0 # attenuation parameters to time derivative of N due to hamiltonians attenuation_hamiltonians = 0 -collision_cfl_factor = 1e-3 -cfl_factor = 0.5 -flavor_cfl_factor = 0.5 +collision_cfl_factor = 1e-1 +cfl_factor = 2 +flavor_cfl_factor = 2 max_adaptive_speedup = 0 maxError = 1e-6 @@ -27,7 +27,7 @@ particle_data_filename = "particle_input.dat" max_grid_size = 16 # Number of steps to run -nsteps = 1000 +nsteps = 10000 # Simulation end time end_time = 5.0e19 @@ -41,10 +41,10 @@ T_MeV = 7.0 Ye = 0 # Write plotfiles -write_plot_every = 100 +write_plot_every = 1000 # Write particle data in plotfiles -write_plot_particles_every = 100 +write_plot_particles_every = 1000 # checkpointing do_restart = 0 @@ -89,10 +89,10 @@ IMFP_method = 1 Do_Pauli_blocking = 0 # If 1, it will multiply the inverse mean free path by 1 / (1 - f_eq); if 0, do nothing. -IMFP_abs0_cm = 2.398082e1 +IMFP_abs0_cm = 1.0e2 IMFP_abs1_cm = 0 IMFP_abs2_cm = 0 -IMFP_abs0bar_cm = 2.29358e0 +IMFP_abs0bar_cm = 1.0e2 IMFP_abs1bar_cm = 0 IMFP_abs2bar_cm = 0 From 032d1d198866e8fac080a321ba0c3980981b1005 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 10:41:28 -0400 Subject: [PATCH 261/276] Move black hole parameter out of the IMFP method 2 so we can run simulations with a black hole in all methods. --- Source/Parameters.H | 17 +++++++++-------- sample_inputs/inputs_bc_periodic_init | 3 ++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index 66c8cf82..dde3b7d1 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -150,6 +150,15 @@ struct TestParams : public amrex::Gpu::Managed pp.get("do_periodic_empty_bc", do_periodic_empty_bc); } + + // Black hole properties + pp.get("do_nsm", do_nsm); + if ( do_nsm == 1 ){ + pp.get("bh_radius", bh_radius); + pp.get("bh_center_x", bh_center_x); + pp.get("bh_center_y", bh_center_y); + pp.get("bh_center_z", bh_center_z); + } if(IMFP_method==0) { // Do nothing @@ -181,14 +190,6 @@ struct TestParams : public amrex::Gpu::Managed //HDF5 table names (with full path) for EoS and NuLib tables pp.get("nuceos_table_name", nuceos_table_name); pp.get("nulib_table_name", nulib_table_name); - pp.get("do_nsm", do_nsm); - - if ( do_nsm == 1 ){ - pp.get("bh_radius", bh_radius); - pp.get("bh_center_x", bh_center_x); - pp.get("bh_center_y", bh_center_y); - pp.get("bh_center_z", bh_center_z); - } } else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0(do not collisions) and 1(do collisions)"); diff --git a/sample_inputs/inputs_bc_periodic_init b/sample_inputs/inputs_bc_periodic_init index aef00757..58a104cf 100644 --- a/sample_inputs/inputs_bc_periodic_init +++ b/sample_inputs/inputs_bc_periodic_init @@ -117,4 +117,5 @@ delta_E = 2.272540842052914 # Mev ################# read_rho_T_Ye_from_table = 0 boundary_condition_type = 0 -do_periodic_empty_bc = 1 \ No newline at end of file +do_periodic_empty_bc = 1 +do_nsm = 0 \ No newline at end of file From 2bc2769aabdb35ac01d430f1ad831792fbe02207 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 10:45:01 -0400 Subject: [PATCH 262/276] Adding do_nsm parameter to all input files --- sample_inputs/inputs_1d_fiducial | 2 +- sample_inputs/inputs_bipolar_test | 2 +- sample_inputs/inputs_coll_equi_test | 3 ++- sample_inputs/inputs_collisional_instability_test | 3 ++- sample_inputs/inputs_fast_flavor | 2 +- sample_inputs/inputs_fast_flavor_nonzerok | 2 +- sample_inputs/inputs_msw_test | 2 +- 7 files changed, 9 insertions(+), 7 deletions(-) diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 6f04c802..ce1fad9c 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -91,5 +91,5 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 - +do_nsm = 0 boundary_condition_type = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 07386d4f..7963160a 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -92,5 +92,5 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 - +do_nsm = 0 boundary_condition_type = 0 diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index 6960a04a..b517cff3 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -117,4 +117,5 @@ delta_E = 0.8339001570751987 # Mev ################# read_rho_T_Ye_from_table = 0 boundary_condition_type = 0 -do_periodic_empty_bc = 0 \ No newline at end of file +do_periodic_empty_bc = 0 +do_nsm = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test index 5f3e5eed..ddc478c9 100644 --- a/sample_inputs/inputs_collisional_instability_test +++ b/sample_inputs/inputs_collisional_instability_test @@ -117,4 +117,5 @@ delta_E = 2.272540842052914 # Mev ################# read_rho_T_Ye_from_table = 0 boundary_condition_type = 0 -do_periodic_empty_bc = 0 \ No newline at end of file +do_periodic_empty_bc = 0 +do_nsm = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index 449d86e6..1057f605 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -92,5 +92,5 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 - +do_nsm = 0 boundary_condition_type = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index c04d4a73..e10825ed 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -92,5 +92,5 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 - +do_nsm = 0 boundary_condition_type = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index 69be3203..03eea002 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -92,5 +92,5 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 - +do_nsm = 0 boundary_condition_type = 0 From ea5221ed59e3833c44d92d19080924f3f6871613 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 10:54:03 -0400 Subject: [PATCH 263/276] Remove if statement so the empty periodic boundary condition can be run independently of the IMFP method --- Source/main.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Source/main.cpp b/Source/main.cpp index a4e08fcc..950c4fa2 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -214,10 +214,8 @@ void evolve_flavor(const TestParams* parms) auto& neutrinos = neutrinos_new; // Do periodic boundary conditions but initialize particles with N=0 and Nbar=0 at the boundary - if ( parms->IMFP_method == 1 || parms->IMFP_method == 2 ){ - if ( parms->do_periodic_empty_bc == 1 ){ - empty_particles_at_boundary_cells(neutrinos, parms); - } + if ( parms->do_periodic_empty_bc == 1 ){ + empty_particles_at_boundary_cells(neutrinos, parms); } const Real current_dt = integrator.get_timestep(); //FIXME: FIXME: Pass this to neutrinos.CreateParticlesAtBoundary. From fa0572fce2855b6096e0b9fc3142067b4a90f00c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 10:56:07 -0400 Subject: [PATCH 264/276] Adding comments --- Source/main.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Source/main.cpp b/Source/main.cpp index 950c4fa2..e5ca0a9c 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -213,7 +213,9 @@ void evolve_flavor(const TestParams* parms) // Use the latest-time neutrino data auto& neutrinos = neutrinos_new; - // Do periodic boundary conditions but initialize particles with N=0 and Nbar=0 at the boundary + // If do_periodic_empty_bc is one. + // Do periodic boundary conditions but initialize particles with N=0 and Nbar=0 at the boundary. + // If a black hole is present in the simulation it will set N=0 and Nbar=0 for all particles inside the black hole. if ( parms->do_periodic_empty_bc == 1 ){ empty_particles_at_boundary_cells(neutrinos, parms); } From d713ae39d6782c4297b75702b70bb96c4ce3ca1e Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 11:02:00 -0400 Subject: [PATCH 265/276] Making the parameter to do empty periodic BC independent of the IMFP method --- Source/Parameters.H | 4 +++- sample_inputs/inputs_1d_fiducial | 3 ++- sample_inputs/inputs_bipolar_test | 1 + sample_inputs/inputs_fast_flavor | 3 ++- sample_inputs/inputs_fast_flavor_nonzerok | 3 ++- sample_inputs/inputs_msw_test | 1 + 6 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index dde3b7d1..602f1899 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -147,10 +147,12 @@ struct TestParams : public amrex::Gpu::Managed if(IMFP_method==1 || IMFP_method==2 ){ pp.get("Do_Pauli_blocking", Do_Pauli_blocking); // If 1, it will multiply the opacities by 1 / (1 - f_eq); if 0, do nothing. - pp.get("do_periodic_empty_bc", do_periodic_empty_bc); } + // Periodic empty boundary condition + pp.get("do_periodic_empty_bc", do_periodic_empty_bc); + // Black hole properties pp.get("do_nsm", do_nsm); if ( do_nsm == 1 ){ diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index ce1fad9c..39d59178 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -92,4 +92,5 @@ IMFP_method = 0 ################# read_rho_T_Ye_from_table = 0 do_nsm = 0 -boundary_condition_type = 0 \ No newline at end of file +boundary_condition_type = 0 +do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 7963160a..49871fa1 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -94,3 +94,4 @@ IMFP_method = 0 read_rho_T_Ye_from_table = 0 do_nsm = 0 boundary_condition_type = 0 +do_periodic_empty_bc = 0 diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index 1057f605..c0cdfdce 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -93,4 +93,5 @@ IMFP_method = 0 ################# read_rho_T_Ye_from_table = 0 do_nsm = 0 -boundary_condition_type = 0 \ No newline at end of file +boundary_condition_type = 0 +do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index e10825ed..9343c19e 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -93,4 +93,5 @@ IMFP_method = 0 ################# read_rho_T_Ye_from_table = 0 do_nsm = 0 -boundary_condition_type = 0 \ No newline at end of file +boundary_condition_type = 0 +do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index 03eea002..3b4533e0 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -94,3 +94,4 @@ IMFP_method = 0 read_rho_T_Ye_from_table = 0 do_nsm = 0 boundary_condition_type = 0 +do_periodic_empty_bc = 0 \ No newline at end of file From 287c47c16ffb841d52bb716eac63953674e5afff Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 11:25:44 -0400 Subject: [PATCH 266/276] Remove unnecessary line --- Source/Evolve.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 744e2158..897358bb 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -204,8 +204,6 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M // Set time derivatives to zero if particles is inside the BH if ( particle_distance_from_bh_center < parms->bh_radius ) { - p.rdata(PIdx::time) = 1.0; // neutrinos move at one second per second! - // set the dx/dt values p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; From 94187c279205e7a4e4d64d1ea0000784bd29736f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 11:27:19 -0400 Subject: [PATCH 267/276] Make the code able to simulate a black hole independently of the IMFP type. --- Source/Evolve.cpp | 52 +++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 897358bb..2a34630e 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -195,35 +195,33 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M { // If statement to avoid computing quantities of particles inside the black hole. - if( parms->IMFP_method==2 ){ - if( parms->do_nsm==1 ){ - - // Compute particle distance from black hole center - double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm + if( parms->do_nsm==1 ){ + + // Compute particle distance from black hole center + double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm - // Set time derivatives to zero if particles is inside the BH - if ( particle_distance_from_bh_center < parms->bh_radius ) { + // Set time derivatives to zero if particles is inside the BH + if ( particle_distance_from_bh_center < parms->bh_radius ) { - // set the dx/dt values - p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; - p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; - p.rdata(PIdx::z) = p.rdata(PIdx::pupz) / p.rdata(PIdx::pupt) * PhysConst::c; - // set the dt/dt = 1. Neutrinos move at one second per second - p.rdata(PIdx::time) = 1.0; - // set the d(pE)/dt values - p.rdata(PIdx::pupx) = 0; - p.rdata(PIdx::pupy) = 0; - p.rdata(PIdx::pupz) = 0; - // set the dE/dt values - p.rdata(PIdx::pupt) = 0; - // set the dVphase/dt values - p.rdata(PIdx::Vphase) = 0; - - // Set the dN/dt and dNbar/dt values to zero - #include "generated_files/Evolve.cpp_dfdt_fill_zeros" - - return; - } + // set the dx/dt values + p.rdata(PIdx::x) = p.rdata(PIdx::pupx) / p.rdata(PIdx::pupt) * PhysConst::c; + p.rdata(PIdx::y) = p.rdata(PIdx::pupy) / p.rdata(PIdx::pupt) * PhysConst::c; + p.rdata(PIdx::z) = p.rdata(PIdx::pupz) / p.rdata(PIdx::pupt) * PhysConst::c; + // set the dt/dt = 1. Neutrinos move at one second per second + p.rdata(PIdx::time) = 1.0; + // set the d(pE)/dt values + p.rdata(PIdx::pupx) = 0; + p.rdata(PIdx::pupy) = 0; + p.rdata(PIdx::pupz) = 0; + // set the dE/dt values + p.rdata(PIdx::pupt) = 0; + // set the dVphase/dt values + p.rdata(PIdx::Vphase) = 0; + + // Set the dN/dt and dNbar/dt values to zero + #include "generated_files/Evolve.cpp_dfdt_fill_zeros" + + return; } } From 4b0410ace614bc62073fb33efd661a24b77f9183 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 11:34:46 -0400 Subject: [PATCH 268/276] Change variable name from do_nsm to do_blackhole. --- Source/Evolve.cpp | 4 ++-- Source/Parameters.H | 6 +++--- sample_inputs/inputs_1d_fiducial | 2 +- sample_inputs/inputs_bc_periodic_init | 2 +- sample_inputs/inputs_bipolar_test | 2 +- sample_inputs/inputs_coll_equi_test | 2 +- sample_inputs/inputs_collisional_instability_test | 2 +- sample_inputs/inputs_fast_flavor | 2 +- sample_inputs/inputs_fast_flavor_nonzerok | 2 +- sample_inputs/inputs_fermi_dirac_test | 2 +- sample_inputs/inputs_msw_test | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 2a34630e..e2ce7641 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -195,7 +195,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M { // If statement to avoid computing quantities of particles inside the black hole. - if( parms->do_nsm==1 ){ + if( parms->do_blackhole==1 ){ // Compute particle distance from black hole center double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm @@ -466,7 +466,7 @@ void empty_particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, con FlavoredNeutrinoContainer::ParticleType& p = pstruct[i]; // Check if the simulation involves a neutron star merger (NSM) - if(parms->do_nsm==1 ){ + if(parms->do_blackhole==1 ){ // Compute particle distance from black hole center double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm diff --git a/Source/Parameters.H b/Source/Parameters.H index 602f1899..78a0dbaf 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -55,7 +55,7 @@ struct TestParams : public amrex::Gpu::Managed Real attenuation_hamiltonians; // Black hole properties - int do_nsm; + int do_blackhole; int do_periodic_empty_bc; double bh_radius; //cm double bh_center_x; //cm @@ -154,8 +154,8 @@ struct TestParams : public amrex::Gpu::Managed pp.get("do_periodic_empty_bc", do_periodic_empty_bc); // Black hole properties - pp.get("do_nsm", do_nsm); - if ( do_nsm == 1 ){ + pp.get("do_blackhole", do_blackhole); + if ( do_blackhole == 1 ){ pp.get("bh_radius", bh_radius); pp.get("bh_center_x", bh_center_x); pp.get("bh_center_y", bh_center_y); diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 39d59178..ac3c556d 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -91,6 +91,6 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 -do_nsm = 0 +do_blackhole = 0 boundary_condition_type = 0 do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_bc_periodic_init b/sample_inputs/inputs_bc_periodic_init index 58a104cf..6864bdbe 100644 --- a/sample_inputs/inputs_bc_periodic_init +++ b/sample_inputs/inputs_bc_periodic_init @@ -118,4 +118,4 @@ delta_E = 2.272540842052914 # Mev read_rho_T_Ye_from_table = 0 boundary_condition_type = 0 do_periodic_empty_bc = 1 -do_nsm = 0 \ No newline at end of file +do_blackhole = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 49871fa1..4f535e32 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -92,6 +92,6 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 -do_nsm = 0 +do_blackhole = 0 boundary_condition_type = 0 do_periodic_empty_bc = 0 diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index b517cff3..04920d85 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -118,4 +118,4 @@ delta_E = 0.8339001570751987 # Mev read_rho_T_Ye_from_table = 0 boundary_condition_type = 0 do_periodic_empty_bc = 0 -do_nsm = 0 \ No newline at end of file +do_blackhole = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test index ddc478c9..a1705714 100644 --- a/sample_inputs/inputs_collisional_instability_test +++ b/sample_inputs/inputs_collisional_instability_test @@ -118,4 +118,4 @@ delta_E = 2.272540842052914 # Mev read_rho_T_Ye_from_table = 0 boundary_condition_type = 0 do_periodic_empty_bc = 0 -do_nsm = 0 \ No newline at end of file +do_blackhole = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index c0cdfdce..d3a28b58 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -92,6 +92,6 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 -do_nsm = 0 +do_blackhole = 0 boundary_condition_type = 0 do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 9343c19e..8bd7726c 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -92,6 +92,6 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 -do_nsm = 0 +do_blackhole = 0 boundary_condition_type = 0 do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index 31953d87..8cd4cb6f 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -88,7 +88,7 @@ boundary_condition_type = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 1 -do_nsm = 0 +do_blackhole = 0 do_periodic_empty_bc = 0 nulib_table_name = "/NuLib/NuLib_SFHo.h5" nuceos_table_name = "/EOS/LS220_234r_136t_50y_analmu_20091212_SVNr26.h5" diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index 3b4533e0..298a4b05 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -92,6 +92,6 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 -do_nsm = 0 +do_blackhole = 0 boundary_condition_type = 0 do_periodic_empty_bc = 0 \ No newline at end of file From 0b686c3a3143e4da8eff63b59dea0bd7e208d6f7 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 11:36:19 -0400 Subject: [PATCH 269/276] Update comments --- Source/Evolve.H | 8 ++++---- Source/Evolve.cpp | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Source/Evolve.H b/Source/Evolve.H index baf53b0d..61e5ada3 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -33,9 +33,9 @@ void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const amrex::MultiFab& state, const amrex::Geometry& geom, const TestParams* parms); /** - * @brief Sets the time derivatives to zero for particles inside the black hole or boundary cells. + * @brief Sets the N and Nbar to zero for particles inside the black hole or boundary cells. * - * This function iterates over all particles in the `FlavoredNeutrinoContainer` and sets their time derivatives to zero if they are inside the black hole or within the boundary cells of the simulation domain. + * This function iterates over all particles in the `FlavoredNeutrinoContainer` and sets N and Nbar to zero if pariticles are inside the black hole or within the boundary cells of the simulation domain. * * @param neutrinos Reference to the container holding the flavored neutrinos. * @param parms Pointer to the structure containing test parameters, including black hole properties and domain dimensions. @@ -43,8 +43,8 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const a * The function performs the following steps: * - Iterates over all particles in the container. * - Computes the distance of each particle from the black hole center. - * - Sets the time derivatives to zero if the particle is inside the black hole radius. - * - Sets the time derivatives to zero if the particle is within the boundary cells of the simulation domain. + * - Sets N and Nbar to zero if the particle is inside the black hole radius. + * - Sets N and Nbar to zero if the particle is within the boundary cells of the simulation domain. * */ void empty_particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index e2ce7641..a5651255 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -439,9 +439,9 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M /** - * @brief Sets the time derivatives to zero for particles inside the black hole or boundary cells. + * @brief Sets the N and Nbar to zero for particles inside the black hole or boundary cells. * - * This function iterates over all particles in the `FlavoredNeutrinoContainer` and sets their time derivatives to zero if they are inside the black hole or within the boundary cells of the simulation domain. + * This function iterates over all particles in the `FlavoredNeutrinoContainer` and sets N and Nbar to zero if pariticles are inside the black hole or within the boundary cells of the simulation domain. * * @param neutrinos Reference to the container holding the flavored neutrinos. * @param parms Pointer to the structure containing test parameters, including black hole properties and domain dimensions. @@ -449,8 +449,8 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M * The function performs the following steps: * - Iterates over all particles in the container. * - Computes the distance of each particle from the black hole center. - * - Sets the time derivatives to zero if the particle is inside the black hole radius. - * - Sets the time derivatives to zero if the particle is within the boundary cells of the simulation domain. + * - Sets N and Nbar to zero if the particle is inside the black hole radius. + * - Sets N and Nbar to zero if the particle is within the boundary cells of the simulation domain. * */ void empty_particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const TestParams* parms) From af8cb9c6d703dadf3ef26ce0fb97c70c4c9f720e Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 17:40:42 -0400 Subject: [PATCH 270/276] Update empty periodic BC test to include a black hole in the domain center and test if the particles inside the black hole are initialized to zero. --- Scripts/tests/bc_empty_init_test.py | 198 ++++++++++++++++++++------ sample_inputs/inputs_bc_periodic_init | 22 +-- 2 files changed, 172 insertions(+), 48 deletions(-) diff --git a/Scripts/tests/bc_empty_init_test.py b/Scripts/tests/bc_empty_init_test.py index 72048937..fd30a830 100644 --- a/Scripts/tests/bc_empty_init_test.py +++ b/Scripts/tests/bc_empty_init_test.py @@ -88,10 +88,16 @@ def apply_custom_settings(ax, leg, log_scale_y=False): ############################################################ # Domain size in 3D index space -ncell = (3, 3, 3) -Lx = 3 # cm -Ly = 3 # cm -Lz = 3 # cm +ncell = (5, 5, 5) +Lx = 5 # cm +Ly = 5 # cm +Lz = 5 # cm + +# Black hole parameters +bh_radius = 0.5 # cm +bh_center_x = 2.5 # cm +bh_center_y = 2.5 # cm +bh_center_z = 2.5 # cm # Contains mesh faces coordinates cell_x_faces = np.linspace(0, Lx, ncell[0] + 1) @@ -104,6 +110,15 @@ def apply_custom_settings(ax, leg, log_scale_y=False): all_files_uubar_ocupation_in_each_cell = np.zeros((len(directories), *ncell)) # number of particles units time = np.zeros(len(directories)) # seconds +x_pos_bh_cell = [] +y_pos_bh_cell = [] +z_pos_bh_cell = [] + +N00_Re_bh_cell = [] +N11_Re_bh_cell = [] +N00_Rebar_bh_cell = [] +N11_Rebar_bh_cell = [] + # Looping over all directories for i in range(len(directories)): @@ -174,6 +189,15 @@ def apply_custom_settings(ax, leg, log_scale_y=False): uu_ocupation_in_each_cell[j,k,l] = np.sum(N11_Re[mask]) uubar_ocupation_in_each_cell[j,k,l] = np.sum(N11_Rebar[mask]) + if j==2 and k==2 and l==2: + x_pos_bh_cell.append( pos_x[mask] ) + y_pos_bh_cell.append( pos_y[mask] ) + z_pos_bh_cell.append( pos_z[mask] ) + N00_Re_bh_cell.append( N00_Re[mask] ) + N11_Re_bh_cell.append( N11_Re[mask] ) + N00_Rebar_bh_cell.append( N00_Rebar[mask] ) + N11_Rebar_bh_cell.append( N11_Rebar[mask] ) + # Store the occupation numbers for the current file in the all_files arrays all_files_ee_ocupation_in_each_cell[i] = ee_ocupation_in_each_cell all_files_eebar_ocupation_in_each_cell[i] = eebar_ocupation_in_each_cell @@ -191,43 +215,70 @@ def apply_custom_settings(ax, leg, log_scale_y=False): # Print the distance traveled by the particles print(f'Distance traveled by particles = {time[-1]*clight} cm') +# Loop over all cells in the x, y, and z directions for i in range(ncell[0]): for j in range(ncell[1]): for k in range(ncell[2]): - if i==1 and j==1 and k==1: - rel_error_ee = np.abs( all_files_ee_ocupation_in_each_cell[-1,i,j,k] - N00_Re_theory ) / N00_Re_theory # Calculate relative error for ee occupation number - rel_error_eebar = np.abs( all_files_eebar_ocupation_in_each_cell[-1,i,j,k] - N00_Rebar_theory ) / N00_Rebar_theory # Calculate relative error for eebar occupation number - rel_error_uu = np.abs( all_files_uu_ocupation_in_each_cell[-1,i,j,k] - N11_Re_theory ) / N11_Re_theory # Calculate relative error for uu occupation number - rel_error_uubar = np.abs( all_files_uubar_ocupation_in_each_cell[-1,i,j,k] - N11_Rebar_theory ) / N11_Rebar_theory # Calculate relative error for uubar occupation number + # Check if the cell is in the second layer next to the boundary + if (i > 0) and (i < ncell[0] - 1) and (j > 0) and (j < ncell[1] - 1) and (k > 0) and (k < ncell[2] - 1): + + # Calculate the relative errors for the central cells + if i == 2 and j == 2 and k == 2: + + # Calculate the distance of particles from the black hole center + particle_distance_from_bh_center = np.sqrt( + (np.array(x_pos_bh_cell[-1]) - bh_center_x)**2 + + (np.array(y_pos_bh_cell[-1]) - bh_center_y)**2 + + (np.array(z_pos_bh_cell[-1]) - bh_center_z)**2 + ) # cm + + # Mask for particles inside the black hole + mask = particle_distance_from_bh_center < bh_radius + particle_ins_bh = np.sum(N00_Re_bh_cell[-1][mask]) + + # Mask for particles outside the black hole + mask = particle_distance_from_bh_center > bh_radius + particle_out_bh = np.sum(N00_Re_bh_cell[-1][mask]) + + # Print the number of particles inside and outside the black hole + print(f'3BoundaryLayer: Cell ({i},{j},{k}) : Number of particles inside the black hole = {particle_ins_bh}') + print(f'3BoundaryLayer: Cell ({i},{j},{k}) : Number of particles outside the black hole = {particle_out_bh}') + + # Assert that the number of particles inside the black hole is less than the maximum relative error + myassert(particle_ins_bh < rel_error_max) + + else: + + # Calculate relative errors for ee and eebar occupation numbers + rel_error_ee = np.abs(all_files_ee_ocupation_in_each_cell[-1, i, j, k] - N00_Re_theory) / N00_Re_theory + rel_error_eebar = np.abs(all_files_eebar_ocupation_in_each_cell[-1, i, j, k] - N00_Rebar_theory) / N00_Rebar_theory - # print(f"{rel_error_ee} ---> relative error in ee : Cell ({i},{j},{k})") - # print(f"{rel_error_eebar} ---> relative error in eebar : Cell ({i},{j},{k})") - # print(f"{rel_error_uu} ---> relative error in uu") - # print(f"{rel_error_uubar} ---> relative error in uubar") + # Print the relative errors for ee and eebar occupation numbers + print(f"2BoundaryLayer: Cell ({i},{j},{k}) : relative error in ee = {rel_error_ee}") + print(f"2BoundaryLayer: Cell ({i},{j},{k}) : relative error in eebar = {rel_error_eebar}") - myassert( rel_error_ee < rel_error_max ) - myassert( rel_error_eebar < rel_error_max ) - # myassert( rel_error_uu < rel_error_max ) - # myassert( rel_error_uubar < rel_error_max ) + # Assert that the relative errors are less than the maximum relative error + myassert(rel_error_ee < rel_error_max) + myassert(rel_error_eebar < rel_error_max) else: - # print(f'Cell ({j},{k},{l})') - rel_error_ee = np.abs( all_files_ee_ocupation_in_each_cell[-1,i,j,k] ) - rel_error_eebar = np.abs( all_files_eebar_ocupation_in_each_cell[-1,i,j,k] ) - rel_error_uu = np.abs( all_files_uu_ocupation_in_each_cell[-1,i,j,k] ) - rel_error_uubar = np.abs( all_files_uubar_ocupation_in_each_cell[-1,i,j,k] ) + # Calculate the relative errors for the boundary cells + rel_error_ee = np.abs(all_files_ee_ocupation_in_each_cell[-1, i, j, k]) + rel_error_eebar = np.abs(all_files_eebar_ocupation_in_each_cell[-1, i, j, k]) - # print(f"{rel_error_ee} ---> relative error in ee : Cell ({i},{j},{k})") - # print(f"{rel_error_eebar} ---> relative error in eebar : Cell ({i},{j},{k})") - # print(f"{rel_error_uu} ---> relative error in uu") - # print(f"{rel_error_uubar} ---> relative error in uubar") + # Print the relative errors for the boundary cells + print(f"1BoundaryLayer: Cell ({i},{j},{k}) : relative error in ee = {rel_error_ee}") + print(f"1BoundaryLayer: Cell ({i},{j},{k}) : relative error in eebar = {rel_error_eebar}") - myassert( rel_error_ee < rel_error_max ) - myassert( rel_error_eebar < rel_error_max ) - # myassert( rel_error_uu < rel_error_max ) - # myassert( rel_error_uubar < rel_error_max ) + # Assert that the relative errors are less than the maximum relative error + myassert(rel_error_ee < rel_error_max) + myassert(rel_error_eebar < rel_error_max) + +########################################################################################## +# PLOTTING +########################################################################################## # Create a figure and axis for plotting electron occupation numbers fig1, ax1 = plt.subplots() @@ -236,11 +287,47 @@ def apply_custom_settings(ax, leg, log_scale_y=False): for i in range(ncell[0]): for j in range(ncell[1]): for k in range(ncell[2]): - # Plot the electron occupation number for the central cell with a different style - if i == 1 and j == 1 and k == 1: - ax1.plot(time, all_files_ee_ocupation_in_each_cell[:, i, j, k], label=f'Cell ({i},{j},{k})', linestyle='--', color='black') - else: - ax1.plot(time, all_files_ee_ocupation_in_each_cell[:, i, j, k], label=f'Cell ({i},{j},{k})') + + # Check if the cell is in the second layer next to the boundary + if (i > 0) and (i < ncell[0] - 1) and (j > 0) and (j < ncell[1] - 1) and (k > 0) and (k < ncell[2] - 1): + + # Plot the central cells + if i == 2 and j == 2 and k == 2: + + particle_ins_bh_time = [] + particle_out_bh_time = [] + + print(f'len(directories) = {len(directories)}') + + print(f'len(x_pos_bh_cell) = {len(x_pos_bh_cell)}') + print(f'len(x_pos_bh_cell[0]) = {len(x_pos_bh_cell[0])}') + + for l in range(len(directories)): + + # Calculate the distance of particles from the black hole center + particle_distance_from_bh_center = np.sqrt( + (np.array(x_pos_bh_cell[l]) - bh_center_x)**2 + + (np.array(y_pos_bh_cell[l]) - bh_center_y)**2 + + (np.array(z_pos_bh_cell[l]) - bh_center_z)**2 + ) # cm + + # Mask for particles inside the black hole + mask = particle_distance_from_bh_center < bh_radius + particle_ins_bh_time.append(np.sum(N00_Re_bh_cell[l][mask])) + + # Mask for particles outside the black hole + mask = particle_distance_from_bh_center > bh_radius + particle_out_bh_time.append(np.sum(N00_Re_bh_cell[l][mask])) + + ax1.plot(time, particle_ins_bh_time, label=f'Particles inside BH', linestyle='solid', color='black') + ax1.plot(time, particle_out_bh_time, label=f"Particles ouside BH but in BH's cell", linestyle='solid', color='gray') + + else: + # Plot cells adjacent to boundary cell + ax1.plot(time, all_files_ee_ocupation_in_each_cell[:, i, j, k], linestyle='dashed', color='orange') + else: + # Plot boundary cells + ax1.plot(time, all_files_ee_ocupation_in_each_cell[:, i, j, k], linestyle='dotted', color='red') # Set the x and y axis labels ax1.set_xlabel('time ($s$)') @@ -263,11 +350,42 @@ def apply_custom_settings(ax, leg, log_scale_y=False): for i in range(ncell[0]): for j in range(ncell[1]): for k in range(ncell[2]): - # Plot the electron occupation number (bar) for the central cell with a different style - if i == 1 and j == 1 and k == 1: - ax2.plot(time, all_files_eebar_ocupation_in_each_cell[:, i, j, k], label=f'Cell ({i},{j},{k})', linestyle='--', color='black') - else: - ax2.plot(time, all_files_eebar_ocupation_in_each_cell[:, i, j, k], label=f'Cell ({i},{j},{k})') + + # Check if the cell is in the second layer next to the boundary + if (i > 0) and (i < ncell[0] - 1) and (j > 0) and (j < ncell[1] - 1) and (k > 0) and (k < ncell[2] - 1): + + # Plot the central cells + if i == 2 and j == 2 and k == 2: + + particle_ins_bh_time = [] + particle_out_bh_time = [] + + for l in range(len(directories)): + + # Calculate the distance of particles from the black hole center + particle_distance_from_bh_center = np.sqrt( + (np.array(x_pos_bh_cell[l]) - bh_center_x)**2 + + (np.array(y_pos_bh_cell[l]) - bh_center_y)**2 + + (np.array(z_pos_bh_cell[l]) - bh_center_z)**2 + ) # cm + + # Mask for particles inside the black hole + mask = particle_distance_from_bh_center < bh_radius + particle_ins_bh_time.append(np.sum(N00_Rebar_bh_cell[l][mask])) + + # Mask for particles outside the black hole + mask = particle_distance_from_bh_center > bh_radius + particle_out_bh_time.append(np.sum(N00_Rebar_bh_cell[l][mask])) + + ax2.plot(time, particle_ins_bh_time, label=f'Particles inside BH', linestyle='solid', color='black') + ax2.plot(time, particle_out_bh_time, label=f"Particles ouside BH but in BH's cell", linestyle='solid', color='gray') + + else: + # Plot cells adjacent to boundary cell + ax2.plot(time, all_files_eebar_ocupation_in_each_cell[:, i, j, k], linestyle='dashed', color='orange') + else: + # Plot boundary cells + ax2.plot(time, all_files_eebar_ocupation_in_each_cell[:, i, j, k], linestyle='dotted', color='red') # Set the x and y axis labels ax2.set_xlabel('time ($s$)') diff --git a/sample_inputs/inputs_bc_periodic_init b/sample_inputs/inputs_bc_periodic_init index 6864bdbe..faf376d6 100644 --- a/sample_inputs/inputs_bc_periodic_init +++ b/sample_inputs/inputs_bc_periodic_init @@ -14,10 +14,10 @@ integration.type = 1 integration.rk.type = 4 # Domain size in 3D index space -ncell = (3, 3, 3) -Lx = 3 # cm -Ly = 3 # cm -Lz = 3 # cm +ncell = (5, 5, 5) +Lx = 5 # cm +Ly = 5 # cm +Lz = 5 # cm # Number of particles per cell nppc = (1, 1, 1) @@ -27,7 +27,7 @@ particle_data_filename = "particle_input.dat" max_grid_size = 16 # Number of steps to run -nsteps = 10000 +nsteps = 30000 # Simulation end time end_time = 5.0e19 @@ -41,10 +41,10 @@ T_MeV = 7.0 Ye = 0 # Write plotfiles -write_plot_every = 1000 +write_plot_every = 3000 # Write particle data in plotfiles -write_plot_particles_every = 1000 +write_plot_particles_every = 3000 # checkpointing do_restart = 0 @@ -118,4 +118,10 @@ delta_E = 2.272540842052914 # Mev read_rho_T_Ye_from_table = 0 boundary_condition_type = 0 do_periodic_empty_bc = 1 -do_blackhole = 0 \ No newline at end of file + +# Blackhole parameters +do_blackhole = 1 +bh_radius = 0.5 # cm +bh_center_x = 2.5 # cm +bh_center_y = 2.5 # cm +bh_center_z = 2.5 # cm \ No newline at end of file From 7c7896cf1a99f395cd83da57c19f5b4960bda645 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 17:43:44 -0400 Subject: [PATCH 271/276] Cleaning up up and adding returns to speed up --- Source/Evolve.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index a5651255..5b5e00ec 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -465,7 +465,7 @@ void empty_particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, con amrex::ParallelFor (np, [=] AMREX_GPU_DEVICE (int i) { FlavoredNeutrinoContainer::ParticleType& p = pstruct[i]; - // Check if the simulation involves a neutron star merger (NSM) + // Check if the simulation involves a black hole somewhere in the domain if(parms->do_blackhole==1 ){ // Compute particle distance from black hole center @@ -474,6 +474,7 @@ void empty_particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, con // Set time derivatives to zero if particles are inside the black hole if ( particle_distance_from_bh_center < parms->bh_radius ) { #include "generated_files/Evolve.cpp_dfdt_fill_zeros" + return; } } @@ -487,7 +488,7 @@ void empty_particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, con p.rdata(PIdx::z) > parms->Lz - parms->Lz / parms->ncell[2] ) { #include "generated_files/Evolve.cpp_dfdt_fill_zeros" - + return; } }); } From 88b53f81ff51fa6d30fe2d9f37206656a2f36f2f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 17:56:17 -0400 Subject: [PATCH 272/276] Reorganize the parameter file --- Source/Parameters.H | 123 ++++++++++++++++++++++---------------------- 1 file changed, 62 insertions(+), 61 deletions(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index 78a0dbaf..d33e59cc 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -14,62 +14,65 @@ using namespace amrex; //=========================================// struct TestParams : public amrex::Gpu::Managed { - IntVect ncell; // num cells in domain - IntVect nppc; // number of particles per cell in each dim - Real Lx, Ly, Lz; - int max_grid_size; - int nsteps; - Real end_time; - int write_plot_every; - int write_plot_particles_every; - Real rho_in, Ye_in, kT_in; // g/ccm, 1, erg - Real cfl_factor, flavor_cfl_factor,collision_cfl_factor; - Real max_adaptive_speedup; - bool do_restart; - std::string restart_dir; + IntVect ncell; // Number of cells in the domain + IntVect nppc; // Number of points of particle emission per cell in each dimension + Real Lx, Ly, Lz; // Length of the domain in each dimension + int max_grid_size; // Maximum grid size subdivisions of the domain for parallelization + int nsteps; // Number of time steps + Real end_time; // End time of the simulation + int write_plot_every; // Write plot files every n steps + int write_plot_particles_every; // Write plot files for particles every n steps + Real rho_in; // Density in g/ccm + Real Ye_in; // Electron fraction (dimensionless quantity) + Real kT_in; // Temperature in erg + Real cfl_factor, flavor_cfl_factor, collision_cfl_factor; + Real max_adaptive_speedup; + bool do_restart; // Flag to restart from a previous simulation + std::string restart_dir; // Directory path to restart from in case do_restart is true Real maxError; - // angular grid - std::string particle_data_filename; - - // neutrino physics parameters. See first column of table 14.7 in http://pdg.lbl.gov/2019/reviews/rpp2019-rev-neutrino-mixing.pdf - Real mass1, mass2, mass3; // neutrino masses in grams - Real theta12, theta13, theta23; // neutrino mixing angles in radians - Real alpha1, alpha2; // Majorana phases, radians - Real deltaCP; // CP violating phases in radians + // File name with particles' initial conditions + // This includes energy, momentum, N, Nbar, phase space volume, and others. + std::string particle_data_filename; + + // Neutrino physics parameters. See the first column of table 14.7 in http://pdg.lbl.gov/2019/reviews/rpp2019-rev-neutrino-mixing.pdf + Real mass1, mass2, mass3; // Neutrino masses in grams + Real theta12, theta13, theta23; // Neutrino mixing angles in radians + Real alpha1, alpha2; // Majorana phases in radians + Real deltaCP; // CP-violating phases in radians // perturbation parameters int perturbation_type; Real perturbation_wavelength_cm; Real perturbation_amplitude; - // absorption opacities and equilibrium neutrino chemical potentials - int IMFP_method; + // Absorption opacities and equilibrium neutrino chemical potentials + int IMFP_method; // Flag to choose the method to calculate the opacities int Do_Pauli_blocking; // If 1, it will multiply the opacities by 1 / (1 - f_eq); if 0, do nothing. Real IMFP_abs[2][NUM_FLAVORS]; Real IMFP_scat[2][NUM_FLAVORS]; Real munu[2][NUM_FLAVORS]; // equilibrium electron neutrino chemical potential in erg (CGS unist) Real delta_E; // erg (CGS unist) - // attenuation to hamiltonians - Real attenuation_hamiltonians; + // Attenuation to Hamiltonians + Real attenuation_hamiltonians; // Multiplication factor to [N,H] // Black hole properties - int do_blackhole; - int do_periodic_empty_bc; - double bh_radius; //cm - double bh_center_x; //cm - double bh_center_y; //cm - double bh_center_z; //cm + int do_blackhole; // Flag to include a black hole in the simulation + int do_periodic_empty_bc; // Flag to include a periodic empty boundary condition + double bh_radius; // Black hole radius (cm) + double bh_center_x; // Black hole x coordinate (cm) + double bh_center_y; // Black hole y coordinate (cm) + double bh_center_z; // Black hole z coordinate (cm) // Background matter rho, Ye, T flag - int read_rho_T_Ye_from_table; + int read_rho_T_Ye_from_table; // Flag to read rho, Ye, T from a table - //HDF5 table names (with full path) for EoS and NuLib tables - std::string nuceos_table_name; - std::string nulib_table_name; - std::string background_rho_Ye_T_table_name; - int boundary_condition_type; + // HDF5 table names (with full path) for EoS and NuLib tables + std::string nuceos_table_name; // EoS table file name + std::string nulib_table_name; // NuLib table file name + std::string background_rho_Ye_T_table_name; // Background matter table file name + int boundary_condition_type; // Boundary condition type flag void Initialize(){ @@ -92,7 +95,8 @@ struct TestParams : public amrex::Gpu::Managed pp.get("restart_dir", restart_dir); pp.get("maxError", maxError); - // angular grid + // File name with particles' initial conditions + // This includes energy, momentum, N, Nbar, phase space volume, and others. pp.get("particle_data_filename",particle_data_filename); // perturbation parameters @@ -128,33 +132,25 @@ struct TestParams : public amrex::Gpu::Managed // Boundary condition type pp.get("boundary_condition_type", boundary_condition_type); + pp.get("do_periodic_empty_bc", do_periodic_empty_bc); - // Background matter rho, Ye, T flag + // Background matter flag pp.get("read_rho_T_Ye_from_table", read_rho_T_Ye_from_table); - if(read_rho_T_Ye_from_table==1){ - pp.get("background_rho_Ye_T_table_name", background_rho_Ye_T_table_name); - } - - // absorption opacities and equilibrium neutrino chemical potentials - pp.get("IMFP_method", IMFP_method); - - if(IMFP_method==0 || IMFP_method==1 ){ - pp.get("rho_g_ccm", rho_in); - pp.get("Ye", Ye_in); - pp.get("T_MeV", kT_in); + // Background matter options + if(read_rho_T_Ye_from_table==0){ + // Set constant matter field + pp.get("rho_g_ccm", rho_in); // Density in g/ccm + pp.get("Ye", Ye_in); // Electron fraction (dimensionless quantity) + pp.get("T_MeV", kT_in); // Temperature in erg kT_in *= 1e6 * CGSUnitsConst::eV; // convert to cgs : erg - } - - if(IMFP_method==1 || IMFP_method==2 ){ - pp.get("Do_Pauli_blocking", Do_Pauli_blocking); // If 1, it will multiply the opacities by 1 / (1 - f_eq); if 0, do nothing. - - } - - // Periodic empty boundary condition - pp.get("do_periodic_empty_bc", do_periodic_empty_bc); + } else if(read_rho_T_Ye_from_table==1){ + // Read background matter from a table + pp.get("background_rho_Ye_T_table_name", background_rho_Ye_T_table_name); + } else AMREX_ASSERT_WITH_MESSAGE(false, "only available read_rho_T_Ye_from_table values are 0 (set rho_g_ccm = ..., Ye = ..., T_MeV = ...) or 1 (provide a table with background data information)"); - // Black hole properties + // Do black hole pp.get("do_blackhole", do_blackhole); + // Black hole properties if ( do_blackhole == 1 ){ pp.get("bh_radius", bh_radius); pp.get("bh_center_x", bh_center_x); @@ -162,6 +158,9 @@ struct TestParams : public amrex::Gpu::Managed pp.get("bh_center_z", bh_center_z); } + // Flag for collision term treatment + pp.get("IMFP_method", IMFP_method); + if(IMFP_method==0) { // Do nothing } else if(IMFP_method==1){ @@ -183,7 +182,8 @@ struct TestParams : public amrex::Gpu::Managed munu[0][i] *= 1e6*CGSUnitsConst::eV; munu[1][i] *= 1e6*CGSUnitsConst::eV; } - + + pp.get("Do_Pauli_blocking", Do_Pauli_blocking); // If 1, it will multiply the opacities by 1 / (1 - f_eq); if 0, do nothing. pp.get("delta_E", delta_E); delta_E*= 1e6*CGSUnitsConst::eV; // erg @@ -192,6 +192,7 @@ struct TestParams : public amrex::Gpu::Managed //HDF5 table names (with full path) for EoS and NuLib tables pp.get("nuceos_table_name", nuceos_table_name); pp.get("nulib_table_name", nulib_table_name); + pp.get("Do_Pauli_blocking", Do_Pauli_blocking); // If 1, it will multiply the opacities by 1 / (1 - f_eq); if 0, do nothing. } else AMREX_ASSERT_WITH_MESSAGE(false, "only available opacity_method is 0(do not collisions) and 1(do collisions)"); From c42dd1d56fbded0cab6fc042190fd0802595589f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 18:00:36 -0400 Subject: [PATCH 273/276] Clean and comment the code --- Jenkinsfile | 2 +- Source/Parameters.H | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9f872bc2..a65e51f6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,7 +25,7 @@ pipeline { stage('MSW'){ steps{ dir('Exec'){ - sh 'python ../Scripts/initial_conditions/st0_msw_test.py' + sh 'python ../Scripts/initial_conditions/st0_msw_test.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_msw_test' sh 'python ../Scripts/tests/msw_test.py' sh 'rm -rf plt*' diff --git a/Source/Parameters.H b/Source/Parameters.H index d33e59cc..c7ff098a 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -164,7 +164,7 @@ struct TestParams : public amrex::Gpu::Managed if(IMFP_method==0) { // Do nothing } else if(IMFP_method==1){ - + for(int i=0;i Date: Fri, 18 Oct 2024 18:07:13 -0400 Subject: [PATCH 274/276] Update and organize input paramter files --- sample_inputs/inputs_1d_fiducial | 8 ++++++++ sample_inputs/inputs_bc_periodic_init | 14 ++++++++++---- sample_inputs/inputs_bipolar_test | 10 +++++++++- sample_inputs/inputs_coll_equi_test | 12 ++++++++++-- sample_inputs/inputs_collisional_instability_test | 12 ++++++++++-- sample_inputs/inputs_fast_flavor | 8 ++++++++ sample_inputs/inputs_fast_flavor_nonzerok | 8 ++++++++ sample_inputs/inputs_fermi_dirac_test | 15 ++++++++++++++- sample_inputs/inputs_msw_test | 8 ++++++++ 9 files changed, 85 insertions(+), 10 deletions(-) diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index ac3c556d..a8148b92 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -91,6 +91,14 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 + +################# +# Blackhole +################# do_blackhole = 0 + +################# +# Boundary conditions +################# boundary_condition_type = 0 do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_bc_periodic_init b/sample_inputs/inputs_bc_periodic_init index faf376d6..bbc2ad0b 100644 --- a/sample_inputs/inputs_bc_periodic_init +++ b/sample_inputs/inputs_bc_periodic_init @@ -116,12 +116,18 @@ delta_E = 2.272540842052914 # Mev # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 -boundary_condition_type = 0 -do_periodic_empty_bc = 1 -# Blackhole parameters +################# +# Blackhole +################# do_blackhole = 1 bh_radius = 0.5 # cm bh_center_x = 2.5 # cm bh_center_y = 2.5 # cm -bh_center_z = 2.5 # cm \ No newline at end of file +bh_center_z = 2.5 # cm + +################# +# Boundary conditions +################# +boundary_condition_type = 0 +do_periodic_empty_bc = 1 \ No newline at end of file diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 4f535e32..9781389a 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -92,6 +92,14 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 + +################# +# Blackhole +################# do_blackhole = 0 + +################# +# Boundary conditions +################# boundary_condition_type = 0 -do_periodic_empty_bc = 0 +do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index 04920d85..b2d603d3 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -116,6 +116,14 @@ delta_E = 0.8339001570751987 # Mev # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 + +################# +# Blackhole +################# +do_blackhole = 0 + +################# +# Boundary conditions +################# boundary_condition_type = 0 -do_periodic_empty_bc = 0 -do_blackhole = 0 \ No newline at end of file +do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test index a1705714..cc56120e 100644 --- a/sample_inputs/inputs_collisional_instability_test +++ b/sample_inputs/inputs_collisional_instability_test @@ -116,6 +116,14 @@ delta_E = 2.272540842052914 # Mev # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 + +################# +# Blackhole +################# +do_blackhole = 0 + +################# +# Boundary conditions +################# boundary_condition_type = 0 -do_periodic_empty_bc = 0 -do_blackhole = 0 \ No newline at end of file +do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index d3a28b58..fdcdebe7 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -92,6 +92,14 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 + +################# +# Blackhole +################# do_blackhole = 0 + +################# +# Boundary conditions +################# boundary_condition_type = 0 do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 8bd7726c..a20c6f00 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -92,6 +92,14 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 + +################# +# Blackhole +################# do_blackhole = 0 + +################# +# Boundary conditions +################# boundary_condition_type = 0 do_periodic_empty_bc = 0 \ No newline at end of file diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index 8cd4cb6f..f980f5f6 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -88,8 +88,21 @@ boundary_condition_type = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 1 +background_rho_Ye_T_table_name = "./rho_Ye_T.hdf5" + +################# +# Blackhole +################# do_blackhole = 0 + +################# +# Boundary conditions +################# +boundary_condition_type = 0 do_periodic_empty_bc = 0 + +################# +# Tables +################# nulib_table_name = "/NuLib/NuLib_SFHo.h5" nuceos_table_name = "/EOS/LS220_234r_136t_50y_analmu_20091212_SVNr26.h5" -background_rho_Ye_T_table_name = "./rho_Ye_T.hdf5" diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index 298a4b05..8e837961 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -92,6 +92,14 @@ IMFP_method = 0 # Background Ye, T and rho ################# read_rho_T_Ye_from_table = 0 + +################# +# Blackhole +################# do_blackhole = 0 + +################# +# Boundary conditions +################# boundary_condition_type = 0 do_periodic_empty_bc = 0 \ No newline at end of file From f1ee95d9ab09aa0ebd894f8201c23de9193a3ad5 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 18 Oct 2024 18:10:28 -0400 Subject: [PATCH 275/276] Delete debbug print line --- Scripts/tests/bc_empty_init_test.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Scripts/tests/bc_empty_init_test.py b/Scripts/tests/bc_empty_init_test.py index fd30a830..81af079b 100644 --- a/Scripts/tests/bc_empty_init_test.py +++ b/Scripts/tests/bc_empty_init_test.py @@ -296,11 +296,6 @@ def apply_custom_settings(ax, leg, log_scale_y=False): particle_ins_bh_time = [] particle_out_bh_time = [] - - print(f'len(directories) = {len(directories)}') - - print(f'len(x_pos_bh_cell) = {len(x_pos_bh_cell)}') - print(f'len(x_pos_bh_cell[0]) = {len(x_pos_bh_cell[0])}') for l in range(len(directories)): From b09c5ba6bf9e9b9cb40587f84495d6002b018117 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 21 Oct 2024 08:55:18 -0400 Subject: [PATCH 276/276] Use sqrt and amrex::Math::powi<2>(x) functions to speed up the calculation of particle distance from the black hole. --- Source/Evolve.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 5b5e00ec..eb939812 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -198,8 +198,9 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M if( parms->do_blackhole==1 ){ // Compute particle distance from black hole center - double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm - + double particle_distance_from_bh_center = sqrt(amrex::Math::powi<2>(p.rdata(PIdx::x) - parms->bh_center_x) + + amrex::Math::powi<2>(p.rdata(PIdx::y) - parms->bh_center_y) + + amrex::Math::powi<2>(p.rdata(PIdx::z) - parms->bh_center_z)); // cm // Set time derivatives to zero if particles is inside the BH if ( particle_distance_from_bh_center < parms->bh_radius ) { @@ -469,7 +470,9 @@ void empty_particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, con if(parms->do_blackhole==1 ){ // Compute particle distance from black hole center - double particle_distance_from_bh_center = pow( pow( p.rdata(PIdx::x) - parms->bh_center_x , 2.0 ) + pow( p.rdata(PIdx::y) - parms->bh_center_y , 2.0 ) + pow( p.rdata(PIdx::z) - parms->bh_center_z , 2.0 ) , 0.5 ); //cm + double particle_distance_from_bh_center = sqrt(amrex::Math::powi<2>(p.rdata(PIdx::x) - parms->bh_center_x) + + amrex::Math::powi<2>(p.rdata(PIdx::y) - parms->bh_center_y) + + amrex::Math::powi<2>(p.rdata(PIdx::z) - parms->bh_center_z)); // cm // Set time derivatives to zero if particles are inside the black hole if ( particle_distance_from_bh_center < parms->bh_radius ) {