clean up solvers but still doesn't build with cmake + FFT (#1947)

* clean up solvers but still doesn't build with cmake + FFT

* fix cmake issue and separate out GMRES solver

* fix for cmake

* comment out unused

Build/cmake_with_fft.sh

@@ -0,0 +1,17 @@
+
+# Example CMake config script for an OSX laptop with OpenMPI
+
+cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \
+      -DCMAKE_CXX_COMPILER:STRING=mpicxx \
+      -DCMAKE_C_COMPILER:STRING=mpicc \
+      -DCMAKE_Fortran_COMPILER:STRING=mpifort \
+      -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \
+      -DCMAKE_BUILD_TYPE:STRING=Release \
+      -DERF_DIM:STRING=3 \
+      -DERF_ENABLE_MPI:BOOL=ON \
+      -DERF_ENABLE_TESTS:BOOL=ON \
+      -DERF_ENABLE_FCOMPARE:BOOL=ON \
+      -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \
+      -DERF_ENABLE_FFT:BOOL=ON \
+      -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \
+      .. && make -j8

CMake/BuildERFExe.cmake

@@ -47,6 +47,12 @@ function(build_erf_lib erf_lib_name)
     target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_EB)
   endif()
 
+  if(ERF_ENABLE_FFT)
+    target_sources(${erf_lib_name} PRIVATE
+                   ${SRC_DIR}/Utils/ERF_solve_with_fft.cpp)
+    target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_FFT)
+  endif()
+
   if(ERF_ENABLE_NETCDF)
     target_sources(${erf_lib_name} PRIVATE
                    ${SRC_DIR}/IO/ERF_NCInterface.cpp
@@ -185,6 +191,8 @@ function(build_erf_lib erf_lib_name)
        ${SRC_DIR}/Utils/ERF_MomentumToVelocity.cpp
        ${SRC_DIR}/Utils/ERF_PoissonSolve.cpp
        ${SRC_DIR}/Utils/ERF_PoissonSolve_tb.cpp
+       ${SRC_DIR}/Utils/ERF_solve_with_gmres.cpp
+       ${SRC_DIR}/Utils/ERF_solve_with_mlmg.cpp
        ${SRC_DIR}/Utils/ERF_TerrainMetrics.cpp
        ${SRC_DIR}/Utils/ERF_VelocityToMomentum.cpp
        ${SRC_DIR}/Utils/ERF_InteriorGhostCells.cpp

CMake/SetAmrexOptions.cmake

@@ -36,6 +36,11 @@ if(ERF_ENABLE_PARTICLES)
   set(AMReX_PARTICLES ON)
 endif()
 
+set(AMReX_FFT OFF)
+if(ERF_ENABLE_FFT)
+  set(AMReX_FFT ON)
+endif()
+
 set(AMReX_EB OFF)
 if(ERF_ENABLE_EB)
   set(AMReX_EB ON)

Exec/Make.ERF

@@ -115,6 +115,10 @@ USE_LINEAR_SOLVERS_INCFLO = FALSE
 USE_LINEAR_SOLVERS_EM     = FALSE
 AMReXdirs                += LinearSolvers
 
+ifeq ($(USE_FFT),TRUE)
+AMReXdirs                += FFT
+endif
+
 ifeq ($(USE_HDF5),TRUE)
 AMReXdirs             += Extern/HDF5
 endif

Source/ERF.H

@@ -171,13 +171,20 @@ public:
     void project_velocities_tb (int lev, amrex::Real dt, amrex::Vector<amrex::MultiFab >& vars,
                                 amrex::MultiFab& p);
 
+#ifdef ERF_USE_FFT
+    amrex::Array<std::pair<amrex::FFT::Boundary,amrex::FFT::Boundary>,AMREX_SPACEDIM> get_fft_bc () const noexcept;
+    void solve_with_fft (int lev, amrex::MultiFab& rhs, amrex::MultiFab& p,
+                         amrex::Array<amrex::MultiFab,AMREX_SPACEDIM>& fluxes);
+#endif
+
+    void solve_with_gmres (int lev, amrex::Vector<amrex::MultiFab>& rhs, amrex::Vector<amrex::MultiFab>& p,
+                           amrex::Vector<amrex::Array<amrex::MultiFab,AMREX_SPACEDIM>>& fluxes);
+    void solve_with_mlmg (int lev, amrex::Vector<amrex::MultiFab>& rhs, amrex::Vector<amrex::MultiFab>& p,
+                          amrex::Vector<amrex::Array<amrex::MultiFab,AMREX_SPACEDIM>>& fluxes);
+
     // Define the projection bc's based on the domain bc types
     amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM>
       get_projection_bc (amrex::Orientation::Side side) const noexcept;
-#ifdef ERF_USE_FFT
-    amrex::Array<std::pair<amrex::FFT::Boundary,amrex::FFT::Boundary>,AMREX_SPACEDIM>
-      get_fft_bc () const noexcept;
-#endif
     bool projection_has_dirichlet (amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> bcs) const;
 
     // Init (NOT restart or regrid)
@@ -187,7 +194,8 @@ public:
     void restart ();
 
     // Is it time to write a plotfile or checkpoint?
-    bool writeNow (const amrex::Real cur_time, const amrex::Real dt, const int nstep, const int plot_int, const amrex::Real plot_per);
+    bool writeNow (const amrex::Real cur_time, const amrex::Real dt, const int nstep,
+                   const int plot_int, const amrex::Real plot_per);
 
     // Called after every level 0 timestep
     void post_timestep (int nstep, amrex::Real time, amrex::Real dt_lev);

Source/ERF.cpp

@@ -1415,6 +1415,11 @@ ERF::ReadParameters ()
         pp.query("v", verbose);
         pp.query("mg_v", mg_verbose);
         pp.query("use_fft", use_fft);
+#ifndef ERF_USE_FFT
+        if (use_fft) {
+            amrex::Abort("You must build with USE_FFT in order to set use_fft = true in your inputs file");
+        }
+#endif
 
         // Frequency of diagnostic output
         pp.query("sum_interval", sum_interval);