Merge branch 'AMReX-Codes:development' into eyoo/multicuts

Src/Base/AMReX_CTOParallelForImpl.H

@@ -44,24 +44,33 @@ namespace detail
         }
     };
 
-    template <class L, class F, typename... As>
+    template <class L, typename... As, class... Fs>
     bool
-    AnyCTO_helper2 (const L& l, const F& f, TypeList<As...>,
-                    std::array<int,sizeof...(As)> const& runtime_options)
+    AnyCTO_helper2 (const L& l, TypeList<As...>,
+                    std::array<int,sizeof...(As)> const& runtime_options, const Fs&...cto_functs)
     {
         if (runtime_options == std::array<int,sizeof...(As)>{As::value...}) {
-            l(CTOWrapper<F, As::value...>{f});
+            if constexpr (sizeof...(cto_functs) != 0) {
+                // Apply the CTOWrapper to each function that was given in cto_functs
+                // and call the CPU function l with all of them
+                l(CTOWrapper<Fs, As::value...>{cto_functs}...);
+            } else {
+                // No functions in cto_functs so we call l directly with the compile time arguments
+                l(As{}...);
+            }
             return true;
         } else {
             return false;
         }
     }
 
-    template <class L, class F, typename... PPs, typename RO>
+    template <class L, typename... PPs, typename RO, class...Fs>
     void
-    AnyCTO_helper1 (const L& l, const F& f, TypeList<PPs...>, RO const& runtime_options)
+    AnyCTO_helper1 (const L& l, TypeList<PPs...>,
+                    RO const& runtime_options, const Fs&...cto_functs)
     {
-        bool found_option = (false || ... || AnyCTO_helper2(l, f, PPs{}, runtime_options));
+        bool found_option = (false || ... ||
+                                AnyCTO_helper2(l, PPs{}, runtime_options, cto_functs...));
         amrex::ignore_unused(found_option);
         AMREX_ASSERT(found_option);
     }
@@ -168,17 +177,18 @@ namespace detail
  * \param list_of_compile_time_options list of all possible values of the parameters.
  * \param runtime_options the run time parameters.
  * \param l a callable object containing a CPU function that launches the provided GPU kernel.
- * \param f a callable object containing the GPU kernel with optimizations.
+ * \param cto_functs a callable object containing the GPU kernel with optimizations.
  */
-template <class L, class F, typename... CTOs>
+template <class L, class... Fs, typename... CTOs>
 void AnyCTO ([[maybe_unused]] TypeList<CTOs...> list_of_compile_time_options,
              std::array<int,sizeof...(CTOs)> const& runtime_options,
-             L&& l, F&& f)
+             L&& l, Fs&&...cto_functs)
 {
 #if (__cplusplus >= 201703L)
-    detail::AnyCTO_helper1(std::forward<L>(l), std::forward<F>(f),
+    detail::AnyCTO_helper1(std::forward<L>(l),
                            CartesianProduct(typename CTOs::list_type{}...),
-                           runtime_options);
+                           runtime_options,
+                           std::forward<Fs>(cto_functs)...);
 #else
     amrex::ignore_unused(runtime_options, l, f);
     static_assert(std::is_integral<F>::value, "This requires C++17");

Src/Base/AMReX_IOFormat.H

@@ -0,0 +1,51 @@
+#ifndef AMREX_IO_FORMAT_H_
+#define AMREX_IO_FORMAT_H_
+
+#include <ios>
+
+namespace amrex {
+
+/*
+ * \brief I/O stream format saver
+ *
+ * This class can be used to save and restore I/O stream format in a RAII
+ * way. It handles fill, fmtflag, precision, and width.
+ */
+template <class CharT, class Traits>
+class IOFormatSaver
+{
+public:
+    using BasicIos = std::basic_ios<CharT, Traits>;
+
+    explicit IOFormatSaver (BasicIos& ios)
+        : m_ios(&ios),
+          m_fill(ios.fill()),
+          m_flags(ios.flags()),
+          m_precision(ios.precision()),
+          m_width(ios.width())
+    {}
+
+    ~IOFormatSaver ()
+    {
+        m_ios->fill(m_fill);
+        m_ios->flags(m_flags);
+        m_ios->precision(m_precision);
+        m_ios->width(m_width);
+    }
+
+    IOFormatSaver (IOFormatSaver const&) = delete;
+    IOFormatSaver (IOFormatSaver &&) noexcept = delete;
+    IOFormatSaver& operator= (IOFormatSaver const&) = delete;
+    IOFormatSaver& operator= (IOFormatSaver &&) noexcept = delete;
+
+private:
+    BasicIos*                   m_ios;
+    CharT                       m_fill;
+    typename BasicIos::fmtflags m_flags;
+    std::streamsize             m_precision;
+    std::streamsize             m_width;
+};
+
+}
+
+#endif

Src/Base/AMReX_ParmParse.H

@@ -1153,6 +1153,29 @@ public:
         }
     }
 
+    /*
+     * \brief Evaluate given string as math expression
+     *
+     * For unknown symbols, ParmParse database will be queried.
+     */
+    template <typename T, std::enable_if_t<std::is_same_v<T,int> ||
+                                           std::is_same_v<T,long> ||
+                                           std::is_same_v<T,long long> ||
+                                           std::is_same_v<T,float> ||
+                                           std::is_same_v<T,double>,int> = 0>
+    T eval (std::string const& expr) const
+    {
+        if constexpr (std::is_integral_v<T>) {
+            auto const parser = this->makeIParser(expr, {});
+            auto const exe = parser.compileHost<0>();
+            return static_cast<T>(exe()); // In the future, we might add safety check.
+        } else {
+            auto const parser = this->makeParser(expr, {});
+            auto const exe = parser.compileHost<0>();
+            return static_cast<T>(exe());
+        }
+    }
+
     /*
      * \brief Query two names.
      *

Src/Base/AMReX_Stack.H

@@ -0,0 +1,24 @@
+#ifndef AMREX_STACK_H_
+#define AMREX_STACK_H_
+
+namespace amrex {
+
+template <typename T, int N>
+struct Stack
+{
+public:
+    constexpr void push (T v) { m_data[m_size++] = v; }
+    constexpr void pop () { --m_size; }
+    [[nodiscard]] constexpr bool empty () const { return m_size == 0; }
+    [[nodiscard]] constexpr int size () const { return m_size; }
+    [[nodiscard]] constexpr T const& top () const { return m_data[m_size-1]; }
+    [[nodiscard]] constexpr T      & top ()       { return m_data[m_size-1]; }
+    [[nodiscard]] constexpr T operator[] (int i) const { return m_data[i]; }
+private:
+    T m_data[N];
+    int m_size = 0;
+};
+
+}
+
+#endif

Src/Base/AMReX_TinyProfiler.cpp

@@ -10,6 +10,7 @@
 #include <AMReX_GpuDevice.H>
 #endif
 #include <AMReX_Print.H>
+#include <AMReX_IOFormat.H>
 
 #ifdef AMREX_USE_OMP
 #include <omp.h>
@@ -380,7 +381,6 @@ TinyProfiler::Finalize (bool bFlushing) noexcept
 
     std::ofstream ofs;
     std::ostream* os = nullptr;
-    std::streamsize oldprec = 0;
     if (ParallelDescriptor::IOProcessor()) {
         auto const& ofile = get_output_file();
         if (ofile.empty()) {
@@ -394,6 +394,8 @@ TinyProfiler::Finalize (bool bFlushing) noexcept
         }
     }
 
+    IOFormatSaver iofmtsaver(amrex::OutStream());
+
     if (os)
     {
         os->precision(4);
@@ -439,8 +441,6 @@ TinyProfiler::Finalize (bool bFlushing) noexcept
         ttstack.clear();
         statsmap.clear();
     }
-
-    if(os) { os->precision(oldprec); }
 }
 
 void
@@ -595,6 +595,8 @@ TinyProfiler::PrintStats (std::map<std::string,Stats>& regstats, double dt_max,
 
     if (ParallelDescriptor::IOProcessor() && os)
     {
+        IOFormatSaver iofmtsaver(*os);
+
         *os << std::setfill(' ') << std::setprecision(4);
         int wt = 9;
 
@@ -878,6 +880,8 @@ TinyProfiler::PrintMemStats (std::map<std::string, MemStat>& memstats,
 
     if (allstatsstr.size() == 1 || !os) { return; }
 
+    IOFormatSaver iofmtsaver(*os);
+
     int lenhline = 0;
     for (auto i : maxlen) {
         lenhline += i;

Src/Base/AMReX_VisMF.cpp

@@ -1,6 +1,7 @@
 
 #include <AMReX_FabArrayUtility.H>
 #include <AMReX_FPC.H>
+#include <AMReX_IOFormat.H>
 #include <AMReX_ParmParse.H>
 #include <AMReX_Utility.H>
 #include <AMReX_VisMF.H>
@@ -277,9 +278,9 @@ operator<< (std::ostream        &os,
     // Up the precision for the Reals in m_min and m_max.
     // Force it to be written in scientific notation to match fParallel code.
     //
-    std::ios::fmtflags oflags = os.flags();
+    IOFormatSaver iofmtsaver(os);
     os.setf(std::ios::floatfield, std::ios::scientific);
-    int oldPrec = static_cast<int>(os.precision(16));
+    os.precision(17);
 
     os << hd.m_vers     << '\n';
     os << int(hd.m_how) << '\n';
@@ -327,9 +328,6 @@ operator<< (std::ostream        &os,
       }
     }
 
-    os.flags(oflags);
-    os.precision(oldPrec);
-
     if( ! os.good()) {
         amrex::Error("Write of VisMF::Header failed");
     }

Src/Base/CMakeLists.txt

@@ -31,6 +31,7 @@ foreach(D IN LISTS AMReX_SPACEDIM)
        AMReX_parmparse_fi.cpp
        AMReX_ParmParse.H
        AMReX_Functional.H
+       AMReX_Stack.H
        AMReX_String.H
        AMReX_String.cpp
        AMReX_Utility.H
@@ -97,6 +98,7 @@ foreach(D IN LISTS AMReX_SPACEDIM)
        AMReX_Print.H
        AMReX_IntConv.H
        AMReX_IntConv.cpp
+       AMReX_IOFormat.H
        # Index space -------------------------------------------------------------
        AMReX_Box.H
        AMReX_Box.cpp

Src/Base/Make.package

@@ -23,6 +23,8 @@ C$(AMREX_BASE)_sources += AMReX_PODVector.cpp
 C$(AMREX_BASE)_headers += AMReX_BlockMutex.H
 C$(AMREX_BASE)_sources += AMReX_BlockMutex.cpp
 
+C$(AMREX_BASE)_headers += AMReX_Stack.H
+
 C$(AMREX_BASE)_headers += AMReX_String.H
 C$(AMREX_BASE)_sources += AMReX_String.cpp
 
@@ -121,6 +123,7 @@ C$(AMREX_BASE)_headers += AMReX_ParReduce.H
 #
 C${AMREX_BASE}_headers += AMReX_ANSIEscCode.H AMReX_FabConv.H AMReX_FPC.H AMReX_Print.H AMReX_IntConv.H AMReX_VectorIO.H
 C${AMREX_BASE}_sources += AMReX_FabConv.cpp AMReX_FPC.cpp AMReX_IntConv.cpp AMReX_VectorIO.cpp
+C${AMREX_BASE}_headers += AMReX_IOFormat.H
 
 #
 # Index space.

Src/Base/Parser/AMReX_IParser_Exe.H

@@ -3,6 +3,7 @@
 #include <AMReX_Config.H>
 
 #include <AMReX_IParser_Y.H>
+#include <AMReX_Stack.H>
 #include <AMReX_Vector.H>
 
 #include <limits>
@@ -226,24 +227,12 @@ struct alignas(8) IParserExeJUMP {
     int offset;
 };
 
-template <int N>
-struct IParserStack
-{
-    long long m_data[N];
-    int m_size = 0;
-    constexpr void push (long long v) { m_data[m_size++] = v; }
-    constexpr void pop () { --m_size; }
-    [[nodiscard]] constexpr long long const& top () const { return m_data[m_size-1]; }
-    [[nodiscard]] constexpr long long      & top ()       { return m_data[m_size-1]; }
-    [[nodiscard]] constexpr long long operator[] (int i) const { return m_data[i]; }
-};
-
 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
 long long iparser_exe_eval (const char* p, long long const* x)
 {
     if (p == nullptr) { return std::numeric_limits<long long>::max(); }
 
-    IParserStack<AMREX_IPARSER_STACK_SIZE> pstack;
+    Stack<long long, AMREX_IPARSER_STACK_SIZE> pstack;
     while (*((iparser_exe_t*)p) != IPARSER_EXE_NULL) {
         switch (*((iparser_exe_t*)p))
         {

Src/Base/Parser/AMReX_Parser_Exe.H

@@ -3,6 +3,7 @@
 #include <AMReX_Config.H>
 
 #include <AMReX_Parser_Y.H>
+#include <AMReX_Stack.H>
 #include <AMReX_Vector.H>
 
 #include <limits>
@@ -217,24 +218,12 @@ struct alignas(8) ParserExeJUMP {
     int offset;
 };
 
-template <int N>
-struct ParserStack
-{
-    double m_data[N];
-    int m_size = 0;
-    constexpr void push (double v) { m_data[m_size++] = v; }
-    constexpr void pop () { --m_size; }
-    [[nodiscard]] constexpr double const& top () const { return m_data[m_size-1]; }
-    [[nodiscard]] constexpr double      & top ()       { return m_data[m_size-1]; }
-    [[nodiscard]] constexpr double operator[] (int i) const { return m_data[i]; }
-};
-
 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
 double parser_exe_eval (const char* p, double const* x)
 {
     if (p == nullptr) { return std::numeric_limits<double>::max(); }
 
-    ParserStack<AMREX_PARSER_STACK_SIZE> pstack;
+    Stack<double, AMREX_PARSER_STACK_SIZE> pstack;
     while (*((parser_exe_t*)p) != PARSER_EXE_NULL) { // NOLINT
         switch (*((parser_exe_t*)p))
         {