From ecbc595ec086227d1120b97efc51d92aa6e14dc4 Mon Sep 17 00:00:00 2001
From: Michael Zingale <michael.zingale@stonybrook.edu>
Date: Fri, 26 Jul 2024 09:07:04 -0400
Subject: [PATCH] add the ability print warnings from GPU (#1629)

this is enabled with USE_GPU_PRINTF
---
 .github/workflows/good_defines.txt    | 1 +
 Make.Microphysics                     | 4 ++++
 integration/integrator_setup_sdc.H    | 4 ++++
 integration/integrator_setup_strang.H | 4 ++++
 4 files changed, 13 insertions(+)

diff --git a/.github/workflows/good_defines.txt b/.github/workflows/good_defines.txt
index c2c60995b8..7f06eea880 100644
--- a/.github/workflows/good_defines.txt
+++ b/.github/workflows/good_defines.txt
@@ -1,3 +1,4 @@
+ALLOW_GPU_PRINTF
 ALLOW_JACOBIAN_CACHING
 AMREX_USE_CUDA
 AMREX_USE_GPU
diff --git a/Make.Microphysics b/Make.Microphysics
index 9b6fa9e526..3cfab8b613 100644
--- a/Make.Microphysics
+++ b/Make.Microphysics
@@ -35,6 +35,10 @@ endif
 # Require C++17
 CXXSTD := c++17
 
+ifeq ($(USE_GPU_PRINTF),TRUE)
+  DEFINES += -DALLOW_GPU_PRINTF
+endif
+
 include $(AMREX_HOME)/Tools/GNUMake/Make.defs
 
 MICROPHYSICS_AUTO_SOURCE_DIR := $(TmpBuildDir)/microphysics_sources/$(optionsSuffix).EXE
diff --git a/integration/integrator_setup_sdc.H b/integration/integrator_setup_sdc.H
index ce49cc5491..f55c46ff62 100644
--- a/integration/integrator_setup_sdc.H
+++ b/integration/integrator_setup_sdc.H
@@ -3,6 +3,7 @@
 
 #include <AMReX_REAL.H>
 #include <AMReX_ANSIEscCode.H>
+#include <AMReX_GpuPrint.H>
 
 #include <iomanip>
 
@@ -261,6 +262,9 @@ void integrator_cleanup (IntegratorT& int_state, BurnT& state,
             }
             std::cout << std::endl;
 #endif
+#elif defined(ALLOW_GPU_PRINTF)
+            AMREX_DEVICE_PRINTF("[ERROR] integration failed in net, istate = %d, time = %g, dt = %g, dens_start = %g, temp_start = %g\n",
+                                istate, state.time, dt, state.rho_orig, state_save.T_in);
 #endif
         } else {
 #ifndef AMREX_USE_GPU
diff --git a/integration/integrator_setup_strang.H b/integration/integrator_setup_strang.H
index 5c5a94469b..e4fcb9eb86 100644
--- a/integration/integrator_setup_strang.H
+++ b/integration/integrator_setup_strang.H
@@ -3,6 +3,7 @@
 
 #include <AMReX_REAL.H>
 #include <AMReX_ANSIEscCode.H>
+#include <AMReX_GpuPrint.H>
 
 #include <iomanip>
 
@@ -217,6 +218,9 @@ void integrator_cleanup (IntegratorT& int_state, BurnT& state,
             }
             std::cout << std::endl;
             std::cout << "energy generated = " << state.e << std::endl;
+#elif defined(ALLOW_GPU_PRINTF)
+            AMREX_DEVICE_PRINTF("[ERROR] integration failed in net, istate = %d, time = %g, dt = %g, dens = %g, temp_start = %g\n",
+                                istate, int_state.t, dt, state.rho, state_save.T_in);
 #endif
         } else {
 #ifndef AMREX_USE_GPU