Merge pull request #2 from CasparJungbacker/OpenACC-timer

Timers

CMakeLists.txt

@@ -83,7 +83,7 @@ elseif("$ENV{SYST}" STREQUAL "NV-OpenACC")
     set(CMAKE_Fortran_COMPILER "mpif90")
     # Includes (temporary) flags for linking to nVidia-built NetCDF
     set(CMAKE_Fortran_FLAGS "-W -Wall -Mpreprocess -Mr8 -Mfree -Werror" CACHE STRING "")
-    set(CMAKE_Fortran_FLAGS_RELEASE "-Munroll -Ofast -g -traceback" CACHE String :"")
+    set(CMAKE_Fortran_FLAGS_RELEASE "-Munroll -Ofast -g -traceback" CACHE STRING "")
     set(CMAKE_Fortran_FLAGS_DEBUG "-Minit-real=snan -Mbounds -traceback -O0 -g -ffpe-trap=invalid,zero,overflow" CACHE STRING "")
 elseif("$ENV{SYST}" STREQUAL "NO_OVERRIDES")
    # don't set any compilation flags here, allows setting them outside CMake
@@ -200,6 +200,12 @@ FIND_LIBRARY(FFTWF_LIB fftw3f
   DOC "FFTW single precision library"
 )
 
+FIND_LIBRARY(NVTX_LIB nvToolsExt
+  PATHS
+  $ENV{NVTX_LIB}
+  DOC "NVTX library"
+)
+
 if(NETCDF_INCLUDE_DIR)
   include_directories(${NETCDF_INCLUDE_DIR})
 else(NETCDF_INCLUDE_DIR)
@@ -230,6 +236,8 @@ OPTION(USE_HYPRE "Also build iterative solver (optional, needs HYPRE)" OFF)
 ### FFTW based poisson solver (FFTW)
 OPTION(USE_FFTW "Also build FFTW based poisson solver (optional, needs FFTW3)" OFF)
 
+### NVTX
+OPTION(USE_NVTX "Enable GPU profiling" OFF)
 
 ### Documentation
 INCLUDE(FindDoxygen)
@@ -266,12 +274,17 @@ if(USE_FFTW)
   list(APPEND OPTIONAL_LIBS ${FFTW_LIB} ${FFTWF_LIB})
 	set(opt_flags "${opt_flags} -DUSE_FFTW")
 endif(USE_FFTW)
+# Flags for linking to custom netcdf
 if(NETCDF_C_LIB)
     set(opt_flags "${opt_flags} -lnetcdf")
 endif(NETCDF_C_LIB)
 if(NETCDF_FORTRAN_LIB)
     set(opt_flags "${opt_flags} -lnetcdff")
 endif(NETCDF_FORTRAN_LIB)
+# Compile with NVTX markers for profiling
+if(USE_NVTX)
+    set(opt_flags "${opt_flags} -DUSE_NVTX -L${NVTX_LIB} -lnvToolsExt")
+endif(USE_NVTX)
 
 ### Precision 
 set(POIS_PRECISION 64 CACHE STRING  "Precision for poisson solver (default 64 bit) [32,64]")

src/modnvtx.f90

@@ -0,0 +1,112 @@
+! -
+!
+! SPDX-FileCopyrightText: Copyright (c) 2022 Pedro Costa. All rights reserved.
+! SPDX-License-Identifier: MIT
+!
+! -
+!
+! NVTX Fortran Module, adapted from https://github.com/maxcuda/NVTX_example (MIT)
+!
+module modnvtx
+#if defined(USE_NVTX)
+  use, intrinsic :: iso_c_binding
+  implicit none
+  private
+  public nvtxStartRange,nvtxEndRange
+  enum, bind(c)
+    enumerator :: COLOR_G = 1 ! green
+    enumerator :: COLOR_B = 2 ! blue
+    enumerator :: COLOR_Y = 3 ! yellow
+    enumerator :: COLOR_M = 4 ! magenta
+    enumerator :: COLOR_C = 5 ! cyan
+    enumerator :: COLOR_R = 6 ! red
+    enumerator :: COLOR_W = 7 ! white
+  end enum
+  integer(kind=C_INT32_T), private :: col(7) = [ &
+                                                 int(Z'0000ff00',kind=C_INT32_T), & ! 1 -> green
+                                                 int(Z'000000ff',kind=C_INT32_T), & ! 2 -> blue
+                                                 int(Z'00ffff00',kind=C_INT32_T), & ! 3 -> yellow
+                                                 int(Z'00ff00ff',kind=C_INT32_T), & ! 4 -> magenta
+                                                 int(Z'0000ffff',kind=C_INT32_T), & ! 5 -> cyan
+                                                 int(Z'00ff0000',kind=C_INT32_T), & ! 6 -> red
+                                                 int(Z'00ffffff',kind=C_INT32_T)  & ! 7 -> white
+                                               ]
+  integer  , parameter :: NAME_LEN_MAX = 256
+  character, target :: tempName(NAME_LEN_MAX)
+  type, bind(C) :: nvtxEventAttributes
+    integer(C_INT16_T) :: version=1
+    integer(C_INT16_T) :: size=48 !
+    integer(C_INT) :: category=0
+    integer(C_INT) :: colorType=1 ! NVTX_COLOR_ARGB = 1
+    integer(C_INT) :: color
+    integer(C_INT) :: payloadType=0 ! NVTX_PAYLOAD_UNKNOWN = 0
+    integer(C_INT) :: reserved0
+    integer(C_INT64_T) :: payload   ! union uint,int,double
+    integer(C_INT) :: messageType=1  ! NVTX_MESSAGE_TYPE_ASCII     = 1
+    type(C_PTR) :: message  ! ascii char
+  end type
+  interface nvtxRangePush
+    ! push range with custom label and standard color
+    subroutine nvtxRangePushA(name) bind(C, name='nvtxRangePushA')
+    use, intrinsic :: iso_c_binding
+    character(kind=C_CHAR) :: name(*)
+    end subroutine
+    ! push range with custom label and custom color
+    subroutine nvtxRangePushEx(event) bind(C, name='nvtxRangePushEx')
+    use, intrinsic :: iso_c_binding
+    import :: nvtxEventAttributes
+    type(nvtxEventAttributes) :: event
+    end subroutine
+  end interface
+  interface nvtxRangePop
+    subroutine nvtxRangePop() bind(C, name='nvtxRangePop')
+    end subroutine
+  end interface
+contains
+  subroutine nvtxStartRange(name,id,color)
+    character(kind=c_char,len=*) :: name
+    integer, optional :: id
+    character(len=1), optional :: color ! g/b/y/m/c/r/w following matplotlib's convention
+    type(nvtxEventAttributes) :: event
+    character(kind=c_char,len=NAME_LEN_MAX) :: trimmed_name
+    integer :: i,icolor
+    trimmed_name=trim(name)//c_null_char
+    ! move scalar trimmed_name into character array tempName
+    do i=1,len(trim(name))+1
+      tempName(i) = trimmed_name(i:i)
+    end do
+    if(present(color)) then
+      select case(color)
+      case('g')
+        icolor = COLOR_G
+      case('b')
+        icolor = COLOR_B
+      case('y')
+        icolor = COLOR_Y
+      case('m')
+        icolor = COLOR_M
+      case('c')
+        icolor = COLOR_C
+      case('r')
+        icolor = COLOR_R
+      case('w')
+        icolor = COLOR_W
+      case default
+        icolor = COLOR_W
+      end select
+    else if(present(id)) then
+      icolor = mod(id-1,size(col))+1
+    end if
+    if (present(id).or.present(color)) then
+      event%color=col(icolor)
+      event%message=c_loc(tempName)
+      call nvtxRangePushEx(event)
+    else
+      call nvtxRangePush(tempName)
+    end if
+  end subroutine
+  subroutine nvtxEndRange
+    call nvtxRangePop
+  end subroutine
+#endif
+end module modnvtx