make.inc.basecamp.pgi.cpu

#==============================================================================
# Exciting-Plus make.inc file for Summit (OLCF) with PGI compilers
# Last edited: May 6, 2020 (WYP)
#==============================================================================

MAKE = make
COMPILER = pgi

# Compiler switch to turn on the preprocessor
CPP_OPTS = -Mpreprocess

# Add your custom preprocessor defines here
#CPP_OPTS += -D_DEBUG_bmegqblh_ # Output debugging info related to bmegqblh
CPP_OPTS += -D_DEBUG_megqblh_ # Output debugging info related to genmegqblh
#CPP_OPTS += -D_DUMP_spinor_ud_ # Dump spinor_ud when spinpol = .TRUE.
CPP_OPTS += -D_DUMP_megqblh_ # Dump matrix elements

# Common compile time options
F90_OPTS = $(CPP_OPTS)

# Add your common compile time options here
F90_OPTS += -Minfo=ftn,loop,opt,mp,par,vect,accel # Show optimizer log
F90_OPTS += -Mstack_arrays # Use stack instead of heap for all arrays

#==============================================================================
# OpenACC
#==============================================================================

F90_OPTS += -ta=tesla:managed:debug # Debug build, managed memory
#F90_OPTS += -ta=tesla:debug # Debug build. non-managed memory

#F90_OPTS += -ta=multicore # Debug build, run on CPU

#==============================================================================
# MPI
#==============================================================================

F90 = mpifort
CC  = mpicc
CXX = mpic++
MPI_CPP_OPTS = -D_MPI_
CPP_OPTS += $(MPI_CPP_OPTS)

#==============================================================================
# OpenMP
#==============================================================================

# These are passed at compile time and link time
# Don't forget to `export OMP_STACKSIZE=2G` at runtime
OMP_OPTS = -mp
F90_OPTS += $(OMP_OPTS)

#==============================================================================
# Serial compiler (for utilities)
#==============================================================================

F90SERIAL = pgfortran
F90_OPTS_SERIAL = $(filter-out $(OMP_OPTS),$(filter-out $(MPI_CPP_OPTS),$(F90_OPTS)))

#==============================================================================
# Compiler and linker options
#==============================================================================

# Debugging
CPP_OPTS += -DEBUG=1
F90_OPTS += -g -Minform=warn
F90_LINK_OPTS = $(F90_OPTS)
EXE_SFX = dbg-acc

# Debugging with extra checks
# (Don't forget to enable core dump generation using `ulimit -c unlimited`)
#CPP_OPTS += -DEBUG=2
#F90_OPTS += -g -O0 -Minform=warn -Mbounds -traceback
#F90_LINK_OPTS = $(F90_OPTS)
#EXE_SFX = dbgchk-acc

# Optimized build with equivalent options to '-O2'
#F90_OPTS += -gopt -O -Munroll -Mlre -Mvect=simd -Mflushz -Mcache_align -Mnoinline -Minform=warn
#F90_LINK_OPTS = $(F90_OPTS) -fpic
#EXE_SFX = opt-acc

# Profiling
# Note: don't use '-Mpfi', it is incompatible with '-mp' and '-fpic'
#F90_OPTS += -Minstrument -Mprof=ccff -O -Munroll -Mlre -Mflushz -Mcache_align -Mnoinline -Minform=warn
#F90_LINK_OPTS = $(F90_OPTS) -fpic
#EXE_SFX = prof-acc

## For PGI 20.1
# Fully optimized build
# Note: with PGI 19.9 and 19.10, for some reason using '-O2' or higher
# (including '-fast' and '-fastsse') chokes on autoradmt.f90:39
#     DO i = -1, 1
#   (see OLCF ticket #419691)
# This has been fixed in PGI 20.1
#F90_OPTS += -fast -Munroll -Mnoinline -Minform=warn
#F90_LINK_OPTS = $(F90_OPTS) -fpic
#EXE_SFX = fast-acc

# Profiling, fully optimized
#F90_OPTS += -Minstrument -Mprof=ccff -fast -Munroll -Mnoinline -Minform=warn
#F90_LINK_OPTS = $(F90_OPTS) -fpic
#EXE_SFX = fastprof-acc

#==============================================================================
# BLAS and LAPACK
#==============================================================================

# Use PGI's bundled OpenBLAS (not threadsafe!)
#LAPACK_LIB = -lblas -llapack

# Use reference BLAS and LAPACK 3.9.0
#LAPACK_LIB = -L${LAPACK_DIR}/lib -lblas -llapack

# Use OpenBLAS
LAPACK_LIB = -L${OPENBLAS_DIR}/lib -lopenblas

#==============================================================================
# FFTW 3
#==============================================================================

# To disable, comment these three lines
FFT_INC  = -I${FFTW_DIR}/include
FFT_LIB  = -L${FFTW_DIR}/lib -lfftw3_omp -lfftw3 -lm
FFT_CPP_OPTS = -D_FFTW3_

CPP_OPTS += $(FFT_CPP_OPTS)
F90_OPTS += $(FFT_INC)

#==============================================================================
# HDF5
#==============================================================================

# To disable, comment these three lines
# Make sure to `module load hdf5`
HDF5_INC = -I{HDF5_DIR}/include
HDF5_LIB = -L{HDF5_DIR}/lib -lhdf5_fortran -lhdf5_hl -lhdf5 -ldl
HDF5_CPP_OPTS = -D_HDF5_

CPP_OPTS += $(HDF5_CPP_OPTS)
F90_OPTS += $(HDF5_INC)

#==============================================================================
# Other libraries (retained here for historical purposes)
#==============================================================================

# ==- compile with libXC support ===
#CPP_OPTS := $(CPP_OPTS) -D_LIBXC_
#XC_LIB =

# ==- compile with NFFT support ===
#CPP_OPTS := $(CPP_OPTS) -D_NFFT_
#NFFT_INC = -I$(HOME)/local/include
#NFFT_LIB = $(HOME)/local/lib/libnfft3.a $(HOME)/local/lib/libfftw3.a

# === compile with Madness API ===
#CPP_OPTS := $(CPP_OPTS) -D_MAD_
#MADNESS_INC = -I$(HOME)/local/include
#MADNESS_LIB = -L$(HOME)/local/lib/ -lMADmra -lMADlinalg -lMADtensor -lMADmisc -lMADmuparser -lMADtinyxml -lMADworld -lmpichcxx -lstdc++

#==============================================================================
# List all libraries to link
#==============================================================================

LIBS = $(LAPACK_LIB) $(FFT_LIB) $(HDF5_LIB)