forked from shedsaw/exciting-plus-rgvw-mod
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathmake.inc.ascent.pgi.acc
200 lines (157 loc) · 7.46 KB
/
make.inc.ascent.pgi.acc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
#==============================================================================
# Exciting-Plus make.inc file for Summit (OLCF) with PGI compilers
# Last edited: Oct 20, 2020 (WYP)
#==============================================================================
MAKE = make
COMPILER = pgi
# Compiler switch to turn on the preprocessor
CPP_OPTS = -Mpreprocess
# Add your custom preprocessor defines here
#CPP_OPTS += -D_DEBUG_bmegqblh_ # Output debugging info related to bmegqblh
#CPP_OPTS += -D_DEBUG_megqblh_ # Output debugging info related to genmegqblh
#CPP_OPTS += -D_DUMP_spinor_ud_ # Dump spinor_ud when spinpol = .TRUE.
#CPP_OPTS += -D_DUMP_megqblh_ # Dump matrix elements
# Common compile time options
F90_OPTS = $(CPP_OPTS)
# Add your common compile time options here
#F90_OPTS += -Minfo=ftn,loop,opt,mp,par,vect,accel # Show optimizer log
F90_OPTS += -Minfo=all # Show optimizer log
F90_OPTS += -Mstack_arrays # Use stack instead of heap for automatic arrays
F90_OPTS += -mcmodel=medium # Use 64-bit memory model to allow data >2GB
#==============================================================================
# OpenACC
#==============================================================================
#F90_OPTS += -ta=tesla:managed:debug # managed memory, debug build
F90_OPTS += -ta=tesla:managed # managed memory, optimized build
#F90_OPTS += -ta=tesla:debug # non-managed memory, debug build
#F90_OPTS += -ta=tesla # non-managed memory, optimized build
#F90_OPTS += -ta=multicore # run on CPU (for debugging purposes)
#F90_OPTS += -acc=verystrict # debug build
F90_OPTS += -acc=autopar # optimized build
#F90_OPTS += -ta=tesla:autocompare # use PCAST (PGI Compiler Assisted Software Testing)
#==============================================================================
# MPI
#==============================================================================
F90 = mpifort
CC = mpicc
CXX = mpic++
MPI_CPP_OPTS = -D_MPI_ -D_GPUDIRECT_
CPP_OPTS += $(MPI_CPP_OPTS)
#==============================================================================
# OpenMP
#==============================================================================
# These are passed at compile time and link time
# Don't forget to `export OMP_STACKSIZE=2G` at runtime
OMP_OPTS = -mp
F90_OPTS += $(OMP_OPTS)
#==============================================================================
# Serial compiler (for utilities)
#==============================================================================
F90SERIAL = pgfortran
F90_OPTS_SERIAL = $(filter-out $(OMP_OPTS),$(filter-out $(MPI_CPP_OPTS),$(F90_OPTS)))
# Note: MPIMOD_PATH no longer needed after Mar 10 2020 stack upgrade
#F90_OPTS_SERIAL = $(filter-out $(OMP_OPTS),$(filter-out -I${MPIMOD_PATH},$(filter-out $(MPI_CPP_OPTS),$(F90_OPTS)))
#==============================================================================
# Compiler and linker options
#==============================================================================
# Debugging
#CPP_OPTS += -DEBUG=1
#F90_OPTS += -g -Minform=warn
#F90_LINK_OPTS = $(F90_OPTS)
#EXE_SFX = dbg-acc
# Debugging with extra checks
# (Don't forget to enable core dump generation using `ulimit -c unlimited`)
#CPP_OPTS += -DEBUG=2 # Set to 3 for kernel debugging info (bounds checking)
#F90_OPTS += -g -O0 -Minform=warn -Mbounds -traceback
#F90_LINK_OPTS = $(F90_OPTS)
#EXE_SFX = dbgchk-acc
# Optimized build with equivalent options to '-O2'
# Note: don't use '-Munroll', it will incorrectly unroll the nstspin loop in genmegqblh()
#F90_OPTS += -gopt -O -Mlre -Mvect=simd -Mflushz -Mcache_align -Mnoinline -Minform=warn
#F90_LINK_OPTS = $(F90_OPTS) -fpic
#EXE_SFX = opt-acc
# Profiling
# Note: don't use '-Mpfi', it is incompatible with '-mp' and '-fpic'
F90_OPTS += -Minstrument -Mprof=ccff -O -Mlre -Mflushz -Mcache_align -Mnoinline -Minform=warn
F90_LINK_OPTS = $(F90_OPTS) -fpic
EXE_SFX = prof-acc
## For PGI 20.1
# Fully optimized build
# Note: with PGI 19.9 and 19.10, for some reason using '-O2' or higher
# (including '-fast' and '-fastsse') chokes on autoradmt.f90:39
# DO i = -1, 1
# (see OLCF ticket #419691)
# This has been fixed in PGI 20.1
#F90_OPTS += -fast -Munroll -Mnoinline -Minform=warn
#F90_LINK_OPTS = $(F90_OPTS) -fpic
#EXE_SFX = fast-acc
# Profiling, fully optimized
#F90_OPTS += -Minstrument -Mprof=ccff -fast -Munroll -Mnoinline -Minform=warn
#F90_LINK_OPTS = $(F90_OPTS) -fpic
#EXE_SFX = fastprof-acc
#==============================================================================
# BLAS and LAPACK
#==============================================================================
# Use PGI's bundled BLAS and LAPACK
# Note: it's OpenBLAS 0.3.7, might not be thread-safe
#LAPACK_LIB = -lblas -llapack
# Use reference BLAS and LAPACK 3.8.0
# Make sure to `module load netlib-lapack` when building
#LAPACK_LIB = -L${OLCF_NETLIB_LAPACK_ROOT}/lib64 -lblas -llapack
# Use OpenBLAS 0.3.10
#OBLAS_PATH = /ccs/proj/mat201/openblas-0.3.10/pgi-19.10
#LAPACK_LIB = -L$(OBLAS_PATH)/lib -lopenblas
# Use IBM ESSL (depends on libxlf90_r)
# This path was extracted from the modules and stored as summit-xlpaths.sh
# Make sure to `source summit-xlpaths.sh` and `module load essl`
# Note: it used to be necessary to also load libxlf90_r path to LD_LIBRARY_PATH
# on run time; this was fixed with the Mar 10 2020 stack upgrade
ESSL_PATH = ${OLCF_ESSL_ROOT}/lib64
#XLF_PATH = ${OLCF_XLF_ROOT}/lib
#XLSMP_PATH = ${OLCF_XLSMP_ROOT}/lib
LAPACK_LIB = -L$(ESSL_PATH) -lessl
#LAPACK_LIB = -L$(ESSL_PATH) -lesslsmp -L$(XLF_PATH) -lxlf90_r -L$(XLSMP_PATH) -lxlsmp
# Use MAGMA
#MAGMA_PATH = /ccsopen/proj/gen148/eecm/magma-2.5.4/pgi20.1-cuda10.1.243-essl
#MAGMA_INC = -I$(MAGMA_PATH)/include
#LAPACK_LIB += -L$(MAGMA_PATH)/lib -lmagma
#CPP_OPTS += -D_MAGMA_ -DNGPUS=1
#F90_OPTS += $(MAGMA_INC)
#==============================================================================
# FFTW 3
#==============================================================================
# To disable, comment these three lines
# Make sure to `module load fftw`
FFT_INC = -I$(OLCF_FFTW_ROOT)/include
FFT_LIB = -L$(OLCF_FFTW_ROOT)/lib -lfftw3_omp -lfftw3 -lm
FFT_CPP_OPTS = -D_FFTW3_
CPP_OPTS += $(FFT_CPP_OPTS)
F90_OPTS += $(FFT_INC)
#==============================================================================
# HDF5
#==============================================================================
# To disable, comment these three lines
# Make sure to `module load hdf5`
HDF5_INC = -I${OLCF_HDF5_ROOT}/include
HDF5_LIB = -L${OLCF_HDF5_ROOT}/lib -lhdf5_fortran -lhdf5_cpp -lhdf5_hl -lhdf5 -ldl
HDF5_CPP_OPTS = -D_HDF5_
CPP_OPTS += $(HDF5_CPP_OPTS)
F90_OPTS += $(HDF5_INC)
#==============================================================================
# Other libraries (retained here for historical purposes)
#==============================================================================
# ==- compile with libXC support ===
#CPP_OPTS := $(CPP_OPTS) -D_LIBXC_
#XC_LIB =
# ==- compile with NFFT support ===
#CPP_OPTS := $(CPP_OPTS) -D_NFFT_
#NFFT_INC = -I$(HOME)/local/include
#NFFT_LIB = $(HOME)/local/lib/libnfft3.a $(HOME)/local/lib/libfftw3.a
# === compile with Madness API ===
#CPP_OPTS := $(CPP_OPTS) -D_MAD_
#MADNESS_INC = -I$(HOME)/local/include
#MADNESS_LIB = -L$(HOME)/local/lib/ -lMADmra -lMADlinalg -lMADtensor -lMADmisc -lMADmuparser -lMADtinyxml -lMADworld -lmpichcxx -lstdc++
#==============================================================================
# List all libraries to link
#==============================================================================
LIBS = $(LAPACK_LIB) $(FFT_LIB) $(HDF5_LIB)