Merge pull request #7 from xuqimen/patch

add makefile for IBM machines; modify scalapack.h; abs type issues

ChangeLog

@@ -4,6 +4,15 @@
 -changes
 
 
+--------------
+Jun 09, 2020
+Name: Qimen Xu
+Changes: 
+1. Add makefile-IBM for compiling SPARC on IBM machines.
+2. Modify scalapack.h for compatibility of ScaLAPACK function declarations on IBM machines.
+3. Resolve some abs data type compatibility issues.
+
+
 --------------
 May 26, 2020
 Name: Qimen Xu

README.md

@@ -73,6 +73,8 @@ There are several options to compile SPARC, depending on the available external
     ```
 **Remark**: make sure in the makefile `USE_MKL = 0` and `USE_SCALAPACK = 1` for option 3.
 
+Once compilation is done, a binary named `sparc` will be created in the `lib/` directory.
+
 ### (3) Input files:
 The required input files to run a simulation with SPARC are (with shared names)  
 

src/eigenSolverKpt.c

@@ -1195,7 +1195,7 @@ void Solve_Generalized_EigenProblem_kpt(SPARC_OBJ *pSPARC, int kpt, int spn_i)
         //** calculate eigenvalues and eigenvectors **//
         // Warning: pdsygvx requires the block sizes in both row and col 
         //          dimension to be the same!
-        lwork = (int) fabs(work[0]);
+        lwork = (int) cabs(work[0]);
         int NB = pSPARC->desc_Hp_BLCYC[4]; // distribution block size
         int NN = max(max(N, NB),2);
         int NP0 = numroc_( &NN, &NB, &ZERO, &ZERO, &nprow );

src/electronicGroundState.c

@@ -1204,7 +1204,7 @@ void printEigen(SPARC_OBJ *pSPARC) {
     int *kpt_displs= (int    *)malloc((pSPARC->npkpt+1) * sizeof(int));
 
     char EigenFilename[L_STRING];
-    snprintf(EigenFilename, L_STRING, pSPARC->EigenFilename);
+    snprintf(EigenFilename, L_STRING, "%s", pSPARC->EigenFilename);
 
     FILE *output_fp;
     // first create an empty file

src/electrostatics.c

@@ -1443,7 +1443,7 @@ void poisson_RHS(SPARC_OBJ *pSPARC, double *rhs) {
 	}
 
     // for charged systems, add a uniform background charge so that total charge is 0
-    if (fabs(pSPARC->NetCharge) > TEMP_TOL) {
+    if (fabs((double)pSPARC->NetCharge) > TEMP_TOL) {
 		double Lx = pSPARC->range_x;
 		double Ly = pSPARC->range_y;
 		double Lz = pSPARC->range_z;

src/include/scalapack.h

@@ -15,6 +15,18 @@
 // #include <complex.h>
 // #include <mkl_scalapack.h> // for numroc_, descinit_, pdgemr2d_, pzgemr2d_
 
+#ifdef IBM
+#define numroc_ numroc
+#define descinit_ descinit
+#define pdlamch_ pdlamch
+#define pdgemr2d_ pdgemr2d
+#define pdgemm_ pdgemm
+#define pdsygvx_ pdsygvx
+#define pzgemm_ pzgemm
+#define pzgemr2d_ pzgemr2d
+#define pzhegvx_ pzhegvx
+#endif
+
 extern void   pdlawrite_();
 extern void   pdelset_();
 extern double pdlamch_();

src/initialization.c

@@ -2172,7 +2172,7 @@ void write_output_init(SPARC_OBJ *pSPARC) {
     }
 
     fprintf(output_fp,"***************************************************************************\n");
-    fprintf(output_fp,"*                       SPARC (version May 26, 2020)                      *\n");  
+    fprintf(output_fp,"*                       SPARC (version Jun 09, 2020)                      *\n");  
     fprintf(output_fp,"*   Copyright (c) 2020 Material Physics & Mechanics Group, Georgia Tech   *\n");
     fprintf(output_fp,"*           Distributed under GNU General Public License 3 (GPL)          *\n");
     fprintf(output_fp,"*                   Start time: %s                  *\n",c_time_str);

src/makefile-IBM

@@ -0,0 +1,115 @@
+MACHINE = IBM
+# Options for MACHOPT: sequential, scalapack, scalapack_essl
+MACHOPT = scalapack
+# Choose whether to use MKL or ScaLAPACK or neither
+# Warning: don't turn on USE_MKL and USE_SCALAPACK simultaneously
+# Set USE_MKL = 1 to compile with MKL BLAS, LAPACK, and ScaLAPACK
+# Set USE_MKL = 0 otherwise
+USE_MKL       = 0
+# Set USE_SCALAPACK = 1 to compile with non-MKL BLAS, LAPACK, and ScaLAPACK
+# Set USE_SCALAPACK = 0 to compile with non-MKL BLAS and LAPACK only
+USE_SCALAPACK = 0
+USE_SCALAPACK_ESSL = 0
+ifeq ($(MACHINE),IBM)
+ifeq ($(MACHOPT),scalapack)
+USE_SCALAPACK = 1
+else ifeq ($(MACHOPT),scalapack_essl)
+USE_SCALAPACK_ESSL = 1
+endif # $(MACHOPT)
+endif # $(MACHINE)
+
+# Set USE_DP_SUBEIG = 1 to use SPARC rather than ScaLAPACK routines for matrix data distribution
+# (Required if not using ScaLAPACK)
+# Set USE_DP_SUBEIG = 0 to use ScaLAPACK rather than SPARC routines
+USE_DP_SUBEIG = 0
+# Set DEBUG_MODE = 1 to run with debug mode and print debug output
+DEBUG_MODE    = 0
+
+# Specify the path MKLROOT if it's not already set to compile with MKL, e.g,
+# MKLROOT = /opt/intel/compilers_and_libraries_2017.4.196/linux/mkl
+#MKLROOT = /usr/tce/packages/mkl/mkl-2020.0/compilers_and_libraries/linux/mkl
+
+# Specify the path to ScaLAPACK, LAPACK and BLAS if necessary, and 
+# add to LDFLAGS. Note that sometimes LDFLAGS already contains the default 
+# path to these libraries, or the libraries are located in the default search
+# path. In those cases, the following is not needed.
+# SCALAPACKROOT = /nv/hp27/qxu78/data/scalapack-2.0.2
+SCALAPACKROOT = /usr/tcetmp/packages/lapack/lapack-3.8.0-xl-2019.08.20
+ESSLLIBROOT = ${ESSLLIBDIR64}
+ESSLINCROOT = ${ESSLHEADERDIR}
+# LDFLAGS += -L$(SCALAPACKROOT)
+# LAPACKROOT = /usr/local/pacerepov1/lapack/3.6.0
+# LDFLAGS += -L$(LAPACKROOT)
+# BLASROOT = /usr/lib64
+# LDFLAGS += -L$(BLASROOT)
+
+CPPFLAGS = -Iinclude/
+LDLIBS   = -lrt
+
+ifeq ($(USE_MKL), 1)
+#CPPFLAGS += -m64 -I${MKLROOT}/include -DUSE_MKL
+CPPFLAGS += -I${MKLROOT}/include -DUSE_MKL
+LDFLAGS   = -L${MKLROOT}/lib/intel64
+#LDLIBS   += -Wl,--no-as-needed -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lmkl_blacs_intelmpi_lp64 -lpthread -lm -ldl
+LDLIBS   += -Wl,-rpath=${MKLROOT}/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lmkl_blacs_intelmpi_lp64 -lpthread -lm -ldl
+else ifeq ($(USE_SCALAPACK), 1)
+CPPFLAGS += -I${SCALAPACKROOT}/include -DUSE_SCALAPACK -DIBM
+LDFLAGS   = -L${SCALAPACKROOT}/lib -L${LAPACK_DIR}
+LDLIBS += -L${LAPACK_DIR} -lcblas -llapacke -llapack -lscalapack
+else ifeq ($(USE_SCALAPACK_ESSL), 1)
+CPPFLAGS += -I${SCALAPACKROOT}/include -DUSE_SCALAPACK -DUSE_ESSL -DIBM
+LDLIBS   += -L${ESSLLIBDIR64} -L${LAPACK_DIR} -lessl -llapackforessl -lscalapack 
+else
+CPPFLAGS += -I${SCALAPACKROOT}/include
+LDFLAGS = -L${LAPACK_DIR} -lcblas -llapacke -llapack
+endif
+
+# if MKL is not used, link BLAS and LAPACK directly
+ifeq ($(USE_MKL), 0)
+# if you are using BLAS instead of OpenBLAS, change -lopenblas to -lblas
+# and move it to after -llapack
+#LDLIBS += -lopenblas -llapacke -llapack -lm
+LDLIBS += -llapacke -lm
+endif
+
+# To use domain parallelization + LAPACK for solving sobspace eigen problem
+ifeq ($(USE_DP_SUBEIG), 1)
+CPPFLAGS += -DUSE_DP_SUBEIG
+endif
+
+# to compile with DEBUG mode
+ifeq ($(DEBUG_MODE), 1)
+CPPFLAGS += -Wall -g -DDEBUG
+endif
+
+# for old Intel compiler, use -qopenmp instead of -fopenmp. ICC 17 and later also accepts -fopenmp. 
+#CFLAGS = -std=gnu99 -O3 -fopenmp
+#CFLAGS = -std=gnu99 -O3 -fopenmp
+CFLAGS = -std=gnu99 -fopenmp
+
+OBJSC = main.o initialization.o readfiles.o atomdata.o parallelization.o relax.o tools.o md.o    \
+        electrostatics.o electronicGroundState.o electronDensity.o orbitalElecDensInit.o         \
+        occupation.o lapVecRoutines.o gradVecRoutines.o gradVecRoutinesKpt.o nlocVecRoutines.o   \
+        hamiltonianVecRoutines.o lapVecOrth.o lapVecOrthKpt.o lapVecNonOrth.o lapVecNonOrthKpt.o \
+        linearSolver.o mixing.o exchangeCorrelation.o eigenSolver.o eigenSolverKpt.o energy.o    \
+        forces.o stress.o pressure.o finalization.o
+
+LIBBASE = ../lib/sparc-${MACHINE}-${MACHOPT}
+TESTBASE = ../.ci
+
+override CC=mpicc
+
+all: sparc
+
+# Note the implicit rule to compile '.c' files into '.o' files is
+# %.o : %.c
+# 	$(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@
+
+sparc: $(OBJSC)
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $(LIBBASE) $^ $(LDLIBS)
+
+.PHONY: clean
+clean:
+	rm -f  $(OBJSC) $(LIBBASE)
+test: ../.ci/run_tests.py
+	cd $(TESTBASE); ./run_tests.py

src/parallelization.c

@@ -1435,8 +1435,8 @@ void ScaLAPACK_Dims_2D_BLCYC(int nproc, int *gridsizes, int *dims)
     double diff1, diff2, last_diff;
     last_diff = 1e20;
     for (i = 0; i < ftors.count/2; i++) {
-        diff1 = fabs(gridsizes[1] / ftors.list[2*i] - gridsizes[0] / ftors.list[2*i+1]);
-        diff2 = fabs(gridsizes[0] / ftors.list[2*i] - gridsizes[1] / ftors.list[2*i+1]);
+        diff1 = fabs((double)gridsizes[1] / ftors.list[2*i] - (double)gridsizes[0] / ftors.list[2*i+1]);
+        diff2 = fabs((double)gridsizes[0] / ftors.list[2*i] - (double)gridsizes[1] / ftors.list[2*i+1]);
         diff = diff1 < diff2 ? diff1 : diff2;
         if (i > 0 && diff > last_diff) {
             ind1 = (i - 1) * 2;
@@ -1450,7 +1450,7 @@ void ScaLAPACK_Dims_2D_BLCYC(int nproc, int *gridsizes, int *dims)
     }
 
     if (ftors.count % 2 && i == ftors.count/2) {
-        diff1 = fabs(gridsizes[1] / ftors.list[ftors.count-1] - gridsizes[0] / ftors.list[ftors.count-1]);
+        diff1 = fabs((double)gridsizes[1] / ftors.list[ftors.count-1] - (double)gridsizes[0] / ftors.list[ftors.count-1]);
         if (diff1 < diff) {
             ind1 = ind2 = ftors.count-1;
         }