Updates

.github/workflows/github_actions.yml

@@ -44,6 +44,15 @@ jobs:
             mpi_impl: mpich
             nwchem_modules: "qmandpw qmd"
         include:
+          - os: ubuntu-22.04
+            experimental: true
+            mpi_impl: openmpi
+            armci_network: MPI-TS
+            nwchem_modules: "nwdft driver solvation"
+            fc: gfortran
+            cc: gcc
+            blas: "libopenblas64-openmp-dev"
+            blas_size: 8
           - os: ubuntu-20.04
             experimental: true
             mpi_impl: openmpi
@@ -371,6 +380,24 @@ jobs:
             armci_network: MPI-TS
             nwchem_modules: "tinyqmpw"
             fc: flang-new-17
+          - os: ubuntu-22.04
+            experimental: true
+            mpi_impl: openmpi
+            armci_network: MPI-TS
+            nwchem_modules: "nwdft driver solvation"
+            fc: gfortran
+            cc: gcc
+            blas: "libopenblas64-pthread-dev"
+            blas_size: 8
+          - os: macos-latest
+            experimental: true
+            mpi_impl: openmpi
+            armci_network: MPI-TS
+            nwchem_modules: "nwdft driver solvation"
+            fc: gfortran
+            cc: gcc
+            blas: "brew_openblas"
+            blas_size: 4
       fail-fast: false
     env:
         MPI_IMPL: ${{ matrix.mpi_impl }}

src/config/makefile.h

@@ -1267,6 +1267,7 @@ ifeq ($(TARGET),MACX64)
     DEFINES   = -DMACX
     DEFINES  += -DEXT_INT
     LINK.f    = $(FC) $(LDFLAGS) -Wl,-flat_namespace
+    LINK.f   += -Wl,-headerpad_max_install_names
     GOTCLANG= $(shell $(CC) -dM -E - </dev/null 2> /dev/null |grep __clang__|head -1|cut -c19)
     ifeq ($(GOTCLANG),1)
         COPTIONS   += -fPIC
@@ -1362,6 +1363,9 @@ ifeq ($(TARGET),MACX64)
             FOPTIONS += -fsanitize=address -fsanitize-recover=address
             LDOPTIONS += -fsanitize=address -fsanitize-recover=address
         endif
+        ifdef OPENBLAS_USES_OPENMP
+            LDOPTIONS += -fopenmp
+        endif
 
         ifdef  USE_FPE
             FOPTIONS += -ffpe-trap=invalid,zero,overflow  -fbacktrace
@@ -2221,6 +2225,9 @@ ifneq ($(TARGET),LINUX)
 #                    DEFINES +=-DUSE_F90_ALLOCATABLE -DUSE_OMP_TEAMS_DISTRIBUTE
                 endif
             endif
+	    ifdef OPENBLAS_USES_OPENMP
+                LDOPTIONS += -fopenmp
+	    endif
         endif
 
 
@@ -2368,6 +2375,9 @@ ifneq ($(TARGET),LINUX)
                         FOPTIONS += -fopenmp-targets=spirv64
                     endif
                 endif
+                ifdef OPENBLAS_USES_OPENMP
+                     LDOPTIONS += -fopenmp
+                endif
                 ifdef IFX_DEBUG
                     # debugging remove at some point
                     FOPTIONS += -std95 -what
@@ -2904,6 +2914,9 @@ ifneq ($(TARGET),LINUX)
                   FOPTIONS  += -fopenmp
                   LDOPTIONS += -fopenmp
                 endif
+                ifdef OPENBLAS_USES_OPENMP
+                     LDOPTIONS += -fopenmp
+                endif
 
                 DEFINES   +=   -DARMFLANG
                 LINK.f = $(FC)  $(LDFLAGS) 
@@ -3419,19 +3432,26 @@ endif
 
 
 ifeq ($(LAPACK_LIB),)
-    lapackliberr:
-        @echo makefile error for lapack definition
-        @exit 1
+       lapackliberr:
+	$(info     )
+	$(info makefile error for lapack definition)
+	$(info please set LAPACK_LIB)
+	$(info     )
+	$(error )
 else
     CORE_LIBS += $(LAPACK_LIB)
 endif
 
 
 ifeq ($(BLASOPT),)
-    blasliberr:
-        @echo makefile error for blas definition
-        @exit 1
-    CORE_LIBS += $(BLASOPT)
+       blasliberr:
+	$(info     )
+	$(info makefile error for blas definition)
+	$(info please set BLASOPT)
+	$(info     )
+	$(error )
+else
+     CORE_LIBS += $(BLASOPT)
 endif
 
 
@@ -3858,11 +3878,12 @@ ifdef BUILD_OPENBLAS
     DEFINES += -DOPENBLAS
 endif
 ifeq ($(shell echo $(BLASOPT) |awk '/openblas/ {print "Y"; exit}'),Y)
+    ifeq ($(shell $(NWCHEM_TOP)/src/config/oblas_ompcheck.sh),1)
+        OPENBLAS_USES_OPENMP = 1
+        LDOPTIONS += -fopenmp
+    endif
     ifdef OPENBLAS_USES_OPENMP
         DEFINES += -DBLAS_OPENMP
-        ifndef USE_OPENMP
-           $(error USE_OPENMP must be set when OPENBLAS_USES_OPENMP is set)
-        endif
     else
         DEFINES += -DOPENBLAS
     endif

src/config/oblas_ompcheck.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# oblas_dir directory where libopenblas is located
+# oblas_name "openblas" suffix in lib"openblas".
+# e.g. for debian pkg libopenblas64-pthread-dev
+# oblas_dir=/usr/lib/x86_64-linux-gnu/openblas64-pthread/
+# oblas_name=openblas64
+#echo "USE_OPENMP is equal to " $USE_OPENMP
+mylog=/tmp/mylog.txt
+file_check=$NWCHEM_TOP/src/oblas_ompcheck_done.txt
+if [ -f $file_check ]; then
+    echo $file_check present >> $mylog
+    cat $file_check
+    exit 0
+fi
+#extract oblas_dir & oblas_name from BLASOPT
+# 
+oblas_dir=$(echo $BLASOPT | awk 'sub(/.*-L */,""){f=1} f{if ( sub(/ * .*/,"") ) f=0; print}')
+oblas_name=$(echo $BLASOPT | awk 'sub(/.* -l */,""){f=1} f{if ( sub(/ * .*/,"") ) f=0; print}')
+echo oblas_dir is $oblas_dir  oblas_name is $oblas_name >> $mylog
+find $oblas_dir -name "lib*openblas*.*"|| true
+if [ $(uname -s) == 'Darwin' ]; then
+    MYLDD='otool -L'
+    SOSUFFIX=dylib
+else
+    MYLDD=ldd
+    SOSUFFIX=so
+fi
+# check first against clang libomp
+gotomp=$($MYLDD $oblas_dir/lib$oblas_name.$SOSUFFIX | grep libomp | wc -l )
+# next check against gcc libgomp
+if [ $gotomp -eq 0 ]
+then
+gotomp=$($MYLDD $oblas_dir/lib$oblas_name.$SOSUFFIX | grep libgomp | wc -l )
+fi
+echo gotomp $gotomp >> $mylog
+#conda packages might use OpenMP to thread OpenBLAS
+if [ $gotomp -ne 0 ]
+then
+    echo openblas built with OpenMP >> $mylog
+    export OPENBLAS_USES_OPENMP=1
+    unset USE_OPENMP
+else
+    echo openblas built without OpenMP >> $mylog
+fi
+echo $gotomp >  $NWCHEM_TOP/src/oblas_ompcheck_done.txt
+echo $gotomp 
+exit 0

travis/build_env.sh

@@ -98,6 +98,13 @@ echo DISTR is "$DISTR"
 	 brew update || true
 	 brew unlink open-mpi && brew install mpich && brew upgrade mpich  && brew link --overwrite  mpich || true
      fi
+     if [[ "$BLAS_ENV" == "brew_openblas" ]]; then
+	 brew install openblas
+	 if [ -z "$HOMEBREW_PREFIX" ] ; then
+	     HOMEBREW_PREFIX=/usr/local
+	 fi
+	 PKG_CONFIG_PATH=$HOMEBREW_PREFIX/opt/openblas/lib/pkgconfig pkg-config --libs openblas
+     fi
 #  if [[ "$MPI_IMPL" == "openmpi" ]]; then
 #      HOMEBREW_NO_INSTALL_CLEANUP=1 HOMEBREW_NO_AUTO_UPDATE=1 brew install scalapack
 #  fi
@@ -162,6 +169,10 @@ if [[ "$os" == "Linux" ]]; then
 	    if [[ "$USE_LIBXC" == "-1" ]]; then
 		pkg_extra+=" libxc-dev"
 	    fi
+	    echo "BLAS_ENV is" $BLAS_ENV
+	    if [[ "$BLAS_ENV" == lib*openblas* ]]; then
+		pkg_extra+=" $BLAS_ENV"
+	    fi
 	    echo pkg to install: gfortran python3-dev  make perl  python3 rsync $mpi_libdev $mpi_bin $pkg_extra
             tries=0 ; until [ "$tries" -ge 10 ] ; do \
 			  $MYSUDO apt-get -y install gfortran python3-dev  make perl  python3 rsync $mpi_libdev $mpi_bin $pkg_extra \

travis/compile_nwchem.sh

@@ -67,6 +67,22 @@ else
 	fi
     fi
 fi
+if [[ "$BLAS_ENV" == lib*openblas* ]] || [[ "$BLAS_ENV" == "brew_openblas" ]]; then
+    if [[ "$BLAS_ENV" == *openblas64* ]]; then
+        myob="openblas64"
+    else
+        myob="openblas"
+    fi
+    if [[ "$BLAS_ENV" == "brew_openblas" ]]; then
+	if [ -z "$HOMEBREW_PREFIX" ] ; then
+	    HOMEBREW_PREFIX=/usr/local
+	fi
+	export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$HOMEBREW_PREFIX/opt/openblas/lib/pkgconfig
+    fi
+    export BLASOPT=$(pkg-config --libs $myob)
+    export LAPACK_LIB=$BLASOPT
+    echo "BLASOPT and LAPACK_LIB are" $BLASOPT $LAPACK_LIB
+fi
 #check linear algebra
 if [[ -z "$BLASOPT" ]] && [[ -z "$BUILD_OPENBLAS" ]] && [[ -z "$USE_INTERNALBLAS" ]] ; then
     echo " no existing BLAS settings, defaulting to BUILD_OPENBLAS=y "

travis/nwchem.bashrc

@@ -132,6 +132,20 @@ elif [[ "$BLAS_ENV" == "accelerate" ]]; then
     export BLAS_LIB=${BLASOPT}
     export LAPACK_LIB=${BLASOPT}
     export BLAS_SIZE=4
+elif [[ "$BLAS_ENV" == lib*openblas* ]] || [[ "$BLAS_ENV" == "brew_openblas" ]]; then
+     if [[ "$BLAS_ENV" == *openblas64* ]]; then
+         myob="openblas64"
+     else
+         myob="openblas"
+     fi
+     if [[ "$BLAS_ENV" == "brew_openblas" ]]; then
+	 if [ -z "$HOMEBREW_PREFIX" ] ; then
+	     HOMEBREW_PREFIX=/usr/local
+	 fi
+	 export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$HOMEBREW_PREFIX/opt/openblas/lib/pkgconfig
+     fi
+     export BLASOPT=$(pkg-config --libs $myob)
+     export LAPACK_LIB=$BLASOPT
 fi
 if [[ "$BLAS_SIZE" == "4" ]]; then
   export SCALAPACK_SIZE=4