Merge pull request NASA-LIS#1414 from jvgeiger/feature/configure_lapa…

…ck_petsc

Rework LAPACK and PETSC handling by the configure scripts

.github/scripts/appimage/compile_lisf

@@ -52,12 +52,11 @@ printf '%s\n' "0" "2" "2" "2" "4" "1" "1" "1" "1" "1" "1" "1" "1" "1" | ./config
 # Use MINPACK? (1-yes, 0-no, default=0): 0
 # Use LIS-CRTM? (1-yes, 0-no, default=0): 0
 # Use LIS-CMEM? (1-yes, 0-no, default=0): 0
-# Use LIS-LAPACK? (1-yes, 0-no, default=0): 0
-# Use LIS-MKL-LAPACK? (1-yes, 0-no, default=0): 0
+# Use LIS-LAPACK? (0-no, 1-mkl, 2-lapack/blas, 3-lapack/refblas, default=0): 0
 # Use PETSc? (1-yes, 0-no, default=0): 0
 
 cd ../lis
-printf '%s\n' "1" "2" "2" "2" "0" "1" "4" "1" "1" "1" "1" "1" "1" "0" "0" "0" "0" "0" "0" | ./configure
+printf '%s\n' "1" "2" "2" "2" "0" "1" "4" "1" "1" "1" "1" "1" "1" "0" "0" "0" "0" "0" | ./configure
 ./compile -j 2
 
 #

.github/scripts/appimage/lisf.env

@@ -11,6 +11,8 @@ LISF_LIBS_GDAL=/home/workdir/lib/gdal/3.5.2_gnu-11.2.0
 LISF_LIBS_FORTRANGIS=/home/workdir/lib/fortrangis/2.6-6_gnu-11.2.0
 LISF_LIBS_LIBGEOTIFF=/home/workdir/lib/geotiff/1.7.0_gnu-11.2.0
 LISF_LIBS_TIFF=/home/workdir/lib/tiff/4.0.9_gnu-11.2.0
+LISF_LIBS_LAPACK=/home/workdir/lib/blaslapack/3.11.0_gnu-11.2.0
+LISF_LIBS_PETSC=/home/workdir/lib/petsc/3.16.1_gnu-11.2.0
 
 LISF_LIBS_PATH="$LISF_LIBS_MPICH/lib"
 LISF_LIBS_PATH+=":$LISF_LIBS_ECCODES/lib"
@@ -23,6 +25,8 @@ LISF_LIBS_PATH+=":$LISF_LIBS_GDAL/lib"
 LISF_LIBS_PATH+=":$LISF_LIBS_LIBGEOTIFF/lib"
 LISF_LIBS_PATH+=":$LISF_LIBS_OPENJPEG/lib"
 LISF_LIBS_PATH+=":$LISF_LIBS_TIFF/lib"
+LISF_LIBS_PATH+=":$LISF_LIBS_LAPACK/lib"
+LISF_LIBS_PATH+=":$LISF_LIBS_PETSC/lib"
 export LISF_LIBS_PATH
 
 def_lis_jpeg=""
@@ -38,7 +42,8 @@ def_lis_minpack=""
 def_lis_crtm=""
 def_lis_crtm_prof=""
 def_lis_cmem=""
-def_lis_lapack=""
+def_lis_lapack=$LISF_LIBS_LAPACK/lib
+def_lis_petsc=$LISF_LIBS_PETSC
 def_lvt_gdal=$LISF_LIBS_GDAL
 def_lvt_fortrangis=$LISF_LIBS_FORTRANGIS
 def_ldt_libgeotiff=$LISF_LIBS_LIBGEOTIFF
@@ -61,6 +66,7 @@ export LIS_MINPACK=$def_lis_minpack
 export LIS_CRTM=$def_lis_crtm
 export LIS_CRTM_PROF=$def_lis_crtm_prof
 export LIS_CMEM=$def_lis_cmem
+export LIS_PETSC=$def_lis_petsc
 export LIS_LAPACK=$def_lis_lapack
 
 export LDT_ARCH=linux_gfortran
@@ -112,6 +118,7 @@ unset def_lis_crtm
 unset def_lis_crtm_prof
 unset def_lis_cmem
 unset def_lis_lapack
+unset def_lis_petsc
 unset def_lvt_gdal
 unset def_lvt_fortrangis
 unset def_ldt_libgeotiff
@@ -128,3 +135,5 @@ unset LISF_LIBS_GDAL
 unset LISF_LIBS_FORTRANGIS
 unset LISF_LIBS_LIBGEOTIFF
 unset LISF_LIBS_TIFF
+unset LISF_LIBS_LAPACK
+unset LISF_LIBS_PETSC

.github/scripts/appimage/prep_appdir

@@ -29,7 +29,7 @@ rm -rf mpich
 
 cd /home/workdir/LISF.AppDir
 patchelf --clear-symbol-version expf --clear-symbol-version logf --clear-symbol-version powf usr/bin/LDT
-patchelf --clear-symbol-version expf --clear-symbol-version glob --clear-symbol-version logf --clear-symbol-version powf usr/bin/LIS
+patchelf --clear-symbol-version expf --clear-symbol-version glob --clear-symbol-version logf --clear-symbol-version powf --clear-symbol-version lgamma usr/bin/LIS
 patchelf --clear-symbol-version expf --clear-symbol-version logf --clear-symbol-version powf usr/bin/LVT
 patchelf --clear-symbol-version powf usr/lib/hdf5/1.12.0_gnu-11.2.0/lib/libhdf5.so
 patchelf --clear-symbol-version logf usr/lib/esmf/8.1.1_gnu-11.2.0_mpich-3.4/lib/libO/Linux.gfortran.64.mpich3.default/libesmf.so

docs/LIS_users_guide/build.adoc

@@ -96,6 +96,7 @@ This script will prompt the user with a series of questions regarding support to
 | `LIS_CRTM_PROF` | path to LIS-CRTM Profile library | optional
 | `LIS_CMEM`      | path to LIS-CMEM library         | optional
 | `LIS_LAPACK`    | path to LAPACK library           | optional
+| `LIS_PETSC`     | path to PETSc library            | optional
 | `LIS_JPEG`      | path to JPEG library             | optional (use system libjpeg by default)
 |====
 
@@ -137,8 +138,7 @@ Use HDFEOS? (1-yes, 0-no, default=1):
 Use MINPACK? (1-yes, 0-no, default=0):
 Use LIS-CRTM? (1-yes, 0-no, default=0):
 Use LIS-CMEM? (1-yes, 0-no, default=0):
-Use LIS-LAPACK? (1-yes, 0-no, default=0):
-Use LIS-MKL-LAPACK? (1-yes, 0-no, default=0):
+Use LIS-LAPACK? (0-no, 1-mkl, 2-lapack/blas, 3-lapack/refblas, default=0):
 Use PETSc? (1-yes, 0-no, default=0):
 -----------------------------------------------------
  configure.lis file generated successfully
@@ -175,6 +175,20 @@ if you wish to enable LIS-CRTM2EM support, then you must also enable LIS-CMEM su
 * for `Use LIS-CMEM? (1-yes, 0-no, default=0):`,
 if you wish to enable LIS-CMEM support, then you must also enable LIS-CRTM.  So for `Use LIS-CRTM? (1-yes, 0-no, default=0):`, you must also select 1.
 
+* for `Use LIS-LAPACK? (0-no, 1-mkl, 2-lapack/blas, 3-lapack/refblas, default=0):`,
+select the type of LAPACK library that you have installed.  This is used with `Use PETSc? (1-yes, 0-no, default=0):`.  If you select 0 for `Use PETSc?`, then select 0 for `Use LIS-LAPACK?`.  If you select 1 for `Use PETSc?`, then select `Use LIS-LAPACK?` according to the following table:
++
+|====
+| Selection | Description
+
+| 0         | No LAPACK needed / Use system LAPACK
+| 1         | LAPACK in Intel MKL
+| 2         | liblapack / libblas
+| 3         | liblapack / librefblas
+|====
++
+For example, on a Cray system using the Cray compilers, select 0 because the linker will find Cray's LAPACK libraries.
+
 Note that due to an issue involving multiple definitions within the NetCDF 3 and HDF 4 libraries, you cannot compile LIS with support for both NetCDF 3 and HDF 4 together.
 
 Note that if you compiled NetCDF 4 without compression, then when specifying `NETCDF version (3 or 4, default=4):`, select 3.  Then you must manually append `-lnetcdff` to the `LDFLAGS` variable in the _make/configure.lis_ file.

env/discover/lisf_7.5_gnu_11.2.0_impi_2021.4.0

@@ -0,0 +1,171 @@
+#%Module1.0###################################################################
+
+proc ModulesHelp { } {
+    puts stderr "\t[module-info name] - loads the LISF_7_5_GNU_11_2_0_IMPI_2021_4_0 env"
+    puts stderr ""
+    puts stderr "This is for use on NCCS' discover system running SLES 12.3."
+    puts stderr ""
+    puts stderr "\tThe following env variables are set:"
+    puts stderr "\t\tDEV_ENV"
+    puts stderr "\t\tLIS_ARCH"
+    puts stderr "\t\tLIS_SPMD"
+    puts stderr "\t\tLIS_FC"
+    puts stderr "\t\tLIS_CC"
+    puts stderr "\t\tLIS_JPEG"
+    puts stderr "\t\tLIS_OPENJPEG"
+    puts stderr "\t\tLIS_ECCODES"
+    puts stderr "\t\tLIS_NETCDF"
+    puts stderr "\t\tLIS_HDF4"
+    puts stderr "\t\tLIS_HDFEOS"
+    puts stderr "\t\tLIS_HDF5"
+    puts stderr "\t\tLIS_MODESMF"
+    puts stderr "\t\tLIS_LIBESMF"
+    puts stderr "\t\tLIS_MINPACK"
+    puts stderr "\t\tLIS_CRTM"
+    puts stderr "\t\tLIS_CRTM_PROF"
+    puts stderr "\t\tLIS_CMEM"
+    puts stderr "\t\tLIS_LAPACK"
+    puts stderr "\t\tLIS_PETSC"
+    puts stderr "\t\tLDT_ARCH"
+    puts stderr "\t\tLDT_FC"
+    puts stderr "\t\tLDT_CC"
+    puts stderr "\t\tLDT_JPEG"
+    puts stderr "\t\tLDT_OPENJPEG"
+    puts stderr "\t\tLDT_ECCODES"
+    puts stderr "\t\tLDT_NETCDF"
+    puts stderr "\t\tLDT_HDF4"
+    puts stderr "\t\tLDT_HDFEOS"
+    puts stderr "\t\tLDT_HDF5"
+    puts stderr "\t\tLDT_MODESMF"
+    puts stderr "\t\tLDT_LIBESMF"
+    puts stderr "\t\tLDT_GDAL"
+    puts stderr "\t\tLDT_FORTRANGIS"
+    puts stderr "\t\tLDT_LIBGEOTIFF"
+    puts stderr "\t\tLVT_ARCH"
+    puts stderr "\t\tLVT_FC"
+    puts stderr "\t\tLVT_CC"
+    puts stderr "\t\tLVT_JPEG"
+    puts stderr "\t\tLVT_OPENJPEG"
+    puts stderr "\t\tLVT_ECCODES"
+    puts stderr "\t\tLVT_NETCDF"
+    puts stderr "\t\tLVT_HDF4"
+    puts stderr "\t\tLVT_HDFEOS"
+    puts stderr "\t\tLVT_HDF5"
+    puts stderr "\t\tLVT_MODESMF"
+    puts stderr "\t\tLVT_LIBESMF"
+    puts stderr "\t\tLVT_GDAL"
+    puts stderr "\t\tLVT_FORTRANGIS"
+    puts stderr ""
+    puts stderr "\tThe following modules are loaded:"
+    puts stderr "\t\tcomp/gcc/11.2.0"
+    puts stderr "\t\tmpi/impi/2021.4.0"
+    puts stderr "\t\ttview/2019.0.4"
+    puts stderr "\t\tgit/2.40.1"
+    puts stderr ""
+}
+
+
+conflict comp mpi
+
+
+module-whatis	"loads the [module-info name] environment"
+
+
+set modname     [module-info name]
+set modmode     [module-info mode]
+
+
+module load comp/gcc/11.2.0
+module load mpi/impi/2021.4.0
+
+module load tview/2019.0.4
+module load git/2.40.1
+
+
+set   def_lis_jpeg        /discover/nobackup/projects/lis/libs/sles-12.3/jpeg/8d
+set   def_lis_hdf5        /discover/nobackup/projects/lis/libs/sles-12.3/hdf5/1.12.1_gnu-11.2.0
+set   def_lis_netcdf      /discover/nobackup/projects/lis/libs/sles-12.3/netcdf/4.8.1_gnu-11.2.0
+set   def_lis_openjpeg    /discover/nobackup/projects/lis/libs/sles-12.3/openjpeg/2.4.0_gnu-11.2.0
+set   def_lis_eccodes     /discover/nobackup/projects/lis/libs/sles-12.3/ecCodes/2.23.0_gnu-11.2.0
+set   def_lis_hdf4        /discover/nobackup/projects/lis/libs/sles-12.3/hdf4/4.2.15_gnu-11.2.0
+set   def_lis_hdfeos      /discover/nobackup/projects/lis/libs/sles-12.3/hdfeos2/3.0_gnu-11.2.0
+set   def_lis_modesmf     /discover/nobackup/projects/lis/libs/sles-12.3/esmf/8.1.1_gnu-11.2.0_impi-2021.4.0/mod/modO/Linux.gfortran.64.intelmpi.default
+set   def_lis_libesmf     /discover/nobackup/projects/lis/libs/sles-12.3/esmf/8.1.1_gnu-11.2.0_impi-2021.4.0/lib/libO/Linux.gfortran.64.intelmpi.default
+#set   def_lis_minpack     /discover/nobackup/projects/lis/libs/minpack/intel_11_1_038
+#set   def_lis_crtm        /discover/nobackup/projects/lis/libs/JCSDA_CRTM/REL-2.0.2.Surface-rev_intel_18_0_3_222
+#set   def_lis_crtm_prof   /discover/nobackup/projects/lis/libs/CRTM_Profile_Utility/intel_18_0_3_222
+#set   def_lis_cmem        /discover/nobackup/projects/lis/libs/LIS-MEM/intel_18_0_3_222
+set   def_lis_lapack      /discover/nobackup/projects/lis/libs/sles-12.3/blaslapack/3.11.0_gnu-11.2.0/lib
+set   def_lvt_proj        /discover/nobackup/projects/lis/libs/sles-12.3/proj/9.1.0_gnu-11.2.0
+set   def_lvt_gdal        /discover/nobackup/projects/lis/libs/sles-12.3/gdal/3.5.2_gnu-11.2.0
+set   def_lvt_fortrangis  /discover/nobackup/projects/lis/libs/sles-12.3/fortrangis/2.6_gdal-3.5.2_gnu-11.2.0
+set   def_ldt_libgeotiff  /discover/nobackup/projects/lis/libs/sles-12.3/geotiff/1.7.0_proj-9.1.0_gnu-11.2.0
+set   def_lis_petsc       /discover/nobackup/projects/lis/libs/sles-12.3/petsc/3.16.1_gnu-11.2.0
+
+setenv   DEV_ENV         LISF_7_5_GNU_11_2_0_IMPI_2021_4_0
+setenv   LIS_ARCH        linux_gfortran
+setenv   LIS_SPMD        parallel
+setenv   LIS_FC          mpif90
+setenv   LIS_CC          mpicc
+setenv   LIS_JPEG        $def_lis_jpeg
+setenv   LIS_OPENJPEG    $def_lis_openjpeg
+setenv   LIS_ECCODES     $def_lis_eccodes
+setenv   LIS_NETCDF      $def_lis_netcdf
+setenv   LIS_HDF4        $def_lis_hdf4
+setenv   LIS_HDFEOS      $def_lis_hdfeos
+setenv   LIS_HDF5        $def_lis_hdf5
+setenv   LIS_MODESMF     $def_lis_modesmf
+setenv   LIS_LIBESMF     $def_lis_libesmf
+#setenv   LIS_MINPACK     $def_lis_minpack
+#setenv   LIS_CRTM        $def_lis_crtm
+#setenv   LIS_CRTM_PROF   $def_lis_crtm_prof
+#setenv   LIS_CMEM        $def_lis_cmem
+setenv   LIS_LAPACK      $def_lis_lapack
+setenv   LIS_PETSC       $def_lis_petsc
+
+
+setenv   LDT_ARCH        linux_gfortran
+setenv   LDT_FC          mpif90
+setenv   LDT_CC          mpicc
+setenv   LDT_JPEG        $def_lis_jpeg
+setenv   LDT_OPENJPEG    $def_lis_openjpeg
+setenv   LDT_ECCODES     $def_lis_eccodes
+setenv   LDT_NETCDF      $def_lis_netcdf
+setenv   LDT_HDF4        $def_lis_hdf4
+setenv   LDT_HDFEOS      $def_lis_hdfeos
+setenv   LDT_HDF5        $def_lis_hdf5
+setenv   LDT_MODESMF     $def_lis_modesmf
+setenv   LDT_LIBESMF     $def_lis_libesmf
+setenv   LDT_GDAL        $def_lvt_gdal
+setenv   LDT_FORTRANGIS  $def_lvt_fortrangis
+setenv   LDT_LIBGEOTIFF  $def_ldt_libgeotiff
+
+
+setenv   LVT_ARCH        linux_gfortran
+setenv   LVT_FC          mpif90
+setenv   LVT_CC          mpicc
+setenv   LVT_JPEG        $def_lis_jpeg
+setenv   LVT_OPENJPEG    $def_lis_openjpeg
+setenv   LVT_ECCODES     $def_lis_eccodes
+setenv   LVT_NETCDF      $def_lis_netcdf
+setenv   LVT_HDF4        $def_lis_hdf4
+setenv   LVT_HDFEOS      $def_lis_hdfeos
+setenv   LVT_HDF5        $def_lis_hdf5
+setenv   LVT_MODESMF     $def_lis_modesmf
+setenv   LVT_LIBESMF     $def_lis_libesmf
+setenv   LVT_GDAL        $def_lvt_gdal
+setenv   LVT_FORTRANGIS  $def_lvt_fortrangis
+
+
+prepend-path   LD_LIBRARY_PATH   "$def_lis_jpeg/lib"
+prepend-path   LD_LIBRARY_PATH   "$def_lis_openjpeg/lib"
+prepend-path   LD_LIBRARY_PATH   "$def_ldt_libgeotiff/lib"
+prepend-path   LD_LIBRARY_PATH   "$def_lvt_proj/lib"
+prepend-path   LD_LIBRARY_PATH   "$def_lvt_gdal/lib"
+prepend-path   LD_LIBRARY_PATH   "$def_lis_hdf4/lib"
+prepend-path   LD_LIBRARY_PATH   "$def_lis_hdf5/lib"
+prepend-path   LD_LIBRARY_PATH   "$def_lis_libesmf"
+prepend-path   LD_LIBRARY_PATH   "$def_lis_netcdf/lib"
+prepend-path   LD_LIBRARY_PATH   "$def_lis_eccodes/lib"
+prepend-path   LD_LIBRARY_PATH   "$def_lis_petsc/lib"
+prepend-path   PATH   "$def_lis_netcdf/bin:$def_lis_eccodes/bin"

lis/arch/Config.pl

@@ -802,15 +802,15 @@
    }
 }
 
-print "Use LIS-LAPACK? (1-yes, 0-no, default=0): ";
+print "Use LIS-LAPACK? (0-no, 1-mkl, 2-lapack/blas, 3-lapack/refblas, default=0): ";
 $use_lapack=<stdin>;
 $use_lapack=~s/ *#.*$//;
 chomp($use_lapack);
 if($use_lapack eq ""){
    $use_lapck=0;
 }
 
-if($use_lapack == 1) {
+if($use_lapack != 0) {
    if(defined($ENV{LIS_LAPACK})){
       $sys_lapack_path = $ENV{LIS_LAPACK};
       $lib = "/";
@@ -826,29 +826,6 @@
    }
 }
 
-print "Use LIS-MKL-LAPACK? (1-yes, 0-no, default=0): ";
-$use_mkllapack=<stdin>;
-chomp($use_mkllapack);
-if($use_mkllapack eq ""){
-   $use_mkllapck=0;
-}
-
-if($use_mkllapack == 1) {
-   if(defined($ENV{LIS_MKL_LAPACK})){
-      $sys_lapack_path = $ENV{LIS_MKL_LAPACK};
-      $lib = "/";
-      $lib_lapack=$sys_lapack_path.$lib;
-   }
-   else {
-      print "--------------ERROR---------------------\n";
-      print "Please specify the MKL-LAPACK path using\n";
-      print "the LIS_MKL_LAPACK variable.\n";
-      print "Configuration exiting ....\n";
-      print "--------------ERROR---------------------\n";
-      exit 1;
-   }
-}
-
 print "Use PETSc? (1-yes, 0-no, default=0): ";
 $use_petsc=<stdin>;
 chomp($use_petsc);
@@ -1024,6 +1001,12 @@
    $lib_flags= $lib_flags." -lmfhdf -ldf ".$libjpeg." -lz";
    $lib_paths= $lib_paths." -L\$(LIB_HDF4)"
 }
+
+if($use_petsc == 1){
+   $fflags = $fflags." -I\$(INC_PETSC)";
+   $ldflags = $ldflags." -L\$(LIB_PETSC) -lpetsc -lm -ldl";
+}
+
 if($use_hdf5 == 1){
    $fflags77 = $fflags77." -I\$(INC_HDF5)";
    $fflags = $fflags." -I\$(INC_HDF5)";
@@ -1058,19 +1041,19 @@
 }
 
 if($use_lapack == 1){
+   $ldflags = $ldflags." -L\$(LIB_LAPACK) -lmkl_rt";
+   $lib_flags= $lib_flags." -lmkl_rt";
+   $lib_paths= $lib_paths." -L\$(LIB_LAPACK)";
+}
+elsif($use_lapack == 2){
    $ldflags = $ldflags." -L\$(LIB_LAPACK) -llapack -lblas";
    $lib_flags= $lib_flags." -llapack -lblas";
    $lib_paths= $lib_paths." -L\$(LIB_LAPACK)";
 }
-
-if($use_mkllapack == 1){
-   #Changed to be able to use mkl Wendy Sharples
-   $ldflags = $ldflags." -L\$(LIB_LAPACK) -lmkl_rt";
-}
-
-if($use_petsc == 1){
-   $fflags = $fflags." -I\$(INC_PETSC)";
-   $ldflags = $ldflags." -L\$(LIB_PETSC) -lpetsc -lm";
+elsif($use_lapack == 3){
+   $ldflags = $ldflags." -L\$(LIB_LAPACK) -llapack -lrefblas";
+   $lib_flags= $lib_flags." -llapack -lrefblas";
+   $lib_paths= $lib_paths." -L\$(LIB_LAPACK)";
 }
 
 if($use_esmf_trace == 1){
@@ -1241,20 +1224,6 @@
    printf misc_file "%s\n","#undef RTMS ";
 }
 
-if($use_lapack == 1) {
-   printf misc_file "%s\n","#define LAPACK ";
-}
-else{
-   printf misc_file "%s\n","#undef LAPACK ";
-}
-
-if($use_mkllapack == 1) {
-   printf misc_file "%s\n","#define MKL_LAPACK ";
-}
-else{
-   printf misc_file "%s\n","#undef MKL_LAPACK ";
-}
-
 if($use_petsc == 1) {
    printf misc_file "%s\n","#define PETSc ";
 }