updates for running pisces case on cori

ORNL-Fusion · Oct 7, 2019 · 2e08c67 · 2e08c67
1 parent 827e380
commit 2e08c67
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 81 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -14,7 +14,7 @@ list (REMOVE_ITEM TEST_FILES /Users/tyounkin/Code/gitr_repo/src/gitr.cpp)
 message("${TEST_FILES}")
 file( GLOB HEADER_FILES include/*.h )
 add_library(YourLib ${TEST_FILES})
-include_directories( ${HEADER_FILES})
+include_directories( include)
 find_package(Thrust REQUIRED)
 find_package(NetCDF COMPONENTS CXX REQUIRED)
 find_package(LibConfig)
@@ -31,6 +31,7 @@ include_directories(${LIBCONFIGPP_INCLUDE_DIR})
 # Simulation defines
 add_definitions(-DUSE_CUDA=${USE_CUDA}
         -DUSE_MPI=${USE_MPI}
+	-DUSE_FILESYSTEM=${USE_FILESYSTEM}
         -DUSEIONIZATION=${USEIONIZATION}
         -DUSERECOMBINATION=${USERECOMBINATION}
         -DUSEPERPDIFFUSION=${USEPERPDIFFUSION}
@@ -93,7 +94,7 @@ if (${OPENMP})
     Message("Enabling OpenMP backend")
     find_package(OpenMP REQUIRED)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -O3 -std=c++1z")#-g -G -Wno-c++11-narrowing -fstack-protector-all -fstack-check -W -Wall") #-O3 -g -G
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -O3 -std=c++17")#-g -G -Wno-c++11-narrowing -fstack-protector-all -fstack-check -W -Wall") #-O3 -g -G
     set_source_files_properties(${SRC_FILES} PROPERTIES LANGUAGE CXX)
     add_executable(GITR ${SRC_FILES})
     add_definitions("-x c++ -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP")

diff --git a/build/makeGITRCori.sh b/build/makeGITRCori.sh
@@ -1,23 +1,20 @@
 source ../env.cori.sh
 
-cmake -DTHRUST_INCLUDE_DIR=/global/homes/t/tyounkin/code/thrust/ \
--DLIBCONFIGPP_INCLUDE_DIR=/global/homes/t/tyounkin/code/libconfigBuild/gnu/include \
--DLIBCONFIGPP_LIBRARY=/global/homes/t/tyounkin/code/libconfigBuild/gnu/lib/libconfig++.so \
+cmake -DTHRUST_INCLUDE_DIR=$THRUST_INCLUDE_DIRS \
+-DLIBCONFIGPP_INCLUDE_DIR=$LIBCONFIG_INCLUDE_DIR \
+-DLIBCONFIGPP_LIBRARY=$LIBCONFIGPP_LIBRARY \
 -DCMAKE_C_COMPILER=gcc \
 -DCMAKE_CXX_COMPILER=g++ \
 -DNETCDF_LIBRARY=$NETCDF_DIR/lib/libnetcdf.so \
 -DNETCDF_INCLUDE_DIR=$NETCDF_DIR/include \
 -DNETCDF_CXX_INCLUDE_DIR=/opt/cray/pe/netcdf/4.6.1.3/GNU/8.2/include \
 -DNETCDF_CXX_LIBRARY=$NETCDF_DIR/lib/libnetcdf_c++4_gnu_82.so \
 -DUSE_CUDA=0 \
-    -DUSEMPI=0 \
     -DUSE_MPI=1 \
-    -DUSE_OPENMP=0 \
-    -DUSE_BOOST=1 \
+    -DUSE_FILESYSTEM=0 \
     -DUSEIONIZATION=1 \
     -DUSERECOMBINATION=1 \
     -DUSEPERPDIFFUSION=1 \
-    -DUSEPARDIFFUSION=1 \
     -DUSECOULOMBCOLLISIONS=1 \
     -DUSEFRICTION=1 \
     -DUSEANGLESCATTERING=1 \
@@ -40,8 +37,8 @@ cmake -DTHRUST_INCLUDE_DIR=/global/homes/t/tyounkin/code/thrust/ \
     -DFIXEDSEEDS=0 \
     -DPARTICLESEEDS=1 \
     -DGEOM_TRACE=0 \
-    -DGEOM_HASH=3 \
-    -DGEOM_HASH_SHEATH=3 \
+    -DGEOM_HASH=1 \
+    -DGEOM_HASH_SHEATH=1 \
     -DPARTICLE_TRACKS=0 \
     -DPARTICLE_SOURCE_SPACE=0 \
     -DPARTICLE_SOURCE_ENERGY=0 \
@@ -56,11 +53,3 @@ cmake -DTHRUST_INCLUDE_DIR=/global/homes/t/tyounkin/code/thrust/ \
     -DFORCE_EVAL=0 \
     -DCHECK_COMPATIBILITY=1 \
 ..
-#-DMPI_C_LIBRARIES=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/lib/libmpich.so \
-#-DMPI_C_LIB_NAMES=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/lib/libmpich.so \
-#-DMPI_C_INCLUDE_PATH=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/include \
-#-DMPI_CXX_LIBRARIES=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/lib/libmpichcxx_gnu_82.so \
-#-DMPI_CXX_LIB_NAMES=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/lib/libmpichcxx_gnu_82.so \
-#-DMPI_CXX_INCLUDE_PATH=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/include \
-#-DMPI_C_COMPILER=mpich \
-#-DMPI_CXX_COMPILER=mpichcxx \
diff --git a/env.cori.sh b/env.cori.sh
@@ -1,70 +1,30 @@
 #!/bin/bash
 module swap PrgEnv-intel/6.0.5 PrgEnv-gnu/6.0.5
-#module load cray-parallel-netcdf
 module load cray-netcdf
-#module load netcdf
-#module load netcdf-cxx
-module unload boost/1.63
-module load boost
 module load openmpi
-#module load libconfig/gcc/64/1.5 
-#module load git
-#module load mpich/ge/gcc/64/3.1
-##module load cuda75/toolkit/7.5.18
-#module load cuda70/toolkit/7.0.28 
-#module load slurm
-##module load netcdf4
-#PATH=$PATH:/home/tqd/code/netcdfBuild/bin:/home/tqd/code/netcdfcxx/bin
-##prepend-path     PKG_CONFIG_PATH /home/dg6/code/netcdf/gnu/lib/pkgconfig 
-#LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/tqd/code/netcdfBuild/bin:/home/tqd/code/netcdfcxx/bin
-#
-export LIBCONFIGDIR=/global/homes/t/tyounkin/code/libconfigBuild/gnu
+
+export GITR_BASE_DIR=/project/projectdirs/m1709/psi-install-cori
+export LIBCONFIGDIR=$GITR_BASE_DIR/libconfig
 export LIBCONFIGLIB=$LIBCONFIGDIR/lib
 export LIBCONFIGPP_LIBRARIES=lconfig++
-export LIBCONFIGPP_LIBRARY=lconfig++
+export LIBCONFIGPP_LIBRARY=$LIBCONFIGLIB/libconfig++.so
 export LIBCONFIGPP_STATIC_LIBRARY=
 export LIBCONFIG_INCLUDE_DIR=$LIBCONFIGDIR/include
 export LIBCONFIGPP_INCLUDE_DIR=$LIBCONFIGDIR/include
 export LIBCONFIG_LIBRARY=lconfig
-#LD_LIBRARY_PATH=LD_RUN_PATH
-export THRUST_INCLUDE_DIRS=/global/homes/t/tyounkin/code/thrust/include
+export THRUST_INCLUDE_DIRS=$GITR_BASE_DIR/thrust
 export NETCDF_CXX_BASEDIR=/opt/cray/pe/netcdf/4.6.1.3/GNU/8.2
 export NETCDF_CXX_INCLUDE_DIR=$NETCDF_CXX_BASEDIR/include
 export NETCDF_CXX_LIBRARY=$NETCDF_CXX_BASEDIR/lib
 export NETCDF_INCLUDE_DIR=$NETCDF_DIR/lib
 export NETCDF_INCLUDE_DIRS=$NETCDF_DIR/include
 export NETCDF_LIBRARIES=$NETCDF_DIR/lib
 export NETCDF_LIBRARY=$NETCDF_DIR/lib
-#export NETCDF=$NETCDF_DIR
-#export NETCDF_CXX_INCLUDE_DIR=$NETCDF/include
-#export NETCDF_CXX_LIBRARY=
-#export NETCDF_INCLUDE_DIR=$NETCDF/include
-#export NETCDF_LIBRARY
-#export  NETCDFCXX4=/home/tqd/code/netcdfcxx  
-#export  NETCDFDIR=/home/tqd/code/netcdfBuild/lib 
-#export  NETCDFCXX4DIR=/home/tqd/code/netcdfcxx/lib 
-#export  NETCDFINCLUDE=/home/tqd/code/netcdfBuild/include 
-#export  NETCDFCXX4INCLUDE=/home/tqd/code/netcdfcxx/include 
-#export  NETCDFLIB=netcdf 
-#export  NETCDFLIB_CPP=netcdf_c++
-#export NETCDF_LIBRARIES=$NETCDF/lib
-#export NETCDF_INCLUDE_DIRS=$NETCDF/include
-#
-##boost vars
-#export Boost_INCLUDE_DIRS=~/code/boost/include
-#export Boost_LIBRARY_DIRS=/global/homes/t/tyounkin/code/boostBuild/lib
-#export BOOST_ROOT=~/code/boost
-#export BOOST_INCLUDEDIR=~/code/boost/include
-#export BOOST_LIBRARYDIR=~/code/boost/lib
-#
-export PATH=/global/homes/t/tyounkin/code/python2.7.14/bin/:$PATH
-export PYTHONPATH=/global/homes/t/tyounkin/code/libconfPython/lib/python2.7/site-packages/:$PYTHONPATH
-export PYTHONPATH=/global/homes/t/tyounkin/code/scipyBuild/lib/python2.7/site-packages:$PYTHONPATH
-export PYTHONPATH=/global/homes/t/tyounkin/atomIPS/atom-install-cori/GITR/python:$PYTHONPATH
-export PYTHONPATH=/global/homes/t/tyounkin/code/netcdfPython/lib.linux-x86_64-2.7:$PYTHONPATH
-#export MPI_C_LIBRARIES=mpicc
-#export MPI_C_INCLUDE_PATH=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/include
-#export MPI_CXX_LIBRARIES=mpichcxx
-#export MPI_CXX_INCLUDE_PATH=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/include
-LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIBCONFIGLIB
-export LD_LIBRARY_PATH
+
+#export PATH=/global/homes/t/tyounkin/code/python2.7.14/bin/:$PATH
+#export PYTHONPATH=/global/homes/t/tyounkin/code/libconfPython/lib/python2.7/site-packages/:$PYTHONPATH
+#export PYTHONPATH=/global/homes/t/tyounkin/code/scipyBuild/lib/python2.7/site-packages:$PYTHONPATH
+#export PYTHONPATH=/global/homes/t/tyounkin/atomIPS/atom-install-cori/GITR/python:$PYTHONPATH
+#export PYTHONPATH=/global/homes/t/tyounkin/code/netcdfPython/lib.linux-x86_64-2.7:$PYTHONPATH
+#LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIBCONFIGLIB
+#export LD_LIBRARY_PATH
diff --git a/src/gitr.cpp b/src/gitr.cpp
@@ -42,8 +42,10 @@
 #ifdef __CUDACC__
 #include <curand.h>
 #include <curand_kernel.h>
-#include <experimental/filesystem>
 #else
+#endif
+
+#if USE_FILESYSTEM > 0
 #include <experimental/filesystem>
 #endif
 
@@ -132,6 +134,7 @@ int main(int argc, char **argv, char **envp) {
   }
 
 // show memory usage of GPU
+#if USE_FILESYSTEM > 0
 #if __CUDACC__
   namespace fsn = std::experimental::filesystem;
 #else
@@ -150,7 +153,7 @@ print_gpu_memory_usage(world_rank);
       std::cout << " Successfully Created " << std::endl;
     }
   }
-
+#endif
   // Background species info
   float background_Z = 0.0, background_amu = 0.0;
   if (world_rank == 0) {
@@ -449,14 +452,8 @@ print_gpu_memory_usage(world_rank);
      nHashPointsTotal = 0;
      for(int j=0;j<nHashes;j++)
     {
-      if(nHashes > 1)
-      {
         nHashPoints[j] =nR_closeGeom[j]*nY_closeGeom[j]*nZ_closeGeom[j];
-      }
-      else
-      {
-        nHashPoints[j] =nR_closeGeom[j]*nZ_closeGeom[j];
-      }
+
       nHashPointsTotal = nHashPointsTotal + nHashPoints[j];
       nGeomHash = nGeomHash + nHashPoints[j]*n_closeGeomElements[j];
       nR_closeGeomTotal = nR_closeGeomTotal + nR_closeGeom[j];
@@ -762,7 +759,9 @@ print_gpu_memory_usage(world_rank);
     for (int i = 0; i < nHashes; i++) {
       NcFile ncFile_hash("output/geomHash" + std::to_string(i) + ".nc",
                          NcFile::replace);
+  std::cout << "opened GeomHash file " << std::endl;
       NcDim hashNR = ncFile_hash.addDim("nR", nR_closeGeom[i]);
+  std::cout << "added first dimension nR " << std::endl;
 #if USE3DTETGEOM > 0
       NcDim hashNY = ncFile_hash.addDim("nY", nY_closeGeom[i]);
 #endif
@@ -775,32 +774,43 @@ print_gpu_memory_usage(world_rank);
 #endif
       geomHashDim.push_back(hashNZ);
       geomHashDim.push_back(hashN);
+  std::cout << "added all dimensions " << std::endl;
       NcVar hash_gridR = ncFile_hash.addVar("gridR", ncFloat, hashNR);
+  std::cout << "added first variable " << std::endl;
 #if USE3DTETGEOM > 0
       NcVar hash_gridY = ncFile_hash.addVar("gridY", ncFloat, hashNY);
 #endif
       NcVar hash_gridZ = ncFile_hash.addVar("gridZ", ncFloat, hashNZ);
+  std::cout << "added gridZ variable " << std::endl;
       NcVar hash = ncFile_hash.addVar("hash", ncInt, geomHashDim);
+  std::cout << "added hash variable " << std::endl;
       int ncIndex = 0;
       if (i > 0)
         ncIndex = nR_closeGeom[i - 1];
       hash_gridR.putVar(&closeGeomGridr[ncIndex]);
+  std::cout << "put gridr variable " << std::endl;
 #if USE3DTETGEOM > 0
       if (i > 0)
         ncIndex = nY_closeGeom[i - 1];
       hash_gridY.putVar(&closeGeomGridy[ncIndex]);
+  std::cout << "put gridy variable " << std::endl;
 #endif
 
       if (i > 0)
         ncIndex = nZ_closeGeom[i - 1];
       hash_gridZ.putVar(&closeGeomGridz[ncIndex]);
+  std::cout << "put gridz variable " << std::endl;
       if (i > 0)
         ncIndex = nR_closeGeom[i - 1] * nY_closeGeom[i - 1] *
                   nZ_closeGeom[i - 1] * n_closeGeomElements[i - 1];
+  std::cout << "ncIndex "<<ncIndex << std::endl;
       hash.putVar(&closeGeom[ncIndex]);
+  std::cout << "put hash variable " << std::endl;
       ncFile_hash.close();
+  std::cout << "closed file " << std::endl;
     }
   }
+  std::cout << "finished GeomHash file " << std::endl;
 #elif GEOM_HASH > 1
   if (world_rank == 0) {
     for (int i = 0; i < nHashes; i++) {

diff --git a/src/utils.cpp b/src/utils.cpp
@@ -922,4 +922,5 @@ int ncdfIO(int rwCode,const std::string& fileName,vector< std::string> dimNames,
     {
        std::cout << "dimension " <<i << " " <<  dims[i] << std::endl;
     }
+    return 0;
 }
-Original file line number
+Diff line change
@@ Expand Up @@
         {
            std::cout << "dimension " <<i << " " <<  dims[i] << std::endl;
         }
+        return 0;
     }