From 2e08c67383da0de293db701bbaa2772131f2e2cc Mon Sep 17 00:00:00 2001 From: Timothy Younkin Date: Mon, 7 Oct 2019 13:17:13 -0700 Subject: [PATCH] updates for running pisces case on cori --- CMakeLists.txt | 5 ++-- build/makeGITRCori.sh | 23 ++++----------- env.cori.sh | 66 +++++++++---------------------------------- src/gitr.cpp | 28 ++++++++++++------ src/utils.cpp | 1 + 5 files changed, 42 insertions(+), 81 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 093ab0fd..f217e5dd 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ list (REMOVE_ITEM TEST_FILES /Users/tyounkin/Code/gitr_repo/src/gitr.cpp) message("${TEST_FILES}") file( GLOB HEADER_FILES include/*.h ) add_library(YourLib ${TEST_FILES}) -include_directories( ${HEADER_FILES}) +include_directories( include) find_package(Thrust REQUIRED) find_package(NetCDF COMPONENTS CXX REQUIRED) find_package(LibConfig) @@ -31,6 +31,7 @@ include_directories(${LIBCONFIGPP_INCLUDE_DIR}) # Simulation defines add_definitions(-DUSE_CUDA=${USE_CUDA} -DUSE_MPI=${USE_MPI} + -DUSE_FILESYSTEM=${USE_FILESYSTEM} -DUSEIONIZATION=${USEIONIZATION} -DUSERECOMBINATION=${USERECOMBINATION} -DUSEPERPDIFFUSION=${USEPERPDIFFUSION} @@ -93,7 +94,7 @@ if (${OPENMP}) Message("Enabling OpenMP backend") find_package(OpenMP REQUIRED) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -O3 -std=c++1z")#-g -G -Wno-c++11-narrowing -fstack-protector-all -fstack-check -W -Wall") #-O3 -g -G + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -O3 -std=c++17")#-g -G -Wno-c++11-narrowing -fstack-protector-all -fstack-check -W -Wall") #-O3 -g -G set_source_files_properties(${SRC_FILES} PROPERTIES LANGUAGE CXX) add_executable(GITR ${SRC_FILES}) add_definitions("-x c++ -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP") diff --git a/build/makeGITRCori.sh b/build/makeGITRCori.sh index 941e7b36..6e1a2874 100755 --- a/build/makeGITRCori.sh +++ b/build/makeGITRCori.sh @@ -1,8 +1,8 @@ source ../env.cori.sh -cmake -DTHRUST_INCLUDE_DIR=/global/homes/t/tyounkin/code/thrust/ \ --DLIBCONFIGPP_INCLUDE_DIR=/global/homes/t/tyounkin/code/libconfigBuild/gnu/include \ --DLIBCONFIGPP_LIBRARY=/global/homes/t/tyounkin/code/libconfigBuild/gnu/lib/libconfig++.so \ +cmake -DTHRUST_INCLUDE_DIR=$THRUST_INCLUDE_DIRS \ +-DLIBCONFIGPP_INCLUDE_DIR=$LIBCONFIG_INCLUDE_DIR \ +-DLIBCONFIGPP_LIBRARY=$LIBCONFIGPP_LIBRARY \ -DCMAKE_C_COMPILER=gcc \ -DCMAKE_CXX_COMPILER=g++ \ -DNETCDF_LIBRARY=$NETCDF_DIR/lib/libnetcdf.so \ @@ -10,14 +10,11 @@ cmake -DTHRUST_INCLUDE_DIR=/global/homes/t/tyounkin/code/thrust/ \ -DNETCDF_CXX_INCLUDE_DIR=/opt/cray/pe/netcdf/4.6.1.3/GNU/8.2/include \ -DNETCDF_CXX_LIBRARY=$NETCDF_DIR/lib/libnetcdf_c++4_gnu_82.so \ -DUSE_CUDA=0 \ - -DUSEMPI=0 \ -DUSE_MPI=1 \ - -DUSE_OPENMP=0 \ - -DUSE_BOOST=1 \ + -DUSE_FILESYSTEM=0 \ -DUSEIONIZATION=1 \ -DUSERECOMBINATION=1 \ -DUSEPERPDIFFUSION=1 \ - -DUSEPARDIFFUSION=1 \ -DUSECOULOMBCOLLISIONS=1 \ -DUSEFRICTION=1 \ -DUSEANGLESCATTERING=1 \ @@ -40,8 +37,8 @@ cmake -DTHRUST_INCLUDE_DIR=/global/homes/t/tyounkin/code/thrust/ \ -DFIXEDSEEDS=0 \ -DPARTICLESEEDS=1 \ -DGEOM_TRACE=0 \ - -DGEOM_HASH=3 \ - -DGEOM_HASH_SHEATH=3 \ + -DGEOM_HASH=1 \ + -DGEOM_HASH_SHEATH=1 \ -DPARTICLE_TRACKS=0 \ -DPARTICLE_SOURCE_SPACE=0 \ -DPARTICLE_SOURCE_ENERGY=0 \ @@ -56,11 +53,3 @@ cmake -DTHRUST_INCLUDE_DIR=/global/homes/t/tyounkin/code/thrust/ \ -DFORCE_EVAL=0 \ -DCHECK_COMPATIBILITY=1 \ .. -#-DMPI_C_LIBRARIES=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/lib/libmpich.so \ -#-DMPI_C_LIB_NAMES=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/lib/libmpich.so \ -#-DMPI_C_INCLUDE_PATH=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/include \ -#-DMPI_CXX_LIBRARIES=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/lib/libmpichcxx_gnu_82.so \ -#-DMPI_CXX_LIB_NAMES=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/lib/libmpichcxx_gnu_82.so \ -#-DMPI_CXX_INCLUDE_PATH=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/include \ -#-DMPI_C_COMPILER=mpich \ -#-DMPI_CXX_COMPILER=mpichcxx \ diff --git a/env.cori.sh b/env.cori.sh index c5a34bd5..a470e6f5 100644 --- a/env.cori.sh +++ b/env.cori.sh @@ -1,33 +1,18 @@ #!/bin/bash module swap PrgEnv-intel/6.0.5 PrgEnv-gnu/6.0.5 -#module load cray-parallel-netcdf module load cray-netcdf -#module load netcdf -#module load netcdf-cxx -module unload boost/1.63 -module load boost module load openmpi -#module load libconfig/gcc/64/1.5 -#module load git -#module load mpich/ge/gcc/64/3.1 -##module load cuda75/toolkit/7.5.18 -#module load cuda70/toolkit/7.0.28 -#module load slurm -##module load netcdf4 -#PATH=$PATH:/home/tqd/code/netcdfBuild/bin:/home/tqd/code/netcdfcxx/bin -##prepend-path PKG_CONFIG_PATH /home/dg6/code/netcdf/gnu/lib/pkgconfig -#LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/tqd/code/netcdfBuild/bin:/home/tqd/code/netcdfcxx/bin -# -export LIBCONFIGDIR=/global/homes/t/tyounkin/code/libconfigBuild/gnu + +export GITR_BASE_DIR=/project/projectdirs/m1709/psi-install-cori +export LIBCONFIGDIR=$GITR_BASE_DIR/libconfig export LIBCONFIGLIB=$LIBCONFIGDIR/lib export LIBCONFIGPP_LIBRARIES=lconfig++ -export LIBCONFIGPP_LIBRARY=lconfig++ +export LIBCONFIGPP_LIBRARY=$LIBCONFIGLIB/libconfig++.so export LIBCONFIGPP_STATIC_LIBRARY= export LIBCONFIG_INCLUDE_DIR=$LIBCONFIGDIR/include export LIBCONFIGPP_INCLUDE_DIR=$LIBCONFIGDIR/include export LIBCONFIG_LIBRARY=lconfig -#LD_LIBRARY_PATH=LD_RUN_PATH -export THRUST_INCLUDE_DIRS=/global/homes/t/tyounkin/code/thrust/include +export THRUST_INCLUDE_DIRS=$GITR_BASE_DIR/thrust export NETCDF_CXX_BASEDIR=/opt/cray/pe/netcdf/4.6.1.3/GNU/8.2 export NETCDF_CXX_INCLUDE_DIR=$NETCDF_CXX_BASEDIR/include export NETCDF_CXX_LIBRARY=$NETCDF_CXX_BASEDIR/lib @@ -35,36 +20,11 @@ export NETCDF_INCLUDE_DIR=$NETCDF_DIR/lib export NETCDF_INCLUDE_DIRS=$NETCDF_DIR/include export NETCDF_LIBRARIES=$NETCDF_DIR/lib export NETCDF_LIBRARY=$NETCDF_DIR/lib -#export NETCDF=$NETCDF_DIR -#export NETCDF_CXX_INCLUDE_DIR=$NETCDF/include -#export NETCDF_CXX_LIBRARY= -#export NETCDF_INCLUDE_DIR=$NETCDF/include -#export NETCDF_LIBRARY -#export NETCDFCXX4=/home/tqd/code/netcdfcxx -#export NETCDFDIR=/home/tqd/code/netcdfBuild/lib -#export NETCDFCXX4DIR=/home/tqd/code/netcdfcxx/lib -#export NETCDFINCLUDE=/home/tqd/code/netcdfBuild/include -#export NETCDFCXX4INCLUDE=/home/tqd/code/netcdfcxx/include -#export NETCDFLIB=netcdf -#export NETCDFLIB_CPP=netcdf_c++ -#export NETCDF_LIBRARIES=$NETCDF/lib -#export NETCDF_INCLUDE_DIRS=$NETCDF/include -# -##boost vars -#export Boost_INCLUDE_DIRS=~/code/boost/include -#export Boost_LIBRARY_DIRS=/global/homes/t/tyounkin/code/boostBuild/lib -#export BOOST_ROOT=~/code/boost -#export BOOST_INCLUDEDIR=~/code/boost/include -#export BOOST_LIBRARYDIR=~/code/boost/lib -# -export PATH=/global/homes/t/tyounkin/code/python2.7.14/bin/:$PATH -export PYTHONPATH=/global/homes/t/tyounkin/code/libconfPython/lib/python2.7/site-packages/:$PYTHONPATH -export PYTHONPATH=/global/homes/t/tyounkin/code/scipyBuild/lib/python2.7/site-packages:$PYTHONPATH -export PYTHONPATH=/global/homes/t/tyounkin/atomIPS/atom-install-cori/GITR/python:$PYTHONPATH -export PYTHONPATH=/global/homes/t/tyounkin/code/netcdfPython/lib.linux-x86_64-2.7:$PYTHONPATH -#export MPI_C_LIBRARIES=mpicc -#export MPI_C_INCLUDE_PATH=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/include -#export MPI_CXX_LIBRARIES=mpichcxx -#export MPI_CXX_INCLUDE_PATH=/opt/cray/pe/mpt/7.7.6/gni/mpich-gnu/8.2/include -LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIBCONFIGLIB -export LD_LIBRARY_PATH + +#export PATH=/global/homes/t/tyounkin/code/python2.7.14/bin/:$PATH +#export PYTHONPATH=/global/homes/t/tyounkin/code/libconfPython/lib/python2.7/site-packages/:$PYTHONPATH +#export PYTHONPATH=/global/homes/t/tyounkin/code/scipyBuild/lib/python2.7/site-packages:$PYTHONPATH +#export PYTHONPATH=/global/homes/t/tyounkin/atomIPS/atom-install-cori/GITR/python:$PYTHONPATH +#export PYTHONPATH=/global/homes/t/tyounkin/code/netcdfPython/lib.linux-x86_64-2.7:$PYTHONPATH +#LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIBCONFIGLIB +#export LD_LIBRARY_PATH diff --git a/src/gitr.cpp b/src/gitr.cpp index 8c4999d0..63c73822 100644 --- a/src/gitr.cpp +++ b/src/gitr.cpp @@ -42,8 +42,10 @@ #ifdef __CUDACC__ #include #include -#include #else +#endif + +#if USE_FILESYSTEM > 0 #include #endif @@ -132,6 +134,7 @@ int main(int argc, char **argv, char **envp) { } // show memory usage of GPU +#if USE_FILESYSTEM > 0 #if __CUDACC__ namespace fsn = std::experimental::filesystem; #else @@ -150,7 +153,7 @@ print_gpu_memory_usage(world_rank); std::cout << " Successfully Created " << std::endl; } } - +#endif // Background species info float background_Z = 0.0, background_amu = 0.0; if (world_rank == 0) { @@ -449,14 +452,8 @@ print_gpu_memory_usage(world_rank); nHashPointsTotal = 0; for(int j=0;j 1) - { nHashPoints[j] =nR_closeGeom[j]*nY_closeGeom[j]*nZ_closeGeom[j]; - } - else - { - nHashPoints[j] =nR_closeGeom[j]*nZ_closeGeom[j]; - } + nHashPointsTotal = nHashPointsTotal + nHashPoints[j]; nGeomHash = nGeomHash + nHashPoints[j]*n_closeGeomElements[j]; nR_closeGeomTotal = nR_closeGeomTotal + nR_closeGeom[j]; @@ -762,7 +759,9 @@ print_gpu_memory_usage(world_rank); for (int i = 0; i < nHashes; i++) { NcFile ncFile_hash("output/geomHash" + std::to_string(i) + ".nc", NcFile::replace); + std::cout << "opened GeomHash file " << std::endl; NcDim hashNR = ncFile_hash.addDim("nR", nR_closeGeom[i]); + std::cout << "added first dimension nR " << std::endl; #if USE3DTETGEOM > 0 NcDim hashNY = ncFile_hash.addDim("nY", nY_closeGeom[i]); #endif @@ -775,32 +774,43 @@ print_gpu_memory_usage(world_rank); #endif geomHashDim.push_back(hashNZ); geomHashDim.push_back(hashN); + std::cout << "added all dimensions " << std::endl; NcVar hash_gridR = ncFile_hash.addVar("gridR", ncFloat, hashNR); + std::cout << "added first variable " << std::endl; #if USE3DTETGEOM > 0 NcVar hash_gridY = ncFile_hash.addVar("gridY", ncFloat, hashNY); #endif NcVar hash_gridZ = ncFile_hash.addVar("gridZ", ncFloat, hashNZ); + std::cout << "added gridZ variable " << std::endl; NcVar hash = ncFile_hash.addVar("hash", ncInt, geomHashDim); + std::cout << "added hash variable " << std::endl; int ncIndex = 0; if (i > 0) ncIndex = nR_closeGeom[i - 1]; hash_gridR.putVar(&closeGeomGridr[ncIndex]); + std::cout << "put gridr variable " << std::endl; #if USE3DTETGEOM > 0 if (i > 0) ncIndex = nY_closeGeom[i - 1]; hash_gridY.putVar(&closeGeomGridy[ncIndex]); + std::cout << "put gridy variable " << std::endl; #endif if (i > 0) ncIndex = nZ_closeGeom[i - 1]; hash_gridZ.putVar(&closeGeomGridz[ncIndex]); + std::cout << "put gridz variable " << std::endl; if (i > 0) ncIndex = nR_closeGeom[i - 1] * nY_closeGeom[i - 1] * nZ_closeGeom[i - 1] * n_closeGeomElements[i - 1]; + std::cout << "ncIndex "< 1 if (world_rank == 0) { for (int i = 0; i < nHashes; i++) { diff --git a/src/utils.cpp b/src/utils.cpp index 7fc5e1a5..4a5d4269 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -922,4 +922,5 @@ int ncdfIO(int rwCode,const std::string& fileName,vector< std::string> dimNames, { std::cout << "dimension " <