Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Need to adapt LOFT to latest LOFS, and the LOFS API has changed #8

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Need to adapt LOFT to latest LOFS, and the LOFS API has changed #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 4 additions & 9 deletions Makefiles/Makefile.frontera.orf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,18 @@ HDF5_DIR=$(TACC_HDF5_DIR)
## this path needs to have $(LOFS_DIR)/include/*.h files
## and $(LOFS_DIR)/lib/liblofs.a
LOFS_DIR=${HOME}/build/git/LOFS-read
#H5ZZFP_DIR=/home1/06818/khalbert/installs/H5Z-ZFP/install
#ZFP_DIR=/home1/06818/khalbert/installs/zfp
H5ZZFP_DIR=/home1/06618/orf/H5Z-ZFP-install/
ZFP_DIR=/work/06618/orf/frontera/build/git/zfp/

#ZFP_DIR=/work/06618/orf/frontera/build/git/zfp
#H5ZZFP_DIR=/home1/06618/orf/H5Z-ZFP-install
H5ZZFP_DIR=/home1/06618/orf/H5Z-ZFP-install
ZFP_DIR=/home1/06618/orf/build/zfp-latest/zfp

## These flags are passed to the C++ compiler of the main program
INCLPATHS = -I$(HDF5_DIR)/include -I$(ZFP_DIR)/include -I$(H5ZZFP_DIR)/include -I$(NETCDF_DIR)/include -I$(LOFS_DIR)/include
LIBPATHS = -L$(HDF5_DIR)/lib -L$(ZFP_DIR)/lib -L$(H5ZZFP_DIR)/lib -L$(NETCDF_DIR)/lib -L$(LOFS_DIR)/lib -L$(CUDA)/lib64

CFLAGS = -O3 -std=c++11 $(INCLPATHS)
CFLAGS = -g -O2 -std=c++11 $(INCLPATHS)
LINKOPTS = $(LIBPATHS) -lh5zzfp -lzfp -lhdf5_hl -lhdf5 -lnetcdf -lnetcdf_c++4 -llofs -lcudart -lm -qopenmp

## These flags are passed to the CUDA compiler
NVFLAGS = -O3 --default-stream per-thread -gencode=arch=compute_61,code=compute_61 --prec-div=true --ftz=true --fmad=true --std=c++11
NVFLAGS = -g -O2 --default-stream per-thread -gencode=arch=compute_61,code=compute_61 --prec-div=true --ftz=true --fmad=true --std=c++11


SOURCEDIR := src
Expand Down
2 changes: 1 addition & 1 deletion buildLOFT-Frontera.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,5 @@ module load cuda
module load hdf5
module load netcdf

ln -s ./Makefiles/Makefile.frontera ./Makefile
#ln -s ./Makefiles/Makefile.frontera ./Makefile
make
14 changes: 7 additions & 7 deletions run/parcel.namelist
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
## Path to the HDF5 data
histpath = ./3D
## The name of the output NetCDF file
basename = 24May2011-ElRe-SVC
basename = ElReno10m-parcel
verbose = 0
debug = 0

Expand All @@ -20,19 +20,19 @@ debug = 0
### PARCEL SEED CONFIG ###
###########################################
## The parcel seed starting position
x0 = 0
y0 = 7500
z0 = 30
x0 = -7000
y0 = 5000
z0 = 10

## The number of parcels desired along each axis
nx = 20
ny = 20
nz = 10

## The spacing of the parcels desired along each axis
dx = 375
dy = 125
dz = 15
dx = 40
dy = 40
dz = 20



Expand Down
11 changes: 11 additions & 0 deletions src/io/datastructs.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ sounding* allocate_sounding_managed( int NZ ) {
cudaMallocManaged(&(snd->th0), (NZ)*sizeof(float));
cudaMallocManaged(&(snd->rho0), (NZ)*sizeof(float));
cudaMallocManaged(&(snd->pres0), (NZ)*sizeof(float));
cudaMallocManaged(&(snd->thv0), (NZ)*sizeof(float));
return snd;
}

Expand All @@ -111,6 +112,8 @@ sounding* allocate_sounding_cpu( int NZ ) {
snd->th0 = new float[NZ];
snd->rho0 = new float[NZ];
snd->pres0 = new float[NZ];
snd->thv0 = new float[NZ];
cout << "Allocate: snd->thv0 = " << snd->thv0 << endl;

return snd;
}
Expand Down Expand Up @@ -171,6 +174,7 @@ void deallocate_sounding_managed(sounding *snd) {
cudaFree(snd->qv0);
cudaFree(snd->rho0);
cudaFree(snd->pres0);
cudaFree(snd->thv0);
cudaFree(snd);
}

Expand All @@ -182,6 +186,13 @@ void deallocate_sounding_cpu(sounding *snd) {
delete[] snd->qv0;
delete[] snd->rho0;
delete[] snd->pres0;
cout << "Deallocate: snd->thv0 = " << snd->thv0 << endl;
cout << "snd->thv0[0] = " << snd->thv0[0] << endl;
cout << "snd->thv0[1] = " << snd->thv0[1] << endl;
cout << "snd->thv0[2] = " << snd->thv0[2] << endl;
cout << "deallocating thv0..." << endl;
delete[] snd->thv0;
cout << "Done." << endl;
delete snd;
}

Expand Down
5 changes: 3 additions & 2 deletions src/io/readlofs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ void lofs_get_dataset_structure(std::string base_dir, dir_meta *dm, hdf_meta *hm
int i,ni,nj,nk,status;
char **argv;
hid_t hdf_file_id;
zfpacc zfpacc;

/* begin */

Expand Down Expand Up @@ -84,10 +85,10 @@ void lofs_get_dataset_structure(std::string base_dir, dir_meta *dm, hdf_meta *hm
//ERROR_STOP("Can't open firstfilename! Weird...");
} // Keep open as we need metadata, 1d sounding data, etc.

get_hdf_metadata(*dm,hm,cmd,argv,&hdf_file_id);
get_hdf_metadata(*dm,hm,cmd,nc,argv,&hdf_file_id,&zfpacc);

if (cmd->debug) {
printf("nx = %i ny = %i nz = %i nodex = %i nodey = %i\n",hm->nx,hm->ny,hm->nz,hm->nodex,hm->nodey);
printf("nx = %i ny = %i nz = %i rankx = %i ranky = %i\n",hm->nx,hm->ny,hm->nz,hm->rankx,hm->ranky);
}

/* Check for idiocy and tweak the span (X0-X1/Y0-Y1/Z0-Z1) as necessary */
Expand Down
12 changes: 10 additions & 2 deletions src/main/run_cm1.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ void parse_cfg(map<string, string> *usrCfg, iocfg *io, string *histpath, string
* When the next chunk of time is read in, check and see where the parcels
* are currently and request a subset that is relevent to those parcels.
*/
/* parcelsAreInDomain = getMeshBounds(base_dir, dm, hm, gd, parcels, verbose, rank); */
int getMeshBounds(string base_dir, dir_meta *dm, hdf_meta *hm, grid *gd, parcel_pos *parcels, int verbose, int rank) {
if (verbose) cout << "Retrieving HDF Metadata" << endl;

Expand Down Expand Up @@ -226,8 +227,9 @@ int getMeshBounds(string base_dir, dir_meta *dm, hdf_meta *hm, grid *gd, parcel_
}
if (verbose) cout << "Finished searching parcel bounds" << endl;
// clear the memory from the temp grid
if (verbose) cout << "Deallocating temporary grid" << endl;
if (verbose) cout << "Deallocating mesh cpu" << endl;
deallocate_mesh_cpu(temp_msh);
if (verbose) cout << "Deallocating sounding cpu" << endl;
deallocate_sounding_cpu(temp_snd);
// If all of our parcels have left the domain, then the
// invalid parcel counter will tell us. This will prevent
Expand Down Expand Up @@ -417,10 +419,11 @@ int main(int argc, char **argv ) {
grid *gd = new grid();
cmdline *cmd = new cmdline();
ncstruct *nc = new ncstruct();
zfpacc *zfpaccuracy = new zfpacc();
mesh *req_msh;
sounding *snd;
readahead *rh = new readahead();
init_structs(cmd,dm,gd,nc,rh);
init_structs(cmd,dm,gd,nc,rh,hm,zfpaccuracy);

cmd->verbose = verbose;
cmd->debug = debug;
Expand Down Expand Up @@ -482,17 +485,22 @@ int main(int argc, char **argv ) {
// CPU and GPU so that the GPU knows something
// about our data for future integration.
if (rank == 0) {
if (verbose) cout << "allocate_mesh_managed..." << endl;
req_msh = allocate_mesh_managed( hm, gd );
if (verbose) cout << "allocate_sounding_managed..." << endl;
snd = allocate_sounding_managed( gd->NZ );
}
// For the other MPI ranks, we only need to
// allocate the grids on the CPU for copying
// data to the MPI_Gather call
else {
if (verbose) cout << "allocate_mesh_cpu..." << endl;
req_msh = allocate_mesh_cpu( hm, gd );
if (verbose) cout << "allocate_sounding_cpu..." << endl;
snd = allocate_sounding_cpu( gd->NZ );
}

if (verbose) cout << "lofs_get_grid..." << endl;
lofs_get_grid(dm, hm, gd, req_msh, snd);
if (verbose) cout << "END METADATA & MESH REQUEST" << endl;

Expand Down