Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

New feature: non-coarray LUT #172

Merged
merged 3 commits into from
Nov 28, 2023
Merged

New feature: non-coarray LUT #172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
469de07
The large LUT coarray has been changed to a regular array and two new…
scrasmussen Aug 28, 2023
6341a98
checking if LUT filepaths are writeable when reading namelist options
scrasmussen Sep 28, 2023
6a31134
Cray preprocessor is now -eT instead of -eZ. No .i files are produced…
scrasmussen Sep 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/io/io_routines.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ subroutine io_read6d(filename, varname, data_in, extradim)
implicit none
! This is the name of the data_in file and variable we will read.
character(len=*), intent(in) :: filename, varname
real, allocatable, intent(inout) :: data_in(:,:,:,:,:,:)[:]
real, allocatable, intent(inout) :: data_in(:,:,:,:,:,:)
integer, optional, intent(in) :: extradim

integer, dimension(io_maxDims) :: diminfo !will hold dimension lengths
Expand All @@ -205,7 +205,7 @@ subroutine io_read6d(filename, varname, data_in, extradim)
call io_getdims(filename,varname,diminfo)

if (allocated(data_in)) deallocate(data_in)
allocate(data_in(diminfo(2),diminfo(3),diminfo(4),diminfo(5),diminfo(6),diminfo(7))[*])
allocate(data_in(diminfo(2),diminfo(3),diminfo(4),diminfo(5),diminfo(6),diminfo(7)))

! Open the file. NF90_NOWRITE tells netCDF we want read-only access to the file.
call check(nf90_open(filename, NF90_NOWRITE, ncid),filename)
Expand Down
4 changes: 2 additions & 2 deletions src/io/lt_lut_io.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ module linear_theory_lut_disk_io
function write_lut(filename, uLUT, vLUT, dz, options) result(error)
implicit none
character(len=*), intent(in) :: filename
real, dimension(:,:,:,:,:,:), intent(in) :: uLUT[*], vLUT[*]
real, dimension(:,:,:,:,:,:), intent(in) :: uLUT, vLUT
real, dimension(:), intent(in) :: dz
type(lt_options_type), intent(in) :: options
integer :: error
Expand Down Expand Up @@ -134,7 +134,7 @@ end function write_lut
function read_LUT(filename, uLUT, vLUT, dz, dims, options) result(error)
implicit none
character(len=*), intent(in) :: filename
real, allocatable, dimension(:,:,:,:,:,:), intent(inout) :: uLUT[:], vLUT[:]
real, allocatable, dimension(:,:,:,:,:,:), intent(inout) :: uLUT, vLUT
real, dimension(:), intent(in) :: dz
integer, dimension(3,2), intent(in) :: dims
type(lt_options_type), intent(in) :: options
Expand Down
18 changes: 7 additions & 11 deletions src/makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ endif
ifeq ($(COMPILER), cray)
COMP= -h omp vector2 -O2 -c -eI -hfp0
LINK= -fopenmp
PREPROC= -eZ
PREPROC= -eT
MODOUTPUT= -J $(BUILD) -em
endif

Expand All @@ -213,7 +213,7 @@ ifeq ($(MODE), debugslow)
ifeq ($(COMPILER), cray)
COMP=-h noomp -c -g -h develop -m 0 -R csp -M 399 -hfp0
LINK=-h noomp
PREPROC=-eZ
PREPROC=-eT
MODOUTPUT=-e m -J $(BUILD)
endif
endif
Expand All @@ -229,7 +229,7 @@ ifeq ($(MODE), debug)
ifeq ($(COMPILER), cray)
COMP=-O2 -h noomp -c -g -hfp0
LINK=-h noomp
PREPROC=-eZ
PREPROC=-eT
MODOUTPUT=-e m -J $(BUILD)
endif
endif
Expand All @@ -246,7 +246,7 @@ ifeq ($(MODE), debugompslow)
ifeq ($(COMPILER), cray)
COMP=-c -g -m 0 -h develop -R csp -M 399 -hfp0
LINK=
PREPROC=-eZ
PREPROC=-eT
MODOUTPUT=-e m -J $(BUILD)
endif
endif
Expand All @@ -263,7 +263,7 @@ ifeq ($(MODE), debugomp)
ifeq ($(COMPILER), cray)
COMP=-O1 -c -g -hfp0
LINK=
PREPROC=-eZ
PREPROC=-eT
MODOUTPUT=-e m -J $(BUILD)
endif
endif
Expand Down Expand Up @@ -454,26 +454,22 @@ TEST_EXECUTABLES= fftshift_test \

icar:${OBJS} $(BUILD)driver.o
${LINKER} $^ -o icar ${LFLAGS}
make move_i

all:icar test

install:icar
mkdir -p ${INSTALLDIR}
${CP} icar ${INSTALLDIR}

move_i:
$(ECHO_MOVE) *.i ${BUILD} 2>/dev/null || true

clean:
${RM} $(BUILD)*.o $(BUILD)*.mod $(BUILD)*.smod *.i *.lst docs/doxygen_sqlite3.db 2>/dev/null ||:
${RM} $(BUILD)*.o $(BUILD)*.mod $(BUILD)*.smod *.lst docs/doxygen_sqlite3.db 2>/dev/null ||:

allclean:cleanall

cleanall:clean
${RM} icar $(TEST_EXECUTABLES) 2>/dev/null ||:

test: $(TEST_EXECUTABLES) move_i
test: $(TEST_EXECUTABLES)

caf_tests: $(CAF_TEST_EXECUTABLES)

Expand Down
3 changes: 3 additions & 0 deletions src/objects/options_obj.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1554,6 +1554,9 @@ subroutine lt_parameters_namelist(filename, options)
endif
endif

! check if directory paths to LUT file strings exist, if not fails before write step
call check_writeable_path(u_LUT_Filename)
call check_writeable_path(v_LUT_Filename)
opt%u_LUT_Filename = u_LUT_Filename
opt%v_LUT_Filename = v_LUT_Filename
opt%overwrite_lt_lut = overwrite_lt_lut
Expand Down
71 changes: 53 additions & 18 deletions src/physics/linear_winds.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,9 @@ module linear_theory_winds
!! Linear wind look up table values and tables
real, allocatable, dimension(:) :: dir_values, nsq_values, spd_values
! Look Up Tables for linear perturbation are nspd x n_dir_values x n_nsq_values x nx x nz x ny
real, allocatable, dimension(:,:,:,:,:,:) :: hi_u_LUT[:], hi_v_LUT[:] !, rev_u_LUT, rev_v_LUT
real, allocatable, dimension(:,:,:,:,:,:) :: hi_u_LUT, hi_v_LUT
real, allocatable, dimension(:,:,:) :: hi_u_LUT_co[:], hi_v_LUT_co[:]
integer, allocatable, dimension(:,:) :: LUT_index_co[:]
! real, pointer, dimension(:,:,:,:,:,:) :: u_LUT, v_LUT
real, allocatable, dimension(:,:) :: linear_mask, nsq_calibration

Expand Down Expand Up @@ -574,28 +576,50 @@ subroutine setup_remote_grids(u_grids, v_grids, terrain, nz)

end subroutine setup_remote_grids

subroutine copy_data_to_remote(wind, grids, LUT, i,j,k, z)
subroutine copy_data_to_remote(wind, grids, LUT, LUT_co, LUT_index_co, i, j, k, z)
implicit none
real, intent(in) :: wind(:,:)
type(grid_t), intent(in) :: grids(:)
real, intent(inout):: LUT(:,:,:,:,:,:)[*]
integer, intent(in) :: i,j,k, z

integer :: img

real, intent(inout):: LUT(:,:,:,:,:,:)
real, intent(inout):: LUT_co(:,:,:)[*]
integer, intent(inout):: LUT_index_co(:,:)[*]
integer, intent(in) :: i, j, k, z
integer :: img, LUT_i, LUT_j, LUT_k, LUT_z

! each image will communicate the wind values it has calculated for an ijkz
! location to the other images, the ijkz values are also sent so the receiving
! images know where to put the incoming values
do img = 1, num_images()
associate(ims => grids(img)%ims, &
ime => grids(img)%ime, &
jms => grids(img)%jms, &
jme => grids(img)%jme &
)
!$omp critical
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

double check that !$omp critical isn't needed. Are there other OpenMP calls in the LUT routines

LUT(k,i,j, 1:ime-ims+1, z, 1:jme-jms+1)[img] = wind(ims:ime,jms:jme)
!$omp end critical
LUT_co(1:ime-ims+1, 1:jme-jms+1, this_image())[img] = wind(ims:ime,jms:jme)

end associate
LUT_index_co(this_image(), 1)[img] = i
LUT_index_co(this_image(), 2)[img] = j
LUT_index_co(this_image(), 3)[img] = k
LUT_index_co(this_image(), 4)[img] = z
end associate
enddo

sync all

associate(ims => grids(this_image())%ims, &
ime => grids(this_image())%ime, &
jms => grids(this_image())%jms, &
jme => grids(this_image())%jme &
)
do img = 1, num_images()
LUT_i = LUT_index_co(img, 1)
LUT_j = LUT_index_co(img, 2)
LUT_k = LUT_index_co(img, 3)
LUT_z = LUT_index_co(img, 4)
LUT(LUT_k, LUT_i, LUT_j, 1:ime-ims+1, LUT_z, 1:jme-jms+1) = LUT_co(1:ime-ims+1, 1:jme-jms+1, img)
end do
end associate
sync all
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can remove this sync all since one is called outside these subroutine calls

end subroutine copy_data_to_remote

!>----------------------------------------------------------
Expand Down Expand Up @@ -670,8 +694,11 @@ subroutine initialize_spatial_winds(domain,options,reverse)

! Allocate the (LARGE) look up tables for both U and V
if (.not.options%lt_options%read_LUT) then
allocate(hi_u_LUT(n_spd_values, n_dir_values, n_nsq_values, nxu, nz, ny)[*], source=0.0)
allocate(hi_v_LUT(n_spd_values, n_dir_values, n_nsq_values, nx, nz, nyv)[*], source=0.0)
allocate(hi_u_LUT(n_spd_values, n_dir_values, n_nsq_values, nxu, nz, ny), source=0.0)
allocate(hi_v_LUT(n_spd_values, n_dir_values, n_nsq_values, nx, nz, nyv), source=0.0)
allocate(hi_u_LUT_co(nxu, ny, num_images())[*], source=0.0)
allocate(hi_v_LUT_co(nx, nyv, num_images())[*], source=0.0)
allocate(LUT_index_co(num_images(), 4)[*], source=0)
error=0
else
if (this_image()==1) write(*,*) " Reading LUT from file: ", trim(LUT_file)
Expand All @@ -680,10 +707,18 @@ subroutine initialize_spatial_winds(domain,options,reverse)
if (error/=0) then
if (this_image()==1) write(*,*) "WARNING: LUT on disk does not match that specified in the namelist or does not exist."
if (this_image()==1) write(*,*) " LUT will be recreated"

if (allocated(hi_u_LUT_co)) deallocate(hi_u_LUT_co)
allocate(hi_u_LUT_co(nxu, ny, num_images())[*], source=0.0)
if (allocated(hi_v_LUT_co)) deallocate(hi_v_LUT_co)
allocate(hi_v_LUT_co(nx, nyv, num_images())[*], source=0.0)

if (allocated(LUT_index_co)) deallocate(LUT_index_co)
allocate(LUT_index_co(num_images(), 4)[*], source=0)
if (allocated(hi_u_LUT)) deallocate(hi_u_LUT)
allocate(hi_u_LUT(n_spd_values, n_dir_values, n_nsq_values, nxu, nz, ny)[*], source=0.0)
allocate(hi_u_LUT(n_spd_values, n_dir_values, n_nsq_values, nxu, nz, ny), source=0.0)
if (allocated(hi_v_LUT)) deallocate(hi_v_LUT)
allocate(hi_v_LUT(n_spd_values, n_dir_values, n_nsq_values, nx, nz, nyv)[*], source=0.0)
allocate(hi_v_LUT(n_spd_values, n_dir_values, n_nsq_values, nx, nz, nyv), source=0.0)
endif
endif

Expand All @@ -694,7 +729,6 @@ subroutine initialize_spatial_winds(domain,options,reverse)
if (this_image()==1) write(*,*) "Stabilities:",exp(nsq_values)
! endif


if (reverse.or.(.not.((options%lt_options%read_LUT).and.(error==0)))) then
! loop over combinations of U, V, and Nsq values
loops_completed = 0
Expand Down Expand Up @@ -780,8 +814,9 @@ subroutine initialize_spatial_winds(domain,options,reverse)
( lt_data_m%v_perturb(1+buffer:fftnx-buffer, buffer:fftny-buffer) &
+ lt_data_m%v_perturb(1+buffer:fftnx-buffer, 1+buffer:fftny-buffer+1)) )) / 2

call copy_data_to_remote(temporary_u, u_grids, hi_u_LUT, i,j,k, z)
call copy_data_to_remote(temporary_v, v_grids, hi_v_LUT, i,j,k, z)
call copy_data_to_remote(temporary_u, u_grids, hi_u_LUT, hi_u_LUT_co, LUT_index_co, i, j, k, z)
call copy_data_to_remote(temporary_v, v_grids, hi_v_LUT, hi_v_LUT_co, LUT_index_co, i, j, k, z)
sync all
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove this sync all


else
stop "ERROR: linear wind LUT creation not set up for non-staggered grids yet"
Expand Down