Skip to content

Commit

Permalink
Merge branch 'lc/gpu-develop' of github.com:benegee/Trixi.jl into lc/…
Browse files Browse the repository at this point in the history
…gpu-develop
benegee committed Dec 18, 2024
2 parents 4a978a5 + d6d946f commit 20e3174
Showing 6 changed files with 2,295 additions and 9 deletions.
30 changes: 30 additions & 0 deletions esiwace/LocalPreferences.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
[CUDA_Runtime_jll]
local = "true"
version = "12.6"

[HDF5]
libhdf5 = "/dss/lrzsys/sys/spack/release/22.2.1/opt/icelake/hdf5/1.10.7-gcc-dgmcmi4/lib/libhdf5.so"
libhdf5_hl = "/dss/lrzsys/sys/spack/release/22.2.1/opt/icelake/hdf5/1.10.7-gcc-dgmcmi4/lib/libhdf5_hl.so"

[HDF5_jll]
libhdf5_hl_path = "/dss/lrzsys/sys/spack/release/22.2.1/opt/icelake/hdf5/1.10.7-gcc-dgmcmi4/lib/libhdf5_hl.so"
libhdf5_path = "/dss/lrzsys/sys/spack/release/22.2.1/opt/icelake/hdf5/1.10.7-gcc-dgmcmi4/lib/libhdf5.so"

[MPIPreferences]
__clear__ = ["preloads_env_switch"]
_format = "1.0"
abi = "OpenMPI"
binary = "system"
cclibs = []
libmpi = "libmpi"
mpiexec = "srun"
preloads = []

[P4est]
libp4est = "/dss/dsstbyfs02/scratch/07/di38qiw/install/t8code/lib/libp4est.so"
libsc = "/dss/dsstbyfs02/scratch/07/di38qiw/install/t8code/lib/libsc.so"

[T8code]
libp4est = "/dss/dsstbyfs02/scratch/07/di38qiw/install/t8code/lib/libp4est.so"
libsc = "/dss/dsstbyfs02/scratch/07/di38qiw/install/t8code/lib/libsc.so"
libt8 = "/dss/dsstbyfs02/scratch/07/di38qiw/install/t8code/lib/libt8.so"
2,236 changes: 2,236 additions & 0 deletions esiwace/Manifest.toml

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions esiwace/Project.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
[deps]
CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267"
OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
P4est = "7d669430-f675-4ae7-b43e-fab78ec5a902"
T8code = "d0cc0030-9a40-4274-8435-baadcfd54fa1"
TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
Trixi = "a7f1ee26-1774-49b1-8366-f1abc58fbfcb"
11 changes: 11 additions & 0 deletions esiwace/profile
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
module purge
module load spack/22.2.1
module load gcc/11.2.0
module load openmpi/4.1.2-gcc11
module load hdf5/1.10.7-gcc11

export CUDA_HOME=/usr/local/cuda-12.6/

export JULIA_DEPOT_PATH="$SCRATCH/julia-depot/"
export JL="julia +1.11"

12 changes: 5 additions & 7 deletions src/solvers/dgsem_p4est/dg_3d.jl
Original file line number Diff line number Diff line change
@@ -13,11 +13,11 @@ function create_cache(mesh::Union{P4estMesh{3}, T8codeMesh{3}}, equations,
fstar_primary_threaded = [Array{uEltype, 4}(undef, nvariables(equations),
nnodes(mortar_l2),
nnodes(mortar_l2), 4)
for _ in 1:Threads.nthreads()]
for _ in 1:Threads.nthreads()] |> VecOfArrays
fstar_secondary_threaded = [Array{uEltype, 4}(undef, nvariables(equations),
nnodes(mortar_l2),
nnodes(mortar_l2), 4)
for _ in 1:Threads.nthreads()]
for _ in 1:Threads.nthreads()] |> VecOfArrays

fstar_tmp_threaded = [Array{uEltype, 3}(undef, nvariables(equations),
nnodes(mortar_l2), nnodes(mortar_l2))
@@ -451,16 +451,14 @@ end

function prolong2mortars!(cache, u,
mesh::Union{P4estMesh{3}, T8codeMesh{3}}, equations,
mortar_l2::LobattoLegendreMortarL2,
surface_integral, dg::DGSEM)
mortar_l2::LobattoLegendreMortarL2, dg::DGSEM)
backend = backend_or_nothing(cache.mortars)
_prolong2mortars!(backend, cache, u, mesh, equations, mortar_l2, surface_integral, dg)
_prolong2mortars!(backend, cache, u, mesh, equations, mortar_l2, dg)
end

@inline function _prolong2mortars!(backend::Nothing, cache, u,
mesh::Union{P4estMesh{3}, T8codeMesh{3}}, equations,
mortar_l2::LobattoLegendreMortarL2,
surface_integral, dg::DGSEM)
mortar_l2::LobattoLegendreMortarL2, dg::DGSEM)
@unpack fstar_tmp_threaded = cache
@unpack neighbor_ids, node_indices = cache.mortars
index_range = eachnode(dg)
4 changes: 2 additions & 2 deletions src/solvers/dgsem_p4est/dg_3d_gpu.jl
Original file line number Diff line number Diff line change
@@ -464,7 +464,7 @@ end
@inline function _prolong2mortars!(backend::Backend, cache, u,
mesh::P4estMesh{3}, equations,
mortar_l2::LobattoLegendreMortarL2,
surface_integral, dg::DGSEM)
dg::DGSEM)
if nmortars(dg, cache) > 0
error("mortars currently not supported by KA.jl P4estMesh solver")
end
@@ -588,4 +588,4 @@ end
add_to_first_axis!(du, du_local, i, j, k, element)
end
end
end # @muladd
end # @muladd

0 comments on commit 20e3174

Please sign in to comment.