From 51c9f083f7494b2ea31b6bc98779deccb64633a0 Mon Sep 17 00:00:00 2001 From: Pere Mato Date: Wed, 4 Dec 2024 11:09:34 +0100 Subject: [PATCH] MT for JuliaAction example --- advanced/EmbedJulia/G4example.cpp | 47 +++++++++++++++++++++++-- advanced/EmbedJulia/JuliaAction.ipynb | 17 ++++----- advanced/EmbedJulia/JuliaAction.jl | 11 +++--- advanced/EmbedJulia/JuliaAction_lit.jl | 19 +++++----- advanced/EmbedJulia/MyCode.jl | 48 +++++++++++++++++++------- 5 files changed, 105 insertions(+), 37 deletions(-) diff --git a/advanced/EmbedJulia/G4example.cpp b/advanced/EmbedJulia/G4example.cpp index 1ec2656..2b6db42 100644 --- a/advanced/EmbedJulia/G4example.cpp +++ b/advanced/EmbedJulia/G4example.cpp @@ -14,6 +14,7 @@ //#include "G4SystemOfUnits.hh" #include "globals.hh" #include "G4VUserActionInitialization.hh" +#include "G4UserWorkerInitialization.hh" #include #include @@ -37,6 +38,7 @@ class DetectorConstruction : public G4VUserDetectorConstruction new G4PVPlacement(0, G4ThreeVector(), logicCore, "Core", logicWorld, false, 0); return physWorld; } + void ConstructSDandField() override {} }; //---Primary generator action class---------------------------------------------------------------- @@ -107,13 +109,32 @@ class RunAction : public G4UserRunAction runaction_f endrun_jl; }; + +class WorkerInitialization : public G4UserWorkerInitialization { + public: + WorkerInitialization() = default; + virtual ~WorkerInitialization() = default; + virtual void WorkerInitialize() const override { + if (jl_get_pgcstack() == NULL) jl_adopt_thread(); + } + virtual void WorkerStart() const override {} + virtual void WorkerRunStart() const override {} + virtual void WorkerRunEnd() const override { + jl_ptls_t ptls = jl_current_task->ptls; + jl_gc_safe_enter(ptls); + } + virtual void WorkerStop() const override {} +}; + //---Action initialization class------------------------------------------------------------------- class ActionInitialization : public G4VUserActionInitialization { public: ActionInitialization() = default; ~ActionInitialization() override = default; - void BuildForMaster() const override {} + void BuildForMaster() const override { + SetUserAction(new RunAction); + } void Build() const override { SetUserAction(new PrimaryGeneratorAction); SetUserAction(new RunAction); @@ -122,11 +143,16 @@ class ActionInitialization : public G4VUserActionInitialization } }; + + JULIA_DEFINE_FAST_TLS // only define this once, in an executable (not in a shared library) if you want fast code. //----Main program--------------------------------------------------------------------------------- int main(int, char**) { + int nthreads = 0; // Default number of threads + if (getenv("G4NUMTHREADS")) nthreads = atoi(getenv("G4NUMTHREADS")); + //--- Required to setup the Julia context jl_init(); /* run Julia commands */ @@ -136,9 +162,19 @@ int main(int, char**) } //---Construct the default run manager - auto runManager = G4RunManagerFactory::CreateRunManager(G4RunManagerType::Serial); + G4RunManager* runManager; + if (nthreads > 0) { + runManager = G4RunManagerFactory::CreateRunManager(G4RunManagerType::MT); + runManager->SetNumberOfThreads(nthreads); + runManager->SetUserInitialization(new WorkerInitialization()); + } else { + runManager = G4RunManagerFactory::CreateRunManager(G4RunManagerType::Serial); + } //---Set mandatory initialization classes + + + // Detector construction runManager->SetUserInitialization(new DetectorConstruction()); @@ -159,7 +195,12 @@ int main(int, char**) //UImanager->ApplyCommand("/tracking/verbose 1"); UImanager->ApplyCommand("/gun/particle e+"); UImanager->ApplyCommand("/gun/energy 100 MeV"); - UImanager->ApplyCommand("/run/beamOn 100000"); + + // Start a run (we need to enter GC safe region here because the worker threads + // will enter a wait state waiting for workers to finish and can block GC + auto state = jl_gc_safe_enter(jl_current_task->ptls); + runManager->BeamOn(100000); + jl_gc_safe_leave(jl_current_task->ptls, state); // Job termination delete runManager; diff --git a/advanced/EmbedJulia/JuliaAction.ipynb b/advanced/EmbedJulia/JuliaAction.ipynb index c3f8c8e..10d1c7c 100644 --- a/advanced/EmbedJulia/JuliaAction.ipynb +++ b/advanced/EmbedJulia/JuliaAction.ipynb @@ -81,7 +81,7 @@ "source": [ "cd(@__DIR__)\n", "g4prefix = Geant4_jll.artifact_dir\n", - "jlprefix = dirname(Sys.BINDIR)" + "jlprefix = dirname(Sys.BINDIR);" ], "metadata": {}, "execution_count": null @@ -103,7 +103,8 @@ "jllibs = read(`$jlprefix/share/julia/julia-config.jl --ldlibs`, String) |> split\n", "append!(jllibs, [\"-L$jlprefix/lib\"])\n", "cflags = read(`$g4prefix/bin/geant4-config --cflags`, String) |> split\n", - "ldflags = [\"-Wl,-rpath,$g4prefix/lib\", \"-Wl,-rpath,$jlprefix/lib\"];" + "ldflags = [\"-Wl,-rpath,$g4prefix/lib\", \"-Wl,-rpath,$jlprefix/lib\"];\n", + "Sys.KERNEL == :Linux && append!(ldflags, [\"-Wl,--no-as-needed\"]);" ], "metadata": {}, "execution_count": null @@ -137,8 +138,8 @@ "outputs": [], "cell_type": "code", "source": [ - "withenv(\"JULIA_PROJECT\" => \"@.\") do\n", - " Base.run(`./G4example`).exitcode == 0 || error(\"Execution failed\");\n", + "withenv(\"JULIA_PROJECT\" => \"@.\", \"G4NUMTHREADS\" => \"8\") do\n", + " Base.run(`./G4example`).exitcode == 0 || error(\"Execution failed\")\n", "end" ], "metadata": {}, @@ -155,7 +156,7 @@ "outputs": [], "cell_type": "code", "source": [ - "display(\"image/png\", read(\"edepHist.png\"))" + "println(\"=====> The file edepHist.png should have been saved\")" ], "metadata": {}, "execution_count": null @@ -163,7 +164,7 @@ { "cell_type": "markdown", "source": [ - "next" + "![](edepHist.png)" ], "metadata": {} }, @@ -183,11 +184,11 @@ "file_extension": ".jl", "mimetype": "application/julia", "name": "julia", - "version": "1.11.1" + "version": "1.11.2" }, "kernelspec": { "name": "julia-1.11", - "display_name": "Julia 1.11.1", + "display_name": "Julia 1.11.2", "language": "julia" } }, diff --git a/advanced/EmbedJulia/JuliaAction.jl b/advanced/EmbedJulia/JuliaAction.jl index 3e01a58..7171242 100644 --- a/advanced/EmbedJulia/JuliaAction.jl +++ b/advanced/EmbedJulia/JuliaAction.jl @@ -2,7 +2,7 @@ using Geant4_jll # Needed to locate the Geant4 installation directory cd(@__DIR__) g4prefix = Geant4_jll.artifact_dir -jlprefix = dirname(Sys.BINDIR) +jlprefix = dirname(Sys.BINDIR); g4libs = read(`$g4prefix/bin/geant4-config --libs`, String) |> split filter!(x -> x != "-lG4gdml", g4libs) @@ -10,14 +10,13 @@ jllibs = read(`$jlprefix/share/julia/julia-config.jl --ldlibs`, String) |> split append!(jllibs, ["-L$jlprefix/lib"]) cflags = read(`$g4prefix/bin/geant4-config --cflags`, String) |> split ldflags = ["-Wl,-rpath,$g4prefix/lib", "-Wl,-rpath,$jlprefix/lib"]; +Sys.KERNEL == :Linux && append!(ldflags, ["-Wl,--no-as-needed"]); Base.run(`c++ -O2 -fPIC $cflags -I$jlprefix/include/julia $ldflags $g4libs $jllibs -o G4example $(@__DIR__)/G4example.cpp`).exitcode == 0 || error("Compilation failed"); -withenv("JULIA_PROJECT" => "@.") do - Base.run(`./G4example`).exitcode == 0 || error("Execution failed"); +withenv("JULIA_PROJECT" => "@.", "G4NUMTHREADS" => "8") do + Base.run(`./G4example`).exitcode == 0 || error("Execution failed") end -display("image/png", read("edepHist.png")) - -display("image/png", read("edepHist.png")) +println("=====> The file edepHist.png should have been saved") diff --git a/advanced/EmbedJulia/JuliaAction_lit.jl b/advanced/EmbedJulia/JuliaAction_lit.jl index 19e45b0..424731d 100644 --- a/advanced/EmbedJulia/JuliaAction_lit.jl +++ b/advanced/EmbedJulia/JuliaAction_lit.jl @@ -44,7 +44,10 @@ #md # You can also download this example as a #md # [Jupyter notebook](JuliaAction.ipynb) and a plain #md # [Julia source file](JuliaAction.jl). -# +#md # +#md # The C++ code is available as a [source file](G4example.cpp) and the +#md # Julia code is available as a [source file](MyCode.jl). +#md # #md # #### Table of contents #md # ```@contents #md # Pages = ["JuliaAction.md"] @@ -63,7 +66,7 @@ using Geant4_jll # Needed to locate the Geant4 installation directory # Sources are in the same location as this script. cd(@__DIR__) g4prefix = Geant4_jll.artifact_dir -jlprefix = dirname(Sys.BINDIR) +jlprefix = dirname(Sys.BINDIR); # We use the executables `geant4-config` and `julia-config.jl` to get the needed # libraries and compiler/linker flags. @@ -74,6 +77,7 @@ jllibs = read(`$jlprefix/share/julia/julia-config.jl --ldlibs`, String) |> split append!(jllibs, ["-L$jlprefix/lib"]) cflags = read(`$g4prefix/bin/geant4-config --cflags`, String) |> split ldflags = ["-Wl,-rpath,$g4prefix/lib", "-Wl,-rpath,$jlprefix/lib"]; +Sys.KERNEL == :Linux && append!(ldflags, ["-Wl,--no-as-needed"]); # Run the compilation and link command Base.run(`c++ -O2 -fPIC $cflags -I$jlprefix/include/julia $ldflags $g4libs $jllibs @@ -81,12 +85,11 @@ Base.run(`c++ -O2 -fPIC $cflags -I$jlprefix/include/julia $ldflags $g4libs $jlli # ## Run the application # We need to set the variable `JULIA_PROJECT` pointing to correctly setup Julia environment. -withenv("JULIA_PROJECT" => "@.") do - Base.run(`./G4example`).exitcode == 0 || error("Execution failed"); +withenv("JULIA_PROJECT" => "@.", "G4NUMTHREADS" => "8") do + Base.run(`./G4example`).exitcode == 0 || error("Execution failed") end # ## Display the results -display("image/png", read("edepHist.png")) -# next -#jl display("image/png", read("edepHist.png")) -#md PNG(read("edepHist.png")) +println("=====> The file edepHist.png should have been saved") +#md # ![](edepHist.png) +#nb # ![](edepHist.png) diff --git a/advanced/EmbedJulia/MyCode.jl b/advanced/EmbedJulia/MyCode.jl index 4bafe50..2fa9c04 100644 --- a/advanced/EmbedJulia/MyCode.jl +++ b/advanced/EmbedJulia/MyCode.jl @@ -1,27 +1,44 @@ #---Load the needed Julia modules------------------------------------------------------------------ - using Geant4 using GeometryBasics using FHist using Plots +using Parameters println("=====> Loading MyCode.jl") -edepHist = H1D("Event total Edep distribution", 100, 0., 110.) -edep = 0.0 +#---Simulation data struct------------------------------------------------------------------------- +@with_kw mutable struct MyData + edep = 0.0 + edepHist = H1D("Event total Edep distribution", 100, 0., 110.) +end +add!(d::MyData, d2::MyData) = (d.edep += d2.edep; merge!(d.edepHist, d2.edepHist)) + +const nthreads = ENV["G4NUMTHREADS"] == nothing ? 0 : parse(Int, ENV["G4NUMTHREADS"]) +const simdata = [MyData() for i in 1:nthreads+1] # #workers + 1(master) + +function getMyData() + tid = G4Threading!G4GetThreadId() + tid < 0 && (tid = -1) # master thread (-2 for without multi-threading support) + simdata[tid+2] +end +#---Actions---------------------------------------------------------------------------------------- function end_of_event_action(event) - push!(edepHist, edep) + data = getMyData() + push!(data.edepHist, data.edep) return # This is mandatory to force to return nothing end function begin_of_event_action(event) - global edep = 0.0 + data = getMyData() + data.edep = 0.0 return # This is mandatory to force to return nothing end function stepping_action(step) - global edep += GetTotalEnergyDeposit(step) + data = getMyData() + data.edep += GetTotalEnergyDeposit(step) return # This is mandatory to force to return nothing end @@ -29,11 +46,18 @@ function begin_of_run_action(run) return # This is mandatory to force to return nothing end -function end_of_run_action(run) - println("=====> End of run") - h = edepHist - img = plot(h.hist, title=h.title) - savefig(img, "edepHist.png") - println("=====> edepHist.png saved") +function end_of_run_action(run) + if G4Threading!G4GetThreadId() < 0 + println("=====> End of run") + data = simdata[1] + ##---This is the master thread, so we need to add all the simulation results----------------- + for d in simdata[2:end] + add!(data, d) + end + h = data.edepHist + img = plot(h.hist, title=h.title) + savefig(img, "edepHist.png") + println("=====> edepHist.png saved") + end return # This is mandatory to force to return nothing end