Merge branch 'development' into stretch_factor

cmake/dependencies/openPMD.cmake

@@ -13,7 +13,7 @@ function(find_openpmd)
     if(HiPACE_openpmd_internal OR HiPACE_openpmd_src)
         set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
 
-        # see https://openpmd-api.readthedocs.io/en/0.15.2/dev/buildoptions.html
+        # see https://openpmd-api.readthedocs.io/en/0.16.0/dev/buildoptions.html
         set(openPMD_USE_MPI         ${HiPACE_openpmd_mpi} CACHE INTERNAL "")
         set(openPMD_USE_PYTHON      OFF                   CACHE INTERNAL "")
         set(openPMD_BUILD_CLI_TOOLS OFF                   CACHE INTERNAL "")
@@ -63,7 +63,7 @@ function(find_openpmd)
         else()
             set(COMPONENT_WMPI NOMPI)
         endif()
-        find_package(openPMD 0.15.2 CONFIG REQUIRED COMPONENTS ${COMPONENT_WMPI})
+        find_package(openPMD 0.16.0 CONFIG REQUIRED COMPONENTS ${COMPONENT_WMPI})
         message(STATUS "openPMD-api: Found version '${openPMD_VERSION}'")
     endif()
 endfunction()
@@ -79,7 +79,7 @@ if(HiPACE_OPENPMD)
     set(HiPACE_openpmd_repo "https://github.com/openPMD/openPMD-api.git"
         CACHE STRING
         "Repository URI to pull and build openPMD-api from if(HiPACE_openpmd_internal)")
-    set(HiPACE_openpmd_branch "0.15.2"
+    set(HiPACE_openpmd_branch "0.16.0"
         CACHE STRING
         "Repository branch for HiPACE_openpmd_repo if(HiPACE_openpmd_internal)")
 

docs/source/building/building.rst

@@ -33,7 +33,7 @@ Please see installation instructions below in the Developers section.
 - a mature `C++17 <https://en.wikipedia.org/wiki/C%2B%2B14>`__ compiler: e.g. GCC 7, Clang 7, NVCC 11.0, MSVC 19.15 or newer
 - `CMake 3.24.0+ <https://cmake.org/>`__
 - `AMReX development <https://amrex-codes.github.io>`__: we automatically download and compile a copy of AMReX
-- `openPMD-api 0.15.1+ <https://github.com/openPMD/openPMD-api>`__: we automatically download and compile a copy of openPMD-api
+- `openPMD-api 0.16.0+ <https://github.com/openPMD/openPMD-api>`__: we automatically download and compile a copy of openPMD-api
 
   - `HDF5 <https://support.hdfgroup.org/HDF5>`__ 1.8.13+ (optional; for ``.h5`` file support)
   - `ADIOS2 <https://github.com/ornladios/ADIOS2>`__ 2.7.0+ (optional; for ``.bp`` file support)
@@ -208,7 +208,7 @@ CMake Option                 Default & Values
 ``HiPACE_openpmd_mpi``       ON/OFF (default is set to value of ``HiPACE_MPI``)  Build openPMD with MPI support, although I/O is always serial
 ``HiPACE_openpmd_src``       *None*                                              Path to openPMD-api source directory (preferred if set)
 ``HiPACE_openpmd_repo``      ``https://github.com/openPMD/openPMD-api.git``      Repository URI to pull and build openPMD-api from
-``HiPACE_openpmd_branch``    ``0.15.2``                                          Repository branch for ``HiPACE_openpmd_repo``
+``HiPACE_openpmd_branch``    ``0.16.0``                                          Repository branch for ``HiPACE_openpmd_repo``
 ``HiPACE_openpmd_internal``  **ON**/OFF                                          Needs a pre-installed openPMD-api library if set to ``OFF``
 ``AMReX_LINEAR_SOLVERS``     ON/**OFF**                                          Compile AMReX multigrid solver.
 ===========================  ==================================================  =============================================================

docs/source/run/parameters.rst

@@ -123,10 +123,17 @@ General parameters
     can do this at once in initialization instead of one after another
     as part of the communication pipeline.
 
+* ``hipace.do_shared_depos`` (`bool`) optional (default `false`)
+    Whether to use shared memory current deposition on GPU.
+
 * ``hipace.do_tiling`` (`bool`) optional (default `true`)
     Whether to use tiling, when running on CPU.
     Currently, this option only affects plasma operations (gather, push and deposition).
-    The tile size can be set with ``plasmas.sort_bin_size``.
+    The tile size can be set with ``hipace.tile_size``.
+
+* ``hipace.tile_size`` (`int`) optional (default `32`)
+    Tile size for beam and plasma current deposition, when running on CPU
+    and tiling is activated (``hipace.do_tiling = 1``).
 
 * ``hipace.depos_order_xy`` (`int`) optional (default `2`)
     Transverse particle shape order. Currently, `0,1,2,3` are implemented.
@@ -451,10 +458,6 @@ When both are specified, the per-species value is used.
 * ``<plasma name> or plasmas.neutralize_background`` (`bool`) optional (default `1`)
     Whether to add a neutralizing background of immobile particles of opposite charge.
 
-* ``plasmas.sort_bin_size`` (`int`) optional (default `32`)
-    Tile size for plasma current deposition, when running on CPU
-    and tiling is activated (``hipace.do_tiling = 1``).
-
 * ``<plasma name>.temperature_in_ev`` (`float`) optional (default `0`)
     | Initializes the plasma particles with a given temperature :math:`k_B T` in eV. Using a temperature, the plasma particle momentum is normally distributed with a variance of :math:`k_B T /(M c^2)` in each dimension, with :math:`M` the particle mass, :math:`k_B` the Boltzmann constant, and :math:`T` the isotropic temperature in Kelvin.
     | Note: Using a temperature can affect the performance since the plasma particles loose their order and thus their favorable memory access pattern. The performance can be mostly recovered by reordering the plasma particles (see ``<plasma name> or plasmas.reorder_period``).
@@ -871,95 +874,98 @@ Parameters starting with ``lasers.`` apply to all laser pulses, parameters start
     Whether to use the most stable discretization for the envelope solver.
 
 * ``<laser name>.init_type`` (list of `string`) optional (default `gaussian`)
-    The initializing method of laser. Possible options are:
-
-      * ``gaussian``(default) the laser is iniliatized with an ideal gaussian pulse.
+    The initialisation method of laser. Possible options are:
 
-      * ``from_file``, the laser is loaded from an openPMD file.
+      Option: ``gaussian`` (default) the laser is initialised with an ideal gaussian pulse.
 
-      *  ``parser``, the laser is initialized with the expression of the complex envelope function.
+      * ``<laser name>.a0`` (`float`) optional (default `0`)
+          Peak normalized vector potential of the laser pulse.
 
-Option: ``gaussian``
+      * ``lasers.lambda0`` (`float`)
+          Wavelength of the laser pulses. Currently, all pulses must have the same wavelength.
 
-* ``<laser name>.a0`` (`float`) optional (default `0`)
-    Peak normalized vector potential of the laser pulse.
+      * ``<laser name>.position_mean`` (3 `float`) optional (default `0 0 0`)
+          The mean position of the laser in `x, y, z`.
 
-* ``lasers.lambda0`` (`float`)
-    Wavelength of the laser pulses. Currently, all pulses must have the same wavelength.
+      * ``<laser name>.w0`` (2 `float`) optional (default `0 0`)
+          The laser waist in `x, y`.
 
-* ``<laser name>.position_mean`` (3 `float`) optional (default `0 0 0`)
-    The mean position of the laser in `x, y, z`.
+      * ``<laser name>.L0`` (`float`) optional (default `0`)
+          The laser pulse length in `z`. Use either the pulse length or the pulse duration ``<laser name>.tau``.
 
-* ``<laser name>.w0`` (2 `float`) optional (default `0 0`)
-    The laser waist in `x, y`.
+      * ``<laser name>.tau`` (`float`) optional (default `0`)
+          The laser pulse duration. The pulse length is set to `laser.tau`:math:`*c_0`.
+          Use either the pulse length or the pulse duration.
 
-* ``<laser name>.L0`` (`float`) optional (default `0`)
-    The laser pulse length in `z`. Use either the pulse length or the pulse duration ``<laser name>.tau``.
+      * ``<laser name>.focal_distance`` (`float`)
+          Distance at which the laser pulse is focused (in the z direction, counted from laser initial position).
 
-* ``<laser name>.tau`` (`float`) optional (default `0`)
-    The laser pulse duration. The pulse length is set to `laser.tau`:math:`*c_0`.
-    Use either the pulse length or the pulse duration.
+      * ``<laser name>.propagation_angle_yz`` (`float`) optional (default `0`)
+          Propagation angle of the pulse in the yz plane (0 is along the z axis)
 
-* ``<laser name>.focal_distance`` (`float`)
-    Distance at which the laser pulse if focused (in the z direction, counted from laser initial position).
 
-* ``<laser name>.propagation_angle_yz`` (`float`) optinal (default `0`)
-    Propagation angle of the pulse in the yz plane (0 is the along the z axis)
+      * ``<laser name>.tau`` (`float`) optional (default `0`)
+          The laser pulse duration. The pulse length is set to `laser.tau`:math:`*c_0`.
+          Use either the pulse length or the pulse duration.
 
-* ``<laser name>.chirp_theta_xy`` (`float`) optional (default `pi/2`)
-    Direction of the linear spatial and angular chirp on x-y plane.
+      * ``<laser name>.focal_distance`` (`float`)
+          Distance at which the laser pulse if focused (in the z direction, counted from laser initial position).
 
-* ``<laser name>.beta`` (`float`) optional (default `0.`)
-    Angular dispersion (or angular chirp) at focus defined by `S. Akturk et al., Optics Express 12, 4399 (2004) <https://doi.org/10.1364/OPEX.12.004399>`__.
+      * ``<laser name>.propagation_angle_yz`` (`float`) optinal (default `0`)
+          Propagation angle of the pulse in the yz plane (0 is the along the z axis)
 
-* ``<laser name>.zeta`` (`float`) optional (default `0.`)
-    Spatial chirp at focus defined by `S. Akturk et al., Optics Express 12, 4399 (2004) <https://doi.org/10.1364/OPEX.12.004399>`__.
+      * ``<laser name>.chirp_theta_xy`` (`float`) optional (default `pi/2`)
+          Direction of the linear spatial and angular chirp on x-y plane.
 
-* ``<laser name>.phi2`` (`float`) optional (default `pi/2`)
-    The amount of temporal chirp :math:`\phi^{(2)}` at focus (in the lab frame).
-    Namely, a wave packet centered on the frequency :math:`(\omega_0 + \delta \omega)` will reach its peak intensity at :math:`z(\delta \omega) = z_0 - c \phi^{(2)} \, \delta \omega`.
-    Thus, a positive :math:`\phi^{(2)}` corresponds to positive chirp, i.e., red part of the spectrum in the front of the pulse and blue part of the spectrum in the back.
-    More specifically, the electric field in the focal plane is of the form:
+      * ``<laser name>.beta`` (`float`) optional (default `0.`)
+          Angular dispersion (or angular chirp) at focus defined by `S. Akturk et al., Optics Express 12, 4399 (2004) <https://doi.org/10.1364/OPEX.12.004399>`__.
 
-    .. math::
+      * ``<laser name>.zeta`` (`float`) optional (default `0.`)
+          Spatial chirp at focus defined by `S. Akturk et al., Optics Express 12, 4399 (2004) <https://doi.org/10.1364/OPEX.12.004399>`__.
 
-        E(\boldsymbol{x},t) \propto Re\left[ \exp\left(  -\frac{(t-t_{peak})^2}{\tau^2 + 2i\phi^{(2)}} + i\omega_0 (t-t_{peak}) + i\phi_0 \right) \right]
+      * ``<laser name>.phi2`` (`float`) optional (default `pi/2`)
+          The amount of temporal chirp :math:`\phi^{(2)}` at focus (in the lab frame).
+          Namely, a wave packet centered on the frequency :math:`(\omega_0 + \delta \omega)` will reach its peak intensity at :math:`z(\delta \omega) = z_0 - c \phi^{(2)} \, \delta \omega`.
+          Thus, a positive :math:`\phi^{(2)}` corresponds to positive chirp, i.e., red part of the spectrum in the front of the pulse and blue part of the spectrum in the back.
+          More specifically, the electric field in the focal plane is of the form:
 
-    where :math:`\tau` is given by ``<laser_name>.tau`` and represents the Fourier-limited duration of the laser pulse. Thus, the actual duration of the chirped laser pulse is:
+          .. math::
 
-    .. math::
+              E(\boldsymbol{x},t) \propto Re\left[ \exp\left(  -\frac{(t-t_{peak})^2}{\tau^2 + 2i\phi^{(2)}} + i\omega_0 (t-t_{peak}) + i\phi_0 \right) \right]
 
-        \tau' = \sqrt{ \tau^2 + 4 (\phi^{(2)})^2/\tau^2 }
+          where :math:`\tau` is given by ``<laser_name>.tau`` and represents the Fourier-limited duration of the laser pulse. Thus, the actual duration of the chirped laser pulse is:
 
-    See also the definition in `S. Akturk et al., Optics Express 12, 4399 (2004) <https://doi.org/10.1364/OPEX.12.004399>`__.
+          .. math::
 
+               \tau' = \sqrt{ \tau^2 + 4 (\phi^{(2)})^2/\tau^2 }
 
+         See also the definition in `S. Akturk et al., Optics Express 12, 4399 (2004) <https://doi.org/10.1364/OPEX.12.004399>`__.
 
-Option: ``from_file``
+      Option: ``from_file`` the laser is loaded from an openPMD file
 
-* ``lasers.input_file`` (`string`) optional (default `""`)
-    Path to an openPMD file containing a laser envelope.
-    The file should comply with the `LaserEnvelope extension of the openPMD-standard <https://github.com/openPMD/openPMD-standard/blob/upcoming-2.0.0/EXT_LaserEnvelope.md>`__, as generated by `LASY <https://github.com/LASY-org/LASY>`__.
-    Currently supported geometries: 3D or cylindrical profiles with azimuthal decomposition.
-    The laser pulse is injected in the HiPACE++ simulation so that the beginning of the temporal profile from the file corresponds to the head of the simulation box, and time (in the file) is converted to space (HiPACE++ longitudinal coordinate) with ``z = -c*t + const``.
-    If this parameter is set, then the file is used to initialize all lasers instead of using a gaussian profile.
+      * ``<laser name>.input_file`` (`string`) optional (default `""`)
+          Path to an openPMD file containing a laser envelope.
+          The file should comply with the `LaserEnvelope extension of the openPMD-standard <https://github.com/openPMD/openPMD-standard/blob/upcoming-2.0.0/EXT_LaserEnvelope.md>`__, as generated by `LASY <https://github.com/LASY-org/LASY>`__.
+          Currently supported geometries: 3D or cylindrical profiles with azimuthal decomposition.
+          The laser pulse is injected in the HiPACE++ simulation so that the beginning of the temporal profile from the file corresponds to the head of the simulation box, and time (in the file) is converted to space (HiPACE++ longitudinal coordinate) with ``z = -c*t + const``.
+          If this parameter is set, then the file is used to initialize all lasers instead of using a gaussian profile.
 
-* ``lasers.openPMD_laser_name`` (`string`) optional (default `laserEnvelope`)
-    Name of the laser envelope field inside the openPMD file to be read in.
+      * ``<laser name>.openPMD_laser_name`` (`string`) optional (default `laserEnvelope`)
+          Name of the laser envelope field inside the openPMD file to be read in.
 
-* ``lasers.iteration`` (`int`) optional (default `0`)
-    Iteration of the openPMD file to be read in.
+      * ``<laser name>.iteration`` (`int`) optional (default `0`)
+          Iteration of the openPMD file to be read in.
 
-Option: ``parser``
+      Option: ``parser``, the laser is initialized with the expression of the complex envelope function.
 
-* ``<laser name>.laser_real(x,y,z)`` (`string`)
-    Expression for the real part of the laser evelope in `x, y, z`.
+      * ``<laser name>.laser_real(x,y,z)`` optional (`string`) (default `""`)
+          Expression for the real part of the laser envelope in `x, y, z`.
 
-* ``<laser name>.laser_imag(x,y,z)`` (`string`)
-    Expression for the imaginary part of the laser evelope `x, y, z`.
+      * ``<laser name>.laser_imag(x,y,z)`` optional (`string`) (default `""`)
+          Expression for the imaginary part of the laser envelope `x, y, z`.
 
-* ``lasers.lambda0`` (`float`)
-    Wavelength of the laser pulses. Currently, all pulses must have the same wavelength.
+      * ``lasers.lambda0`` (`float`)
+          Wavelength of the laser pulses. Currently, all pulses must have the same wavelength.
 
 Diagnostic parameters
 ---------------------

examples/linear_wake/inputs_ion_motion_SI

@@ -34,7 +34,7 @@ beam.profile = gaussian
 beam.position_mean = "0.25*kp_inv" "(z-2*kp_inv)*0.2" "2*kp_inv"
 beam.position_std = 0.4*kp_inv 0.4*kp_inv 1.41*kp_inv
 
-plasmas.sort_bin_size = 8
+hipace.tile_size = 8
 plasmas.names = elec ions
 
 elec.mass = m_e

src/Hipace.H

@@ -236,6 +236,10 @@ public:
 #else
     inline static bool m_do_tiling = false;
 #endif
+    /** Tile size for particle operations when using tiling */
+    inline static int m_tile_size = 32;
+    /** Whether to use shared memory for current deposition */
+    inline static bool m_do_shared_depos = false;
     /** Whether the explicit field solver is used */
     inline static bool m_explicit = true;
     /** Relative tolerance for the multigrid solver, when using the explicit solver */

src/Hipace.cpp

@@ -155,7 +155,9 @@ Hipace::Hipace () :
     queryWithParser(pph, "MG_tolerance_abs", m_MG_tolerance_abs);
     queryWithParser(pph, "MG_verbose", m_MG_verbose);
     queryWithParser(pph, "use_amrex_mlmg", m_use_amrex_mlmg);
+    queryWithParser(pph, "do_shared_depos", m_do_shared_depos);
     queryWithParser(pph, "do_tiling", m_do_tiling);
+    queryWithParser(pph, "tile_size", m_tile_size);
 #ifdef AMREX_USE_GPU
     AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_do_tiling==0, "Tiling must be turned off to run on GPU.");
 #endif
@@ -451,9 +453,6 @@ Hipace::Evolve ()
 
         // deposit neutralizing background
         if (m_interpolate_neutralizing_background) {
-            if (m_do_tiling) {
-                m_multi_plasma.TileSort(m_slice_geom[0].Domain(), m_slice_geom[0]);
-            }
             // Store charge density of (immobile) ions into WhichSlice::RhomJzIons of level 0
             m_multi_plasma.DepositNeutralizingBackground(
                 m_fields, WhichSlice::RhomJzIons, m_3D_geom, 0);
@@ -466,9 +465,6 @@ Hipace::Evolve ()
                 m_multi_plasma.TagByLevel(m_N_level, m_3D_geom);
             }
             for (int lev=0; lev<m_N_level; ++lev) {
-                if (m_do_tiling) {
-                    m_multi_plasma.TileSort(m_slice_geom[lev].Domain(), m_slice_geom[lev]);
-                }
                 // Store charge density of (immobile) ions into WhichSlice::RhomJzIons
                 m_multi_plasma.DepositNeutralizingBackground(
                     m_fields, WhichSlice::RhomJzIons, m_3D_geom, lev);
@@ -595,7 +591,7 @@ Hipace::SolveOneSlice (int islice, int step)
         m_multi_plasma.TagByLevel(current_N_level, m_3D_geom);
     }
 
-    // reorder plasma before TileSort
+    // reorder plasma
     m_multi_plasma.ReorderParticles(islice);
 
     // prepare/initialize fields
@@ -608,9 +604,6 @@ Hipace::SolveOneSlice (int islice, int step)
 
     // deposit current
     for (int lev=0; lev<current_N_level; ++lev) {
-        // tiling used by plasma current deposition
-        if (m_do_tiling) m_multi_plasma.TileSort(m_slice_geom[lev].Domain(), m_slice_geom[lev]);
-
         if (m_explicit) {
             // deposit jx, jy, chi and rhomjz for all plasmas
             m_multi_plasma.DepositCurrent(m_fields, WhichSlice::This, true, false,
@@ -981,7 +974,6 @@ Hipace::PredictorCorrectorLoopToSolveBxBy (const int islice, const int current_N
         }
 
         for (int lev=0; lev<current_N_level; ++lev) {
-            if (m_do_tiling) m_multi_plasma.TileSort(m_slice_geom[lev].Domain(), m_slice_geom[lev]);
             // plasmas deposit jx jy to next temp slice
             m_multi_plasma.DepositCurrent(m_fields, WhichSlice::Next,
                 true, false, false, false, false, m_3D_geom, lev);