diff --git a/.gitignore b/.gitignore index bde6d386..714ed394 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,3 @@ -build*/ -bin -lib -thirdparty/ -include/revision.hpp .venv *.bak *.gcda @@ -14,6 +9,11 @@ include/revision.hpp **/*.fld **/*.opt *~ + +# Covers spack build symlinks and other common manual build locations +build* + +# spack output .spack-env spack-build-*.txt spack-configure-args.txt @@ -21,14 +21,13 @@ spack-build-*/ install-time-test-log.txt spack.lock views/ -builds/ # CMake cmake.check_cache CMakeCache.txt # Directory generated by scripts/run_eg.sh -example-runs +runs # IDE settings .vscode diff --git a/README.md b/README.md index f8b0cb1c..a450e90c 100644 --- a/README.md +++ b/README.md @@ -69,23 +69,19 @@ packages and their dependencies will be installed in the usual Spack locations. They will also be linked into ["filesystem views"](https://spack.readthedocs.io/en/latest/environments.html#filesystem-views) `view/gcc-hipsycl` and `view/oneapi-dpcpp`. The NESO builds will be -done in directories called something like `spack-build-abc1234` (the +done in directories called something like `build-arch-abc1234` (the hashes at the end will differ). If you change your spack installation in some way (e.g., upgrading the version of a dependency) then the hash will change and NESO and/or Nektar++ will be rebuilt. The activation provides the convenience command `cleanup` to delete these old builds. -In order to tell which build is which, symlinks `builds/gcc-hipsycl` -and `builds/oneapi-dpcpp` are provided. As Nektar++ is being built -from the submodule, its build trees are located at -`nektar/spack-build-abc1234` (the hashes at the end will differ) and -can be accessed with symlinks `nektar/builds/gcc` and -`nektar/builds/oneapi`. Test binaries will be contained within -these build directories. The activation script launches a background -task which regularly checks whether the hashes of your NESO and -Nektar++ builds has changed. If they have, it will update the -symlinks. They will also be checked whenever the environment is +In order to tell which NESO build is which, symlinks are generated at +`builds/gcc-` and `builds/oneapi-`. Similar links are generated for +Nektar++ and neso-particles in their own `builds` subdirectories. The activation +script also launches a background task which regularly checks whether the hashes +of your NESO, Nektar++ and neso-particles builds have changed. If they have, it +will update the symlinks. They will also be checked whenever the environment is activated or deactivated. #### Developing @@ -97,7 +93,7 @@ configuration for NESO (as specificed in the NESO [package repository](https://github.com/ExCALIBUR-NEPTUNE/NESO-Spack) and is the same as if you were doing a traditional Spack installation of a named version of NESO. This has the particular advantage of building -with all toolchaings (i.e., GCC, OneAPI) at one time. It also works +with all toolchains (i.e., GCC, OneAPI) at one time. It also works well if you are developing NESO and Nektar++ simultaneously, as it will rebuild both. The main disadvantage of this approach is that Spack hides the output of CMake during the build process and will only @@ -302,7 +298,9 @@ To run a solver example: ``` ./scripts/run_eg.sh [solver_name] [example_name] <-n num_MPI> <-b build_dir> ``` -which will look for the solver executable in the most recently modified spack-build-* directory, unless one is supplied with `-b`. Output is generated in `example-runs//`. +which will look for the solver executable in the most recently modified build +directory (or symlink) in `builds`, unless an alternative location is supplied +with `-b`. Output is generated in `runs//`. ## Address Sanitizers @@ -319,6 +317,16 @@ One pipeline for creating simple Nektar++ meshes is through Gmsh and NekMesh. .msh -> .xml with NekMesh A .geo file can be created using the Gmsh UI (each command adds a new line to the .geo file). For simple meshes it may be easier to produce the .geo file in a text editor. .geo files can also be loaded into the UI to edit. + +The script at `scripts/geo_to_xml.sh` can be used to convert .geo files to +Nektar++ XML meshes. Execute the script with no arguments to see the available +options. Note that no `` node is generated; users need to add one +themselves, either to the mesh file itself, or to another xml file that gets +passed to Nektar++. Alternatively, an xml mesh can be generated by running Gmsh +and NekMesh separately, using the instructions below. + + +An example .geo file is shown below:
Expand mesh.geo @@ -364,8 +372,9 @@ Recombine Surface "*"; ```
+Loading the above file in the Gmsh GUI, then selecting `2D mesh` and saving will produce a .msh file. +Alternatively for simple meshes one can jump straight to the next step by writing the .msh file directly in a text editor. -Selecting 2D mesh in Gmsh and saving will produce a .msh file. The mesh should be visible in the UI to check before saving. Alternatively for simple meshes one can jump straight to this step by writing the .msh file in a text editor.
Expand mesh.msh diff --git a/activate b/activate index 42957ae1..1f2e7f27 100644 --- a/activate +++ b/activate @@ -27,59 +27,57 @@ deactivate() { unset NESO_DEV_DIR # Undeclare all the functions in this file - unset -f deactivate - unset -f list-neso-builds - unset -f list-builds + unset -f cleanup unset -f cleanup-deprecated-builds unset -f create-link + unset -f deactivate + unset -f gen-build-link-maps + unset -f link-builds unset -f link-nektar-builds unset -f link-neso-builds unset -f link-neso-particles-builds - unset -f setup unset -f monitor-build-dirs + unset -f setup spack env deactivate } -# Get a list of the NESO builds, including both compiler and SYCL -# implementation -list-neso-builds() { - spack find --format "{compiler.name}-{^sycl.name}/{hash:7}" neso - return $? -} - -# Get a list of the builds of a package (specified as the first -# argument), with the name of the compiler. -list-builds() { - spack find --format "{compiler.name}/{hash:7}" $1 - return $? -} - -# Deletes spack build directories for hashes which are no longer +# Deletes build directory links for hashes which are no longer # installed in the environment. First argument is the package name and -# second is the directory in which to cleanup installs. +# second is the directory in which to cleanup links. cleanup-deprecated-builds() { - local package="$1" - local basedir="$2" + local pkg_name="$1" + local pkg_root_dir="$2" + local links_dir="$pkg_root_dir/builds" + + # Use the build-link-map to compile a list of valid link paths + local valid_hashes=() valid_link_paths=() + for entry in $(gen-build-link-maps "$pkg_name" "{compiler.name}-{hash:7}"); do + IFS="|" read -ra _split <<<"$entry" + valid_link_paths+=("${links_dir}/${_split[1]}") + IFS="-" read -ra _split2 <<<"${_split[1]}" + valid_hashes+=("${_split2[1]}") + done - local good_hashes=() - local h _vals - for h in $(list-builds "${package}"); do - IFS="/" read -ra _vals <<<"$h" - good_hashes+=("${_vals[1]}") + # Remove links from links_dir if they don't appear in valid_link_paths + link_paths=$(find "$links_dir" -maxdepth 1 -type l -regextype posix-egrep -regex '.*-[a-z0-9]{7}$') + for link_path in $link_paths; do + if ! [[ " ${valid_link_paths[*]} " =~ " ${link_path} " ]] + then + echo "Cleaning up stale ${pkg_name} build link at ${link_path}" + rm -Rf "$link_path" > /dev/null + fi done - local f - for f in "$basedir"/spack-build-???????; do - if [[ -d "$f" ]] + # Also cleanup top-level build links created by spack install + spack_link_paths=$(find "$pkg_root_dir" -type l -regextype posix-egrep -regex "${pkg_root_dir}/build-.*-[a-z0-9]{7}$") + for link_path in $spack_link_paths; do + local hash="${link_path:(-7)}" + if ! [[ " ${valid_hashes[*]} " =~ " ${hash} " ]] then - local hash="${f:(-7)}" - if ! [[ " ${good_hashes[*]} " =~ " ${hash} " ]] - then - echo "Removing build directory $f" - rm -Rf "$f" > /dev/null - fi - fi + echo "Cleaning up stale ${pkg_name} SPACK build link at ${link_path}" + rm -Rf "$link_path" > /dev/null + fi done } @@ -92,68 +90,58 @@ create-link() { link="$2" if ! [[ -L "$link" && $(readlink "$link") == "$target" ]] then - echo " Linking $(realpath -s --relative-to="$(pwd)" "$link") => $target" + echo " Linking $(realpath -s --relative-to="$NESO_DEV_DIR" "$link") => $target" rm -f "$link" > /dev/null ln -s "$target" "$link" fi } -# Creates symlinks to the nektar build directories which have -# meaningful names -link-nektar-builds() { - mkdir -p "${NESO_DEV_DIR}/nektar/builds" - local build _vals builds - builds=$(list-builds nektar) # Need to declare builds already or else declaration would overwrite $? - local list_err=$? - if [[ $list_err != 0 ]] - then - return $list_err - fi +# Find all instances of a package installed in the environment and return a +# string for each instance of the form "package_name/hash|package_link" +gen-build-link-maps() { + local package="$1" + local link_fmt="$2" - for build in $builds; do - IFS="/" read -ra _vals <<<"$build" - create-link "../spack-build-${_vals[1]}" "${NESO_DEV_DIR}/nektar/builds/${_vals[0]}" + local identifier_fmt="{package.name}/{hash:7}" + spack find --format "$identifier_fmt|$link_fmt" "$package" + return $? +} + + +# Generate soft links to the spack build directories for all instances of a +# package installed in the environment +link-builds() { + local package="$1" + local link_fmt="$2" + local link_dir="$3" + + local build_dir + for entry in $(gen-build-link-maps "$package" "$link_fmt"); do + IFS="|" read -ra _split <<<"$entry" + build_dir=$(spack location -b "${_split[0]}") + create-link "${build_dir}" "${link_dir}/${_split[1]}" done + +} + +# Creates symlinks to the nektar build directories which have meaningful names +link-nektar-builds() { + link-builds "nektar" "{compiler.name}-{hash:7}" "${NESO_DEV_DIR}/nektar/builds" } # Creates symlinks to the neso-particles build directories which have # meaningful names link-neso-particles-builds() { - mkdir -p "${NESO_DEV_DIR}/neso-particles/builds" - local build _vals builds - builds=$(list-builds neso-particles) # Need to declare builds already or else declaration would overwrite $? - local list_err=$? - if [[ $list_err != 0 ]] - then - return $list_err - fi - - for build in $builds; do - IFS="/" read -ra _vals <<<"$build" - create-link "../spack-build-${_vals[1]}" "${NESO_DEV_DIR}/neso-particles/builds/${_vals[0]}" - done + link-builds "neso-particles" "{compiler.name}-{hash:7}" "${NESO_DEV_DIR}/neso-particles/builds" } -# Creates symlinks to the NESO build directories which have -# meaningful names +# Creates symlinks to the NESO build directories which have meaningful names link-neso-builds() { - mkdir -p "${NESO_DEV_DIR}/builds" - local build _vals builds - builds=$(list-neso-builds) # Need to declare builds already or else declaration would overwrite $? - list_err=$? - if [[ $list_err != 0 ]] - then - return $list_err - fi - - for build in ${builds}; do - IFS="/" read -ra _vals <<<"$build" - create-link "../spack-build-${_vals[1]}" "${NESO_DEV_DIR}/builds/${_vals[0]}" - done + link-builds "neso" "{compiler.name}-{hash:7}" "${NESO_DEV_DIR}/builds" } -# Create symlink to the build directories +# Create symlinks to the build directories setup() { echo "Checking links to NESO build directories" link-neso-builds @@ -183,7 +171,7 @@ cleanup() { cleanup-deprecated-builds neso-particles "${NESO_DEV_DIR}/neso-particles" } -spack env activate -p -d ${NESO_DEV_DIR} +spack env activate -p -d "${NESO_DEV_DIR}" setup ( monitor-build-dirs ) & __MONITOR_PROCESS=$! diff --git a/scripts/geo_to_xml.sh b/scripts/geo_to_xml.sh index 999790c3..83d0c21e 100755 --- a/scripts/geo_to_xml.sh +++ b/scripts/geo_to_xml.sh @@ -33,8 +33,14 @@ check_exec() { echo_usage() { echo "Usage:" - echo " $0 [path_to_geo_file] <-g gmsh_path> <-m NekMesh_path> <-o output_filename> <-x x_bndry_compIDs> <-y y_bndry_compIDs> <-z z_bndry_compIDs>" - echo " If periodic BCs will be used, provide boundary composite IDs via the -x, -y and -z args; e.g. -x 1,2 -y 3,4 -z 5,6" + echo " $0 [path_to_geo_file] " + echo " Options include" + echo ' - Specifying a path and/or arguments for gmsh: <-g gmsh_path> <--gmsh_args "arg1 arg2...">' + echo ' - Specifying a path and/or XML file properties for NekMesh: <-m NekMesh_path> <-p "prop1 prop2...">' + echo ' (e.g. -x "uncompress" to generate a human-readable, uncompressed XML file)' + echo " - Setting an output filename <-o output_filename>" + echo " - Specifying composite IDs for periodic boundary conditions <-x x_bndry_compIDs> <-y y_bndry_compIDs> <-z z_bndry_compIDs>" + echo " (e.g. -x 1,2 -y 3,4 -z 5,6)" } parse_args() { @@ -45,10 +51,18 @@ parse_args() { gm_exec="$2" shift 2 ;; + --gmsh_args) + gm_args="$gm_args $2" + shift 2 + ;; -m|--nekmesh) nm_exec="$2" shift 2 ;; + -p|--xml_props) + nm_xml_props="$2" + shift 2 + ;; -n|--ndims) ndims="$2" case $ndims in @@ -135,13 +149,26 @@ set_peralign_opts() { done } +set_xml_opts(){ + xml_opts="xml" + for prop in $nm_xml_props; do + xml_opts="$xml_opts:$prop" + done +} + report_options() { - echo "Options:" - echo " path to .geo : $geo_path" - echo " output path : $xml_path" - echo " n dims : $ndims" - echo " gmsh exec : $gm_exec" - echo " NekMesh exec : $nm_exec" + echo "Using options" + echo " path to .geo : $geo_path" + echo " output path : $xml_path" + echo " n dims : $ndims" + echo " gmsh exec : $gm_exec" + if [ -n "$gm_args" ]; then + echo " gmsh args : [$gm_args]" + fi + echo " NekMesh exec : $nm_exec" + if [ -n "$nm_xml_props" ]; then + echo " NekMesh xml props : [$nm_xml_props]" + fi echo "" } #------------------------------------------------------------------------------ @@ -149,24 +176,28 @@ report_options() { # Default options geo_path="Not set" gm_exec="gmsh" +gm_args="" ndims="3" set_nm_default_exec nm_args="-v" +nm_xml_props="" # Parse command line args and report resulting options parse_args "$@" report_options set_peralign_opts +set_xml_opts # Check gmsh, NekMesh can be found check_exec "$gm_exec" "g" check_exec "$nm_exec" "m" # Run gmsh -gmsh_cmd="$gm_exec -$ndims $geo_path" +gm_args="-$ndims $gm_args" +gmsh_cmd="$gm_exec $gm_args $geo_path" echo "Running [$gmsh_cmd]" gmsh_output=$($gmsh_cmd) -gmsh_ret_code=$? +gmsh_ret_code=$? if [ $gmsh_ret_code -ne 0 ] then echo "gmsh returned $gmsh_ret_code. Output was: " @@ -180,12 +211,11 @@ echo # Remove any existing .xml file \rm -f "$xml_path" - # Run NekMesh -nm_cmd="$nm_exec $nm_args $msh_path $xml_path:xml:uncompress" +nm_cmd="$nm_exec $nm_args $msh_path $xml_path:$xml_opts" echo "Running [$nm_cmd]" nm_output=$($nm_cmd) -nm_ret_code=$? +nm_ret_code=$? if [ $nm_ret_code -ne 0 ] then echo "NekMesh returned $nm_ret_code. Output was: " @@ -201,4 +231,5 @@ echo # Remove EXPANSIONS node from mesh xml sed -i '//,/<\/EXPANSIONS>/d' "$xml_path" -echo "Generated Nektar xml mesh at $xml_path" \ No newline at end of file +echo "Generated Nektar xml mesh at $xml_path" +echo "(remember to add an node to one of the xml files passed to nektar)" \ No newline at end of file diff --git a/scripts/run_eg.sh b/scripts/run_eg.sh index 5f65508c..d789faa0 100755 --- a/scripts/run_eg.sh +++ b/scripts/run_eg.sh @@ -4,7 +4,7 @@ # Helper functions echo_usage() { echo "Usage:" - echo " $0 [solver_name] [example_name] <-n num_MPI> <-b build_dir>" + echo " $0 [solver_name] [example_name] <-n num_MPI> <-b build_dir/install_dir>" } execute() { @@ -21,7 +21,7 @@ execute() { generate_run_dir() { local eg_dir="$1" local run_dir="$2" - run_dir="$REPO_ROOT/example-runs/$solver_name/$eg_name" + run_dir="$REPO_ROOT/runs/$solver_name/$eg_name" if [ -e "$run_dir" ]; then read -p "Overwrite existing run directory at $run_dir? (Y/N): " choice && [[ $choice == [yY] || $choice == [yY][eE][sS] ]] || exit 5 \rm -rf "$run_dir" @@ -30,8 +30,8 @@ generate_run_dir() { cp -r "$eg_dir" "$run_dir" } -set_default_build_dir() { - build_dir=$(find "$REPO_ROOT" -maxdepth 1 -type d -name "spack-build*" -printf "%TY-%Tm-%Td %TT %p\n" | sort -n|tail -1|cut -d " " -f 3) +set_default_exec_loc() { + solver_exec=$(find -L "$REPO_ROOT/views" -maxdepth 3 -name "$solver_name" -printf "%TY-%Tm-%Td %TT %p\n" | sort -n|tail -1|cut -d " " -f 3) } parse_args() { @@ -43,14 +43,14 @@ parse_args() { while [[ $# -gt 0 ]]; do case $1 in -b|--build-dir) - build_dir=$(realpath "$2") + exec_loc=$(realpath "$2") shift 2 ;; -n|--num_mpi) nmpi="$2" shift 2 ;; - -*|--*) + -*) echo "Unknown option $1" exit 2 ;; @@ -91,13 +91,33 @@ set_run_cmd() { fi } -validate_paths() { - local solver_exec=$1 +set_exec_paths_and_validate() { + local exec_loc=$1 local eg_dir=$2 - if [ ! -f "$solver_exec" ]; then - echo "No solver found at $solver_exec" - exit 3 + + + if [ -z "$exec_loc" ]; then + # If no executable location was specified, set solver_exec to the most + # recently modified executable in the views + set_default_exec_loc + if [ -z "$solver_exec" ]; then + echo "No installed solver found in ./views; run spack install or pass -b " + exit 3 + fi + else + # Else check build and install locations relative to the specified $exec_loc + solver_build_exec="$exec_loc/solvers/$solver_name/$solver_name" + solver_install_exec="$exec_loc/bin/$solver_name" + if [ -f "$solver_build_exec" ]; then + solver_exec="$solver_build_exec" + elif [ -f "$solver_install_exec" ]; then + solver_exec="$solver_install_exec" + else + echo "No solver found at [$solver_build_exec] or [$solver_install_exec]" + exit 3 + fi fi + if [ ! -d "$eg_dir" ]; then echo "No example directory found at $eg_dir" exit 4 @@ -111,21 +131,19 @@ REPO_ROOT=$( cd -- "$(realpath $( dirname -- "${BASH_SOURCE[0]}" )/..)" &> /dev/ solver_name='Not set' eg_name='Not set' nmpi='4' -build_dir='Not set' -set_default_build_dir +exec_loc='' # Parse command line args and report resulting options parse_args $* report_options -# Set paths to the solver executable and example directory -solver_exec="$build_dir/solvers/$solver_name/$solver_name" +solver_exec='Not set' eg_dir="$REPO_ROOT/examples/$solver_name/$eg_name" -# Validate exec, examples paths -validate_paths "$solver_exec" "$eg_dir" +# Find the executable inside $exec_loc and validate the examples paths +set_exec_paths_and_validate "$exec_loc" "$eg_dir" # Set up run directory, confirming overwrite if it already exists -run_dir="$REPO_ROOT/example-runs/$solver_name/$eg_name" +run_dir="$REPO_ROOT/runs/$solver_name/$eg_name" generate_run_dir "$eg_dir" "$run_dir" # Read run command template and populate it