-
Notifications
You must be signed in to change notification settings - Fork 44
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'development' into kess_micro
- Loading branch information
Showing
113 changed files
with
1,730 additions
and
1,242 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -56,46 +56,3 @@ jobs: | |
| -DERF_ENABLE_MPI:BOOL=ON \ | ||
| -DERF_ENABLE_CUDA:BOOL=ON . | ||
| cmake --build build-${{matrix.cuda_pkg}} -- -j $(nproc) | ||
| cuda-moisture-build: | ||
| runs-on: ubuntu-20.04 | ||
| name: (MOISTURE ON) CUDA v${{matrix.cuda_ver}} | ||
| strategy: | ||
| matrix: | ||
| cuda_pkg: [11-0] | ||
| include: | ||
| - cuda_ver: "11.0" | ||
| cuda_pkg: 11-0 | ||
| cuda_extra: libcurand-dev-11-0 cuda-cupti-dev-11-0 libcusolver-dev-11-0 libcublas-dev-11-0 libcusparse-dev-11-0 | ||
| steps: | ||
| - name: Cancel previous runs | ||
| uses: styfle/[email protected] | ||
| with: | ||
| access_token: ${{github.token}} | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| submodules: true | ||
| - name: Prepare CUDA environment | ||
| run: | | ||
| export DEBIAN_FRONTEND=noninteractive | ||
| sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub | ||
| sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu2004/x86_64/7fa2af80.pub | ||
| echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 /" | sudo tee /etc/apt/sources.list.d/cuda.list | ||
| echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu2004/x86_64 /" | sudo tee /etc/apt/sources.list.d/nvidia-ml.list | ||
| sudo apt-get update | ||
| sudo apt-get install -y --no-install-recommends \ | ||
| libopenmpi-dev cuda-command-line-tools-${{matrix.cuda_pkg}} cuda-compiler-${{matrix.cuda_pkg}} cuda-minimal-build-${{matrix.cuda_pkg}} cuda-nvml-dev-${{matrix.cuda_pkg}} cuda-nvtx-${{matrix.cuda_pkg}} ${{matrix.cuda_extra}} | ||
| - name: Configure and build | ||
| run: | | ||
| export PATH=/usr/local/nvidia/bin:/usr/local/cuda-${{matrix.cuda_ver}}/bin:${PATH} | ||
| export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda-${{matrix.cuda_ver}}/lib:${LD_LIBRARY_PATH} | ||
| cmake -Bbuild-${{matrix.cuda_pkg}} \ | ||
| -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \ | ||
| -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ | ||
| -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON \ | ||
| -DERF_DIM:STRING=3 \ | ||
| -DERF_ENABLE_MPI:BOOL=OFF \ | ||
| -DERF_ENABLE_MOISTURE:BOOL=ON \ | ||
| -DERF_ENABLE_CUDA:BOOL=ON . | ||
| cmake --build build-${{matrix.cuda_pkg}} -- -j $(nproc) | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -119,105 +119,3 @@ jobs: | |
| name: build-and-test | ||
| path: | | ||
| ${{runner.workspace}}/ERF/regressioncov.xml | ||
| Build-And-Test-HIP-Moisture: | ||
| name: (MOISTURE ON) HIP ROCm [email protected] C++17 [tests] | ||
| runs-on: ubuntu-20.04 | ||
| # Have to have -Wno-deprecated-declarations due to deprecated atomicAddNoRet | ||
| # Have to have -Wno-gnu-zero-variadic-macro-arguments to avoid | ||
| # amrex/Src/Base/AMReX_GpuLaunchGlobal.H:15:5: error: must specify at least one argument for '...' parameter of variadic macro [-Werror,-Wgnu-zero-variadic-macro-arguments] | ||
| # __launch_bounds__(amrex_launch_bounds_max_threads) | ||
| # ^ | ||
| # /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:178:71: note: expanded from macro '__launch_bounds__' | ||
| # select_impl_(__VA_ARGS__, launch_bounds_impl1, launch_bounds_impl0)(__VA_ARGS__) | ||
| # ^ | ||
| # /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:176:9: note: macro 'select_impl_' defined here | ||
| # #define select_impl_(_1, _2, impl_, ...) impl_ | ||
| # NOTE: -Werror was removed because ERF specifically had a lot of warnings. It will be a separate task to go through and fix them all... | ||
| env: {CXXFLAGS: "-Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wno-deprecated-declarations -Wno-gnu-zero-variadic-macro-arguments -Wno-pass-failed"} | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| submodules: true | ||
|
|
||
| - name: Dependencies | ||
| run: .github/workflows/dependencies/dependencies_hip.sh | ||
|
|
||
| - name: Build & Install | ||
| run: | | ||
| source /etc/profile.d/rocm.sh | ||
| hipcc --version | ||
| which clang | ||
| which clang++ | ||
| # "mpic++ --showme" forgets open-pal in Ubuntu 20.04 + OpenMPI 4.0.3 | ||
| # https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786 | ||
| # https://github.com/open-mpi/ompi/issues/9317 | ||
| export LDFLAGS="-lopen-pal" | ||
| cmake \ | ||
| -B${{runner.workspace}}/ERF/build-${{matrix.os}} \ | ||
| -DCMAKE_VERBOSE_MAKEFILE=ON \ | ||
| -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/ERF/install \ | ||
| -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \ | ||
| -DERF_DIM:STRING=3 \ | ||
| -DERF_ENABLE_MPI:BOOL=ON \ | ||
| -DERF_ENABLE_HIP:BOOL=ON \ | ||
| -DAMReX_AMD_ARCH=gfx908 \ | ||
| -DERF_ENABLE_TESTS:BOOL=ON \ | ||
| -DERF_ENABLE_ALL_WARNINGS:BOOL=ON \ | ||
| -DERF_ENABLE_FCOMPARE:BOOL=ON \ | ||
| -DERF_ENABLE_MOISTURE:BOOL=ON \ | ||
| -DCODECOV:BOOL=ON \ | ||
| -DCMAKE_C_COMPILER=$(which clang) \ | ||
| -DCMAKE_CXX_COMPILER=$(which clang++) \ | ||
| -DCMAKE_CXX_STANDARD=17 \ | ||
| ${{github.workspace}}; | ||
| # ${{matrix.mpipreflags}} \ | ||
| # for some reason this cmake command fails to build the code, | ||
| # and we need to use the make command that follows instead ... | ||
| # cmake --build ${{runner.workspace}}/ERF/build-${{matrix.os}} --parallel ${{env.NPROCS}}; | ||
| pushd ${{runner.workspace}}/ERF/build-${{matrix.os}}; | ||
| # make -j ${{env.NPROCS}}; | ||
| make -j 2; | ||
| # - name: Regression Tests | ||
| # run: | | ||
| # ctest -L regression -VV | ||
| # working-directory: ${{runner.workspace}}/ERF/build-${{matrix.os}} | ||
|
|
||
| # - name: Generate coverage report | ||
| # working-directory: ${{runner.workspace}}/ERF/build-${{matrix.os}} | ||
| # run: | | ||
| # find . -type f -name '*.gcno' -path "**Source**" -exec gcov -pb {} + | ||
| # cd .. | ||
| # gcovr -g -k -r . --xml regressioncov.xml # -v | ||
|
|
||
| # - name: Upload coverage to Codecov | ||
| # uses: codecov/codecov-action@v3 | ||
| # with: | ||
| # dry_run: false | ||
| # # token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos | ||
| # files: ./regressioncov.xml # optional | ||
| # flags: regtests # optional | ||
| # # name: codecov-umbrella # optional | ||
| # fail_ci_if_error: true # optional (default = false) | ||
| # verbose: true # optional (default = false) | ||
| # directory: ${{runner.workspace}}/ERF | ||
|
|
||
| - name: Success artifacts | ||
| uses: actions/upload-artifact@v3 | ||
| if: success() | ||
| with: | ||
| name: build-and-test | ||
| path: | | ||
| ${{runner.workspace}}/ERF/regressioncov.xml | ||
| - name: Failing test artifacts | ||
| uses: actions/upload-artifact@v3 | ||
| if: failure() | ||
| with: | ||
| name: build-and-test | ||
| path: | | ||
| ${{runner.workspace}}/ERF/regressioncov.xml | ||
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -85,8 +85,6 @@ or if using tcsh, | |
| +--------------------+------------------------------+------------------+-------------+ | ||
| | USE_HDF5 | Whether to enable HDF5 | TRUE / FALSE | FALSE | | ||
| +--------------------+------------------------------+------------------+-------------+ | ||
| | USE_MOISTURE | Whether to enable moisture | TRUE / FALSE | FALSE | | ||
| +--------------------+------------------------------+------------------+-------------+ | ||
| | USE_WARM_NO_PRECIP | Whether to use warm moisture | TRUE / FALSE | FALSE | | ||
| +--------------------+------------------------------+------------------+-------------+ | ||
| | USE_MULTIBLOCK | Whether to enable multiblock | TRUE / FALSE | FALSE | | ||
|
|
@@ -176,8 +174,6 @@ Analogous to GNU Make, the list of cmake directives is as follows: | |
| +---------------------------+------------------------------+------------------+-------------+ | ||
| | ERF_ENABLE_HDF5 | Whether to enable HDF5 | TRUE / FALSE | FALSE | | ||
| +---------------------------+------------------------------+------------------+-------------+ | ||
| | ERF_ENABLE_MOISTURE | Whether to enable moisture | TRUE / FALSE | FALSE | | ||
| +---------------------------+------------------------------+------------------+-------------+ | ||
| | ERF_ENABLE_WARM_NO_PRECIP | Whether to use warm moisture | TRUE / FALSE | FALSE | | ||
| +---------------------------+------------------------------+------------------+-------------+ | ||
| | ERF_ENABLE_MULTIBLOCK | Whether to enable multiblock | TRUE / FALSE | FALSE | | ||
|
|
@@ -190,6 +186,58 @@ Analogous to GNU Make, the list of cmake directives is as follows: | |
| +---------------------------+------------------------------+------------------+-------------+ | ||
|
|
||
|
|
||
| Mac with CMake | ||
| ~~~~~~~~~~~~~~ | ||
| Tested with macOS 12.7 (Monterey) using cmake (3.27.8), open-mpi (5.0.0), and | ||
| pkg-config (0.29.2) installed with the homebrew package manager. HDF5 and | ||
| NetCDF will be compiled from source. The instructions below should be version | ||
| agnostic. | ||
|
|
||
| HDF5 (tested with v1.14.3) | ||
|
|
||
| #. Download latest source package from `hdfgroup.org`_ | ||
| #. Extract source code ``tar xzf hdf5-<version>.tar.gz`` | ||
| #. Create build directory ``cd hdf5-<version> && mkdir build && cd build`` | ||
| #. Configure for your system ``../configure --prefix=/usr/local --enable-parallel`` | ||
| #. Build ``make -j8`` and ``sudo make install`` | ||
|
|
||
| .. _hdfgroup.org: https://www.hdfgroup.org/downloads/hdf5/source-code/ | ||
|
|
||
| NetCDF (tested with v4.9.2) | ||
|
|
||
| #. Download latest source package from `ucar.edu`_ | ||
| #. (Optional) install Zstd compression library ``brew install zstd`` | ||
| #. Create build directory ``cd netcdf-c-4.9.2 && mkdir build && cd build`` | ||
| #. Configure for your system ``../configure --enable-parallel CC=mpicc CXX=mpicxx LDFLAGS="-L/opt/homebrew/Cellar/zstd/1.5.5/lib" CPPFLAGS="-I/opt/homebrew/Cellar/zstd/1.5.5/include"`` | ||
| (omit the LDFLAGS and CPPFLAGS if you do not have Zstd installed) -- note | ||
| that you may encounter cmake errors if you do not have pkg-config installed | ||
| #. Build ``make -j8`` and ``sudo make install`` | ||
|
|
||
| .. _ucar.edu: https://downloads.unidata.ucar.edu/netcdf/ | ||
|
|
||
| ERF (tested with commit ``40e64ed35ebc080ad61d08aea828330dfbdbc162``) | ||
|
|
||
| #. Get latest source code ``git clone --recursive [email protected]:erf-model/ERF.git`` | ||
| #. Create build directory ``cd ERF && mkdir MyBuild && cd MyBuild`` | ||
| #. Configure with cmake and build | ||
|
|
||
| :: | ||
|
|
||
| cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \ | ||
| -DCMAKE_CXX_COMPILER:STRING=mpicxx \ | ||
| -DCMAKE_C_COMPILER:STRING=mpicc \ | ||
| -DCMAKE_Fortran_COMPILER:STRING=mpifort \ | ||
| -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \ | ||
| -DERF_DIM:STRING=3 \ | ||
| -DERF_ENABLE_MPI:BOOL=ON \ | ||
| -DERF_ENABLE_TESTS:BOOL=ON \ | ||
| -DERF_ENABLE_FCOMPARE:BOOL=ON \ | ||
| -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \ | ||
| -DERF_ENABLE_NETCDF:BOOL=ON \ | ||
| -DERF_ENABLE_HDF5:BOOL=ON \ | ||
| -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ | ||
| .. && make -j8 | ||
|
|
||
| Perlmutter (NERSC) | ||
| ~~~~~~~~~~~~~~~~~~ | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.