From 5827647389147cc2fd0162a818b99c77102d0a21 Mon Sep 17 00:00:00 2001 From: Bruce Perry <53018946+baperry2@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:16:09 -0700 Subject: [PATCH] Updates for PelePhysics reorganization. (#325) * use amrex and sundials from pp * also remove PeleMP submod * changes for GNUmake after pp reorg * delete commented PeleC stuff from CI * update some documentation for PeleMP move * use recursive clone in ci * account for mechanism move in tutorials docs * update unitTest for submodules reorg * update convergence and nightly tests for reorged submods * start updating ci.yml for submod reorg * first pass at cmake except soot for pp reorg * update pp * add soot changes for cmake (untested) * update pp * update pp again and try something * use fewer procs for clang-tidy * update pp again --- .clang-tidy | 2 +- .github/dependabot.yml | 1 - .github/workflows/ci.yml | 155 ++----------------- .github/workflows/convergenceTesting.yml | 16 +- .github/workflows/linux.yml | 8 +- .github/workflows/nightlyTest.yml | 31 ++-- .github/workflows/unitTest.yml | 25 +-- .gitmodules | 9 -- CMake/BuildPeleLMeXExe.cmake | 92 ++++++----- CMake/BuildPeleLMeXLib.cmake | 132 +++++++++------- CMake/PeleLMeXUtils.cmake | 2 +- CMakeLists.txt | 4 +- Docs/sphinx/manual/Model.rst | 2 +- Docs/sphinx/manual/Tutorials_BFSFlame.rst | 2 +- Docs/sphinx/manual/Tutorials_FlameSheet.rst | 2 +- Docs/sphinx/manual/Tutorials_FlowPastCyl.rst | 2 +- Docs/sphinx/manual/Tutorials_HotBubble.rst | 2 +- Docs/sphinx/manual/Tutorials_TripleFlame.rst | 2 +- Exec/Make.PeleLMeX | 52 +++---- README.md | 10 +- Submodules/PeleMP | 1 - Submodules/PelePhysics | 2 +- Submodules/amrex | 1 - Submodules/sundials | 1 - Tests/CMakeLists.txt | 4 +- Tests/CTestNightlyScript.cmake | 8 +- Utils/CloneDeps.sh | 3 +- 27 files changed, 208 insertions(+), 363 deletions(-) delete mode 160000 Submodules/PeleMP delete mode 160000 Submodules/amrex delete mode 160000 Submodules/sundials diff --git a/.clang-tidy b/.clang-tidy index aa9a8f1b..daa36fc7 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -21,7 +21,7 @@ Checks: 'bugprone-*, -readability-function-cognitive-complexity, -readability-function-size' WarningsAsErrors: '' -HeaderFilterRegex: 'PeleLMeX\/Source|PeleLMeX\/Exec|PeleLMeX\/Submodules\/Pele' +HeaderFilterRegex: 'PeleLMeX\/Source|PeleLMeX\/Exec|PeleLMeX\/Submodules\/PelePhysics\/Source|PeleLMeX\/Submodules\/PelePhysics\/Mechanism' AnalyzeTemporaryDtors: false FormatStyle: none User: user diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ecd2ccdf..4e1554b6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,6 +6,5 @@ updates: interval: "weekly" directory: / allow: - - dependency-name: "Submodules/amrex" - dependency-name: "Submodules/AMReX-Hydro" - dependency-name: "Submodules/PelePhysics" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02f62804..bd6aed08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: - name: Clone uses: actions/checkout@v3 with: - submodules: true + submodules: recursive - name: Setup working-directory: ./Exec/RegTests/FlameSheet run: | @@ -40,8 +40,6 @@ jobs: echo "CCACHE_COMPRESSLEVEL=1" >> $GITHUB_ENV echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV echo "CCACHE_MAXSIZE=200M" >> $GITHUB_ENV - #printf "CC=gcc-10\nCXX=g++-10\nFC=gfortran-10\nF90=gfortran-10" \ - # >> ${{github.workspace}}/Submodules/amrex/Tools/GNUMake/Make.local make COMP=gnu USE_MPI=FALSE DEBUG=TRUE TPL make COMP=llvm USE_MPI=FALSE DEBUG=TRUE PRECISION=FLOAT TPL - name: Install Ccache @@ -81,100 +79,6 @@ jobs: cat warnings.txt export return=$(tail -n 1 warnings.txt | awk '{print $2}') exit ${return} - #- name: EB debug build - # working-directory: ./Exec/RegTests/EB-C10 - # run: | - # echo "::add-matcher::.github/problem-matchers/gcc.json" - # set -eu -o pipefail - # ccache -z - # make -j ${{env.NPROCS}} KEEP_BUILDINFO_CPP=TRUE USE_CCACHE=TRUE COMP=gnu USE_MPI=FALSE DEBUG=TRUE 2>&1 | tee -a build-output.txt - #- name: EB debug ccache report - # working-directory: ./Exec/RegTests/EB-C10 - # run: | - # ccache -s - # du -hs ~/.cache/ccache - #- name: EB debug test - # working-directory: ./Exec/RegTests/EB-C10 - # run: ./PeleLMeX3d.gnu.DEBUG.ex eb-c10.inp max_step=2 amr.plot_files_output=0 amr.checkpoint_files_output=0 - #- name: EB debug report - # working-directory: ./Exec/RegTests/EB-C10 - # run: | - # egrep "warning:|error:" build-output.txt \ - # | egrep -v "Submodules/amrex|Submodules/sundials|Submodules/AMReX-Hydro|GNUmakefile" \ - # | egrep -v "ld: warning:" | sort | uniq \ - # | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt - # cat warnings.txt - # export return=$(tail -n 1 warnings.txt | awk '{print $2}') - # exit ${return} - #- name: TG debug single precision build - # working-directory: ./Exec/RegTests/TG - # run: | - # echo "::add-matcher::.github/problem-matchers/gcc.json" - # set -eu -o pipefail - # ccache -z - # make -j ${{env.NPROCS}} KEEP_BUILDINFO_CPP=TRUE USE_CCACHE=TRUE COMP=llvm USE_MPI=FALSE DEBUG=TRUE PRECISION=FLOAT 2>&1 | tee -a build-output.txt - #- name: TG debug single precision ccache report - # working-directory: ./Exec/RegTests/TG - # run: | - # ccache -s - # du -hs ~/.cache/ccache - #- name: TG debug single precision test - # working-directory: ./Exec/RegTests/TG - # run: ./PeleLMeX3d.llvm.FLOAT.DEBUG.ex tg-2.inp max_step=2 amr.plot_files_output=0 amr.checkpoint_files_output=0 - #- name: TG debug single precision report - # working-directory: ./Exec/RegTests/TG - # run: | - # egrep "warning:|error:" build-output.txt \ - # | egrep -v "Submodules/amrex|Submodules/sundials|Submodules/AMReX-Hydro|GNUmakefile" \ - # | egrep -v "ld: warning:" | sort | uniq \ - # | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt - # cat warnings.txt - # export return=$(tail -n 1 warnings.txt | awk '{print $2}') - # exit ${return} - #- name: Soot debug build - # working-directory: ./Exec/RegTests/Soot-Flame - # run: | - # echo "::add-matcher::.github/problem-matchers/gcc.json" - # set -eu -o pipefail - # ccache -z - # make -j ${{env.NPROCS}} KEEP_BUILDINFO_CPP=TRUE USE_CCACHE=TRUE COMP=gnu USE_MPI=FALSE DEBUG=TRUE 2>&1 | tee -a build-output.txt - #- name: Soot debug ccache report - # working-directory: ./Exec/RegTests/Soot-Flame - # run: | - # ccache -s - # du -hs ~/.cache/ccache - #- name: Soot debug report - # working-directory: ./Exec/RegTests/Soot-Flame - # run: | - # egrep "warning:|error:" build-output.txt \ - # | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|Submodules/PeleMP|Submodules/AMReX-Hydro" \ - # | egrep -v "Submodules/PelePhysics/Support/Mechanism/Models" | sort | uniq \ - # | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt - # cat warnings.txt - # export return=$(tail -n 1 warnings.txt | awk '{print $2}') - # exit ${return} - #- name: Spray debug build - # working-directory: ./Exec/RegTests/Spray-Conv - # run: | - # echo "::add-matcher::.github/problem-matchers/gcc.json" - # set -eu -o pipefail - # ccache -z - # make -j ${{env.NPROCS}} KEEP_BUILDINFO_CPP=TRUE USE_CCACHE=TRUE COMP=gnu USE_MPI=FALSE DEBUG=TRUE 2>&1 | tee -a build-output.txt - #- name: Spray debug ccache report - # working-directory: ./Exec/RegTests/Soot-Flame - # run: | - # ccache -s - # du -hs ~/.cache/ccache - #- name: Spray debug report - # working-directory: ./Exec/RegTests/Spray-Conv - # run: | - # egrep "warning:|error:" build-output.txt \ - # | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|Submodules/PeleMP|Submodules/AMReX-Hydro" \ - # | egrep -v "Submodules/PelePhysics/Support/Mechanism/Models" | sort | uniq \ - # | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt - # cat warnings.txt - # export return=$(tail -n 1 warnings.txt | awk '{print $2}') - # exit ${return} CPU-CMake: needs: Formatting runs-on: ${{matrix.os}} @@ -206,7 +110,7 @@ jobs: - name: Clone uses: actions/checkout@v3 with: - submodules: true + submodules: recursive - name: Python uses: actions/setup-python@v4 with: @@ -220,7 +124,7 @@ jobs: echo "CCACHE_MAXSIZE=${{matrix.ccache_size}}" >> $GITHUB_ENV cmake -E make_directory ${{runner.workspace}}/deps if [ "${RUNNER_OS}" != "macOS" ]; then - ${{github.workspace}}/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh + ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh fi - name: Dependencies run: | @@ -305,7 +209,7 @@ jobs: - name: Clone uses: actions/checkout@v3 with: - submodules: true + submodules: recursive - name: Setup run: | echo "NPROCS=$(nproc)" >> $GITHUB_ENV @@ -315,7 +219,7 @@ jobs: echo "CCACHE_MAXSIZE=300M" >> $GITHUB_ENV - name: Dependencies run: | - ${{github.workspace}}/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh + ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb sudo dpkg -i cuda-keyring_1.0-1_all.deb sudo apt-get update @@ -368,7 +272,7 @@ jobs: - name: Clone uses: actions/checkout@v3 with: - submodules: true + submodules: recursive - name: Setup run: | echo "NPROCS=$(nproc)" >> $GITHUB_ENV @@ -378,7 +282,7 @@ jobs: echo "CCACHE_MAXSIZE=80M" >> $GITHUB_ENV - name: Dependencies run: | - ${{github.workspace}}/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh + ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh curl -O https://repo.radeon.com/rocm/rocm.gpg.key sudo apt-key add rocm.gpg.key echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/5.6.1 ubuntu main' \ @@ -438,7 +342,7 @@ jobs: - name: Clone uses: actions/checkout@v3 with: - submodules: true + submodules: recursive - name: Setup run: | echo "NPROCS=$(nproc)" >> $GITHUB_ENV @@ -449,7 +353,7 @@ jobs: echo "CCACHE_DEPEND=1" >> $GITHUB_ENV - name: Dependencies run: | - ${{github.workspace}}/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh + ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB echo "deb https://apt.repos.intel.com/oneapi all main" \ @@ -496,41 +400,6 @@ jobs: run: | ccache -s du -hs ~/.cache/ccache -# Lint-cppcheck: -# needs: Formatting -# runs-on: macos-12 -# steps: -# - name: Clone -# uses: actions/checkout@v3 -# with: -# submodules: true -# - name: Dependencies -# run: | -# export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=TRUE -# wget https://raw.githubusercontent.com/Homebrew/homebrew-core/9b0e3d32f590458ae9e5dbf01943b39845b5315e/Formula/cppcheck.rb -# brew install cppcheck.rb -# - name: Configure -# run: | -# cmake -B${{runner.workspace}}/build-cppcheck \ -# -DPELELMEX_ENABLE_MPI:BOOL=OFF \ -# -DPELELMEX_ENABLE_FCOMPARE_FOR_TESTS:BOOL=OFF \ -# -DPELELMEX_ENABLE_MASA:BOOL=OFF \ -# -DPELELMEX_ENABLE_CPPCHECK:BOOL=ON \ -# ${{github.workspace}} -# - name: Check -# working-directory: ${{runner.workspace}}/build-cppcheck -# run: make cppcheck -# - name: Full report -# working-directory: ${{runner.workspace}}/build-cppcheck/cppcheck -# run: cat cppcheck-full-report.txt -# - name: Short report -# working-directory: ${{runner.workspace}}/build-cppcheck/cppcheck -# run: | -# echo "::add-matcher::.github/problem-matchers/cppcheck.json" -# awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' < cppcheck-report.txt > cppcheck-warnings.txt -# cat cppcheck-warnings.txt -# export return=$(tail -n 1 cppcheck-warnings.txt | awk '{print $2}') -# exit ${return} Lint-clang-tidy: needs: Formatting runs-on: ubuntu-latest @@ -554,7 +423,7 @@ jobs: - name: Clone uses: actions/checkout@v3 with: - submodules: true + submodules: recursive - name: Install Ccache run: | wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz @@ -586,7 +455,7 @@ jobs: working-directory: ${{runner.workspace}}/build-clang-tidy run: | #set -eu -o pipefail - cmake --build . --parallel ${{env.NPROCS}} 2>&1 | tee -a clang-tidy-full-report.txt + cmake --build . --parallel 2 2>&1 | tee -a clang-tidy-full-report.txt egrep "warning:|error:" clang-tidy-full-report.txt \ | egrep -v "Submodules/amrex|Submodules/sundials|Submodules/AMReX-Hydro" \ | egrep -v "ld: warning:" | sort | uniq \ @@ -612,7 +481,7 @@ jobs: - name: Clone uses: actions/checkout@v3 with: - submodules: true + submodules: recursive - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: diff --git a/.github/workflows/convergenceTesting.yml b/.github/workflows/convergenceTesting.yml index db32269d..85d732cd 100644 --- a/.github/workflows/convergenceTesting.yml +++ b/.github/workflows/convergenceTesting.yml @@ -19,6 +19,8 @@ jobs: with: access_token: ${{github.token}} - uses: actions/checkout@v3 + with: + submodules: recursive - name: System Dependencies run: | .github/workflows/dependencies/dependencies.sh @@ -26,21 +28,11 @@ jobs: python3 -m pip install --user numpy python3 -m pip install --user argparse python3 -m pip install --user matplotlib - - name: Repo Dependencies - run: Utils/CloneDeps.sh - name: Build AMReX Tools - env: - AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex - working-directory: ./build/amrex/Tools/Plotfile + working-directory: ./Submodules/PelePhysics/Submodules/amrex/Tools/Plotfile run: | make - name: Build PeleLMeX - env: - AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex - PELE_PHYSICS_HOME: ${GITHUB_WORKSPACE}/build/PelePhysics - AMREX_HYDRO_HOME: ${GITHUB_WORKSPACE}/build/AMReX-Hydro - SUNDIALS_HOME: ${GITHUB_WORKSPACE}/build/sundials - PELELMEX_HOME: ${GITHUB_WORKSPACE} working-directory: ./Exec/RegTests/PeriodicCases run: | make -j 2 TPL COMP=gnu @@ -48,7 +40,7 @@ jobs: - name: Run working-directory: ./Exec/RegTests/PeriodicCases run: | - cp ${GITHUB_WORKSPACE}/build/amrex/Tools/Plotfile/fcompare.gnu.ex . + cp ${GITHUB_WORKSPACE}/Submodules/PelePhysics/Submodules/amrex/Tools/Plotfile/fcompare.gnu.ex . sed -i "34s/.*/ resolution = [32,64,128,256,512]/" multiRuns.py sed -i "s/mpiexec -n 1/mpiexec -n 4 --oversubscribe/g" multiRuns.py ./multiRuns.py --test_name GH_CI_CoVo --input_file input.2d_CoVo diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index fbf8a1fc..d4e39738 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -178,7 +178,7 @@ jobs: run: Utils/CloneDeps.sh - name: GenerateTurbFile env: - AMREX_HOME: ${GITHUB_WORKSPACE}/Submodules/amrex + AMREX_HOME: ${GITHUB_WORKSPACE}/Submodules/PelePhysics/Submodules/amrex working-directory: ./Exec/RegTests/TurbInflow/TurbFileHIT run: | make -j 2 COMP=gnu @@ -214,8 +214,8 @@ jobs: run: Utils/CloneDeps.sh - name: Build AMReX Tools env: - AMREX_HOME: ${GITHUB_WORKSPACE}/Submodules/amrex - working-directory: ./Submodules/amrex/Tools/Plotfile + AMREX_HOME: ${GITHUB_WORKSPACE}/Submodules/PelePhysics/Submodules/amrex + working-directory: ./Submodules/PelePhysics/Submodules/amrex/Tools/Plotfile run: | make - name: Build PeleLMeX @@ -226,7 +226,7 @@ jobs: - name: Run working-directory: ./Exec/RegTests/SprayTest/ run: | - cp ${GITHUB_WORKSPACE}/Submodules/amrex/Tools/Plotfile/fcompare.gnu.ex . + cp ${GITHUB_WORKSPACE}/Submodules/PelePhysics/Submodules/amrex/Tools/Plotfile/fcompare.gnu.ex . sed -i "s/mpiexec -n 1/mpiexec -n 2/g" multiRuns.py ./multiRuns.py ./compareOutput.py diff --git a/.github/workflows/nightlyTest.yml b/.github/workflows/nightlyTest.yml index b0c3402c..1af43566 100644 --- a/.github/workflows/nightlyTest.yml +++ b/.github/workflows/nightlyTest.yml @@ -17,7 +17,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - submodules: true + submodules: recursive - name: System Dependencies run: | .github/workflows/dependencies/dependencies_gcc10.sh @@ -28,53 +28,42 @@ jobs: - name: LMeX Reference run: | mkdir buildRef - git clone --recursive -b development https://github.com/AMReX-Combustion/PeleLMeX.git buildRef/PeleLMeX - - name: Set Environment Variables - run: | - echo "AMREX_HOME=${{github.workspace}}/buildRef/PeleLMeX/Submodules/amrex" >> $GITHUB_ENV - echo "PELE_PHYSICS_HOME=${{github.workspace}}/buildRef/PeleLMeX/Submodules/PelePhysics" >> $GITHUB_ENV - echo "AMREX_HYDRO_HOME=${{github.workspace}}/buildRef/PeleLMeX/Submodules/AMReX-Hydro" >> $GITHUB_ENV - echo "SUNDIALS_HOME=${{github.workspace}}/buildRef/PeleLMeX/Submodules/sundials" >> $GITHUB_ENV - echo "PELELMEX_HOME=${{github.workspace}}/buildRef/PeleLMeX/" >> $GITHUB_ENV + git clone --recursive --single-branch --shallow-submodules -b development https://github.com/AMReX-Combustion/PeleLMeX.git buildRef/PeleLMeX - name: Build Reference PeleLMeX - working-directory: ${{env.PELELMEX_HOME}}/Exec/UnitTests/DodecaneLu + working-directory: ${{github.workspace}}/buildRef/PeleLMeX/Exec/UnitTests/DodecaneLu run: | make -j2 TPL COMP=gnu DEBUG=FALSE make -j2 COMP=gnu DEBUG=FALSE - name: Run Reference Case - working-directory: ${{env.PELELMEX_HOME}}/Exec/UnitTests/DodecaneLu + working-directory: ${{github.workspace}}/buildRef/PeleLMeX/Exec/UnitTests/DodecaneLu run: | ./PeleLMeX3d.gnu.MPI.ex inputs.3d - name: Update PeleLMeX Submodules working-directory: ${{github.workspace}} run: | - cd ${{github.workspace}}/Submodules/amrex - git checkout development - cd - cd ${{github.workspace}}/Submodules/AMReX-Hydro git checkout development cd - cd ${{github.workspace}}/Submodules/PelePhysics git checkout development cd - + cd ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex + git checkout development + cd - - name: Reset Environment Variables run: | - echo "AMREX_HOME=${{github.workspace}}/Submodules/amrex" >> $GITHUB_ENV - echo "PELE_PHYSICS_HOME=${{github.workspace}}/Submodules/PelePhysics" >> $GITHUB_ENV - echo "AMREX_HYDRO_HOME=${{github.workspace}}/Submodules/AMReX-Hydro" >> $GITHUB_ENV - echo "SUNDIALS_HOME=${{github.workspace}}/Submodules/sundials" >> $GITHUB_ENV - echo "PELELMEX_HOME=${{github.workspace}}" >> $GITHUB_ENV + echo "AMREX_HOME=${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex" >> $GITHUB_ENV - name: Build AMReX Tools working-directory: ${{env.AMREX_HOME}}/Tools/Plotfile run: | make -j2 VERBOSE=FALSE - name: Build PeleLMeX - working-directory: ${{env.PELELMEX_HOME}}/Exec/UnitTests/DodecaneLu + working-directory: ${{github.workspace}}/Exec/UnitTests/DodecaneLu run: | make -j2 TPL COMP=gnu DEBUG=FALSE make -j2 COMP=gnu DEBUG=FALSE - name: Run Case - working-directory: ${{env.PELELMEX_HOME}}/Exec/UnitTests/DodecaneLu + working-directory: ${{github.workspace}}/Exec/UnitTests/DodecaneLu run: | cp ${{env.AMREX_HOME}}/Tools/Plotfile/fcompare.gnu.ex . ./PeleLMeX3d.gnu.MPI.ex inputs.3d diff --git a/.github/workflows/unitTest.yml b/.github/workflows/unitTest.yml index ac22b82b..49b3f4fc 100644 --- a/.github/workflows/unitTest.yml +++ b/.github/workflows/unitTest.yml @@ -19,7 +19,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - submodules: true + submodules: recursive - name: System Dependencies run: | .github/workflows/dependencies/dependencies_gcc10.sh @@ -30,41 +30,30 @@ jobs: - name: LMeX Reference run: | mkdir buildRef - git clone --recursive -b development https://github.com/AMReX-Combustion/PeleLMeX.git buildRef/PeleLMeX - - name: Set Environment Variables - run: | - echo "AMREX_HOME=${{github.workspace}}/buildRef/PeleLMeX/Submodules/amrex" >> $GITHUB_ENV - echo "PELE_PHYSICS_HOME=${{github.workspace}}/buildRef/PeleLMeX/Submodules/PelePhysics" >> $GITHUB_ENV - echo "AMREX_HYDRO_HOME=${{github.workspace}}/buildRef/PeleLMeX/Submodules/AMReX-Hydro" >> $GITHUB_ENV - echo "SUNDIALS_HOME=${{github.workspace}}/buildRef/PeleLMeX/Submodules/sundials" >> $GITHUB_ENV - echo "PELELMEX_HOME=${{github.workspace}}/buildRef/PeleLMeX/" >> $GITHUB_ENV + git clone --recursive --single-branch --shallow-submodules -b development https://github.com/AMReX-Combustion/PeleLMeX.git buildRef/PeleLMeX - name: Build Reference PeleLMeX - working-directory: ${{env.PELELMEX_HOME}}/Exec/UnitTests/DodecaneLu + working-directory: ${{github.workspace}}/buildRef/PeleLMeX/Exec/UnitTests/DodecaneLu run: | make -j2 TPL COMP=gnu DEBUG=FALSE make -j2 COMP=gnu DEBUG=FALSE - name: Run Reference Case - working-directory: ${{env.PELELMEX_HOME}}/Exec/UnitTests/DodecaneLu + working-directory: ${{github.workspace}}/buildRef/PeleLMeX/Exec/UnitTests/DodecaneLu run: | ./PeleLMeX3d.gnu.MPI.ex inputs.3d - name: Reset Environment Variables run: | - echo "AMREX_HOME=${{github.workspace}}/Submodules/amrex" >> $GITHUB_ENV - echo "PELE_PHYSICS_HOME=${{github.workspace}}/Submodules/PelePhysics" >> $GITHUB_ENV - echo "AMREX_HYDRO_HOME=${{github.workspace}}/Submodules/AMReX-Hydro" >> $GITHUB_ENV - echo "SUNDIALS_HOME=${{github.workspace}}/Submodules/sundials" >> $GITHUB_ENV - echo "PELELMEX_HOME=${{github.workspace}}" >> $GITHUB_ENV + echo "AMREX_HOME=${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex" >> $GITHUB_ENV - name: Build AMReX Tools working-directory: ${{env.AMREX_HOME}}/Tools/Plotfile run: | make -j2 VERBOSE=FALSE - name: Build PeleLMeX - working-directory: ${{env.PELELMEX_HOME}}/Exec/UnitTests/DodecaneLu + working-directory: ${{github.workspace}}/Exec/UnitTests/DodecaneLu run: | make -j2 TPL COMP=gnu DEBUG=FALSE make -j2 COMP=gnu DEBUG=FALSE - name: Run Case - working-directory: ${{env.PELELMEX_HOME}}/Exec/UnitTests/DodecaneLu + working-directory: ${{github.workspace}}/Exec/UnitTests/DodecaneLu run: | cp ${{env.AMREX_HOME}}/Tools/Plotfile/fcompare.gnu.ex . ./PeleLMeX3d.gnu.MPI.ex inputs.3d diff --git a/.gitmodules b/.gitmodules index db8d7cef..63369cb8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,15 +1,6 @@ -[submodule "Submodules/amrex"] - path = Submodules/amrex - url = https://github.com/AMReX-Codes/amrex.git [submodule "Submodules/AMReX-Hydro"] path = Submodules/AMReX-Hydro url = https://github.com/AMReX-Fluids/AMReX-Hydro.git [submodule "Submodules/PelePhysics"] path = Submodules/PelePhysics url = https://github.com/AMReX-Combustion/PelePhysics.git -[submodule "Submodules/PeleMP"] - path = Submodules/PeleMP - url = https://github.com/AMReX-Combustion/PeleMP.git -[submodule "Submodules/sundials"] - path = Submodules/sundials - url = https://github.com/LLNL/sundials diff --git a/CMake/BuildPeleLMeXExe.cmake b/CMake/BuildPeleLMeXExe.cmake index f0c8b485..50e8eff7 100644 --- a/CMake/BuildPeleLMeXExe.cmake +++ b/CMake/BuildPeleLMeXExe.cmake @@ -5,7 +5,7 @@ function(build_pelelmex_exe pelelmex_exe_name pelelmex_lib_name) add_executable(${pelelmex_exe_name} "") if(CLANG_TIDY_EXE) - set_target_properties(${pelelmex_exe_name} PROPERTIES CXX_CLANG_TIDY + set_target_properties(${pelelmex_exe_name} PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_EXE};--config-file=${CMAKE_SOURCE_DIR}/.clang-tidy") endif() @@ -15,57 +15,46 @@ function(build_pelelmex_exe pelelmex_exe_name pelelmex_lib_name) pelelmex_prob.H pelelmex_prob.cpp ) - + #PeleLMeX include directories target_include_directories(${pelelmex_exe_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(${pelelmex_exe_name} PRIVATE ${SRC_DIR}) target_include_directories(${pelelmex_exe_name} PRIVATE ${CMAKE_BINARY_DIR}) #target_include_directories(${pelelmex_exe_name} PRIVATE ${CMAKE_SOURCE_DIR}/Source/Params/param_includes) - # Set PeleMP flags - set(PELEMP_SRC_DIR ${CMAKE_SOURCE_DIR}/Submodules/PeleMP/Source) - if(PELELMEX_ENABLE_AMREX_PARTICLES AND PELEMP_SPRAY_FUEL_NUM GREATER 0) - target_compile_definitions(${pelelmex_exe_name} PRIVATE PELELMEX_USE_SPRAY) - target_compile_definitions(${pelelmex_exe_name} PRIVATE SPRAY_FUEL_NUM=${PELEMP_SPRAY_FUEL_NUM}) - target_sources(${pelelmex_exe_name} PRIVATE - SprayParticlesInitInsert.cpp - ${SRC_DIR}/PeleLMeX_SprayParticles.cpp - ${PELEMP_SRC_DIR}/PP_Spray/SprayParticles.cpp - ${PELEMP_SRC_DIR}/PP_Spray/SprayParticles.H - ${PELEMP_SRC_DIR}/PP_Spray/SprayFuelData.H - ${PELEMP_SRC_DIR}/PP_Spray/SprayInterpolation.H - ${PELEMP_SRC_DIR}/PP_Spray/Drag.H - ${PELEMP_SRC_DIR}/PP_Spray/SprayInjection.H - ${PELEMP_SRC_DIR}/PP_Spray/SpraySetup.cpp - ${PELEMP_SRC_DIR}/PP_Spray/SprayDerive.cpp - ${PELEMP_SRC_DIR}/PP_Spray/SprayJet.H - ${PELEMP_SRC_DIR}/PP_Spray/SprayJet.cpp - ${PELEMP_SRC_DIR}/PP_Spray/SprayIO.cpp - ${PELEMP_SRC_DIR}/PP_Spray/WallFunctions.H - ${PELEMP_SRC_DIR}/PP_Spray/Distribution/DistBase.H - ${PELEMP_SRC_DIR}/PP_Spray/Distribution/Distributions.H - ${PELEMP_SRC_DIR}/PP_Spray/Distribution/Distributions.cpp) - target_include_directories(${pelelmex_exe_name} PRIVATE ${PELEMP_SRC_DIR}/PP_Spray) - target_include_directories(${pelelmex_exe_name} PRIVATE ${PELEMP_SRC_DIR}/PP_Spray/Distribution) - endif() - if(PELELMEX_ENABLE_SOOT) - target_compile_definitions(${pelelmex_exe_name} PRIVATE PELELMEX_USE_SOOT) - target_compile_definitions(${pelelmex_exe_name} PRIVATE NUM_SOOT_MOMENTS=${PELEMP_NUM_SOOT_MOMENTS}) - set(SOOT_MOMENTS_VALUES 3 6) - if(NOT PELEMP_NUM_SOOT_MOMENTS IN_LIST SOOT_MOMENTS_VALUES) - message(FATAL_ERROR "NUM_SOOT_MOMENTS must be either 3 or 6") - endif() - target_sources(${pelelmex_exe_name} PRIVATE - ${SRC_DIR}/PeleLMeX_Soot.cpp - ${PELEMP_SRC_DIR}/Soot_Models/SootModel.cpp - ${PELEMP_SRC_DIR}/Soot_Models/SootModel_react.cpp - ${PELEMP_SRC_DIR}/Soot_Models/SootModel_derive.cpp - ${PELEMP_SRC_DIR}/Soot_Models/Constants_Soot.H - ${PELEMP_SRC_DIR}/Soot_Models/SootData.H - ${PELEMP_SRC_DIR}/Soot_Models/SootReactions.H - ${PELEMP_SRC_DIR}/Soot_Models/SootModel.H - ${PELEMP_SRC_DIR}/Soot_Models/SootModel_derive.H) - target_include_directories(${pelelmex_exe_name} PRIVATE ${PELEMP_SRC_DIR}/Soot_Models) + # Spray + set(PELE_PHYSICS_SPRAY_DIR ${CMAKE_SOURCE_DIR}/Submodules/PelePhysics/Source/Spray) + + if(PELELMEX_ENABLE_AMREX_PARTICLES AND PELE_SPRAY_FUEL_NUM GREATER 0) + target_compile_definitions(${pelelmex_exe_name} PRIVATE PELELMEX_USE_SPRAY) + target_compile_definitions(${pelelmex_exe_name} PRIVATE SPRAY_FUEL_NUM=${PELE_SPRAY_FUEL_NUM}) + target_sources(${pelelmex_exe_name} PRIVATE + SprayParticlesInitInsert.cpp + ${SRC_DIR}/Particle.cpp + ${PELE_PHYSICS_SPRAY_DIR}/Drag.H + ${PELE_PHYSICS_SPRAY_DIR}/SprayDerive.cpp + ${PELE_PHYSICS_SPRAY_DIR}/SprayFuelData.H + ${PELE_PHYSICS_SPRAY_DIR}/SprayIO.cpp + ${PELE_PHYSICS_SPRAY_DIR}/SprayInjection.H + ${PELE_PHYSICS_SPRAY_DIR}/SprayInterpolation.H + ${PELE_PHYSICS_SPRAY_DIR}/SprayJet.H + ${PELE_PHYSICS_SPRAY_DIR}/SprayJet.cpp + ${PELE_PHYSICS_SPRAY_DIR}/SprayParticles.H + ${PELE_PHYSICS_SPRAY_DIR}/SprayParticles.cpp + ${PELE_PHYSICS_SPRAY_DIR}/SpraySB.cpp + ${PELE_PHYSICS_SPRAY_DIR}/SpraySetup.cpp + ${PELE_PHYSICS_SPRAY_DIR}/WallFunctions.H + ${PELE_PHYSICS_SPRAY_DIR}/BreakupSplash/AhamedSplash.H + ${PELE_PHYSICS_SPRAY_DIR}/BreakupSplash/ReitzKHRT.H + ${PELE_PHYSICS_SPRAY_DIR}/BreakupSplash/SBData.H + ${PELE_PHYSICS_SPRAY_DIR}/BreakupSplash/TABBreakup.H + ${PELE_PHYSICS_SPRAY_DIR}/BreakupSplash/WallFilm.H + ${PELE_PHYSICS_SPRAY_DIR}/Distribution/DistBase.H + ${PELE_PHYSICS_SPRAY_DIR}/Distribution/Distributions.H + ${PELE_PHYSICS_SPRAY_DIR}/Distribution/Distributions.cpp) + target_include_directories(${pelelmex_exe_name} PUBLIC ${PELE_PHYSICS_SPRAY_DIR}) + target_include_directories(${pelelmex_exe_name} PUBLIC ${PELE_PHYSICS_SPRAY_DIR}/Distribution) + target_include_directories(${pelelmex_exe_name} PUBLIC ${PELE_PHYSICS_SPRAY_DIR}/BreakupSplash) endif() target_sources(${pelelmex_exe_name} @@ -113,6 +102,13 @@ function(build_pelelmex_exe pelelmex_exe_name pelelmex_lib_name) ${SRC_DIR}/main.cpp ) + # Soot + if(PELELMEX_ENABLE_SOOT) + target_sources(${pelelmex_exe_name} + PRIVATE + ${SRC_DIR}/Soot.cpp) + endif() + #if(PELELMEX_ENABLE_ASCENT) # target_sources(${pelelmex_exe_name} # PRIVATE @@ -147,10 +143,10 @@ function(build_pelelmex_exe pelelmex_exe_name pelelmex_lib_name) set_target_properties(${pelelmex_exe_name} PROPERTIES CUDA_SEPARABLE_COMPILATION ON) target_compile_options(${pelelmex_exe_name} PRIVATE $<$:-Xptxas --disable-optimizer-constants>) endif() - + target_link_libraries(${pelelmex_exe_name} PRIVATE ${pelelmex_lib_name} AMReX-Hydro::amrex_hydro_api AMReX::amrex) - #Define what we want to be installed during a make install + #Define what we want to be installed during a make install install(TARGETS ${pelelmex_exe_name} RUNTIME DESTINATION bin ARCHIVE DESTINATION lib diff --git a/CMake/BuildPeleLMeXLib.cmake b/CMake/BuildPeleLMeXLib.cmake index 140c9df1..591577ea 100644 --- a/CMake/BuildPeleLMeXLib.cmake +++ b/CMake/BuildPeleLMeXLib.cmake @@ -3,9 +3,12 @@ function(build_pelelmex_lib pelelmex_lib_name) add_library(${pelelmex_lib_name} OBJECT) set(PELE_PHYSICS_SRC_DIR ${CMAKE_SOURCE_DIR}/Submodules/PelePhysics) - set(PELE_PHYSICS_TRANSPORT_DIR "${PELE_PHYSICS_SRC_DIR}/Transport") - set(PELE_PHYSICS_EOS_DIR "${PELE_PHYSICS_SRC_DIR}/Eos") - set(PELE_PHYSICS_MECHANISM_DIR "${PELE_PHYSICS_SRC_DIR}/Support/Mechanism/Models/${PELELMEX_CHEMISTRY_MODEL}") + set(PELE_PHYSICS_TRANSPORT_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Transport") + set(PELE_PHYSICS_EOS_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Eos") + set(PELE_PHYSICS_MECHANISM_DIR "${PELE_PHYSICS_SRC_DIR}/Mechanisms/${PELELMEX_CHEMISTRY_MODEL}") + set(PELE_PHYSICS_UTILITY_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Utility") + set(PELE_PHYSICS_REACTIONS_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Reactions") + set(PELE_PHYSICS_SOOT_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Soot") set(AMREX_SUNDIALS_DIR ${AMREX_SUBMOD_LOCATION}/Src/Extern/SUNDIALS) if(CLANG_TIDY_EXE) @@ -14,47 +17,47 @@ function(build_pelelmex_lib pelelmex_lib_name) endif() include(SetPeleLMeXCompileFlags) - + target_sources(${pelelmex_lib_name} PRIVATE - ${PELE_PHYSICS_SRC_DIR}/Utility/TurbInflow/turbinflow.cpp - ${PELE_PHYSICS_SRC_DIR}/Utility/TurbInflow/turbinflow.H + ${PELE_PHYSICS_UTILITY_DIR}/TurbInflow/turbinflow.cpp + ${PELE_PHYSICS_UTILITY_DIR}/TurbInflow/turbinflow.H ) - target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_SRC_DIR}/Utility/TurbInflow) + target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_UTILITY_DIR}/TurbInflow) target_sources(${pelelmex_lib_name} PRIVATE - ${PELE_PHYSICS_SRC_DIR}/Utility/Diagnostics/DiagBase.H - ${PELE_PHYSICS_SRC_DIR}/Utility/Diagnostics/DiagBase.cpp - ${PELE_PHYSICS_SRC_DIR}/Utility/Diagnostics/DiagConditional.H - ${PELE_PHYSICS_SRC_DIR}/Utility/Diagnostics/DiagConditional.cpp - ${PELE_PHYSICS_SRC_DIR}/Utility/Diagnostics/DiagFilter.H - ${PELE_PHYSICS_SRC_DIR}/Utility/Diagnostics/DiagFilter.cpp - ${PELE_PHYSICS_SRC_DIR}/Utility/Diagnostics/DiagFramePlane.H - ${PELE_PHYSICS_SRC_DIR}/Utility/Diagnostics/DiagFramePlane.cpp - ${PELE_PHYSICS_SRC_DIR}/Utility/Diagnostics/DiagPDF.H - ${PELE_PHYSICS_SRC_DIR}/Utility/Diagnostics/DiagPDF.cpp + ${PELE_PHYSICS_UTILITY_DIR}/Diagnostics/DiagBase.H + ${PELE_PHYSICS_UTILITY_DIR}/Diagnostics/DiagBase.cpp + ${PELE_PHYSICS_UTILITY_DIR}/Diagnostics/DiagConditional.H + ${PELE_PHYSICS_UTILITY_DIR}/Diagnostics/DiagConditional.cpp + ${PELE_PHYSICS_UTILITY_DIR}/Diagnostics/DiagFilter.H + ${PELE_PHYSICS_UTILITY_DIR}/Diagnostics/DiagFilter.cpp + ${PELE_PHYSICS_UTILITY_DIR}/Diagnostics/DiagFramePlane.H + ${PELE_PHYSICS_UTILITY_DIR}/Diagnostics/DiagFramePlane.cpp + ${PELE_PHYSICS_UTILITY_DIR}/Diagnostics/DiagPDF.H + ${PELE_PHYSICS_UTILITY_DIR}/Diagnostics/DiagPDF.cpp ) - target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_SRC_DIR}/Utility/Diagnostics) - + target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_UTILITY_DIR}/Diagnostics) + target_sources(${pelelmex_lib_name} PRIVATE - ${PELE_PHYSICS_SRC_DIR}/Utility/PltFileManager/PltFileManager.cpp - ${PELE_PHYSICS_SRC_DIR}/Utility/PltFileManager/PltFileManager.H - ${PELE_PHYSICS_SRC_DIR}/Utility/PltFileManager/PltFileManagerBCFill.H + ${PELE_PHYSICS_UTILITY_DIR}/PltFileManager/PltFileManager.cpp + ${PELE_PHYSICS_UTILITY_DIR}/PltFileManager/PltFileManager.H + ${PELE_PHYSICS_UTILITY_DIR}/PltFileManager/PltFileManagerBCFill.H ) - target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_SRC_DIR}/Utility/PltFileManager) + target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_UTILITY_DIR}/PltFileManager) target_sources(${pelelmex_lib_name} PRIVATE - ${PELE_PHYSICS_SRC_DIR}/Utility/PMF/PMF.H - ${PELE_PHYSICS_SRC_DIR}/Utility/PMF/PMFData.cpp - ${PELE_PHYSICS_SRC_DIR}/Utility/PMF/PMFData.H + ${PELE_PHYSICS_UTILITY_DIR}/PMF/PMF.H + ${PELE_PHYSICS_UTILITY_DIR}/PMF/PMFData.cpp + ${PELE_PHYSICS_UTILITY_DIR}/PMF/PMFData.H ) - target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_SRC_DIR}/Utility/PMF) - + target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_UTILITY_DIR}/PMF) + target_sources(${pelelmex_lib_name} - PRIVATE + PRIVATE ${AMREX_SUNDIALS_DIR}/AMReX_Sundials.H ${AMREX_SUNDIALS_DIR}/AMReX_Sundials_Core.cpp ${AMREX_SUNDIALS_DIR}/AMReX_Sundials_Core.H @@ -116,36 +119,55 @@ function(build_pelelmex_lib pelelmex_lib_name) target_sources(${pelelmex_lib_name} PRIVATE - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorArkode.H - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorArkode.cpp - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorBase.H - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorBase.cpp - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorCvode.H - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorCvode.cpp - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorCvodeCustomLinSolver.H - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorCvodeCustomLinSolver.cpp - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorCvodeJacobian.H - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorCvodeJacobian.cpp - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorCvodePreconditioner.H - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorCvodePreconditioner.cpp - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorCvodeUtils.H - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorCvodeUtils.cpp - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorNull.H - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorNull.cpp - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorRK64.H - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorRK64.cpp - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorTypes.H - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorUtils.H - ${PELE_PHYSICS_SRC_DIR}/Reactions/ReactorUtils.cpp + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorArkode.H + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorArkode.cpp + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorBase.H + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorBase.cpp + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorCvode.H + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorCvode.cpp + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorCvodeCustomLinSolver.H + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorCvodeCustomLinSolver.cpp + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorCvodeJacobian.H + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorCvodeJacobian.cpp + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorCvodePreconditioner.H + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorCvodePreconditioner.cpp + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorCvodeUtils.H + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorCvodeUtils.cpp + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorNull.H + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorNull.cpp + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorRK64.H + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorRK64.cpp + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorTypes.H + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorUtils.H + ${PELE_PHYSICS_REACTIONS_DIR}/ReactorUtils.cpp ) - target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_SRC_DIR}/Reactions) + target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_REACTIONS_DIR}) + + if(PELELMEX_ENABLE_SOOT) + target_compile_definitions(${pelelmex_lib_name} PUBLIC PELELMEX_USE_SOOT) + target_compile_definitions(${pelelmex_lib_name} PUBLIC NUM_SOOT_MOMENTS=${PELE_NUM_SOOT_MOMENTS}) + set(SOOT_MOMENTS_VALUES 3 6) + if(NOT PELE_NUM_SOOT_MOMENTS IN_LIST SOOT_MOMENTS_VALUES) + message(FATAL_ERROR "NUM_SOOT_MOMENTS must be either 3 or 6") + endif() + target_sources(${pelelmex_lib_name} PRIVATE + ${PELE_PHYSICS_SOOT_DIR}/SootModel.H + ${PELE_PHYSICS_SOOT_DIR}/SootModel.cpp + ${PELE_PHYSICS_SOOT_DIR}/SootModel_react.cpp + ${PELE_PHYSICS_SOOT_DIR}/SootModel_derive.H + ${PELE_PHYSICS_SOOT_DIR}/SootModel_derive.cpp + ${PELE_PHYSICS_SOOT_DIR}/Constants_Soot.H + ${PELE_PHYSICS_SOOT_DIR}/SootData.H + ${PELE_PHYSICS_SOOT_DIR}/SootReactions.H) + target_include_directories(${pele=lmex_lib_name} PUBLIC ${PELE_PHYSICS_SOOT_DIR}) + endif() include(AMReXBuildInfo) generate_buildinfo(${pelelmex_lib_name} ${CMAKE_SOURCE_DIR}) target_include_directories(${pelelmex_lib_name} SYSTEM PUBLIC ${AMREX_SUBMOD_LOCATION}/Tools/C_scripts) - + target_link_libraries(${pelelmex_lib_name} PUBLIC sundials_arkode sundials_cvode) - + if(PELELMEX_ENABLE_CUDA) target_link_libraries(${pelelmex_lib_name} PUBLIC sundials_nveccuda sundials_sunlinsolcusolversp sundials_sunmatrixcusparse) elseif(PELELMEX_ENABLE_HIP) @@ -153,11 +175,11 @@ function(build_pelelmex_lib pelelmex_lib_name) elseif(PELELMEX_ENABLE_SYCL) target_link_libraries(${pelelmex_lib_name} PUBLIC sundials_nvecsycl) endif() - + if(PELELMEX_ENABLE_MPI) target_link_libraries(${pelelmex_lib_name} PUBLIC $<$:MPI::MPI_CXX>) endif() - + #Link to amrex libraries target_link_libraries(${pelelmex_lib_name} PUBLIC AMReX-Hydro::amrex_hydro_api AMReX::amrex) diff --git a/CMake/PeleLMeXUtils.cmake b/CMake/PeleLMeXUtils.cmake index c20f1265..1c509624 100644 --- a/CMake/PeleLMeXUtils.cmake +++ b/CMake/PeleLMeXUtils.cmake @@ -44,7 +44,7 @@ macro(init_code_checks) COMMAND sed "s/isystem /I/g" compile_commands.json > cppcheck/cppcheck_compile_commands.json COMMAND ${CPPCHECK_EXE} --version COMMAND ${CPPCHECK_EXE} --template=gcc --inline-suppr --suppress=unusedFunction --suppress=useStlAlgorithm --suppress=missingIncludeSystem --std=c++17 --language=c++ --enable=all --project=cppcheck/cppcheck_compile_commands.json --output-file=cppcheck/cppcheck-full-report.txt -j ${NP} - COMMAND egrep "information:|error:|performance:|portability:|style:|warning:" cppcheck/cppcheck-full-report.txt | egrep -v "Submodules/amrex|Submodules/sundials|Submodules/GoogleTest|Submodules/PelePhysics/Support/Mechanism/Models" | sort | uniq > cppcheck/cppcheck-report.txt + COMMAND egrep "information:|error:|performance:|portability:|style:|warning:" cppcheck/cppcheck-full-report.txt | egrep -v "Submodules/amrex|Submodules/sundials|Submodules/PelePhysics/Mechanisms" | sort | uniq > cppcheck/cppcheck-report.txt COMMAND wc -l cppcheck/cppcheck-report.txt COMMENT "Run cppcheck on project compile_commands.json" BYPRODUCTS cppcheck/cppcheck-report.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index e4f79edb..99e55414 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,7 +66,7 @@ endif() ########################### AMReX ##################################### message(STATUS "AMReX Configure Section") -set(AMREX_SUBMOD_LOCATION "${CMAKE_SOURCE_DIR}/Submodules/amrex") +set(AMREX_SUBMOD_LOCATION "${CMAKE_SOURCE_DIR}/Submodules/PelePhysics/Submodules/amrex") include(SetAmrexOptions) list(APPEND CMAKE_MODULE_PATH "${AMREX_SUBMOD_LOCATION}/Tools/CMake") add_subdirectory(${AMREX_SUBMOD_LOCATION}) @@ -80,7 +80,7 @@ add_subdirectory(${AMREX_HYDRO_SUBMOD_LOCATION}) ########################### SUNDIALS ################################## message(STATUS "Sundials Configure Section") -set(SUNDIALS_SUBMOD_LOCATION "${CMAKE_SOURCE_DIR}/Submodules/sundials") +set(SUNDIALS_SUBMOD_LOCATION "${CMAKE_SOURCE_DIR}/Submodules/PelePhysics/Submodules/sundials") include(SetSundialsOptions) #BUILD_TESTING is an old CMake keyword so don't clear it in sundials configure set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) diff --git a/Docs/sphinx/manual/Model.rst b/Docs/sphinx/manual/Model.rst index 50f23575..9ef481cb 100644 --- a/Docs/sphinx/manual/Model.rst +++ b/Docs/sphinx/manual/Model.rst @@ -35,7 +35,7 @@ In a nutshell, `PeleLMeX` features include: * Several higher-order Godunov integration schemes for advection * Temporally implicit viscosity, species mass diffusion, thermal conductivity, chemical kinetics * Closed chamber algorithm enable time-varying background pressure changes - * Lagrangian spray description using `PeleMP `_ + * Lagrangian spray description and Hybrid Method of Moments Soot modeling using `PelePhysics `_ (formerly these were part of `PeleMP `_). Mathematical background ----------------------- diff --git a/Docs/sphinx/manual/Tutorials_BFSFlame.rst b/Docs/sphinx/manual/Tutorials_BFSFlame.rst index 43ee4570..cce52c60 100644 --- a/Docs/sphinx/manual/Tutorials_BFSFlame.rst +++ b/Docs/sphinx/manual/Tutorials_BFSFlame.rst @@ -219,7 +219,7 @@ The user also needs to make sure the additional C++ header employed to define th In `PeleLMeX`, the chemistry model (set of species, their thermodynamic and transport properties as well as the description of their of chemical interactions) is specified at compile time. Chemistry models available in -`PelePhysics` can used in `PeleLMeX` by specifying the name of the folder in `PelePhysics/Support/Mechanisms/Models` containing +`PelePhysics` can used in `PeleLMeX` by specifying the name of the folder in `PelePhysics/Mechanisms` containing the relevant files, for example: :: Chemistry_Model = drm19 diff --git a/Docs/sphinx/manual/Tutorials_FlameSheet.rst b/Docs/sphinx/manual/Tutorials_FlameSheet.rst index 4446045f..074f1624 100644 --- a/Docs/sphinx/manual/Tutorials_FlameSheet.rst +++ b/Docs/sphinx/manual/Tutorials_FlameSheet.rst @@ -233,7 +233,7 @@ The compiler (``gnu``) and the parallelism paradigm (in the present case only MP platform, please set ``USE_MPI = FALSE``. Note that on OSX platform, one should update the compiler to ``llvm``. -In `PeleLMeX`, the chemistry model (set of species, their thermodynamic and transport properties as well as the description of their of chemical interactions) is specified at compile time. Chemistry models available in `PelePhysics` can used in `PeleLMeX` by specifying the name of the folder in `PelePhysics/Support/Mechanisms/Models` containing the relevant files, for example: :: +In `PeleLMeX`, the chemistry model (set of species, their thermodynamic and transport properties as well as the description of their of chemical interactions) is specified at compile time. Chemistry models available in `PelePhysics` can used in `PeleLMeX` by specifying the name of the folder in `PelePhysics/Mechanisms` containing the relevant files, for example: :: Chemistry_Model = drm19 diff --git a/Docs/sphinx/manual/Tutorials_FlowPastCyl.rst b/Docs/sphinx/manual/Tutorials_FlowPastCyl.rst index c36e9594..0ee4793e 100644 --- a/Docs/sphinx/manual/Tutorials_FlowPastCyl.rst +++ b/Docs/sphinx/manual/Tutorials_FlowPastCyl.rst @@ -167,7 +167,7 @@ Next comes the build configuration block: :: It allows the user to specify the number of spatial dimensions (2D), trigger the compilation of the EB source code, the compiler (``gnu``) and the parallelism paradigm (in the present case only MPI is used). The other options can be activated for debugging and profiling purposes. Note that on OSX platform, one should update the compiler to ``llvm``. -In `PeleLMeX`, the chemistry model (set of species, their thermodynamic and transport properties as well as the description of their of chemical interactions) is specified at compile time. Chemistry models available in `PelePhysics` can used in `PeleLMeX` by specifying the name of the folder in `PelePhysics/Support/Mechanisms/Models` containing the relevant files, for example: :: +In `PeleLMeX`, the chemistry model (set of species, their thermodynamic and transport properties as well as the description of their of chemical interactions) is specified at compile time. Chemistry models available in `PelePhysics` can used in `PeleLMeX` by specifying the name of the folder in `PelePhysics/Mechanisms` containing the relevant files, for example: :: Chemistry_Model = air diff --git a/Docs/sphinx/manual/Tutorials_HotBubble.rst b/Docs/sphinx/manual/Tutorials_HotBubble.rst index bf823937..4bf2df40 100644 --- a/Docs/sphinx/manual/Tutorials_HotBubble.rst +++ b/Docs/sphinx/manual/Tutorials_HotBubble.rst @@ -177,7 +177,7 @@ trigger debug compilation and other AMReX options. The compiler (``gnu``) and th In `PeleLMeX`, the chemistry model (set of species, their thermodynamic and transport properties as well as the description of their of chemical interactions) is specified at compile time. Chemistry models available in -`PelePhysics` can used in `PeleLMeX` by specifying the name of the folder in `PelePhysics/Support/Mechanisms/Models` containing +`PelePhysics` can used in `PeleLMeX` by specifying the name of the folder in `PelePhysics/Mechanisms` containing the relevant files, for example: :: Chemistry_Model = air diff --git a/Docs/sphinx/manual/Tutorials_TripleFlame.rst b/Docs/sphinx/manual/Tutorials_TripleFlame.rst index e6c7eec4..d5a4f67b 100644 --- a/Docs/sphinx/manual/Tutorials_TripleFlame.rst +++ b/Docs/sphinx/manual/Tutorials_TripleFlame.rst @@ -210,7 +210,7 @@ The next few lines specify AMReX compilation options and compiler selection: :: In `PeleLMeX`, the chemistry model (set of species, their thermodynamic and transport properties as well as the description of their of chemical interactions) is specified at compile time. Chemistry models available -in `PelePhysics` can used in `PeleLMeX` by specifying the name of the folder in `PelePhysics/Support/Mechanisms/Models` containing +in `PelePhysics` can used in `PeleLMeX` by specifying the name of the folder in `PelePhysics/Mechanisms` containing the relevant files, for example: :: Chemistry_Model = drm19 diff --git a/Exec/Make.PeleLMeX b/Exec/Make.PeleLMeX index f2136af0..ff38b408 100644 --- a/Exec/Make.PeleLMeX +++ b/Exec/Make.PeleLMeX @@ -1,9 +1,8 @@ PELELMEX_HOME ?= ../ -AMREX_HOME ?= $(abspath $(lastword ${PELELMEX_HOME}/Submodules/amrex)) -PELE_PHYSICS_HOME ?= $(abspath $(lastword ${PELELMEX_HOME}/Submodules/PelePhysics)) AMREX_HYDRO_HOME ?= $(abspath $(lastword ${PELELMEX_HOME}/Submodules/AMReX-Hydro)) -SUNDIALS_HOME ?= $(abspath $(lastword ${PELELMEX_HOME}/Submodules/sundials)) -PELEMP_HOME ?= $(abspath $(lastword ${PELELMEX_HOME}/Submodules/PeleMP)) +PELE_PHYSICS_HOME ?= $(abspath $(lastword ${PELELMEX_HOME}/Submodules/PelePhysics)) +AMREX_HOME ?= $(abspath $(lastword ${PELE_PHYSICS_HOME}/Submodules/amrex)) +SUNDIALS_HOME ?= $(abspath $(lastword ${PELE_PHYSICS_HOME}/Submodules/sundials)) USE_SUNDIALS = TRUE BL_NO_FORT = TRUE @@ -103,15 +102,34 @@ ifeq ($(PELE_COMPILE_AJACOBIAN), TRUE) DEFINES += -DPELE_COMPILE_AJACOBIAN endif -ChemDir = Support/Mechanism/Models/$(Chemistry_Model) - -PPdirs := Utility/PMF Utility/TurbInflow Utility/PltFileManager Utility/Diagnostics -PPdirs += Source $(ChemDir) Reactions Eos Transport +ChemDir = Mechanisms/$(Chemistry_Model) +PPdirs := Source/Utility/PMF Source/Utility/TurbInflow Source/Utility/PltFileManager Source/Utility/Diagnostics +PPdirs += Source $(ChemDir) Source/Reactions Source/Eos Source/Transport Bpack += $(foreach dir, $(PPdirs), $(PELE_PHYSICS_HOME)/$(dir)/Make.package) Blocs += $(foreach dir, $(PPdirs), $(PELE_PHYSICS_HOME)/$(dir)) include $(PELE_PHYSICS_HOME)/ThirdParty/Make.ThirdParty +# Spray +ifeq ($(USE_PARTICLES), TRUE) + ifdef SPRAY_FUEL_NUM + MPdirs := ./ ./Distribution ./BreakupSplash + SPRAY_HOME = $(PELE_PHYSICS_HOME)/Source/Spray + Bpack += $(foreach dir, $(MPdirs), $(SPRAY_HOME)/$(dir)/Make.package) + Blocs += $(foreach dir, $(MPdirs), $(SPRAY_HOME)/$(dir)) + endif +endif + +# Soot +ifeq ($(USE_SOOT), TRUE) + SOOT_HOME = $(PELE_PHYSICS_HOME)/Source/Soot + Bpack += $(SOOT_HOME)/Make.package + Blocs += $(SOOT_HOME)/Source/Soot_Models +endif +ifeq ($(USE_PELERAD), TRUE) + Blocs += $(PELERAD_HOME)/src +endif + #--------------- # AMReX-Hydro sources #--------------- @@ -135,24 +153,6 @@ endif Bpack += $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package) -#--------------- -# PeleMP sources -#--------------- -ifeq ($(USE_PARTICLES), TRUE) - ifdef SPRAY_FUEL_NUM - MPdirs := PP_Spray PP_Spray/Distribution - Bpack += $(foreach dir, $(MPdirs), $(PELEMP_HOME)/Source/$(dir)/Make.package) - Blocs += $(foreach dir, $(MPdirs), $(PELEMP_HOME)/Source/$(dir)) - endif -endif -ifeq ($(USE_SOOT), TRUE) - Bpack += $(PELEMP_HOME)/Source/Soot_Models/Make.package - Blocs += $(PELEMP_HOME)/Source/Soot_Models -endif -ifeq ($(USE_PELERAD), TRUE) - Blocs += $(PELERAD_HOME)/src -endif - #--------------- # Includes #--------------- diff --git a/README.md b/README.md index 481861e0..430d2abd 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,10 @@ Finally, when building with GPU support, CUDA >= 11 is required with NVIDIA GPUs ### Download The preferred method consists of cloning *PeleLMeX* and its submodules -([amrex](https://github.com/AMReX-Codes/amrex), [AMReX-Hydro](https://github.com/AMReX-Fluids/AMReX-Hydro), [PelePhysics](https://github.com/AMReX-Combustion/PelePhysics), [PeleMP](https://github.com/AMReX-Combustion/PeleMP)), and [SUNDIALS](https://github.com/LLNL/sundials) using a recursive `git clone`: +([PelePhysics](https://github.com/AMReX-Combustion/PelePhysics), +[amrex](https://github.com/AMReX-Codes/amrex), +[AMReX-Hydro](https://github.com/AMReX-Fluids/AMReX-Hydro), and +[SUNDIALS](https://github.com/LLNL/sundials) using a recursive `git clone`: ``` git clone --recursive --shallow-submodules --single-branch https://github.com/AMReX-Combustion/PeleLMeX.git @@ -82,11 +85,10 @@ setup the following environment variables (e.g. using bash) to any other locatio ``` export PELELMEX_HOME= -export AMREX_HOME=${PELELMEX_HOME}/Submodules/amrex export AMREX_HYDRO_HOME=${PELELMEX_HOME}/Submodules/AMReX-Hydro export PELE_PHYSICS_HOME=${PELELMEX_HOME}/Submodules/PelePhysics -export PELEMP_HOME=${PELELMEX_HOME}/Submodules/PeleMP -export SUNDIALS_HOME=${PELELMEX_HOME}/Submodules/sundials +export AMREX_HOME=${PELE_PHYSICS_HOME}/Submodules/amrex +export SUNDIALS_HOME=${PELE_PHYSICS_HOME}/Submodules/sundials ``` ### Compilation diff --git a/Submodules/PeleMP b/Submodules/PeleMP deleted file mode 160000 index 7264baa0..00000000 --- a/Submodules/PeleMP +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7264baa0c1250beb5f0cc1cc71fa2f753772fe16 diff --git a/Submodules/PelePhysics b/Submodules/PelePhysics index 769ae634..22637b33 160000 --- a/Submodules/PelePhysics +++ b/Submodules/PelePhysics @@ -1 +1 @@ -Subproject commit 769ae634f5841212fe6ffd8fd19b36e63cecf185 +Subproject commit 22637b33515d5fe27593aa9517db5d232891b11b diff --git a/Submodules/amrex b/Submodules/amrex deleted file mode 160000 index efd77ffa..00000000 --- a/Submodules/amrex +++ /dev/null @@ -1 +0,0 @@ -Subproject commit efd77ffab39c4e32e2b9e4a4fd446a49e22f0c9a diff --git a/Submodules/sundials b/Submodules/sundials deleted file mode 160000 index a4553b50..00000000 --- a/Submodules/sundials +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a4553b5021b7089b036f1edcfdd9457a7764d5a0 diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index d9879908..6c80b248 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -45,7 +45,7 @@ macro(setup_test) set(PLOT_TEST ${CURRENT_TEST_BINARY_DIR}/plt00010) # Find fcompare if(PELELMEX_ENABLE_FCOMPARE_FOR_TESTS) - set(FCOMPARE ${CMAKE_BINARY_DIR}/Submodules/AMReX/Tools/Plotfile/fcompare) + set(FCOMPARE ${CMAKE_BINARY_DIR}/Submodules/PelePhysics/Submodules/amrex/Tools/Plotfile/fcompare) endif() # Make working directory for test file(MAKE_DIRECTORY ${CURRENT_TEST_BINARY_DIR}) @@ -196,7 +196,7 @@ function(add_test_spray TEST_EXE_DIR) set(PELELMEX_NP 1) unset(MPI_COMMANDS) endif() - set(FCOMPARE ${CMAKE_BINARY_DIR}/Submodules/AMReX/Tools/Plotfile/fcompare) + set(FCOMPARE ${CMAKE_BINARY_DIR}/Submodules/PelePhysics/Submodules/amrex/Tools/Plotfile/fcompare) set(MULTIRUN_FLAGS "--test_dir ${CURRENT_TEST_BINARY_DIR} --run_cmd '${MPI_COMMANDS}' --pele_exec PeleLMeX-${TEST_EXE_DIR} --fcomp_exec ${FCOMPARE} --input_file ${TEST_NAME}.inp") file(GLOB TEST_FILES "${CURRENT_TEST_SOURCE_DIR}/*.dat" "${CURRENT_TEST_SOURCE_DIR}/*.py") # Copy files to test working directory diff --git a/Tests/CTestNightlyScript.cmake b/Tests/CTestNightlyScript.cmake index 75dc85c3..41a93058 100644 --- a/Tests/CTestNightlyScript.cmake +++ b/Tests/CTestNightlyScript.cmake @@ -55,7 +55,7 @@ set(CTEST_BUILD_COMMAND "${MAKE} ${CTEST_BUILD_FLAGS}") # ----------------------------------------------------------- #ctest_empty_binary_directory(${CTEST_BINARY_DIRECTORY}) - + message("\n -- Start dashboard - ${CTEST_BUILD_NAME} --") ctest_start("Nightly" TRACK "Nightly") @@ -69,15 +69,15 @@ if(USE_LATEST_AMREX) message("\n -- Update AMReX- ${CTEST_BUILD_NAME} --") execute_process ( COMMAND ${CTEST_GIT_COMMAND} checkout development - WORKING_DIRECTORY ${CTEST_SOURCE_DIRECTORY}/Submodules/amrex + WORKING_DIRECTORY ${CTEST_SOURCE_DIRECTORY}/Submodules/PelePhysics/Submodules/amrex ) execute_process ( COMMAND ${CTEST_GIT_COMMAND} pull origin development - WORKING_DIRECTORY ${CTEST_SOURCE_DIRECTORY}/Submodules/amrex + WORKING_DIRECTORY ${CTEST_SOURCE_DIRECTORY}/Submodules/PelePhysics/Submodules/amrex ) execute_process ( COMMAND ${CTEST_GIT_COMMAND} log -1 --pretty=oneline - WORKING_DIRECTORY ${CTEST_SOURCE_DIRECTORY}/Submodules/amrex + WORKING_DIRECTORY ${CTEST_SOURCE_DIRECTORY}/Submodules/PelePhysics/Submodules/amrex ) endif() diff --git a/Utils/CloneDeps.sh b/Utils/CloneDeps.sh index df124a3e..170df4cd 100755 --- a/Utils/CloneDeps.sh +++ b/Utils/CloneDeps.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash echo "Getting PeleLMeX dependencies - tests ... " -git submodule init -git submodule update +git submodule update --init --recursive