Skip to content

Commit

Permalink
Remove multiblock code and do not include main.cpp in library (#1874)
Browse files Browse the repository at this point in the history 
* remove multiblock container and evolve stuff

* remove multiblock container files from build, include main.cpp only in executable build and not in the lib build
  • Loading branch information
@mukul1992
mukul1992 authored Oct 12, 2024
1 parent 57e722d commit 4ce47d3
Show file tree
Hide file tree
Showing 4 changed files with 18 additions and 228 deletions.
33 changes: 13 additions & 20 deletions CMake/BuildERFExe.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,8 @@ function(build_erf_lib erf_lib_name)
target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_MOISTURE)

if(ERF_ENABLE_MULTIBLOCK)
target_sources(${erf_lib_name} PRIVATE
${SRC_DIR}/MultiBlock/ERF_MultiBlockContainer.cpp)
target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_MULTIBLOCK)
# if(NOT ERF_MB_EXTERN)
target_sources(${erf_lib_name} PRIVATE
${SRC_DIR}/MultiBlock/ERF_MultiBlockContainer.cpp)
target_include_directories(${erf_lib_name} PRIVATE $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/MultiBlock>)
# endif()
endif()
endif()

if(ERF_ENABLE_WARM_NO_PRECIP)
target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_WARM_NO_PRECIP)
Expand Down Expand Up @@ -91,10 +84,10 @@ function(build_erf_lib erf_lib_name)
${CMAKE_SOURCE_DIR}/Submodules/RRTMGP/cpp/extensions/fluxes_byband/mo_fluxes_byband_kernels.cpp
)

# The interface code needs to know about the RRTMGP includes
# The interface code needs to know about the RRTMGP includes
target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_RRTMGP)

target_include_directories(${erf_lib_name} SYSTEM PUBLIC
target_include_directories(${erf_lib_name} SYSTEM PUBLIC
${CMAKE_SOURCE_DIR}/Submodules/RRTMGP/cpp/extensions/fluxes_byband
${CMAKE_SOURCE_DIR}/Submodules/RRTMGP/cpp/extensions/cloud_optics
${CMAKE_SOURCE_DIR}/Submodules/RRTMGP/cpp/examples
Expand Down Expand Up @@ -161,7 +154,7 @@ function(build_erf_lib erf_lib_name)
${SRC_DIR}/PBL/ERF_ComputeDiffusivityMYNN25.cpp
${SRC_DIR}/PBL/ERF_ComputeDiffusivityYSU.cpp
${SRC_DIR}/SourceTerms/ERF_ApplySpongeZoneBCs.cpp
${SRC_DIR}/SourceTerms/ERF_ApplySpongeZoneBCs_ReadFromFile.cpp
${SRC_DIR}/SourceTerms/ERF_ApplySpongeZoneBCs_ReadFromFile.cpp
${SRC_DIR}/SourceTerms/ERF_make_buoyancy.cpp
${SRC_DIR}/SourceTerms/ERF_add_thin_body_sources.cpp
${SRC_DIR}/SourceTerms/ERF_make_mom_sources.cpp
Expand Down Expand Up @@ -205,13 +198,6 @@ function(build_erf_lib erf_lib_name)
${SRC_DIR}/LandSurfaceModel/MM5/ERF_MM5.cpp
)

if(NOT "${erf_exe_name}" STREQUAL "erf_unit_tests")
target_sources(${erf_lib_name}
PRIVATE
${SRC_DIR}/main.cpp
)
endif()

include(AMReXBuildInfo)
generate_buildinfo(${erf_lib_name} ${CMAKE_SOURCE_DIR})
if (${ERF_USE_INTERNAL_AMREX})
Expand Down Expand Up @@ -250,7 +236,7 @@ endif()
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/Microphysics>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/Microphysics/Null>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/Microphysics/SAM>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/Microphysics/Kessler>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/Microphysics/Kessler>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/WindFarmParametrization>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/WindFarmParametrization/Null>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/WindFarmParametrization/Fitch>)
Expand All @@ -261,7 +247,7 @@ endif()
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/LandSurfaceModel/Null>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/LandSurfaceModel/SLM>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/LandSurfaceModel/MM5>)

if(ERF_ENABLE_RRTMGP)
target_link_libraries(${erf_lib_name} PUBLIC yakl)
target_link_libraries(${erf_lib_name} PUBLIC rrtmgp)
Expand Down Expand Up @@ -296,6 +282,13 @@ function(build_erf_exe erf_exe_name)

set(SRC_DIR ${CMAKE_SOURCE_DIR}/Source)

if(NOT "${erf_exe_name}" STREQUAL "erf_unit_tests")
target_sources(${erf_exe_name}
PRIVATE
${SRC_DIR}/main.cpp
)
endif()

target_link_libraries(${erf_exe_name} PUBLIC ${erf_lib_name})
include(${CMAKE_SOURCE_DIR}/CMake/SetERFCompileFlags.cmake)
set_erf_compile_flags(${erf_exe_name})
Expand Down
10 changes: 5 additions & 5 deletions Source/ERF.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -368,9 +368,6 @@ public:
// Advance a block specified number of time steps
void Evolve_MB (int MBstep, int max_block_step);

// Advance a block specified number of time steps
void Evolve_MB (MultiBlockContainer* mbc, int MBstep, int max_block_step);

// get the current time values
amrex::Real get_t_old() {return t_old[0];}
amrex::Real get_t_new() {return t_new[0];}
Expand All @@ -383,15 +380,14 @@ public:

// Public data copy for MB
std::vector<amrex::Box> domain_p;
MultiBlockContainer *m_mbc = nullptr;
amrex::Vector<amrex::Vector<amrex::MultiFab> > vars_new;
amrex::Vector<amrex::Vector<amrex::MultiFab> > vars_old;

// Velocity time averaged field
amrex::Vector<std::unique_ptr<amrex::MultiFab>> vel_t_avg;
amrex::Vector<amrex::Real> t_avg_cnt;

#endif

std::string pp_prefix {"erf"};

void fill_from_bndryregs (const amrex::Vector<amrex::MultiFab*>& mfs,
Expand Down Expand Up @@ -947,6 +943,10 @@ private:

#endif

#ifdef ERF_USE_MULTIBLOCK
MultiBlockContainer *m_mbc = nullptr;
#endif

static int verbose;
static int mg_verbose;
static bool use_heffte;
Expand Down
99 changes: 0 additions & 99 deletions Source/ERF.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,6 @@
#include <ERF_TerrainMetrics.H>
#include <memory>

#ifdef ERF_USE_MULTIBLOCK
#ifndef ERF_MB_EXTERN // enter only if multiblock does not involve an external class
#include <ERF_MultiBlockContainer.H>
#else
#include <MultiBlockContainer.H>
#endif
#endif

using namespace amrex;

Real ERF::startCPUTime = 0.0;
Expand Down Expand Up @@ -568,17 +560,6 @@ ERF::InitData ()
{
BL_PROFILE_VAR("ERF::InitData()", InitData);
InitData_pre();
#if 0
#ifdef ERF_USE_MULTIBLOCK
#ifndef ERF_MB_EXTERN // enter only if multiblock does not involve an external class
// Multiblock: hook to set BL & comms once ba/dm are known
if(domain_p[0].bigEnd(0) < 500 ) {
m_mbc->SetBoxLists();
m_mbc->SetBlockCommMetaData();
}
#endif
#endif
#endif
InitData_post();
BL_PROFILE_VAR_STOP(InitData);
}
Expand Down Expand Up @@ -2010,86 +1991,6 @@ ERF::ERF (const RealBox& rb, int max_level_in,
}
#endif

#ifdef ERF_USE_MULTIBLOCK
// advance solution over specified block steps
void
ERF::Evolve_MB (int MBstep, int max_block_step)
{
Real cur_time = t_new[0];

int step;

// Take one coarse timestep by calling timeStep -- which recursively calls timeStep
// for finer levels (with or without subcycling)
for (int Bstep(0); Bstep < max_block_step && cur_time < stop_time; ++Bstep)
{
step = Bstep + MBstep - 1;

Print() << "\nCoarse STEP " << step+1 << " starts ..." << std::endl;

ComputeDt(step);

// Make sure we have read enough of the boundary plane data to make it through this timestep
if (input_bndry_planes)
{
m_r2d->read_input_files(cur_time,dt[0],m_bc_extdir_vals);
}

int lev = 0;
int iteration = 1;
timeStep(lev, cur_time, iteration);

#ifndef ERF_MB_EXTERN
// DEBUG
// Multiblock: hook for erf2 to fill from erf1
if(domain_p[0].bigEnd(0) < 500) {
for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx)
m_mbc->FillPatchBlocks(var_idx,var_idx);
}
#endif

cur_time += dt[0];

Print() << "Coarse STEP " << step+1 << " ends." << " TIME = " << cur_time
<< " DT = " << dt[0] << std::endl;

post_timestep(step, cur_time, dt[0]);

if (writeNow(cur_time, dt[0], step+1, m_plot_int_1, m_plot_per_1)) {
last_plot_file_step_1 = step+1;
WritePlotFile(1,plot_var_names_1);
}

if (writeNow(cur_time, dt[0], step+1, m_plot_int_2, m_plot_per_2)) {
last_plot_file_step_2 = step+1;
WritePlotFile(2,plot_var_names_2);
}

if (writeNow(cur_time, dt[0], step+1, m_check_int, m_check_per)) {
last_check_file_step = step+1;
#ifdef ERF_USE_NETCDF
if (check_type == "netcdf") {
WriteNCCheckpointFile();
}
#endif
if (check_type == "native") {
WriteCheckpointFile();
}
}

#ifdef AMREX_MEM_PROFILING
{
std::ostringstream ss;
ss << "[STEP " << step+1 << "]";
MemProfiler::report(ss.str());
}
#endif

if (cur_time >= stop_time - 1.e-6*dt[0]) break;
}
}
#endif

bool
ERF::writeNow(const Real cur_time, const Real dt_lev, const int nstep, const int plot_int, const Real plot_per)
{
Expand Down
104 changes: 0 additions & 104 deletions Source/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,6 @@
//#include "IO.H"
#include "ERF.H"

#ifdef ERF_USE_MULTIBLOCK
#ifndef ERF_MB_EXTERN // enter only if multiblock does not involve an external class
#include <ERF_MultiBlockContainer.H>
#else
#include <MultiBlockContainer.H>
#endif
#endif

#ifdef ERF_USE_WW3_COUPLING
#include <mpi.h>
#include <AMReX_MPMD.H>
Expand Down Expand Up @@ -124,101 +116,6 @@ int main (int argc, char* argv[])
// wallclock time
const Real strt_total = amrex::second();

#ifdef ERF_USE_MULTIBLOCK
{
// Vector of constructor parameters for MultiBlock
std::vector<RealBox> rb_v;
std::vector<int> max_level_v;
std::vector<int> coord_v;
std::vector<amrex::Vector<int>> n_cell_v;
std::vector<amrex::Array<int,AMREX_SPACEDIM>> is_per_v;
std::vector<amrex::Vector<amrex::IntVect>> ref_rat_v;
std::vector<std::string> prefix_v;
int max_step{1};

// Local constructor parameters for vector
RealBox rb;
int max_level{0};
int coord{0};
amrex::Vector<int> n_cell = {1,1,1};
amrex::Array<int,AMREX_SPACEDIM> is_per = {1,1,1};
amrex::Vector<amrex::IntVect> ref_rat = {amrex::IntVect(1,1,1)};

// Parse max steps for the block
{
ParmParse pp;
pp.query("max_step", max_step);
}

// Parse data for erf1 constructor
{
ParmParse pp("erf1");
amrex::Vector<Real> lo = {0.,0.,0.};
amrex::Vector<Real> hi = {0.,0.,0.};
amrex::Vector<int> periodicity = {1,1,1};
pp.queryarr("prob_lo",lo);
pp.queryarr("prob_hi",hi);
rb.setLo(lo);
rb.setHi(hi);
pp.query("max_level",max_level);
pp.query("coord",coord);
pp.queryarr("n_cell",n_cell);
pp.queryarr("is_periodic",periodicity);
{
for( int i(0); i<AMREX_SPACEDIM; i++ ) is_per[i] = periodicity[i];
}
pp.queryarr("ref_ratio",ref_rat);

rb_v.push_back(rb);
max_level_v.push_back(max_level);
coord_v.push_back(coord);
n_cell_v.push_back(n_cell);
is_per_v.push_back(is_per);
ref_rat_v.push_back(ref_rat);
prefix_v.push_back("erf1");
}

// Parse data for erf2 constructor
{
ParmParse pp("erf2");
amrex::Vector<Real> lo = {0.,0.,0.};
amrex::Vector<Real> hi = {0.,0.,0.};
amrex::Vector<int> periodicity = {1,1,1};
pp.queryarr("prob_lo",lo);
pp.queryarr("prob_hi",hi);
rb.setLo(lo);
rb.setHi(hi);
pp.query("max_level",max_level);
pp.query("coord",coord);
pp.queryarr("n_cell",n_cell);
pp.queryarr("is_periodic",periodicity);
{
for( int i(0); i<AMREX_SPACEDIM; i++ ) is_per[i] = periodicity[i];
}
pp.queryarr("ref_ratio",ref_rat);

rb_v.push_back(rb);
max_level_v.push_back(max_level);
coord_v.push_back(coord);
n_cell_v.push_back(n_cell);
is_per_v.push_back(is_per);
ref_rat_v.push_back(ref_rat);
prefix_v.push_back("erf2");

}

// Construct a MultiBlockContainer
MultiBlockContainer mbc(rb_v, max_level_v, n_cell_v,
coord_v, ref_rat_v, is_per_v,
prefix_v, max_step);

// Initialize data
mbc.InitializeBlocks();

// Advance blocks a timestep
mbc.AdvanceBlocks();
}
#else
{
// constructor - reads in parameters from inputs file
// - sizes multilevel arrays and data structures
Expand All @@ -239,7 +136,6 @@ int main (int argc, char* argv[])
amrex::Print() << "\nTotal Time: " << end_total << '\n';
}
}
#endif

// destroy timer for profiling
BL_PROFILE_VAR_STOP(pmain);
Expand Down

0 comments on commit 4ce47d3

Please sign in to comment.