Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

deprecate write_tree_flag in favor of direct method arguments #163

Merged
merged 19 commits into from
Apr 22, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
be8661c
remove commented out code
joeljonsson Jan 9, 2022
8e80969
remove unused structs
joeljonsson Jan 9, 2022
0ca4eed
remove use of with_tree_flag in open_time_point
joeljonsson Jan 9, 2022
186fe16
rename parameter tree -> with_tree in create_time_point
joeljonsson Jan 9, 2022
f73ea40
remove use of with_tree_flag in LazyData::init_file
joeljonsson Jan 9, 2022
2267678
add write_tree argument to write_apr
joeljonsson Jan 9, 2022
c1c4927
remove use of set_read_write_tree in bench_apr_io
joeljonsson Jan 9, 2022
efabf3e
remove usage of set_read_write_tree, add argument to write_apr where …
joeljonsson Jan 9, 2022
7f84da5
remove automatic tree initialization from read_apr, add check to APR:…
joeljonsson Mar 23, 2022
db8443e
add dense LinearAccess tree initialization method
joeljonsson Apr 8, 2022
2f1991a
refactor tree initialization method names
joeljonsson Apr 8, 2022
bc530f1
inline dense tree init to avoid multiple definitions errors with cuda
joeljonsson Apr 8, 2022
ec7f3ab
seems dense random tree initialization is broken
joeljonsson Apr 8, 2022
0524726
add template initializations for additional data types
joeljonsson Apr 8, 2022
2d96379
update submodule vcpkg
joeljonsson Apr 8, 2022
17eea48
make initialize_tree_linear public, allow choosing between sparse and…
joeljonsson Apr 8, 2022
5c59bbf
refactor initialization methods, move checks from iterators
joeljonsson Apr 8, 2022
f98c4fc
remove duplicate checks
joeljonsson Apr 8, 2022
b5c34a0
set linearAccess.genInfo in random->linear initialization
joeljonsson Apr 9, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions benchmarks/BenchIO.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -106,13 +106,11 @@ inline void bench_apr_io(APR& apr,ParticleData<partsType>& parts,int num_rep,Ana

aprFile.open(file_name,"WRITE");

aprFile.set_read_write_tree(false);

for (int r = 0; r < num_rep; ++r) {

timer_steps.start_timer("write_apr");

aprFile.write_apr(apr,r);
aprFile.write_apr(apr, r, "t", false);

timer_steps.stop_timer();

Expand All @@ -132,8 +130,6 @@ inline void bench_apr_io(APR& apr,ParticleData<partsType>& parts,int num_rep,Ana

aprFile.open(file_name,"READ");

aprFile.set_read_write_tree(false);

for (int r = 0; r < num_rep; ++r) {

timer_steps.start_timer("read_apr");
Expand Down
4 changes: 1 addition & 3 deletions benchmarks/BenchPipeline.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,6 @@ inline void bench_apr_pipeline(APR& apr,ParticleData<partsType>& parts,int num_r

aprFile.open("file_name","WRITE");

aprFile.set_read_write_tree(false);

for (int r = 0; r < num_rep; ++r) {

timer_steps.start_timer("get_apr");
Expand All @@ -104,7 +102,7 @@ inline void bench_apr_pipeline(APR& apr,ParticleData<partsType>& parts,int num_r

timer_steps.start_timer("write_apr");

aprFile.write_apr(apr_vec[r],r);
aprFile.write_apr(apr_vec[r], r, "t", false);

timer_steps.stop_timer();

Expand Down
4 changes: 1 addition & 3 deletions examples/Example_get_apr.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,7 @@ int runAPR(cmdLineOptions options) {

aprFile.open(save_loc + file_name + ".apr");

aprFile.set_read_write_tree(false); //not writing tree to file.

aprFile.write_apr(apr);
aprFile.write_apr(apr, 0, "t", options.store_tree);
aprFile.write_particles("particles",particle_intensities);

float apr_file_size = aprFile.current_file_size_MB();
Expand Down
3 changes: 1 addition & 2 deletions examples/Example_get_apr_by_block.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,8 @@ int runAPR(cmdLineOptions options) {
timer.start_timer("write output to file");
APRFile aprFile;
aprFile.open(save_loc + file_name + ".apr");
aprFile.set_read_write_tree(options.store_tree);

aprFile.write_apr(apr);
aprFile.write_apr(apr, 0, "t", options.store_tree);
aprFile.write_particles("particles", parts);
timer.stop_timer();

Expand Down
1 change: 0 additions & 1 deletion examples/Example_lazy_access.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ int main(int argc, char **argv) {

// open APR file for reading
APRFile aprFile;
aprFile.set_read_write_tree(false);
aprFile.open(file_name, "READ");

// initialize lazy access and iterator for spatial information
Expand Down
192 changes: 125 additions & 67 deletions src/data_structures/APR/APR.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,16 @@ class APR {

protected:

//APR Tree function
void initialize_apr_tree_sparse();
void initialize_apr_tree_sparse_linear();
void initialize_apr_tree();
void initialize_linear_access(LinearAccess& aprAccess,APRIterator& it);
// initialize tree RandomAccess
void initialize_tree_random_sparse();
void initialize_tree_random_dense(); // appears to be broken #TODO: remove or fix
joeljonsson marked this conversation as resolved.
Show resolved Hide resolved

// initialize tree LinearAccess
void initialize_tree_sparse();
void initialize_tree_dense();

void initialize_linear_access_from_random(LinearAccess& aprAccess, APRIterator& it);
void initialize_random_access_from_linear();

//New Access
LinearAccess linearAccess;
Expand Down Expand Up @@ -62,14 +67,9 @@ class APR {

GPUAccessHelper gpuAPRHelper(){
if(!apr_initialized){
if(!apr_initialized_random){
std::cerr << "No APR initialized" << std::endl;

} else {
initialize_linear();
apr_initialized = true;
}
initialize_linear();
}

return GPUAccessHelper(gpuAccess,linearAccess);
}

Expand Down Expand Up @@ -104,7 +104,12 @@ class APR {
inline uint64_t total_number_particles() const { return aprInfo.total_number_particles; }
uint64_t org_dims(int dim) const { return aprInfo.org_dims[dim]; }

inline uint64_t total_number_tree_particles() const { return treeInfo.total_number_particles; }
inline uint64_t total_number_tree_particles() {
if(!tree_initialized && !tree_initialized_random) {
initialize_tree_linear();
}
return treeInfo.total_number_particles;
}

inline int number_dimensions() const {
return aprInfo.number_dimensions;
Expand All @@ -118,36 +123,23 @@ class APR {
APRIterator random_iterator() {

if(!apr_initialized_random){
initialize_random_access();
apr_initialized_random = true;
initialize_random();
}

return APRIterator(apr_access,aprInfo);
}

LinearIterator iterator() {
// Just checking if its initialized
if(!apr_initialized){
if(!apr_initialized_random){
std::cerr << "No APR initialized" << std::endl;

} else {
initialize_linear();
apr_initialized = true;
}
if(!apr_initialized){
initialize_linear();
}

return LinearIterator(linearAccess,aprInfo);
}

APRTreeIterator random_tree_iterator() {

if(!apr_initialized_random){
//the random tree iterator and the apr iteratator are joint at the hip.
initialize_random_access();
apr_initialized_random = true;
}

if(!tree_initialized_random){
initialize_tree_random();
}
Expand All @@ -156,7 +148,7 @@ class APR {
}

LinearIterator tree_iterator() {
// Checking if initialized.

if(!tree_initialized){
initialize_tree_linear();
}
Expand Down Expand Up @@ -191,54 +183,64 @@ class APR {

}


protected:

bool initialize_tree_random(){
if(!tree_initialized_random){

initialize_apr_tree_sparse();
tree_initialized_random = true;

void initialize_linear(){
if(!apr_initialized){
if(!apr_initialized_random) {
std::cerr << "APR::initialize_linear - No APR initialized" << std::endl;
} else {
auto it = random_iterator();
initialize_linear_access_from_random(linearAccess, it);
apr_initialized = true;
}
}
return tree_initialized_random;
}

void initialize_linear(){
if(!apr_initialized){
auto it = random_iterator();
initialize_linear_access(linearAccess,it);
apr_initialized = true;
void initialize_random() {
if(!apr_initialized_random) {
if(!apr_initialized) {
std::cerr << "APR::initialize_random - No APR initialized" << std::endl;
} else {
initialize_random_access_from_linear();
apr_initialized_random = true;
}
}
}

void initialize_tree_linear(){
void initialize_tree_linear(bool sparse=false){

if(!tree_initialized){
initialize_apr_tree_sparse_linear();
if(sparse) {
initialize_tree_sparse();
} else {
initialize_tree_dense();
}
tree_initialized = true;
}
}

void initialize_random_access();
void initialize_tree_random(){

if(!tree_initialized_random){
initialize_tree_random_sparse();
tree_initialized_random = true;
}
}

};

#define INTERIOR_PARENT 9


/**
* Initializes linear access apr structures, that require more memory, but are faster. However, the do not allow the same neighbour access as the random iterators
*/
void APR::initialize_linear_access(LinearAccess& aprAccess,APRIterator& it){

// TODO: Should be renamed.. random-> linear access. also need the reverse function

auto& lin_a = aprAccess;
* Initializes linear access apr structures from the random access data. The linear structure is faster in most
* cases, but require more memory and do not allow the same neighbor access as the random access iterators.
*/
void APR::initialize_linear_access_from_random(LinearAccess& lin_a, APRIterator& it){

uint64_t counter = 0;
uint64_t counter_xz = 1;

lin_a.genInfo = &aprInfo;
lin_a.initialize_xz_linear();

lin_a.y_vec.resize(it.total_number_particles());
Expand All @@ -256,21 +258,17 @@ void APR::initialize_linear_access(LinearAccess& aprAccess,APRIterator& it){
counter++;
}


lin_a.xz_end_vec[counter_xz] = (counter);
lin_a.xz_end_vec[counter_xz] = counter;
counter_xz++;
}
}

}


}

/**
* Initializes linear access apr structures, that require more memory, but are faster. However, the do not allow the same neighbour access as the random iterators
*/
void APR::initialize_random_access(){
void APR::initialize_random_access_from_linear(){
//
// TODO: Note this is not performance orientataed, and should be depreciated in the future. (the whole random access iterations should be removed.)
//
Expand Down Expand Up @@ -335,17 +333,16 @@ void APR::initialize_random_access(){
}

apr_access.initialize_structure_from_particle_cell_tree_sparse(parameters,particle_cell_tree);

}


/**
* Initializes the APR tree datastructures using a dense structure for memory, these are all particle cells that are parents of particles in the APR
* , alternatively can be thought of as the interior nodes of an APR represented as a binary tree.
*/
void APR::initialize_apr_tree() {
void APR::initialize_tree_random_dense() {

APRTimer timer(true);
APRTimer timer(false);

auto apr_iterator = iterator();

Expand Down Expand Up @@ -465,7 +462,7 @@ void APR::initialize_apr_tree() {

}

void APR::initialize_apr_tree_sparse_linear() {
void APR::initialize_tree_sparse() {

APRTimer timer(false);

Expand Down Expand Up @@ -607,15 +604,76 @@ void APR::initialize_apr_tree_sparse_linear() {

}


void APR::initialize_tree_dense() {

APRTimer timer(false);
auto apr_iterator = iterator();

treeInfo.init_tree(org_dims(0),org_dims(1),org_dims(2));
linearAccessTree.genInfo = &treeInfo;

std::vector<PixelData<uint8_t>> particle_cell_parent_tree(treeInfo.l_max+1);

timer.start_timer("init tree - allocate dense tree structure");
for (int l = treeInfo.l_min; l <= treeInfo.l_max; ++l) {

particle_cell_parent_tree[l].initWithValue(treeInfo.y_num[l],
treeInfo.x_num[l],
treeInfo.z_num[l],
0);
}
timer.stop_timer();

timer.start_timer("init tree - fill particle parents");
// fill in parents of APR particles
for(int level = apr_iterator.level_max(); level >= apr_iterator.level_min(); --level) {
#ifdef HAVE_OPENMP
#pragma omp parallel for schedule(dynamic, 2) firstprivate(apr_iterator)
#endif
for(int z = 0; z < apr_iterator.z_num(level); ++z) {
for(int x = 0; x < apr_iterator.x_num(level); ++x) {
for(apr_iterator.begin(level, z, x); apr_iterator < apr_iterator.end(); ++apr_iterator) {
particle_cell_parent_tree[level-1].at(apr_iterator.y() / 2, x / 2, z / 2) = 1;
}
}
}
}
timer.stop_timer();

timer.start_timer("init tree - fill tree recursive");
// fill rest of tree, level by level
for(int level = treeInfo.l_max-1; level >= treeInfo.l_min; --level) {
#ifdef HAVE_OPENMP
#pragma omp parallel for schedule(static)
#endif
for(int z = 0; z < treeInfo.z_num[level]; ++z) {
for (int x = 0; x < treeInfo.x_num[level]; ++x) {
for(int y = 0; y < treeInfo.y_num[level]; ++y) {
// suffices to check one child
particle_cell_parent_tree[level].at(y, x, z) = particle_cell_parent_tree[level].at(y, x, z) | particle_cell_parent_tree[level+1].at(2*y, 2*x, 2*z);
}
}
}
}
timer.stop_timer();

timer.start_timer("create sparse data structure");
linearAccessTree.initialize_tree_access_dense(particle_cell_parent_tree);
timer.stop_timer();
}



/**
* Initializes the APR tree datastructures using a sparse structure for reduced memory, these are all particle cells that are parents of particles in the APR
* , alternatively can be thought of as the interior nodes of an APR represented as a binary tree.
*/
void APR::initialize_apr_tree_sparse() {
void APR::initialize_tree_random_sparse() {

APRTimer timer(false);

auto apr_iterator = iterator();
auto apr_iterator = random_iterator();

//need to create a local copy

Expand Down
Loading