From be8661c517fe323c3ec0f74f3f09b0282d61cd13 Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Sun, 9 Jan 2022 19:42:28 +0100 Subject: [PATCH 01/19] remove commented out code --- src/io/APRWriter.hpp | 571 ------------------------------------------- 1 file changed, 571 deletions(-) diff --git a/src/io/APRWriter.hpp b/src/io/APRWriter.hpp index 7f576000..63a7ba6a 100644 --- a/src/io/APRWriter.hpp +++ b/src/io/APRWriter.hpp @@ -578,577 +578,6 @@ class APRWriter { } -// void read_apr(APR& apr, const std::string &file_name,bool build_tree = false, int max_level_delta = 0,unsigned int time_point = UINT32_MAX) { -// -// APRTimer timer; -// timer.verbose_flag = false; -// -// APRTimer timer_f; -// timer_f.verbose_flag = false; -// -// FileStructure::Operation op; -// -// if(build_tree){ -// op = FileStructure::Operation::READ_WITH_TREE; -// } else { -// op = FileStructure::Operation::READ; -// } -// -// unsigned int t = 0; -// -// if(time_point!=UINT32_MAX){ -// t = time_point; -// } -// -// FileStructure f(file_name, op); -// -// hid_t meta_data = f.groupId; -// -// //check if old or new file, for location of the properties. (The metadata moved to the time point.) -// bool old = attribute_exists(f.objectId,AprTypes::MaxLevelType.typeName); -// -// if(old) { -// meta_data = f.objectId; -// } -// -// if (!f.isOpened()) return; -// -// // ------------- read metadata -------------------------- -// char string_out[100] = {0}; -// hid_t attr_id = H5Aopen(meta_data,"name",H5P_DEFAULT); -// hid_t atype = H5Aget_type(attr_id); -// hid_t atype_mem = H5Tget_native_type(atype, H5T_DIR_ASCEND); -// H5Aread(attr_id, atype_mem, string_out) ; -// H5Aclose(attr_id); -// apr.name= string_out; -// -// // check if the APR structure has already been read in, the case when a partial load has been done, now only the particles need to be read -// uint64_t old_particles = apr.apr_access.total_number_particles; -// uint64_t old_gaps = apr.apr_access.total_number_gaps; -// -// //read in access information -// read_access_info(meta_data,apr.apr_access); -// -// int compress_type; -// readAttr(AprTypes::CompressionType, meta_data, &compress_type); -// float quantization_factor; -// readAttr(AprTypes::QuantizationFactorType, meta_data, &quantization_factor); -// -// bool read_structure = true; -// -// if((old_particles == apr.apr_access.total_number_particles) && (old_gaps == apr.apr_access.total_number_gaps) && (build_tree)){ -// read_structure = false; -// } -// -// //incase you ask for to high a level delta -// max_level_delta = std::max(max_level_delta,0); -// max_level_delta = std::min(max_level_delta,(int)apr.apr_access.level_max()); -// -// if(read_structure) { -// //read in pipeline parameters -// read_apr_parameters(meta_data,apr.parameters); -// -// // ------------- map handling ---------------------------- -// -// timer.start_timer("map loading data"); -// -// auto map_data = std::make_shared(); -// -// map_data->global_index.resize(apr.apr_access.total_number_non_empty_rows); -// -// timer_f.start_timer("index"); -// std::vector index_delta(apr.apr_access.total_number_non_empty_rows); -// readData(AprTypes::MapGlobalIndexType, f.objectId, index_delta.data()); -// std::vector index_delta_big(apr.apr_access.total_number_non_empty_rows); -// std::copy(index_delta.begin(), index_delta.end(), index_delta_big.begin()); -// std::partial_sum(index_delta_big.begin(), index_delta_big.end(), map_data->global_index.begin()); -// -// timer_f.stop_timer(); -// -// timer_f.start_timer("y_b_e"); -// map_data->y_end.resize(apr.apr_access.total_number_gaps); -// readData(AprTypes::MapYendType, f.objectId, map_data->y_end.data()); -// map_data->y_begin.resize(apr.apr_access.total_number_gaps); -// readData(AprTypes::MapYbeginType, f.objectId, map_data->y_begin.data()); -// -// timer_f.stop_timer(); -// -// -// timer_f.start_timer("zxl"); -// map_data->number_gaps.resize(apr.apr_access.total_number_non_empty_rows); -// readData(AprTypes::MapNumberGapsType, f.objectId, map_data->number_gaps.data()); -// map_data->level.resize(apr.apr_access.total_number_non_empty_rows); -// readData(AprTypes::MapLevelType, f.objectId, map_data->level.data()); -// map_data->x.resize(apr.apr_access.total_number_non_empty_rows); -// readData(AprTypes::MapXType, f.objectId, map_data->x.data()); -// map_data->z.resize(apr.apr_access.total_number_non_empty_rows); -// readData(AprTypes::MapZType, f.objectId, map_data->z.data()); -// timer_f.stop_timer(); -// -// timer.stop_timer(); -// -// timer.start_timer("map building"); -// -// apr.apr_access.rebuild_map(*map_data); -// -// timer.stop_timer(); -// } -// -//// uint64_t max_read_level = apr.apr_access.level_max()-max_level_delta; -//// uint64_t max_read_level_tree = std::min(apr.apr_access.level_max()-1,max_read_level); -//// uint64_t prev_read_level = 0; -// -//// if(build_tree){ -//// -//// -//// if(read_structure) { -//// -//// -//// timer.start_timer("build tree - map"); -//// -//// apr.apr_tree.tree_access.l_max = apr.level_max() - 1; -//// apr.apr_tree.tree_access.l_min = apr.level_min() - 1; -//// -//// apr.apr_tree.tree_access.x_num.resize(apr.apr_tree.tree_access.level_max() + 1); -//// apr.apr_tree.tree_access.z_num.resize(apr.apr_tree.tree_access.level_max() + 1); -//// apr.apr_tree.tree_access.y_num.resize(apr.apr_tree.tree_access.level_max() + 1); -//// -//// for (int i = apr.apr_tree.tree_access.level_min(); i <= apr.apr_tree.tree_access.level_max(); ++i) { -//// apr.apr_tree.tree_access.x_num[i] = apr.spatial_index_x_max(i); -//// apr.apr_tree.tree_access.y_num[i] = apr.spatial_index_y_max(i); -//// apr.apr_tree.tree_access.z_num[i] = apr.spatial_index_z_max(i); -//// } -//// -//// apr.apr_tree.tree_access.x_num[apr.level_min() - 1] = ceil(apr.spatial_index_x_max(apr.level_min()) / 2.0f); -//// apr.apr_tree.tree_access.y_num[apr.level_min() - 1] = ceil(apr.spatial_index_y_max(apr.level_min()) / 2.0f); -//// apr.apr_tree.tree_access.z_num[apr.level_min() - 1] = ceil(apr.spatial_index_z_max(apr.level_min()) / 2.0f); -//// -//// readAttr(AprTypes::TotalNumberOfParticlesType, f.objectIdTree, -//// &apr.apr_tree.tree_access.total_number_particles); -//// readAttr(AprTypes::TotalNumberOfGapsType, f.objectIdTree, &apr.apr_tree.tree_access.total_number_gaps); -//// readAttr(AprTypes::TotalNumberOfNonEmptyRowsType, f.objectIdTree, -//// &apr.apr_tree.tree_access.total_number_non_empty_rows); -//// -//// auto map_data_tree = std::make_shared(); -//// -//// map_data_tree->global_index.resize(apr.apr_tree.tree_access.total_number_non_empty_rows); -//// -//// -//// std::vector index_delta(apr.apr_tree.tree_access.total_number_non_empty_rows); -//// readData(AprTypes::MapGlobalIndexType, f.objectIdTree, index_delta.data()); -//// std::vector index_delta_big(apr.apr_tree.tree_access.total_number_non_empty_rows); -//// std::copy(index_delta.begin(), index_delta.end(), index_delta_big.begin()); -//// std::partial_sum(index_delta_big.begin(), index_delta_big.end(), map_data_tree->global_index.begin()); -//// -//// -//// map_data_tree->y_end.resize(apr.apr_tree.tree_access.total_number_gaps); -//// readData(AprTypes::MapYendType, f.objectIdTree, map_data_tree->y_end.data()); -//// map_data_tree->y_begin.resize(apr.apr_tree.tree_access.total_number_gaps); -//// readData(AprTypes::MapYbeginType, f.objectIdTree, map_data_tree->y_begin.data()); -//// -//// map_data_tree->number_gaps.resize(apr.apr_tree.tree_access.total_number_non_empty_rows); -//// readData(AprTypes::MapNumberGapsType, f.objectIdTree, map_data_tree->number_gaps.data()); -//// map_data_tree->level.resize(apr.apr_tree.tree_access.total_number_non_empty_rows); -//// readData(AprTypes::MapLevelType, f.objectIdTree, map_data_tree->level.data()); -//// map_data_tree->x.resize(apr.apr_tree.tree_access.total_number_non_empty_rows); -//// readData(AprTypes::MapXType, f.objectIdTree, map_data_tree->x.data()); -//// map_data_tree->z.resize(apr.apr_tree.tree_access.total_number_non_empty_rows); -//// readData(AprTypes::MapZType, f.objectIdTree, map_data_tree->z.data()); -//// -//// apr.apr_tree.tree_access.rebuild_map_tree(*map_data_tree,apr.apr_access); -//// -//// //Important needs linking to the APR -//// apr.apr_tree.APROwn = &apr; -//// -//// timer.stop_timer(); -//// } -//// -//// if(!read_structure) { -//// uint64_t current_parts_size = apr.apr_tree.particles_ds_tree.data.size(); -//// -//// for (int j = apr.level_min(); j 0){ -//// parts_start = apr.apr_tree.tree_access.global_index_by_level_end[prev_read_level] + 1; -//// } -//// uint64_t parts_end = apr.apr_tree.tree_access.global_index_by_level_end[max_read_level_tree] + 1; -//// -//// apr.apr_tree.particles_ds_tree.data.resize(parts_end); -//// -//// if ( apr.apr_tree.particles_ds_tree.data.size() > 0) { -//// readData(AprTypes::ParticleIntensitiesType, f.objectIdTree, apr.apr_tree.particles_ds_tree.data.data() + parts_start,parts_start,parts_end); -//// } -//// -//// APRCompress apr_compress; -//// apr_compress.set_compression_type(1); -//// apr_compress.set_quantization_factor(2); -//// apr_compress.decompress(apr, apr.apr_tree.particles_ds_tree,parts_start); -//// -//// timer.stop_timer(); -//// -//// } -// -//// uint64_t parts_start = 0; -//// uint64_t parts_end = apr.apr_access.global_index_by_level_end[max_read_level] + 1; -//// -//// prev_read_level = 0; -//// if(!read_structure) { -//// uint64_t current_parts_size = apr.total_number_particles(); -//// -//// for (int j = apr.level_min(); j 0){ -//// parts_start = apr.apr_access.global_index_by_level_end[prev_read_level] + 1; -//// } -// -// //apr.apr_access.level_max = max_read_level; -// -//// timer.start_timer("Read intensities"); -//// // ------------- read data ------------------------------ -//// apr.particles_intensities.data.resize(parts_end); -//// if (apr.particles_intensities.data.size() > 0) { -//// readData(AprTypes::ParticleIntensitiesType, f.objectId, apr.particles_intensities.data.data() + parts_start,parts_start,parts_end); -//// } -// -// -// timer.stop_timer(); -// -// //std::cout << "Data rate intensities: " << (apr.particles_intensities.data.size()*2)/(timer.timings.back()*1000000.0f) << " MB/s" << std::endl; -// -// -// timer.start_timer("decompress"); -// // ------------ decompress if needed --------------------- -// if (compress_type > 0) { -//// APRCompress apr_compress; -//// apr_compress.set_compression_type(compress_type); -//// apr_compress.set_quantization_factor(quantization_factor); -//// apr_compress.decompress(apr, apr.particles_intensities,parts_start); -// } -// timer.stop_timer(); -// } - - -// FileSizeInfo write_apr(APR& apr, const std::string &save_loc, const std::string &file_name) { -// APRCompress apr_compressor; -// apr_compressor.set_compression_type(0); -// return write_apr(apr, save_loc, file_name, apr_compressor); -// } -// -// FileSizeInfo write_apr_append(APR& apr, const std::string &save_loc, const std::string &file_name) { -// APRCompress apr_compressor; -// apr_compressor.set_compression_type(0); -// return write_apr(apr, save_loc, file_name, apr_compressor,BLOSC_ZSTD,4,1,false,true); -// } - - - - /** - * Writes the APR to the particle cell structure sparse format, using the p_map for reconstruction - */ - -// FileSizeInfo write_apr(APR &apr, const std::string &save_loc, const std::string &file_name, APRCompress &apr_compressor, unsigned int blosc_comp_type = BLOSC_ZSTD, unsigned int blosc_comp_level = 2, unsigned int blosc_shuffle=1,bool write_tree = false,bool append_apr_time = true) { -// APRTimer write_timer; -// write_timer.verbose_flag = true; -// -// std::string hdf5_file_name = save_loc + file_name + ".h5"; -// -// -// FileStructure::Operation op; -// -// if(write_tree){ -// op = FileStructure::Operation::WRITE_WITH_TREE; -// } else { -// op = FileStructure::Operation::WRITE; -// } -// -// uint64_t t = 0; -// -// if(append_apr_time){ -// t = current_t; -// current_t++; -// } -// -// FileStructure f(hdf5_file_name, op); -// -// FileSizeInfo fileSizeInfo1; -// if (!f.isOpened()) return fileSizeInfo1; -// -// hid_t meta_location = f.groupId; -// -// if(append_apr_time){ -// meta_location = f.objectId; -// } -// -// //global property -// writeAttr(AprTypes::TimeStepType, f.groupId, &t); -// -// // ------------- write metadata ------------------------- -// //per time step -// -// writeAttr(AprTypes::NumberOfXType, meta_location, &apr.apr_access.org_dims[1]); -// writeAttr(AprTypes::NumberOfYType, meta_location, &apr.apr_access.org_dims[0]); -// writeAttr(AprTypes::NumberOfZType, meta_location, &apr.apr_access.org_dims[2]); -// writeAttr(AprTypes::TotalNumberOfGapsType, meta_location, &apr.apr_access.total_number_gaps); -// writeAttr(AprTypes::TotalNumberOfNonEmptyRowsType, meta_location, &apr.apr_access.total_number_non_empty_rows); -// -// writeString(AprTypes::NameType,meta_location, (apr.name.size() == 0) ? "no_name" : apr.name); -// writeString(AprTypes::GitType, meta_location, ConfigAPR::APR_GIT_HASH); -// writeAttr(AprTypes::TotalNumberOfParticlesType, meta_location, &apr.apr_access.total_number_particles); -// writeAttr(AprTypes::MaxLevelType, meta_location, &apr.apr_access.l_max); -// writeAttr(AprTypes::MinLevelType, meta_location, &apr.apr_access.l_min); -// -// int compress_type_num = apr_compressor.get_compression_type(); -// writeAttr(AprTypes::CompressionType, meta_location, &compress_type_num); -// float quantization_factor = apr_compressor.get_quantization_factor(); -// writeAttr(AprTypes::QuantizationFactorType, meta_location, &quantization_factor); -// writeAttr(AprTypes::LambdaType, meta_location, &apr.parameters.lambda); -// writeAttr(AprTypes::SigmaThType, meta_location, &apr.parameters.sigma_th); -// writeAttr(AprTypes::SigmaThMaxType, meta_location, &apr.parameters.sigma_th_max); -// writeAttr(AprTypes::IthType, meta_location, &apr.parameters.Ip_th); -// writeAttr(AprTypes::DxType, meta_location, &apr.parameters.dx); -// writeAttr(AprTypes::DyType, meta_location, &apr.parameters.dy); -// writeAttr(AprTypes::DzType, meta_location, &apr.parameters.dz); -// writeAttr(AprTypes::PsfXType, meta_location, &apr.parameters.psfx); -// writeAttr(AprTypes::PsfYType, meta_location, &apr.parameters.psfy); -// writeAttr(AprTypes::PsfZType, meta_location, &apr.parameters.psfz); -// writeAttr(AprTypes::RelativeErrorType, meta_location, &apr.parameters.rel_error); -// writeAttr(AprTypes::NoiseSdEstimateType, meta_location, &apr.parameters.noise_sd_estimate); -// writeAttr(AprTypes::BackgroundIntensityEstimateType, meta_location, -// &apr.parameters.background_intensity_estimate); -// -// write_timer.start_timer("access_data"); -// MapStorageData map_data; -// apr.apr_access.flatten_structure( map_data); -// -// std::vector index_delta; -// index_delta.resize(map_data.global_index.size()); -// std::adjacent_difference(map_data.global_index.begin(),map_data.global_index.end(),index_delta.begin()); -// writeData(AprTypes::MapGlobalIndexType, f.objectId, index_delta, blosc_comp_type, blosc_comp_level, blosc_shuffle); -// -// writeData(AprTypes::MapYendType, f.objectId, map_data.y_end, blosc_comp_type, blosc_comp_level, blosc_shuffle); -// writeData(AprTypes::MapYbeginType, f.objectId, map_data.y_begin, blosc_comp_type, blosc_comp_level, blosc_shuffle); -// writeData(AprTypes::MapNumberGapsType, f.objectId, map_data.number_gaps, blosc_comp_type, blosc_comp_level, blosc_shuffle); -// writeData(AprTypes::MapLevelType, f.objectId, map_data.level, blosc_comp_type, blosc_comp_level, blosc_shuffle); -// writeData(AprTypes::MapXType, f.objectId, map_data.x, blosc_comp_type, blosc_comp_level, blosc_shuffle); -// writeData(AprTypes::MapZType, f.objectId, map_data.z, blosc_comp_type, blosc_comp_level, blosc_shuffle); -// -// // #TODO: New write file APRTree -// -//// bool store_tree = write_tree; -//// -//// if(store_tree){ -//// -//// if(apr.apr_tree.particles_ds_tree.data.size()==0) { -//// apr.apr_tree.init(apr); -//// apr.apr_tree.fill_tree_mean_downsample(apr.particles_intensities); -//// } -//// -//// writeAttr(AprTypes::TotalNumberOfGapsType, f.objectIdTree, &apr.apr_tree.tree_access.total_number_gaps); -//// writeAttr(AprTypes::TotalNumberOfNonEmptyRowsType, f.objectIdTree, &apr.apr_tree.tree_access.total_number_non_empty_rows); -//// writeAttr(AprTypes::TotalNumberOfParticlesType, f.objectIdTree, &apr.apr_tree.tree_access.total_number_particles); -//// -//// MapStorageData map_data_tree; -//// apr.apr_tree.tree_access.flatten_structure( map_data_tree); -//// -//// std::vector index_delta; -//// index_delta.resize(map_data_tree.global_index.size()); -//// std::adjacent_difference(map_data_tree.global_index.begin(),map_data_tree.global_index.end(),index_delta.begin()); -//// writeData(AprTypes::MapGlobalIndexType, f.objectIdTree, index_delta, blosc_comp_type, blosc_comp_level, blosc_shuffle); -//// -//// writeData(AprTypes::MapYendType, f.objectIdTree, map_data_tree.y_end, blosc_comp_type, blosc_comp_level, blosc_shuffle); -//// writeData(AprTypes::MapYbeginType, f.objectIdTree, map_data_tree.y_begin, blosc_comp_type, blosc_comp_level, blosc_shuffle); -//// writeData(AprTypes::MapNumberGapsType, f.objectIdTree, map_data_tree.number_gaps, blosc_comp_type, blosc_comp_level, blosc_shuffle); -//// writeData(AprTypes::MapLevelType, f.objectIdTree, map_data_tree.level, blosc_comp_type, blosc_comp_level, blosc_shuffle); -//// writeData(AprTypes::MapXType, f.objectIdTree, map_data_tree.x, blosc_comp_type, blosc_comp_level, blosc_shuffle); -//// writeData(AprTypes::MapZType, f.objectIdTree, map_data_tree.z, blosc_comp_type, blosc_comp_level, blosc_shuffle); -//// -//// APRCompress tree_compress; -//// tree_compress.set_compression_type(1); -//// tree_compress.set_quantization_factor(2); -//// -//// tree_compress.compress(apr, apr.apr_tree.particles_ds_tree); -//// -//// unsigned int tree_blosc_comp_type = 6; -//// -//// hid_t type = Hdf5Type::type(); -//// writeData({type, AprTypes::ParticleIntensitiesType}, f.objectIdTree, apr.apr_tree.particles_ds_tree.data, tree_blosc_comp_type, blosc_comp_level, blosc_shuffle); -//// -//// } -// -// write_timer.stop_timer(); -// -// for (size_t i = apr.level_min(); i 0){ -//// apr_compressor.compress(apr,apr.particles_intensities); -//// } -//// hid_t type = Hdf5Type::type(); -//// writeData({type, AprTypes::ParticleIntensitiesType}, meta_location, apr.particles_intensities.data, blosc_comp_type, blosc_comp_level, blosc_shuffle); -//// write_timer.stop_timer(); -// -// // ------------- output the file size ------------------- -// file_size = f.getFileSize(); -// double sizeMB = file_size / 1e6; -// -// fileSizeInfo.total_file_size = sizeMB; -// fileSizeInfo.intensity_data = fileSizeInfo.total_file_size - fileSizeInfo.access_data; -// -// std::cout << "HDF5 Total Filesize: " << sizeMB << " MB\n" << "Writing Complete" << std::endl; -// return fileSizeInfo; -// } -// -// -// -// -// template -// void write_apr_paraview(APR &apr, const std::string &save_loc, const std::string &file_name, const ParticleData &parts,std::vector previous_num = {0}) { -// std::string hdf5_file_name = save_loc + file_name + ".h5"; -// -// bool write_time = false; -// unsigned int t = 0; -// if(previous_num[0]!=0){ -// //time series write -// write_time = true; -// t = (previous_num.size()-1); -// } -// -// -// FileStructure f(hdf5_file_name, FileStructure::Operation::WRITE); -// if (!f.isOpened()) return; -// -// // ------------- write metadata ------------------------- -// -// writeString(AprTypes::NameType, f.objectId, (apr.name.size() == 0) ? "no_name" : apr.name); -// writeString(AprTypes::GitType, f.objectId, ConfigAPR::APR_GIT_HASH); -// writeAttr(AprTypes::MaxLevelType, f.objectId, &apr.apr_access.l_max); -// writeAttr(AprTypes::MinLevelType, f.objectId, &apr.apr_access.l_min); -// writeAttr(AprTypes::TotalNumberOfParticlesType, f.objectId, &apr.apr_access.total_number_particles); -// -// // ------------- write data ---------------------------- -// writeDataStandard({(Hdf5Type::type()), AprTypes::ParticlePropertyType}, f.objectId, parts.data); -// -// auto apr_iterator = apr.iterator(); -// std::vector xv(apr_iterator.total_number_particles()); -// std::vector yv(apr_iterator.total_number_particles()); -// std::vector zv(apr_iterator.total_number_particles()); -// std::vector levelv(apr_iterator.total_number_particles()); -// //std::vector typev(apr_iterator.total_number_particles()); -// -// for (unsigned int level = apr_iterator.level_min(); level <= apr_iterator.level_max(); ++level) { -// int z = 0; -// int x = 0; -// -//#ifdef HAVE_OPENMP -//#pragma omp parallel for schedule(dynamic) private(z, x) firstprivate(apr_iterator) -//#endif -// for (z = 0; z < apr_iterator.spatial_index_z_max(level); z++) { -// for (x = 0; x < apr_iterator.spatial_index_x_max(level); ++x) { -// for (apr_iterator.set_new_lzx(level, z, x); apr_iterator.global_index() < apr_iterator.end_index; -// apr_iterator.set_iterator_to_particle_next_particle()) { -// xv[apr_iterator.global_index()] = apr_iterator.x_global(); -// yv[apr_iterator.global_index()] = apr_iterator.y_global(); -// zv[apr_iterator.global_index()] = apr_iterator.z_global(); -// levelv[apr_iterator.global_index()] = apr_iterator.level(); -// } -// } -// } -// } -// -// writeDataStandard(AprTypes::ParaviewXType, f.objectId, xv); -// writeDataStandard(AprTypes::ParaviewYType, f.objectId, yv); -// writeDataStandard(AprTypes::ParaviewZType, f.objectId, zv); -// writeDataStandard(AprTypes::ParaviewLevelType, f.objectId, levelv); -// //writeDataStandard(AprTypes::ParaviewTypeType, f.objectId, typev); -// -// // TODO: This needs to be able extended to handle more general type, currently it is assuming uint16 -// if(write_time){ -// write_main_paraview_xdmf_xml_time(save_loc, hdf5_file_name, file_name,previous_num); -// } else { -// write_main_paraview_xdmf_xml(save_loc, hdf5_file_name, file_name, apr_iterator.total_number_particles()); -// } -// -// // ------------- output the file size ------------------- -// hsize_t file_size; -// H5Fget_filesize(f.fileId, &file_size); -// std::cout << "HDF5 Filesize: " << file_size*1.0/1000000.0 << " MB" << std::endl; -// std::cout << "Writing Complete" << std::endl; -// } -// -// -// /** -// * Writes only the particle data, requires the same APR to be read in correctly. -// */ -// template -// float write_particles_only(const std::string &save_loc, const std::string &file_name, const ParticleData &parts_extra) { -// std::string hdf5_file_name = save_loc + file_name + ".h5"; -// -// FileStructure f{hdf5_file_name, FileStructure::Operation::WRITE}; -// if (!f.isOpened()) return 0; -// -// // ------------- write metadata ------------------------- -// uint64_t total_number_parts = parts_extra.data.size(); -// writeAttr(AprTypes::TotalNumberOfParticlesType, f.groupId, &total_number_parts); -// writeString(AprTypes::GitType, f.groupId, ConfigAPR::APR_GIT_HASH); -// -// // ------------- write data ---------------------------- -// unsigned int blosc_comp_type = 6; -// unsigned int blosc_comp_level = 9; -// unsigned int blosc_shuffle = 1; -// hid_t type = Hdf5Type::type(); -// writeData({type, AprTypes::ExtraParticleDataType}, f.objectId, parts_extra.data, blosc_comp_type, blosc_comp_level, blosc_shuffle); -// -// // ------------- output the file size ------------------- -// hsize_t file_size = f.getFileSize(); -// std::cout << "HDF5 Filesize: " << file_size/1e6 << " MB" << std::endl; -// std::cout << "Writing ExtraPartCellData Complete" << std::endl; -// -// return file_size/1e6; //returns file size in MB -// } -// -// template -// void read_parts_only(const std::string &aFileName, ParticleData& extra_parts) { -// FileStructure f{aFileName, FileStructure::Operation::READ}; -// if (!f.isOpened()) return; -// -// // ------------- read metadata -------------------------- -// uint64_t numberOfParticles; -// readAttr(AprTypes::TotalNumberOfParticlesType, f.groupId, &numberOfParticles); -// -// // ------------- read data ----------------------------- -// extra_parts.data.resize(numberOfParticles); -// readData(AprTypes::ExtraParticleDataType, f.objectId, extra_parts.data.data()); -// } - template float write_mesh_to_hdf5(PixelData& input_mesh,const std::string &save_loc, const std::string &file_name,unsigned int blosc_comp_type = BLOSC_ZSTD, unsigned int blosc_comp_level = 2, unsigned int blosc_shuffle=1){ std::string hdf5_file_name = save_loc + file_name + "_pixels.h5"; From 8e80969e235e52826f6c7a9940a9efc6a56dca22 Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Sun, 9 Jan 2022 19:42:55 +0100 Subject: [PATCH 02/19] remove unused structs --- src/io/APRWriter.hpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/io/APRWriter.hpp b/src/io/APRWriter.hpp index 63a7ba6a..217b1a0f 100644 --- a/src/io/APRWriter.hpp +++ b/src/io/APRWriter.hpp @@ -15,22 +15,6 @@ #include "numerics/APRCompress.hpp" #include "data_structures/APR/particles/ParticleData.hpp" -struct FileSizeInfo { - float total_file_size=0; - float intensity_data=0; - float access_data=0; -}; - -struct FileSizeInfoTime { -// float total_file_size=0; -// float update_fp = 0; -// float update_index = 0; -// float add_index = 0; -// float add_fp = 0; -// float remove_index = 0; -// float remove_fp = 0; -}; - /* * Class to handle generation of datasets using hdf5 using blosc From 0ca4eed049978f51c9b5c83a19811f7b8929e816 Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Sun, 9 Jan 2022 20:14:22 +0100 Subject: [PATCH 03/19] remove use of with_tree_flag in open_time_point - now takes a parameter `require_tree` instead of `tree` - prints and returns false if APR data group does not exist - prints and returns false if `require_tree=true` but tree group does not exist - if tree data exists, it is opened regardless of `require_tree` - refactor usages (this change should not break anything) --- src/io/APRFile.hpp | 76 ++++++++++++++++++++++---------------------- src/io/APRWriter.hpp | 30 +++++++++-------- 2 files changed, 55 insertions(+), 51 deletions(-) diff --git a/src/io/APRFile.hpp b/src/io/APRFile.hpp index 89bc8b11..5a088eb4 100644 --- a/src/io/APRFile.hpp +++ b/src/io/APRFile.hpp @@ -286,7 +286,7 @@ bool APRFile::write_particles(const std::string particles_name,ParticleData 0){ - max_level_delta_tree = max_level_delta - 1; - } - APRWriter::read_linear_access( fileStructure.objectIdTree, apr.linearAccessTree,max_level_delta_tree); + if(max_level_delta > 0){ + max_level_delta_tree = max_level_delta - 1; } - + APRWriter::read_linear_access( fileStructure.objectIdTree, apr.linearAccessTree,max_level_delta_tree); } - } timer.stop_timer(); @@ -480,7 +475,7 @@ bool APRFile::read_metadata(GenInfo& aprInfo, uint64_t t, std::string channel_na return false; } - if(!fileStructure.open_time_point(t, with_tree_flag,channel_name)) { + if(!fileStructure.open_time_point(t, false, channel_name)) { std::cerr << "Error reading APR file: could not open time point t=" << t << " in channel '" << channel_name << "'" << std::endl; return false; } @@ -518,9 +513,14 @@ bool APRFile::read_metadata_tree(GenInfo& treeInfo, uint64_t t, std::string chan * @param apr_or_tree (Default = True (APR), false = APR Tree) */ template -bool APRFile::read_particles(APR &apr,std::string particles_name,ParticleData& particles,bool apr_or_tree,uint64_t t,std::string channel_name){ +bool APRFile::read_particles(APR &apr, + std::string particles_name, + ParticleData& particles, + bool apr_or_tree, + uint64_t t, + std::string channel_name){ - hid_t part_location = open_dataset_location(particles_name, apr_or_tree,t,channel_name); + hid_t part_location = open_dataset_location(particles_name, apr_or_tree, t, channel_name); if(part_location == 0){ return false; @@ -658,7 +658,7 @@ hid_t APRFile::open_dataset_location(std::string& particles_name, bool apr_or_tr return 0; } - if (!fileStructure.open_time_point(t, with_tree_flag, channel_name)) { + if (!fileStructure.open_time_point(t, !apr_or_tree, channel_name)) { std::cerr << "Error reading APR file: could not open time point t=" << t << " in channel '" << channel_name << "'" << std::endl; return 0; @@ -730,7 +730,7 @@ bool APRFile::read_particles(std::string particles_name, ParticleData& return false; } - if(!fileStructure.open_time_point(t, with_tree_flag,channel_name)) { + if(!fileStructure.open_time_point(t, !apr_or_tree, channel_name)) { std::cerr << "Error reading APR file: could not open time point t=" << t << " in channel '" << channel_name << "'" << std::endl; return false; } @@ -904,7 +904,7 @@ std::vector APRFile::get_channel_names(){ */ std::vector APRFile::get_particles_names(bool apr_or_tree,uint64_t t,std::string channel_name){ - fileStructure.open_time_point(t,with_tree_flag,channel_name); + fileStructure.open_time_point(t, !apr_or_tree, channel_name); const int max_name_size = 1024; diff --git a/src/io/APRWriter.hpp b/src/io/APRWriter.hpp index 217b1a0f..d581384b 100644 --- a/src/io/APRWriter.hpp +++ b/src/io/APRWriter.hpp @@ -610,9 +610,9 @@ class APRWriter { } - bool open_time_point(const unsigned int t,bool tree,std::string t_string = "t"){ + bool open_time_point(const unsigned int t, bool require_tree, std::string t_string = "t") { - //first close existing time point + // first close existing time point close_time_point(); if(t != 0){ @@ -622,27 +622,31 @@ class APRWriter { subGroup1 = ("ParticleRepr/" + t_string); subGroupTree1 = "ParticleRepr/" + t_string + "/Tree"; - const char * const subGroup = subGroup1.c_str(); - const char * const subGroupTree = subGroupTree1.c_str(); + const char * const subGroup = subGroup1.c_str(); + const char * const subGroupTree = subGroupTree1.c_str(); - //need to check if the tree exists - if(!group_exists(fileId,subGroup)){ - //group does not exist + // check if APR data exists + if(!group_exists(fileId, subGroup)){ + std::cerr << "Error reading APR file: could not open time point t=" << t << std::endl; return false; } // open group objectId = H5Gopen2(fileId, subGroup, H5P_DEFAULT); - if(tree){ - if(!group_exists(fileId,subGroupTree)) { - // tree group does not exist - return false; - } + // check if tree data exists + const bool tree_exists = group_exists(fileId, subGroupTree); - // open tree group + if(require_tree && !tree_exists) { + std::cerr << "Error reading APR file: requested tree data does not exist" << std::endl; + return false; + } + + // open tree group if it exists + if(tree_exists) { objectIdTree = H5Gopen2(fileId, subGroupTree, H5P_DEFAULT); } + return true; } From 186fe1621c778fba4deb816563ef4ce763d79e7b Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Sun, 9 Jan 2022 20:27:19 +0100 Subject: [PATCH 04/19] rename parameter tree -> with_tree in create_time_point --- src/io/APRWriter.hpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/io/APRWriter.hpp b/src/io/APRWriter.hpp index d581384b..a6006370 100644 --- a/src/io/APRWriter.hpp +++ b/src/io/APRWriter.hpp @@ -652,7 +652,7 @@ class APRWriter { - void create_time_point(const unsigned int t,bool tree,std::string t_string = "t"){ + void create_time_point(const unsigned int t, bool with_tree, std::string t_string = "t"){ close_time_point(); @@ -663,23 +663,22 @@ class APRWriter { subGroup1 = ("ParticleRepr/" + t_string); subGroupTree1 = "ParticleRepr/" + t_string + "/Tree"; - const char * const subGroup = subGroup1.c_str(); - const char * const subGroupTree = subGroupTree1.c_str(); + const char * const subGroup = subGroup1.c_str(); + const char * const subGroupTree = subGroupTree1.c_str(); - if(!group_exists(fileId,subGroup)) { + if(!group_exists(fileId, subGroup)) { objectId = H5Gcreate2(fileId, subGroup, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); } else { objectId = H5Gopen2(fileId, subGroup, H5P_DEFAULT); } - if(tree){ - if(!group_exists(fileId,subGroupTree)) { + if(with_tree){ + if(!group_exists(fileId, subGroupTree)) { objectIdTree = H5Gcreate2(fileId, subGroupTree, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); } else { objectIdTree = H5Gopen2(fileId, subGroupTree, H5P_DEFAULT); } } - } bool init(const std::string &aFileName, const Operation aOp){ From f73ea40599d19fa8b4cfbeeed1c00548735a2f24 Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Sun, 9 Jan 2022 21:19:44 +0100 Subject: [PATCH 05/19] remove use of with_tree_flag in LazyData::init_file --- src/data_structures/APR/particles/LazyData.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data_structures/APR/particles/LazyData.hpp b/src/data_structures/APR/particles/LazyData.hpp index 2097a354..5f59f511 100644 --- a/src/data_structures/APR/particles/LazyData.hpp +++ b/src/data_structures/APR/particles/LazyData.hpp @@ -160,7 +160,7 @@ class LazyData { parts_name = std::move(particles_name); } fileStructure = file.get_fileStructure(); - fileStructure->create_time_point(0, file.get_read_write_tree(), channel_name); + fileStructure->create_time_point(0, !apr_or_tree, channel_name); if(apr_or_tree) { group_id = fileStructure->objectId; From 2267678ac2602c36230a797df7aad18c6a32674f Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Sun, 9 Jan 2022 21:23:04 +0100 Subject: [PATCH 06/19] add write_tree argument to write_apr Deprecate use of write_tree_flag and related methods. Note that a side effect of this is that in order to write an APR file without the tree data, one has to specify t and channel_name, due to the order of arguments --- src/io/APRFile.hpp | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/src/io/APRFile.hpp b/src/io/APRFile.hpp index 5a088eb4..4f8c426d 100644 --- a/src/io/APRFile.hpp +++ b/src/io/APRFile.hpp @@ -29,7 +29,7 @@ class APRFile { void close(); // write - bool write_apr(APR& apr,uint64_t t = 0,std::string channel_name = "t"); + bool write_apr(APR& apr,uint64_t t = 0,std::string channel_name = "t", bool write_tree = true); bool write_apr_append(APR& apr); template bool write_particles(std::string particles_name,ParticleData& particles,bool apr_or_tree = true,uint64_t t = 0,std::string channel_name = "t"); @@ -52,16 +52,13 @@ class APRFile { template bool read_particles(std::string particles_name, ParticleData& particles, bool apr_or_tree=true, uint64_t t=0, std::string channel_name="t"); - //set helpers + + [[deprecated("this now controlled via arguments to affected methods")]] bool get_read_write_tree(){ return with_tree_flag; } - //set helperslts - /** - * Set whether the internal APR Tree internal access should also be written and read. - * @param write_with_tree_flag_ indicate whether the APRTree should be written and read. (True = save both APR and APR Tree) - */ + [[deprecated("this now controlled via arguments to affected methods")]] void set_read_write_tree(bool with_tree_flag_){ with_tree_flag = with_tree_flag_; } @@ -121,7 +118,7 @@ class APRFile { //Basic Properties. uint64_t current_t=0; - bool with_tree_flag = true; + bool with_tree_flag = true; //usage is deprecated, keeping for backward compatibility std::string file_name = "noname"; APRWriter::FileStructure fileStructure; @@ -178,7 +175,7 @@ void APRFile::close(){ * @param APR to be written * @param t the time point to be written (default will be to append to the end of the file, starting with 0) */ -bool APRFile::write_apr(APR &apr,uint64_t t,std::string channel_name){ +bool APRFile::write_apr(APR &apr, uint64_t t, std::string channel_name, bool write_tree){ current_t = t; @@ -189,7 +186,7 @@ bool APRFile::write_apr(APR &apr,uint64_t t,std::string channel_name){ timer.start_timer("setup"); - fileStructure.create_time_point(t,with_tree_flag,channel_name); + fileStructure.create_time_point(t, write_tree, channel_name); hid_t meta_location = fileStructure.objectId; @@ -228,7 +225,7 @@ bool APRFile::write_apr(APR &apr,uint64_t t,std::string channel_name){ } - if(with_tree_flag){ + if(write_tree){ if(write_linear_tree) { @@ -286,11 +283,8 @@ bool APRFile::write_particles(const std::string particles_name,ParticleData Date: Sun, 9 Jan 2022 21:46:26 +0100 Subject: [PATCH 07/19] remove use of set_read_write_tree in bench_apr_io NOTE: this change affects read times since read_apr now initializes the tree structure --- benchmarks/BenchIO.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/benchmarks/BenchIO.cpp b/benchmarks/BenchIO.cpp index c0db8aa5..e1d43103 100644 --- a/benchmarks/BenchIO.cpp +++ b/benchmarks/BenchIO.cpp @@ -106,13 +106,11 @@ inline void bench_apr_io(APR& apr,ParticleData& parts,int num_rep,Ana aprFile.open(file_name,"WRITE"); - aprFile.set_read_write_tree(false); - for (int r = 0; r < num_rep; ++r) { timer_steps.start_timer("write_apr"); - aprFile.write_apr(apr,r); + aprFile.write_apr(apr, r, "t", false); timer_steps.stop_timer(); @@ -132,8 +130,6 @@ inline void bench_apr_io(APR& apr,ParticleData& parts,int num_rep,Ana aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); - for (int r = 0; r < num_rep; ++r) { timer_steps.start_timer("read_apr"); From efabf3eac75a7843b3484cf5ed6648c1e0ba6c0c Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Sun, 9 Jan 2022 22:12:49 +0100 Subject: [PATCH 08/19] remove usage of set_read_write_tree, add argument to write_apr where appropriate --- benchmarks/BenchPipeline.cpp | 4 +--- examples/Example_get_apr.cpp | 4 +--- examples/Example_get_apr_by_block.cpp | 3 +-- examples/Example_lazy_access.cpp | 1 - test/APRTest.cpp | 14 -------------- test/APRTestCuda.cpp | 12 ------------ test/DenoiseTest.cpp | 3 --- 7 files changed, 3 insertions(+), 38 deletions(-) diff --git a/benchmarks/BenchPipeline.cpp b/benchmarks/BenchPipeline.cpp index 7f4820ae..329ed520 100644 --- a/benchmarks/BenchPipeline.cpp +++ b/benchmarks/BenchPipeline.cpp @@ -86,8 +86,6 @@ inline void bench_apr_pipeline(APR& apr,ParticleData& parts,int num_r aprFile.open("file_name","WRITE"); - aprFile.set_read_write_tree(false); - for (int r = 0; r < num_rep; ++r) { timer_steps.start_timer("get_apr"); @@ -104,7 +102,7 @@ inline void bench_apr_pipeline(APR& apr,ParticleData& parts,int num_r timer_steps.start_timer("write_apr"); - aprFile.write_apr(apr_vec[r],r); + aprFile.write_apr(apr_vec[r], r, "t", false); timer_steps.stop_timer(); diff --git a/examples/Example_get_apr.cpp b/examples/Example_get_apr.cpp index 705ea302..bfbaf31d 100644 --- a/examples/Example_get_apr.cpp +++ b/examples/Example_get_apr.cpp @@ -103,9 +103,7 @@ int runAPR(cmdLineOptions options) { aprFile.open(save_loc + file_name + ".apr"); - aprFile.set_read_write_tree(false); //not writing tree to file. - - aprFile.write_apr(apr); + aprFile.write_apr(apr, 0, "t", options.store_tree); aprFile.write_particles("particles",particle_intensities); float apr_file_size = aprFile.current_file_size_MB(); diff --git a/examples/Example_get_apr_by_block.cpp b/examples/Example_get_apr_by_block.cpp index 67f07e59..70805a4c 100644 --- a/examples/Example_get_apr_by_block.cpp +++ b/examples/Example_get_apr_by_block.cpp @@ -103,9 +103,8 @@ int runAPR(cmdLineOptions options) { timer.start_timer("write output to file"); APRFile aprFile; aprFile.open(save_loc + file_name + ".apr"); - aprFile.set_read_write_tree(options.store_tree); - aprFile.write_apr(apr); + aprFile.write_apr(apr, 0, "t", options.store_tree); aprFile.write_particles("particles", parts); timer.stop_timer(); diff --git a/examples/Example_lazy_access.cpp b/examples/Example_lazy_access.cpp index f0ed087b..2adf78f3 100644 --- a/examples/Example_lazy_access.cpp +++ b/examples/Example_lazy_access.cpp @@ -28,7 +28,6 @@ int main(int argc, char **argv) { // open APR file for reading APRFile aprFile; - aprFile.set_read_write_tree(false); aprFile.open(file_name, "READ"); // initialize lazy access and iterator for spatial information diff --git a/test/APRTest.cpp b/test/APRTest.cpp index 9209c29c..5a4f2d49 100644 --- a/test/APRTest.cpp +++ b/test/APRTest.cpp @@ -1259,8 +1259,6 @@ bool test_linear_access_io(TestData& test_data) { APR apr_random; - writeFile.set_read_write_tree(true); - writeFile.open(file_name,"READ"); writeFile.read_apr(apr_random); @@ -1290,8 +1288,6 @@ bool test_linear_access_io(TestData& test_data) { APR apr_lin; - writeFile.set_read_write_tree(true); - writeFile.open(file_name,"READ"); writeFile.read_apr(apr_lin); @@ -1598,8 +1594,6 @@ bool test_apr_file(TestData& test_data){ readFile.open(file_name,"READ"); - readFile.set_read_write_tree(false); - APR aprRead; readFile.read_apr(aprRead); @@ -3460,7 +3454,6 @@ void CreateBigBigData::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"parts",test_data.particles_intensities); aprFile.close(); @@ -3476,7 +3469,6 @@ void CreateSmallSphereTest::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -3507,7 +3499,6 @@ void CreateDiffDimsSphereTest::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -3538,7 +3529,6 @@ void Create210SphereTestAPROnly::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -3555,7 +3545,6 @@ void CreateGTSmall2DTestProperties::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -3585,7 +3574,6 @@ void CreateGTSmall1DTestProperties::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -4403,7 +4391,6 @@ bool compare_lazy_iterator(LazyIterator& lazy_it, LinearIterator& apr_it) { bool test_lazy_iterators(TestData& test_data) { // write APR to file using new format (required by LazyAccess) #TODO: update the test files APRFile aprFile; - aprFile.set_read_write_tree(true); std::string file_name = "lazy_access_test.apr"; aprFile.open(file_name,"WRITE"); aprFile.write_apr(test_data.apr); @@ -4462,7 +4449,6 @@ bool test_reconstruct_lazy(TestData& test_data, ReconPatch& patch) { // write APR and tree data to file APRFile aprFile; std::string file_name = "lazy_recon_test.apr"; - aprFile.set_read_write_tree(true); aprFile.open(file_name, "WRITE"); aprFile.write_apr(test_data.apr); aprFile.write_particles("particles", test_data.particles_intensities, true); diff --git a/test/APRTestCuda.cpp b/test/APRTestCuda.cpp index 9e768968..618e2d34 100644 --- a/test/APRTestCuda.cpp +++ b/test/APRTestCuda.cpp @@ -134,7 +134,6 @@ void CreateSmallSphereTest::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -165,7 +164,6 @@ void CreatDiffDimsSphereTest::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -198,7 +196,6 @@ void CreateCR1::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -212,7 +209,6 @@ void CreateCR3::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -226,7 +222,6 @@ void CreateCR5::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -240,7 +235,6 @@ void CreateCR10::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -254,7 +248,6 @@ void CreateCR15::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -268,7 +261,6 @@ void CreateCR20::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -282,7 +274,6 @@ void CreateCR30::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -296,7 +287,6 @@ void CreateCR54::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -310,7 +300,6 @@ void CreateCR124::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); @@ -324,7 +313,6 @@ void CreateCR1000::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(test_data.apr); aprFile.read_particles(test_data.apr,"particle_intensities",test_data.particles_intensities); aprFile.close(); diff --git a/test/DenoiseTest.cpp b/test/DenoiseTest.cpp index 8f8019e3..0ebe1901 100644 --- a/test/DenoiseTest.cpp +++ b/test/DenoiseTest.cpp @@ -113,7 +113,6 @@ void Stencil2D::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(testData.apr); aprFile.read_particles(testData.apr,"particle_intensities",testData.parts); aprFile.close(); @@ -129,7 +128,6 @@ void Stencil1D::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(testData.apr); aprFile.read_particles(testData.apr,"particle_intensities",testData.parts); aprFile.close(); @@ -148,7 +146,6 @@ void Stencil3D::SetUp(){ APRFile aprFile; aprFile.open(file_name,"READ"); - aprFile.set_read_write_tree(false); aprFile.read_apr(testData.apr); aprFile.read_particles(testData.apr,"particle_intensities",testData.parts); aprFile.close(); From 7f84da5d7f600965ca1108a030ccc00932c9c279 Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Wed, 23 Mar 2022 18:53:04 +0100 Subject: [PATCH 09/19] remove automatic tree initialization from read_apr, add check to APR::total_number_tree_particles --- src/data_structures/APR/APR.hpp | 7 ++++++- src/io/APRFile.hpp | 5 +---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/data_structures/APR/APR.hpp b/src/data_structures/APR/APR.hpp index b3e1f8a1..5dceaaca 100644 --- a/src/data_structures/APR/APR.hpp +++ b/src/data_structures/APR/APR.hpp @@ -104,7 +104,12 @@ class APR { inline uint64_t total_number_particles() const { return aprInfo.total_number_particles; } uint64_t org_dims(int dim) const { return aprInfo.org_dims[dim]; } - inline uint64_t total_number_tree_particles() const { return treeInfo.total_number_particles; } + inline uint64_t total_number_tree_particles() { + if(!tree_initialized && !tree_initialized_random) { + initialize_tree_linear(); + } + return treeInfo.total_number_particles; + } inline int number_dimensions() const { return aprInfo.number_dimensions; diff --git a/src/io/APRFile.hpp b/src/io/APRFile.hpp index 4f8c426d..94dde0cf 100644 --- a/src/io/APRFile.hpp +++ b/src/io/APRFile.hpp @@ -427,10 +427,7 @@ bool APRFile::read_apr(APR &apr,uint64_t t,std::string channel_name){ bool tree_exists = stored_linear_tree || stored_random_tree; - if(!tree_exists){ - // if tree access does not exist in file, initialize it from the APR access - apr.initialize_tree_linear(); - } else { + if(tree_exists){ APRWriter::readAttr(AprTypes::TotalNumberOfParticlesType, fileStructure.objectIdTree, &apr.treeInfo.total_number_particles); From db8443edcb2a63e9f43de47499e697089ab94a51 Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Fri, 8 Apr 2022 14:39:50 +0200 Subject: [PATCH 10/19] add dense LinearAccess tree initialization method --- src/data_structures/APR/APR.hpp | 66 ++++++++++++++++++- .../APR/access/LinearAccess.hpp | 60 +++++++++++++++++ 2 files changed, 125 insertions(+), 1 deletion(-) diff --git a/src/data_structures/APR/APR.hpp b/src/data_structures/APR/APR.hpp index 5dceaaca..21371843 100644 --- a/src/data_structures/APR/APR.hpp +++ b/src/data_structures/APR/APR.hpp @@ -30,6 +30,7 @@ class APR { void initialize_apr_tree_sparse_linear(); void initialize_apr_tree(); void initialize_linear_access(LinearAccess& aprAccess,APRIterator& it); + void initialize_tree_dense(); //New Access LinearAccess linearAccess; @@ -219,7 +220,8 @@ class APR { void initialize_tree_linear(){ if(!tree_initialized){ - initialize_apr_tree_sparse_linear(); + //initialize_apr_tree_sparse_linear(); + initialize_tree_dense(); tree_initialized = true; } } @@ -612,6 +614,68 @@ void APR::initialize_apr_tree_sparse_linear() { } + +void APR::initialize_tree_dense() { + + APRTimer timer(false); + auto apr_iterator = iterator(); + + + treeInfo.init_tree(org_dims(0),org_dims(1),org_dims(2)); + linearAccessTree.genInfo = &treeInfo; + + std::vector> particle_cell_parent_tree(treeInfo.l_max+1); + + timer.start_timer("init tree - allocate dense tree structure"); + for (int l = treeInfo.l_min; l <= treeInfo.l_max; ++l) { + + particle_cell_parent_tree[l].initWithValue(treeInfo.y_num[l], + treeInfo.x_num[l], + treeInfo.z_num[l], + 0); + } + timer.stop_timer(); + + timer.start_timer("init tree - fill particle parents"); + // fill in parents of APR particles + for(int level = apr_iterator.level_max(); level >= apr_iterator.level_min(); --level) { +#ifdef HAVE_OPENMP +#pragma omp parallel for schedule(dynamic, 2) firstprivate(apr_iterator) +#endif + for(int z = 0; z < apr_iterator.z_num(level); ++z) { + for(int x = 0; x < apr_iterator.x_num(level); ++x) { + for(apr_iterator.begin(level, z, x); apr_iterator < apr_iterator.end(); ++apr_iterator) { + particle_cell_parent_tree[level-1].at(apr_iterator.y() / 2, x / 2, z / 2) = 1; + } + } + } + } + timer.stop_timer(); + + timer.start_timer("init tree - fill tree recursive"); + // fill rest of tree, level by level + for(int level = treeInfo.l_max-1; level >= treeInfo.l_min; --level) { +#ifdef HAVE_OPENMP +#pragma omp parallel for schedule(static) +#endif + for(int z = 0; z < treeInfo.z_num[level]; ++z) { + for (int x = 0; x < treeInfo.x_num[level]; ++x) { + for(int y = 0; y < treeInfo.y_num[level]; ++y) { + // suffices to check one child + particle_cell_parent_tree[level].at(y, x, z) = particle_cell_parent_tree[level].at(y, x, z) | particle_cell_parent_tree[level+1].at(2*y, 2*x, 2*z); + } + } + } + } + timer.stop_timer(); + + timer.start_timer("create sparse data structure"); + linearAccessTree.initialize_tree_access_dense(particle_cell_parent_tree); + timer.stop_timer(); +} + + + /** * Initializes the APR tree datastructures using a sparse structure for reduced memory, these are all particle cells that are parents of particles in the APR * , alternatively can be thought of as the interior nodes of an APR represented as a binary tree. diff --git a/src/data_structures/APR/access/LinearAccess.hpp b/src/data_structures/APR/access/LinearAccess.hpp index 6367c4ba..2aba3605 100644 --- a/src/data_structures/APR/access/LinearAccess.hpp +++ b/src/data_structures/APR/access/LinearAccess.hpp @@ -58,9 +58,69 @@ class LinearAccess : public GenAccess { void initialize_tree_access_sparse(std::vector> &p_map); void initialize_linear_structure_sparse(APRParameters& apr_parameters,SparseGaps& p_map); + + void initialize_tree_access_dense(std::vector> &p_map); + }; +void LinearAccess::initialize_tree_access_dense(std::vector> &p_map) { + APRTimer timer(false); + + initialize_xz_linear(); + + // fill number of particles in each row + timer.start_timer("init tree dense - fill number particles per row"); + for(int level = level_max(); level >= level_min(); --level) { +#ifdef HAVE_OPENMP +#pragma omp parallel for schedule(static) +#endif + for(int z = 0; z < z_num(level); ++z) { + for(int x = 0; x < x_num(level); ++x) { + size_t offset = (size_t) z * x_num(level) * y_num(level) + (size_t) x * y_num(level); + size_t counter = 0; + + for(int y = 0; y < y_num(level); ++y) { + counter += p_map[level].mesh[offset + y]; + } + size_t row_index = level_xz_vec[level] + z * x_num(level) + x; + xz_end_vec[row_index] = counter; + } + } + } + timer.stop_timer(); + + timer.start_timer("init tree dense - cumulative sum"); + std::partial_sum(xz_end_vec.begin(),xz_end_vec.end(),xz_end_vec.begin()); + timer.stop_timer(); + + // set total number of particles + genInfo->total_number_particles = xz_end_vec.back(); + + timer.start_timer("init tree dense - allocate and fill y_vec"); + y_vec.resize(genInfo->total_number_particles); + + for(int level = level_max(); level >= level_min(); --level) { +#ifdef HAVE_OPENMP +#pragma omp parallel for schedule(static) +#endif + for(int z = 0; z < z_num(level); ++z) { + for (int x = 0; x < x_num(level); ++x) { + size_t offset = (size_t) z * x_num(level) * y_num(level) + (size_t) x * y_num(level); + size_t row_index = level_xz_vec[level] + z * x_num(level) + x; + size_t particle_index = xz_end_vec[row_index-1]; + + for(int y = 0; y < y_num(level); ++y) { + if(p_map[level].mesh[offset + y]) { + y_vec[particle_index++] = y; + } + } + } + } + } + timer.stop_timer(); +} + inline void LinearAccess::initialize_tree_access_sparse(std::vector> &p_map) { APRTimer apr_timer(false); From 2f1991a1cd15c5b0ab3a6d0345b64d0949826d7e Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Fri, 8 Apr 2022 15:29:40 +0200 Subject: [PATCH 11/19] refactor tree initialization method names --- src/data_structures/APR/APR.hpp | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/data_structures/APR/APR.hpp b/src/data_structures/APR/APR.hpp index 21371843..a6b5df71 100644 --- a/src/data_structures/APR/APR.hpp +++ b/src/data_structures/APR/APR.hpp @@ -25,13 +25,16 @@ class APR { protected: - //APR Tree function - void initialize_apr_tree_sparse(); - void initialize_apr_tree_sparse_linear(); - void initialize_apr_tree(); - void initialize_linear_access(LinearAccess& aprAccess,APRIterator& it); + // initialize tree RandomAccess + void initialize_tree_random_sparse(); + void initialize_tree_random_dense(); + + // initialize tree LinearAccess + void initialize_tree_sparse(); void initialize_tree_dense(); + void initialize_linear_access(LinearAccess& aprAccess, APRIterator& it); + //New Access LinearAccess linearAccess; LinearAccess linearAccessTree; @@ -203,7 +206,7 @@ class APR { bool initialize_tree_random(){ if(!tree_initialized_random){ - initialize_apr_tree_sparse(); + initialize_tree_random_sparse(); tree_initialized_random = true; } @@ -220,7 +223,6 @@ class APR { void initialize_tree_linear(){ if(!tree_initialized){ - //initialize_apr_tree_sparse_linear(); initialize_tree_dense(); tree_initialized = true; } @@ -350,9 +352,9 @@ void APR::initialize_random_access(){ * Initializes the APR tree datastructures using a dense structure for memory, these are all particle cells that are parents of particles in the APR * , alternatively can be thought of as the interior nodes of an APR represented as a binary tree. */ -void APR::initialize_apr_tree() { +void APR::initialize_tree_random_dense() { - APRTimer timer(true); + APRTimer timer(false); auto apr_iterator = iterator(); @@ -472,7 +474,7 @@ void APR::initialize_apr_tree() { } -void APR::initialize_apr_tree_sparse_linear() { +void APR::initialize_tree_sparse() { APRTimer timer(false); @@ -680,7 +682,7 @@ void APR::initialize_tree_dense() { * Initializes the APR tree datastructures using a sparse structure for reduced memory, these are all particle cells that are parents of particles in the APR * , alternatively can be thought of as the interior nodes of an APR represented as a binary tree. */ -void APR::initialize_apr_tree_sparse() { +void APR::initialize_tree_random_sparse() { APRTimer timer(false); From bc530f156cca5093d4fcf300ba40d5982b2968ab Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Fri, 8 Apr 2022 16:08:35 +0200 Subject: [PATCH 12/19] inline dense tree init to avoid multiple definitions errors with cuda --- src/data_structures/APR/access/LinearAccess.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data_structures/APR/access/LinearAccess.hpp b/src/data_structures/APR/access/LinearAccess.hpp index 2aba3605..5f92c0ef 100644 --- a/src/data_structures/APR/access/LinearAccess.hpp +++ b/src/data_structures/APR/access/LinearAccess.hpp @@ -64,7 +64,7 @@ class LinearAccess : public GenAccess { }; -void LinearAccess::initialize_tree_access_dense(std::vector> &p_map) { +inline void LinearAccess::initialize_tree_access_dense(std::vector> &p_map) { APRTimer timer(false); initialize_xz_linear(); From ec7f3abca70a60b0ae74fd5f1e63d396ddc8b104 Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Fri, 8 Apr 2022 16:09:39 +0200 Subject: [PATCH 13/19] seems dense random tree initialization is broken --- src/data_structures/APR/APR.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data_structures/APR/APR.hpp b/src/data_structures/APR/APR.hpp index a6b5df71..8098e573 100644 --- a/src/data_structures/APR/APR.hpp +++ b/src/data_structures/APR/APR.hpp @@ -27,7 +27,7 @@ class APR { // initialize tree RandomAccess void initialize_tree_random_sparse(); - void initialize_tree_random_dense(); + void initialize_tree_random_dense(); // appears to be broken #TODO: remove or fix // initialize tree LinearAccess void initialize_tree_sparse(); From 0524726b5b62f38292c0b2065f03c816aa2424b0 Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Fri, 8 Apr 2022 16:10:37 +0200 Subject: [PATCH 14/19] add template initializations for additional data types --- src/data_structures/APR/access/GPUAccess.cu | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/data_structures/APR/access/GPUAccess.cu b/src/data_structures/APR/access/GPUAccess.cu index 5aaa41f2..94de9ec3 100644 --- a/src/data_structures/APR/access/GPUAccess.cu +++ b/src/data_structures/APR/access/GPUAccess.cu @@ -158,6 +158,11 @@ template class ParticleDataGpu; template class ParticleDataGpu; template class ParticleDataGpu; +template class ParticleDataGpu; +template class ParticleDataGpu; +template class ParticleDataGpu; +template class ParticleDataGpu; + __global__ void fill_y_vec_max_level(const uint64_t* level_xz_vec, const uint64_t* xz_end_vec, uint16_t* y_vec, From 2d963797b1bf4f605ac82a90a284df245047f5ad Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Fri, 8 Apr 2022 16:53:12 +0200 Subject: [PATCH 15/19] update submodule vcpkg --- vcpkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcpkg b/vcpkg index 57bd7102..d72783cb 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit 57bd7102d9fd880daa1b0958692294c4a125f6d8 +Subproject commit d72783cb3aeddfd667861caef1060e54ca6fa7a9 From 17eea483f667d3802ba9b8e552331c5238c43750 Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Fri, 8 Apr 2022 17:08:06 +0200 Subject: [PATCH 16/19] make initialize_tree_linear public, allow choosing between sparse and dense methods --- src/data_structures/APR/APR.hpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/data_structures/APR/APR.hpp b/src/data_structures/APR/APR.hpp index 8098e573..fc0b5921 100644 --- a/src/data_structures/APR/APR.hpp +++ b/src/data_structures/APR/APR.hpp @@ -201,6 +201,17 @@ class APR { } + void initialize_tree_linear(bool sparse=false){ + if(!tree_initialized){ + if(sparse) { + initialize_tree_sparse(); + } else { + initialize_tree_dense(); + } + tree_initialized = true; + } + } + protected: bool initialize_tree_random(){ @@ -221,13 +232,6 @@ class APR { } } - void initialize_tree_linear(){ - if(!tree_initialized){ - initialize_tree_dense(); - tree_initialized = true; - } - } - void initialize_random_access(); @@ -622,7 +626,6 @@ void APR::initialize_tree_dense() { APRTimer timer(false); auto apr_iterator = iterator(); - treeInfo.init_tree(org_dims(0),org_dims(1),org_dims(2)); linearAccessTree.genInfo = &treeInfo; From 5c59bbf36045cc1942b64d62a2ee687a45a27a47 Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Fri, 8 Apr 2022 17:53:58 +0200 Subject: [PATCH 17/19] refactor initialization methods, move checks from iterators --- src/data_structures/APR/APR.hpp | 71 +++++++++++++++------------------ src/io/APRFile.hpp | 2 +- 2 files changed, 34 insertions(+), 39 deletions(-) diff --git a/src/data_structures/APR/APR.hpp b/src/data_structures/APR/APR.hpp index fc0b5921..3ef4ee5c 100644 --- a/src/data_structures/APR/APR.hpp +++ b/src/data_structures/APR/APR.hpp @@ -33,7 +33,8 @@ class APR { void initialize_tree_sparse(); void initialize_tree_dense(); - void initialize_linear_access(LinearAccess& aprAccess, APRIterator& it); + void initialize_linear_access_from_random(LinearAccess& aprAccess, APRIterator& it); + void initialize_random_access_from_linear(); //New Access LinearAccess linearAccess; @@ -127,23 +128,16 @@ class APR { APRIterator random_iterator() { if(!apr_initialized_random){ - initialize_random_access(); - apr_initialized_random = true; + initialize_random(); } return APRIterator(apr_access,aprInfo); } LinearIterator iterator() { - // Just checking if its initialized - if(!apr_initialized){ - if(!apr_initialized_random){ - std::cerr << "No APR initialized" << std::endl; - } else { - initialize_linear(); - apr_initialized = true; - } + if(!apr_initialized){ + initialize_linear(); } return LinearIterator(linearAccess,aprInfo); @@ -151,12 +145,6 @@ class APR { APRTreeIterator random_tree_iterator() { - if(!apr_initialized_random){ - //the random tree iterator and the apr iteratator are joint at the hip. - initialize_random_access(); - apr_initialized_random = true; - } - if(!tree_initialized_random){ initialize_tree_random(); } @@ -165,7 +153,7 @@ class APR { } LinearIterator tree_iterator() { - // Checking if initialized. + if(!tree_initialized){ initialize_tree_linear(); } @@ -200,8 +188,31 @@ class APR { } + void initialize_linear(){ + if(!apr_initialized){ + if(!apr_initialized_random) { + std::cerr << "APR::initialize_linear - No APR initialized" << std::endl; + } else { + auto it = random_iterator(); + initialize_linear_access_from_random(linearAccess, it); + apr_initialized = true; + } + } + } + + void initialize_random() { + if(!apr_initialized_random) { + if(!apr_initialized) { + std::cerr << "APR::initialize_random - No APR initialized" << std::endl; + } else { + initialize_random_access_from_linear(); + apr_initialized_random = true; + } + } + } void initialize_tree_linear(bool sparse=false){ + if(!tree_initialized){ if(sparse) { initialize_tree_sparse(); @@ -212,29 +223,14 @@ class APR { } } -protected: + void initialize_tree_random(){ - bool initialize_tree_random(){ if(!tree_initialized_random){ - initialize_tree_random_sparse(); tree_initialized_random = true; - } - return tree_initialized_random; } - void initialize_linear(){ - if(!apr_initialized){ - auto it = random_iterator(); - initialize_linear_access(linearAccess,it); - apr_initialized = true; - } - } - - void initialize_random_access(); - - }; #define INTERIOR_PARENT 9 @@ -243,7 +239,7 @@ class APR { /** * Initializes linear access apr structures, that require more memory, but are faster. However, the do not allow the same neighbour access as the random iterators */ -void APR::initialize_linear_access(LinearAccess& aprAccess,APRIterator& it){ +void APR::initialize_linear_access_from_random(LinearAccess& aprAccess, APRIterator& it){ // TODO: Should be renamed.. random-> linear access. also need the reverse function @@ -283,7 +279,7 @@ void APR::initialize_linear_access(LinearAccess& aprAccess,APRIterator& it){ /** * Initializes linear access apr structures, that require more memory, but are faster. However, the do not allow the same neighbour access as the random iterators */ -void APR::initialize_random_access(){ +void APR::initialize_random_access_from_linear(){ // // TODO: Note this is not performance orientataed, and should be depreciated in the future. (the whole random access iterations should be removed.) // @@ -348,7 +344,6 @@ void APR::initialize_random_access(){ } apr_access.initialize_structure_from_particle_cell_tree_sparse(parameters,particle_cell_tree); - } @@ -689,7 +684,7 @@ void APR::initialize_tree_random_sparse() { APRTimer timer(false); - auto apr_iterator = iterator(); + auto apr_iterator = random_iterator(); //need to create a local copy diff --git a/src/io/APRFile.hpp b/src/io/APRFile.hpp index 94dde0cf..dcfdd27b 100644 --- a/src/io/APRFile.hpp +++ b/src/io/APRFile.hpp @@ -217,7 +217,7 @@ bool APRFile::write_apr(APR &apr, uint64_t t, std::string channel_name, bool wri } else { timer.start_timer("write_apr_access_data"); - apr.initialize_random_access(); //check that it is initialized. + apr.initialize_random(); //check that it is initialized. APRWriter::write_random_access(meta_location, fileStructure.objectId, apr.apr_access, blosc_comp_type_access, blosc_comp_level_access, blosc_shuffle_access); From f98c4fc7f4edb8f2976cbddeca8e7e70ff68051a Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Fri, 8 Apr 2022 17:57:28 +0200 Subject: [PATCH 18/19] remove duplicate checks --- src/data_structures/APR/APR.hpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/data_structures/APR/APR.hpp b/src/data_structures/APR/APR.hpp index 3ef4ee5c..699fa80e 100644 --- a/src/data_structures/APR/APR.hpp +++ b/src/data_structures/APR/APR.hpp @@ -67,14 +67,9 @@ class APR { GPUAccessHelper gpuAPRHelper(){ if(!apr_initialized){ - if(!apr_initialized_random){ - std::cerr << "No APR initialized" << std::endl; - - } else { - initialize_linear(); - apr_initialized = true; - } + initialize_linear(); } + return GPUAccessHelper(gpuAccess,linearAccess); } From b5c34a073adf39d989fced19bd562f7805db6938 Mon Sep 17 00:00:00 2001 From: joeljonsson Date: Sat, 9 Apr 2022 03:45:54 +0200 Subject: [PATCH 19/19] set linearAccess.genInfo in random->linear initialization --- src/data_structures/APR/APR.hpp | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/data_structures/APR/APR.hpp b/src/data_structures/APR/APR.hpp index 699fa80e..336f91f1 100644 --- a/src/data_structures/APR/APR.hpp +++ b/src/data_structures/APR/APR.hpp @@ -232,17 +232,15 @@ class APR { /** - * Initializes linear access apr structures, that require more memory, but are faster. However, the do not allow the same neighbour access as the random iterators - */ -void APR::initialize_linear_access_from_random(LinearAccess& aprAccess, APRIterator& it){ - - // TODO: Should be renamed.. random-> linear access. also need the reverse function - - auto& lin_a = aprAccess; + * Initializes linear access apr structures from the random access data. The linear structure is faster in most + * cases, but require more memory and do not allow the same neighbor access as the random access iterators. + */ +void APR::initialize_linear_access_from_random(LinearAccess& lin_a, APRIterator& it){ uint64_t counter = 0; uint64_t counter_xz = 1; + lin_a.genInfo = &aprInfo; lin_a.initialize_xz_linear(); lin_a.y_vec.resize(it.total_number_particles()); @@ -260,15 +258,11 @@ void APR::initialize_linear_access_from_random(LinearAccess& aprAccess, APRItera counter++; } - - lin_a.xz_end_vec[counter_xz] = (counter); + lin_a.xz_end_vec[counter_xz] = counter; counter_xz++; } } - } - - } /**