diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f1f268f03c..1878ec9ce3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,6 +102,7 @@ option(ENABLE_LIBUNWIND option(ENABLE_PAPI "Enables use of PAPI" OFF) option(ENABLE_LIBXC "Enables use of the libxc library of density functionals" ON) option(ENABLE_PCM "Enables use of the polarizable continuum model library" ON) +option(ENABLE_HDF5 "Enables use of the HDF5 library support" OFF) ####### preferred task backend set(MADNESS_TASK_VALID_BACKENDS Pthreads TBB PaRSEC) @@ -632,6 +633,7 @@ include(external/gperftools.cmake) include(external/tbb.cmake) include(external/parsec.cmake) include(external/boost.cmake) +include(external/hdf5.cmake) if (NOT MADNESS_BUILD_MADWORLD_ONLY) include(external/lapack.cmake) include(external/libxc.cmake) @@ -642,6 +644,7 @@ if (DEFINED ELEMENTAL_TAG) include(external/elemental.cmake) endif (DEFINED ELEMENTAL_TAG) + # Add project subdirectories =================================================== # predefine targets diff --git a/external/hdf5.cmake b/external/hdf5.cmake new file mode 100644 index 00000000000..c769c2a6cf9 --- /dev/null +++ b/external/hdf5.cmake @@ -0,0 +1,17 @@ + +#Find HDF5 support +if (ENABLE_HDF5) + set(HDF5_PREFER_PARALLEL FALSE) + find_package(HDF5 COMPONENTS C) + + if (HDF5_FOUND) + message(STATUS "Found HDF5: ${HDF5_LIBRARIES}") + set(HAVE_HDF5 ON CACHE INTERNAL "Have HDF5" FORCE) + endif () +# if (HDF5_IS_PARALLEL) +# set(HAVE_HDF5 0) +# message(STATUS "Found HDF5 Parallel - Disable") +# list(REMOVE_ITEM HDF5_C_LIBRARY_NAMES) +# endif () + +endif () diff --git a/src/madness/world/CMakeLists.txt b/src/madness/world/CMakeLists.txt index 1a5d3ff271a..171822738c9 100644 --- a/src/madness/world/CMakeLists.txt +++ b/src/madness/world/CMakeLists.txt @@ -12,14 +12,14 @@ set(MADWORLD_HEADERS text_fstream_archive.h worlddc.h mem_func_wrapper.h taskfn.h group.h dist_cache.h distributed_id.h type_traits.h function_traits.h stubmpi.h bgq_atomics.h binsorter.h parsec.h meta.h worldinit.h thread_info.h - cloud.h test_utilities.h timing_utilities.h) + cloud.h test_utilities.h timing_utilities.h h5_archive.h ) set(MADWORLD_SOURCES madness_exception.cc world.cc timers.cc future.cc redirectio.cc archive_type_names.cc info.cc debug.cc print.cc worldmem.cc worldrmi.cc safempi.cc worldpapi.cc worldref.cc worldam.cc worldprofile.cc thread.cc world_task_queue.cc worldgop.cc deferred_cleanup.cc worldmutex.cc binary_fstream_archive.cc text_fstream_archive.cc lookup3.c worldmpi.cc - group.cc parsec.cc archive.cc) + group.cc parsec.cc archive.cc h5_archive.cc ) if(MADNESS_ENABLE_CEREAL) set(MADWORLD_HEADERS ${MADWORLD_HEADERS} "cereal_archive.h") diff --git a/src/madness/world/h5_archive.cc b/src/madness/world/h5_archive.cc new file mode 100644 index 00000000000..c0b1e7fa213 --- /dev/null +++ b/src/madness/world/h5_archive.cc @@ -0,0 +1,162 @@ +/* + This file is part of MADNESS. + + Copyright (C) 2007,2010 Oak Ridge National Laboratory + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + For more information please contact: + + Robert J. Harrison + Oak Ridge National Laboratory + One Bethel Valley Road + P.O. Box 2008, MS-6367 + + email: harrisonrj@ornl.gov + tel: 865-241-3937 + fax: 865-572-0680 +*/ + +/** + \file text_fstream_archive.cc + \brief Implements an archive wrapping text filestream. + \ingroup serialization +*/ + +#include + + +namespace madness { + namespace archive { + +//vama void TextFstreamOutputArchive::store(const char* t, long /*n*/) const { +//vama // store character string, escaping &, < and > along the way +//vama while (*t) { +//vama char c = *t++; +//vama if (c == '\\') { +//vama os.put('\\'); +//vama os.put('\\'); +//vama } +//vama else if (c == '<') { +//vama os.put('\\'); +//vama os.put('l'); +//vama } +//vama else if (c == '>') { +//vama os.put('\\'); +//vama os.put('r'); +//vama } +//vama else { +//vama os.put(c); +//vama } +//vama } +//vama os << std::endl; +//vama } +//vama +//vama void TextFstreamOutputArchive::open(const char* filename, +//vama std::ios_base::openmode mode) +//vama { +//vama os.open(filename, mode); +//vama os.setf(std::ios::scientific); +//vama os.precision(17); +//vama char tag[256]; +//vama os << "" << std::endl; +//vama sprintf(tag,"", +//vama ARCHIVE_MAJOR_VERSION, ARCHIVE_MINOR_VERSION); +//vama os << tag << std::endl; +//vama os << "" << std::endl; +//vama for (int i=0; i<256; ++i) { +//vama sprintf(tag,"%d \"%s\"",i,archive_type_names[i]); +//vama store(tag,strlen(tag)); // Must use store to escape characters +//vama } +//vama os << "" << std::endl; +//vama } +//vama +//vama +//vama void TextFstreamOutputArchive::close() { +//vama if (os.is_open()) { +//vama os << "" << std::endl; +//vama os.close(); +//vama } +//vama } +//vama +//vama +//vama // Eat EOL after each entry to enable char-by-char read of strings +//vama void TextFstreamInputArchive::eat_eol() const { +//vama char eol; +//vama is.get(eol); +//vama if (eol != '\n') +//vama MADNESS_EXCEPTION("TextFstreamInputArchive: eat_eol: indigestion", static_cast(eol)); +//vama } +//vama +//vama +//vama void TextFstreamInputArchive::load(unsigned char* t, long n) const { +//vama for (long i=0; i> x; +//vama t[i] = (unsigned char) x; +//vama } +//vama eat_eol(); +//vama } +//vama +//vama void TextFstreamInputArchive::load(char* t, long n) const { +//vama for (long i=0; i(c1)); +//vama } +//vama } +//vama else { +//vama t[i] = c0; +//vama } +//vama } +//vama eat_eol(); +//vama } +//vama +//vama void TextFstreamInputArchive::open(const char* filename, std::ios_base::openmode mode) { +//vama is.open(filename, mode); +//vama char buf[256]; +//vama is.getline(buf,256); // skip xml header +//vama is.getline(buf,256); +//vama +//vama char tag[256]; +//vama sprintf(tag,"", +//vama ARCHIVE_MAJOR_VERSION, ARCHIVE_MINOR_VERSION); +//vama if (strcmp(buf,tag)) { +//vama std::cout << "TextFstreamInputArchive: not an archive/bad version?" << std::endl; +//vama std::cout << "Found this: " << buf; +//vama std::cout << "Expected : " << tag; +//vama MADNESS_EXCEPTION("TextFstreamInputArchive: not an archive/bad version?", 1); +//vama } +//vama +//vama // For now just skip over typemap +//vama for (int i=0; i<258; ++i) is.getline(buf,256); +//vama } +//vama + } +} diff --git a/src/madness/world/h5_archive.h b/src/madness/world/h5_archive.h new file mode 100644 index 00000000000..6884c4cf005 --- /dev/null +++ b/src/madness/world/h5_archive.h @@ -0,0 +1,240 @@ +/* + This file is part of MADNESS. + + Copyright (C) 2007,2010 Oak Ridge National Laboratory + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + For more information please contact: + + Robert J. Harrison + Oak Ridge National Laboratory + One Bethel Valley Road + P.O. Box 2008, MS-6367 + + email: harrisonrj@ornl.gov + tel: 865-241-3937 + fax: 865-572-0680 +*/ + +#ifndef MADNESS_WORLD_H5_ARCHIVE_H__INCLUDED +#define MADNESS_WORLD_H5_ARCHIVE_H__INCLUDED + +/** + \file h5_archive.h + \brief Implements an archive wrapping HDF5 filestream. + \ingroup serialization +*/ + + +#ifdef HAVE_HDF5 +# if ! __has_include("hdf5.h") +# error "HAVE_HDF5 is on, but hdf5.h was not found." +# endif + +#include +#include +#include +#include +#include // this injects operator<<(std::ostream,T) for common Ts +#include +#include "hdf5.h" + + + +//#include "hdf5.h" + +namespace madness { + namespace archive { + + /// \addtogroup serialization + /// @{ + + /// Wraps an archive around a text filestream for output. + class H5OutputArchive : public BaseOutputArchive { + mutable std::ofstream os; ///< The filestream. + hid_t file_id; /* identifiers */ + hid_t dataset_id; /* identifiers */ + hid_t dataspace_id; /* identifiers */ + herr_t status; + mutable std::vector v; + madness::archive::VectorOutputArchive var; + + public: + /// Default constructor. + + /// The filename and open modes are optional here; they can be + /// specified later by calling \c open(). + /// \param[in] filename Name of the file to write to. + /// \param[in] mode I/O attributes for opening the file. + H5OutputArchive(const char* filename = nullptr + ): v(), var(v) + { + if (filename) + open(filename); + }; + void open(const char* filename) { + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + } + + + /// Store data to the filestream. + + /// The function only appears (due to \c enable_if) if \c T is + /// serializable. + /// \tparam T The type of data to be written. + /// \param[in] t Location of the data to be written. + /// \param[in] n The number of data items to be written. + template + typename std::enable_if< madness::is_ostreammable_v >::type + store(const T* t, long n) const { + if (n > 0) { + var.store(t, n); + //here flush, maybe + } + } + + void close(){ + // flush(); + status = H5Dclose(dataset_id); + status = H5Sclose(dataspace_id); + status = H5Fclose(file_id); + } + + /// Flush the filestream. + void flush() { + int dims = 1 ; + hsize_t dimens_1d = v.size(); + + dataspace_id = H5Screate_simple(dims, &dimens_1d, NULL); + +// /* Create the dataset. */ + dataset_id = + H5Dcreate2(file_id, "/dset", H5T_NATIVE_UCHAR, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + status = H5Dwrite(dataset_id, H5T_NATIVE_UCHAR, H5S_ALL, H5S_ALL, H5P_DEFAULT, v.data()); + } + + /// Destructor. +// ~H5OutputArchive() { +// close(); +// } + }; // class H5OutputArchive + + + /// Wraps an archive around a text filestream for input. + class H5InputArchive : public BaseInputArchive { + hid_t file_id; /* identifiers */ + hid_t dataset_id; /* identifiers */ + hid_t dataspace_id; /* identifiers */ + herr_t status; + mutable std::vector v; + madness::archive::VectorInputArchive var; + mutable std::size_t i; ///< Current input location. + + public: + /// Default constructor. + + /// The filename and open modes are optional here; they can be + /// specified later by calling \c open(). + /// \param[in] filename Name of the file to read from. + /// \param[in] mode I/O attributes for opening the file. + H5InputArchive(const char* filename = nullptr + ): v(), var(v) + { + if (filename) + open(filename); + } + void open(const char* filename) { + + file_id = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); +// + dataset_id = H5Dopen2(file_id, "/dset", H5P_DEFAULT); +// + hid_t datatype = H5Dget_type(dataset_id); + size_t size = H5Tget_size(datatype); + H5T_class_t t_class = H5Tget_class(datatype); + + hid_t dataspace_id = H5Dget_space(dataset_id); /* dataspace handle */ + int rank = H5Sget_simple_extent_ndims(dataspace_id); + hsize_t dims_out[1]; + status = H5Sget_simple_extent_dims(dataspace_id, dims_out, NULL); + v.resize((unsigned long)(dims_out[0])); + status = H5Dread(dataset_id, H5T_NATIVE_UCHAR, H5S_ALL, H5S_ALL, H5P_DEFAULT, v.data()); + } + /// The function only appears (due to \c enable_if) if \c T is + /// serializable. + /// \tparam T The type of data to be read. + /// \param[out] t Where to put the loaded data. + /// \param[in] n The number of data items to be loaded. + template + typename std::enable_if< madness::is_istreammable_v >::type + load(T* t, long n) const { + var.load(t,n); + } + /// Close the filestream. + void close() { + status = H5Dclose(dataset_id); +// status = H5Sclose(dataspace_id); + status = H5Fclose(file_id); + } + /// Destructor. +// ~H5InputArchive() { +// close(); +// } + }; // class H5InputArchive + + /// Implement pre/postamble storage routines for a \c TextFstreamOutputArchive. + + /// \tparam T The type to be stored. + template + struct ArchivePrePostImpl { + /// Write the preamble to the archive. + + /// \param[in] ar The archive. + static void preamble_store(const H5OutputArchive& ar) { + } + + /// Write the postamble to the archive. + + /// \param[in] ar The archive. + static inline void postamble_store(const H5OutputArchive& ar) { + } + }; // struct ArchivePrePostImpl + + /// Implement pre/postamble load routines for a \c TextFstreamInputArchive. + + /// \tparam T The expected type to be loaded. + template + struct ArchivePrePostImpl { + /// Load the preamble and perform type-checking. + + /// \param[in] ar The archive. + static inline void preamble_load(const H5InputArchive& ar) { + } + + /// Load the postamble and perform type-checking. + + /// \param[in] ar The archive. + static inline void postamble_load(const H5InputArchive& ar) { + } + }; // struct ArchivePrePostImpl + + /// @} + } +} + +#endif // HAVE_HDF5 + +#endif // MADNESS_WORLD_H5_ARCHIVE_H__INCLUDED diff --git a/src/madness/world/h5p_archive.h b/src/madness/world/h5p_archive.h new file mode 100644 index 00000000000..bdccf7f41da --- /dev/null +++ b/src/madness/world/h5p_archive.h @@ -0,0 +1,331 @@ +/* + This file is part of MADNESS. + + Copyright (C) 2007,2010 Oak Ridge National Laboratory + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + For more information please contact: + + Robert J. Harrison + Oak Ridge National Laboratory + One Bethel Valley Road + P.O. Box 2008, MS-6367 + + email: harrisonrj@ornl.gov + tel: 865-241-3937 + fax: 865-572-0680 +*/ + +#ifdef H5_HAVE_PARALLEL + +#ifndef MADNESS_WORLD_H5P_ARCHIVE_H__INCLUDED +#define MADNESS_WORLD_H5P_ARCHIVE_H__INCLUDED + +/** + \file text_fstream_archive.h + \brief Implements an archive wrapping text filestream. + \ingroup serialization +*/ + +#include +#include +#include +#include +#include // this injects operator<<(std::ostream,T) for common Ts +#include + +#include "hdf5.h" + +namespace madness { + namespace archive { + + /// \addtogroup serialization + /// @{ + + /// Wraps an archive around a text filestream for output. + class H5POutputArchive : public BaseOutputArchive { + mutable std::ofstream os; ///< The filestream. + hid_t file_id; /* HDF5 file IDs */ + hid_t acc_tpl1; /* File access templates */ + hid_t dataset_id; /* Dataset transfer properties list */ + hid_t dataspace_id; /* Dataspace ID */ + hid_t xfer_plist; /* Dataset transfer properties list */ + hid_t file_dataspace; /* File dataspace ID */ + hid_t mem_dataspace; /* memory dataspace ID */ + hid_t dataset1, dataset2; /* Dataset ID */ + + herr_t status; + mutable std::vector v; + madness::archive::VectorOutputArchive var; + mutable World* world; ///< The world. + + public: + /// Default constructor. + + /// The filename and open modes are optional here; they can be + /// specified later by calling \c open(). + /// \param[in] filename Name of the file to write to. + /// \param[in] mode I/O attributes for opening the file. + H5POutputArchive(World& world, const char* filename = nullptr + ):world(&world), v(), var(v) + { + if (filename) + open( world, filename); + }; + void open(World& world,const char* filename) { + std::cout << "h5p this open " << filename << std::endl; + auto iocomm = world.mpi.comm().Get_mpi_comm(); + std::cout << "h5p this open comm " << iocomm << std::endl; + auto ioinfo = world.mpi.comm().Get_info(); + + herr_t ret; +// MPI_Info info = MPI_INFO_NULL; +// /* ------------------- +// * START AN HDF5 FILE +// * -------------------*/ +// /* setup file access template with parallel IO access. */ + acc_tpl1 = H5Pcreate(H5P_FILE_ACCESS); + std::cout << "H5Pcreate access succeed" << std::endl; +// +// /* set Parallel access with communicator */ + ret = H5Pset_fapl_mpio(acc_tpl1, iocomm, ioinfo); + std::cout << "H5Pset_fapl_mpio succeed" << std::endl; +// +// /* create the file collectively */ + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl1); + std::cout << "H5Fcreate succeed" << std::endl; +// + /* Release file-access template */ + ret = H5Pclose(acc_tpl1); + +// file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + + } + + + /// Store data to the filestream. + + /// The function only appears (due to \c enable_if) if \c T is + /// serializable. + /// \tparam T The type of data to be written. + /// \param[in] t Location of the data to be written. + /// \param[in] n The number of data items to be written. + template + typename std::enable_if< madness::is_ostreammable_v >::type + store(const T* t, long n) const { + //using madness::operators::operator<<; + std::cout << "h5p this store 1 v " << v.size() << std::endl; + std::cout << "h5p this store 1 n " << n << std::endl; + if (n > 0) { + var.store(t, n); +// //here flush + } + } + + void close(){ + // flush(); + std::cout << "h5p this close " << std::endl; +// status = H5Dclose(dataset_id); +// status = H5Sclose(dataspace_id); + status = H5Fclose(file_id); + } + + /// Flush the filestream. + void flush() { + std::cout << "h5p this flush " << std::endl; + int dims = 1 ; + hsize_t dimens_1d = v.size(); + std::cout << "h5p this flush 1 v " << v.size() << std::endl; + + herr_t ret; + std::cout <<"h5p flush my rank " <rank() << std::endl; + int my =2 ; + int total =2 ; + + +// +//// dataspace_id = H5Screate_simple(dims, &dimens_1d, NULL); +// dataspace_id = H5Screate_simple(dims, &dimens_1d, NULL); +// std::cout << "h5p this dataspace created " << std::endl; +// +//// /* Create the dataset. */ +// +// dataset_id = +// H5Dcreate2(file_id, "/dset", H5T_NATIVE_UCHAR, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); +//// H5Dcreate2(file_id, "/dset", H5T_NATIVE_UCHAR, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); +//// status = H5Dwrite(dataset_id, H5T_NATIVE_UCHAR, H5S_ALL, H5S_ALL, H5P_DEFAULT, v.data()); +// file_dataspace = H5Dget_space (dataset_id); +// xfer_plist = H5Pcreate(H5P_DATASET_XFER); +// ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); +// std::cout << "h5p this flush H5Pcreate xfer succeed created " << std::endl; +// +// +// +// hsize_t stride = 1; +// hsize_t count = dimens_1d / world.mpi.comm().Get_rank(); +// hsize_t start = mpi_rank * count[0]; + +// ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, &file_offset, &stride, &dsize, NULL); //stride=NULL? +// ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, NULL); +// +// ret = H5Dwrite(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, data_array1); + } + + /// Destructor. +// ~H5POutputArchive() { +// close(); +// } + }; // class H5POutputArchive + + + /// Wraps an archive around a text filestream for input. + class H5PInputArchive : public BaseInputArchive { + hid_t file_id; /* identifiers */ + hid_t dataset_id; /* identifiers */ + hid_t dataspace_id; /* identifiers */ + herr_t status; + mutable std::vector v; + madness::archive::VectorInputArchive var; + mutable std::size_t i; ///< Current input location. + + public: + /// Default constructor. + + /// The filename and open modes are optional here; they can be + /// specified later by calling \c open(). + /// \param[in] filename Name of the file to read from. + /// \param[in] mode I/O attributes for opening the file. + H5PInputArchive(const char* filename = nullptr + ): v(), var(v) + { + if (filename) + open(filename); + } + void open(const char* filename) { + std::cout << "h5p input this open " << filename << std::endl; + //file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file_id = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); +// + dataset_id = H5Dopen2(file_id, "/dset", H5P_DEFAULT); +// + hid_t datatype = H5Dget_type(dataset_id); + size_t size = H5Tget_size(datatype); + H5T_class_t t_class = H5Tget_class(datatype); + if(t_class == H5T_INTEGER) + std::cout << "h5p input ajua" << (int)t_class<< std::endl; + std::cout << "h5p input size "<< (int)size << std::endl; + hid_t dataspace_id = H5Dget_space(dataset_id); /* dataspace handle */ + int rank = H5Sget_simple_extent_ndims(dataspace_id); + std::cout << "h5p input rank "<< (int)rank << std::endl; + hsize_t dims_out[1]; + status = H5Sget_simple_extent_dims(dataspace_id, dims_out, NULL); + printf("h5p input rank %d, dimensions %lu x %lu \n", rank, (unsigned long)(dims_out[0]), + (unsigned long)(dims_out[0])); + v.resize((unsigned long)(dims_out[0])); + status = H5Dread(dataset_id, H5T_NATIVE_UCHAR, H5S_ALL, H5S_ALL, H5P_DEFAULT, v.data()); +// status = H5Dread(dataset_id, H5T_NATIVE_UCHAR, H5S_ALL, H5S_ALL, H5P_DEFAULT, v.data()); +// +// int dims = 1 ; +// hsize_t dimens_1d = v->size(); +// +// dataspace_id = H5Screate_simple(dims, &dimens_1d, NULL); +// +// /* Create the dataset. */ +// dataset_id = +// H5Dcreate2(file_id, "/dset", H5T_NATIVE_UCHAR, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + } + /// The function only appears (due to \c enable_if) if \c T is + /// serializable. + /// \tparam T The type of data to be read. + /// \param[out] t Where to put the loaded data. + /// \param[in] n The number of data items to be loaded. + template + typename std::enable_if< madness::is_istreammable_v >::type + load(T* t, long n) const { + std::cout << "h5p input this load " << std::endl; + std::cout << "h5p input this load 1 n " << n << std::endl; + std::cout << "h5p this store 1 v " << v.size() << std::endl; + var.load(t,n); + } + /// Close the filestream. + void close() { + std::cout << "h5p inp this close " << std::endl; + status = H5Dclose(dataset_id); +// status = H5Sclose(dataspace_id); + status = H5Fclose(file_id); + } + /// Destructor. +// ~H5PInputArchive() { +// close(); +// } + }; // class H5PInputArchive + + /// Implement pre/postamble storage routines for a \c TextFstreamOutputArchive. + + /// \tparam T The type to be stored. + template + struct ArchivePrePostImpl { + /// Write the preamble to the archive. + + /// \param[in] ar The archive. + static void preamble_store(const H5POutputArchive& ar) { + } + + /// Write the postamble to the archive. + + /// \param[in] ar The archive. + static inline void postamble_store(const H5POutputArchive& ar) { + } + }; // struct ArchivePrePostImpl + + /// Implement pre/postamble load routines for a \c TextFstreamInputArchive. + + /// \tparam T The expected type to be loaded. + template + struct ArchivePrePostImpl { + /// Load the preamble and perform type-checking. + + /// \param[in] ar The archive. + static inline void preamble_load(const H5PInputArchive& ar) { + } + + /// Load the postamble and perform type-checking. + + /// \param[in] ar The archive. + static inline void postamble_load(const H5PInputArchive& ar) { + } + }; // struct ArchivePrePostImpl + + /// @} + } + + template <> + struct is_archive : std::true_type {}; + template <> + struct is_archive : std::true_type {}; + template <> + struct is_output_archive : std::true_type {}; + template <> + struct is_input_archive : std::true_type {}; + template + struct is_default_serializable_helper || std::is_function_v || is_any_function_pointer_v>> : std::true_type {}; + template + struct is_default_serializable_helper || is_any_function_pointer_v>> : std::true_type {}; + +} //namespace + +#endif // MADNESS_WORLD_H5P_ARCHIVE_H__INCLUDED +#endif // H5_HAVE_PARALLEL diff --git a/src/madness/world/safempi.h b/src/madness/world/safempi.h index a6e42633f97..3f57ce92555 100644 --- a/src/madness/world/safempi.h +++ b/src/madness/world/safempi.h @@ -495,6 +495,7 @@ namespace SafeMPI { struct Impl { MPI_Comm comm; + MPI_Info info; int me; int numproc; bool owner; @@ -601,6 +602,7 @@ namespace SafeMPI { MADNESS_MPI_TEST(MPI_Comm_create(pimpl->comm, group.group(), &group_comm)); int me; MADNESS_MPI_TEST(MPI_Comm_rank(group_comm, &me)); int nproc; MADNESS_MPI_TEST(MPI_Comm_size(group_comm, &nproc)); + //MPI_Info info; MADNESS_MPI_TEST(MPI_Info_create(&info)); return Intracomm(std::shared_ptr(new Impl(group_comm, me, nproc, true))); } @@ -709,6 +711,11 @@ namespace SafeMPI { return pimpl->numproc; } + MPI_Info& Get_info() const { + MADNESS_ASSERT(pimpl); + return pimpl->info; + } + Request Isend(const void* buf, const int count, const MPI_Datatype datatype, const int dest, const int tag) const { MADNESS_ASSERT(pimpl); SAFE_MPI_GLOBAL_MUTEX; @@ -777,6 +784,11 @@ namespace SafeMPI { SAFE_MPI_GLOBAL_MUTEX; MADNESS_MPI_TEST(MPI_Allreduce(const_cast(sendbuf), recvbuf, count, datatype, op, pimpl->comm)); } + void Scan(const void* sendbuf, void* recvbuf, const int count, const MPI_Datatype datatype, const MPI_Op op) const { + MADNESS_ASSERT(pimpl); + SAFE_MPI_GLOBAL_MUTEX; + MADNESS_MPI_TEST(MPI_Scan(const_cast(sendbuf), recvbuf, count, datatype, op, pimpl->comm)); + } bool Get_attr(int key, void* value) const { MADNESS_ASSERT(pimpl); int flag = 0; @@ -839,6 +851,7 @@ namespace SafeMPI { MADNESS_MPI_TEST(MPI_Comm_rank(COMM_WORLD.pimpl->comm, & COMM_WORLD.pimpl->me)); MADNESS_MPI_TEST(MPI_Comm_size(COMM_WORLD.pimpl->comm, & COMM_WORLD.pimpl->numproc)); MADNESS_MPI_TEST(MPI_Comm_set_errhandler(COMM_WORLD.pimpl->comm, MPI_ERRORS_RETURN)); + //MADNESS_MPI_TEST(MPI_Info_create(&info)); } } // namespace detail diff --git a/src/madness/world/stubmpi.h b/src/madness/world/stubmpi.h index fe7c189cd51..d6246092c3c 100644 --- a/src/madness/world/stubmpi.h +++ b/src/madness/world/stubmpi.h @@ -181,6 +181,10 @@ inline int MPI_Allreduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype, if(sendbuf != MPI_IN_PLACE) std::memcpy(recvbuf, sendbuf, count); return MPI_SUCCESS; } +inline int MPI_Scan(void *sendbuf, void *recvbuf, int count, MPI_Datatype, MPI_Op, MPI_Comm) { + if(sendbuf != MPI_IN_PLACE) std::memcpy(recvbuf, sendbuf, count); + return MPI_SUCCESS; +} inline int MPI_Comm_get_attr(MPI_Comm, int, void*, int*) { return MPI_ERR_COMM; } diff --git a/src/madness/world/type_traits.h b/src/madness/world/type_traits.h index 6aae10ab845..7d0daecbb1e 100644 --- a/src/madness/world/type_traits.h +++ b/src/madness/world/type_traits.h @@ -473,6 +473,10 @@ namespace madness { class VectorInputArchive; class TextFstreamOutputArchive; class TextFstreamInputArchive; + class H5OutputArchive; + class H5PInputArchive; + class H5POutputArchive; + class H5InputArchive; class MPIRawOutputArchive; class MPIRawInputArchive; class MPIOutputArchive; @@ -554,6 +558,11 @@ namespace madness { struct is_default_serializable_helper || std::is_function_v || is_any_function_pointer_v>> : std::true_type {}; template struct is_default_serializable_helper || is_any_function_pointer_v>> : std::true_type {}; + //vama fix argumments + template + struct is_default_serializable_helper || std::is_function_v || is_any_function_pointer_v>> : std::true_type {}; + template + struct is_default_serializable_helper || is_any_function_pointer_v>> : std::true_type {}; template struct is_default_serializable_helper::value>> : std::true_type {}; template @@ -590,6 +599,10 @@ namespace madness { template <> struct is_archive : std::true_type {}; template <> + struct is_archive : std::true_type {}; + template <> + struct is_archive : std::true_type {}; + template <> struct is_archive : std::true_type {}; template <> struct is_archive : std::true_type {}; @@ -615,6 +628,8 @@ namespace madness { template <> struct is_output_archive : std::true_type {}; template <> + struct is_output_archive : std::true_type {}; + template <> struct is_output_archive : std::true_type {}; template <> struct is_output_archive : std::true_type {}; @@ -632,6 +647,8 @@ namespace madness { template <> struct is_input_archive : std::true_type {}; template <> + struct is_input_archive : std::true_type {}; + template <> struct is_input_archive : std::true_type {}; template <> struct is_input_archive : std::true_type {}; @@ -678,6 +695,9 @@ namespace madness { template <> struct is_text_archive : std::true_type {}; +//vama template <> +//vama struct is_text_archive : std::true_type {}; + /// \brief \c is_text_archive_v is a shorthand for \c is_text_archive::value /// \tparam Archive an archive type template diff --git a/src/madness/world/worldgop.h b/src/madness/world/worldgop.h index 00fbd65ee4a..9c2a2b0d025 100644 --- a/src/madness/world/worldgop.h +++ b/src/madness/world/worldgop.h @@ -904,6 +904,89 @@ namespace madness { else return std::vector(); } + template + std::vector concat1(const std::vector& v, size_t bufsz=1024*1024) { + SafeMPI::Request req0, req1; + ProcessID parent, child0, child1; + world_.mpi.binary_tree_info(0, parent, child0, child1); + Tag gsum_tag = world_.mpi.unique_tag(); + + MADNESS_ASSERT(bufsz <= std::numeric_limits::max()); + + unsigned char* buf0 = new unsigned char[bufsz]; + unsigned char* buf1 = new unsigned char[bufsz]; + + if (child0 != -1) req0 = world_.mpi.Irecv(buf0, bufsz, MPI_BYTE, child0, gsum_tag); + if (child1 != -1) req1 = world_.mpi.Irecv(buf1, bufsz, MPI_BYTE, child1, gsum_tag); + + std::vector left, right; + if (child0 != -1) { + World::await(req0); + archive::BufferInputArchive ar(buf0, bufsz); + ar & left; + } + if (child1 != -1) { + World::await(req1); + archive::BufferInputArchive ar(buf1, bufsz); + ar & right; + for (unsigned int i=0; i + void myscan(T* buf, T* off) { + SafeMPI::Request req0, req1; + ProcessID parent, child0, child1; + world_.mpi.binary_tree_info(0, parent, child0, child1); + Tag gsum_tag = world_.mpi.unique_tag(); + WorldSumOp op; + + int nelem = 1; + + T* buf0 = new T[nelem]; + T* buf1 = new T[nelem]; + + if (child0 != -1) req0 = world_.mpi.Irecv(buf0, nelem*sizeof(T), MPI_BYTE, child0, gsum_tag); + if (child1 != -1) req1 = world_.mpi.Irecv(buf1, nelem*sizeof(T), MPI_BYTE, child1, gsum_tag); + + if (child0 != -1) { + World::await(req0); + for (long i=0; i<(long)nelem; ++i) off[i] = op(0,buf0[i]); + //for (long i=0; i<(long)nelem; ++i) off[i] = op(buf[i],buf0[i]); + std::cout << "chi a " << std::endl; + } + if (child1 != -1) { + World::await(req1); + //for (long i=0; i<(long)nelem; ++i) off[i] = op(buf[i],buf1[i]); + for (long i=0; i<(long)nelem; ++i) off[i] = op(0,buf0[i]); + std::cout << "chi b " << std::endl; + } + + + if (parent != -1) { + req0 = world_.mpi.Isend(buf, sizeof(T), MPI_BYTE, parent, gsum_tag); + World::await(req0); + } + + delete [] buf0; + delete [] buf1; + + } + + /// Receive data from \c source /// \tparam valueT The data type stored in cache diff --git a/src/madness/world/worldmpi.h b/src/madness/world/worldmpi.h index 71361d8868d..0803e030d18 100644 --- a/src/madness/world/worldmpi.h +++ b/src/madness/world/worldmpi.h @@ -433,6 +433,12 @@ namespace madness { SafeMPI::Intracomm::Bcast(&buffer, sizeof(T), MPI_BYTE, root); } +// template +// typename std::enable_if::value, void>::type +// Scan(T& buffer, int root) const { +// SafeMPI::Intracomm::Scan(&buffer, sizeof(T), MPI_BYTE,root); +// } + /// Access the rank of this process. /// \return The rank of this process. @@ -447,6 +453,9 @@ namespace madness { /// \return The number of processes. int size() const { return SafeMPI::Intracomm::Get_size(); } + + /// \return The number of processes. + auto info() const { return SafeMPI::Intracomm::Get_info(); } }; // class WorldMpiInterface }