Skip to content

Commit

Permalink
[Draft] Python
Browse files Browse the repository at this point in the history
  • Loading branch information
@ax3l
ax3l committed Sep 1, 2020
1 parent b2717a3 commit 69f91f9
Show file tree
Hide file tree
Showing 4 changed files with 94 additions and 39 deletions.
18 changes: 16 additions & 2 deletions include/openPMD/binding/python/Numpy.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,12 @@ namespace openPMD
return Datatype::ULONG;
else if( dt.is(pybind11::dtype("ulonglong")) )
return Datatype::ULONGLONG;
//else if( dt.is(pybind11::dtype("clongdouble")) )
// return Datatype::CLONG_DOUBLE;
else if( dt.is(pybind11::dtype("cdouble")) )
return Datatype::CDOUBLE;
else if( dt.is(pybind11::dtype("csingle")) )
return Datatype::CFLOAT;
else if( dt.is(pybind11::dtype("longdouble")) )
return Datatype::LONG_DOUBLE;
else if( dt.is(pybind11::dtype("double")) )
Expand All @@ -65,8 +71,10 @@ namespace openPMD
return Datatype::FLOAT;
else if( dt.is(pybind11::dtype("bool")) )
return Datatype::BOOL;
else
else {
pybind11::print(dt);
throw std::runtime_error("Datatype '...' not known in 'dtype_from_numpy'!"); // _s.format(dt)
}
}

/** Return openPMD::Datatype from py::buffer_info::format
Expand All @@ -79,7 +87,7 @@ namespace openPMD
// refs:
// https://docs.scipy.org/doc/numpy-1.15.0/reference/arrays.interface.html
// https://docs.python.org/3/library/struct.html#format-characters
// std::cout << " scalar type '" << buf.format << "'" << std::endl;
std::cout << " scalar type '" << fmt << "'" << std::endl;
// typestring: encoding + type + number of bytes
if( fmt.find("?") != std::string::npos )
return DT::BOOL;
Expand All @@ -103,6 +111,12 @@ namespace openPMD
return DT::ULONG;
else if( fmt.find("Q") != std::string::npos )
return DT::ULONGLONG;
else if( fmt.find("Zf") != std::string::npos )
return DT::CFLOAT;
else if( fmt.find("Zd") != std::string::npos )
return DT::CDOUBLE;
else if( fmt.find("Zg") != std::string::npos )
return DT::CLONG_DOUBLE;
else if( fmt.find("f") != std::string::npos )
return DT::FLOAT;
else if( fmt.find("d") != std::string::npos )
Expand Down
31 changes: 30 additions & 1 deletion src/binding/python/Attributable.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,10 @@
#include "openPMD/auxiliary/Variant.hpp"
#include "openPMD/binding/python/Numpy.hpp"

#include <array>
#include <complex>
#include <string>
#include <vector>
#include <array>


// std::variant
Expand Down Expand Up @@ -113,6 +114,15 @@ bool setAttributeFromBufferInfo(
case DT::LONG_DOUBLE:
return attr.setAttribute( key, *static_cast<long double*>(buf.ptr) );
break;
case DT::CFLOAT:
return attr.setAttribute( key, *static_cast<std::complex<float>*>(buf.ptr) );
break;
case DT::CDOUBLE:
return attr.setAttribute( key, *static_cast<std::complex<double>*>(buf.ptr) );
break;
//case DT::CLONG_DOUBLE:
// return attr.setAttribute( key, *static_cast<std::complex<long double>*>(buf.ptr) );
// break;
default:
throw std::runtime_error("set_attribute: Unknown "
"Python type '" + buf.format +
Expand Down Expand Up @@ -155,6 +165,7 @@ bool setAttributeFromBufferInfo(
static_cast<bool*>(buf.ptr) + buf.size
) );
else */
std::cout << "+++++++++++ BUFFER: " << buf.format << std::endl;
if( buf.format.find("b") != std::string::npos )
return attr.setAttribute( key,
std::vector<char>(
Expand Down Expand Up @@ -215,6 +226,24 @@ bool setAttributeFromBufferInfo(
static_cast<unsigned long long*>(buf.ptr),
static_cast<unsigned long long*>(buf.ptr) + buf.size
) );
else if( buf.format.find("Zf") != std::string::npos )
return attr.setAttribute( key,
std::vector<std::complex<float>>(
static_cast<std::complex<float>*>(buf.ptr),
static_cast<std::complex<float>*>(buf.ptr) + buf.size
) );
else if( buf.format.find("Zd") != std::string::npos )
return attr.setAttribute( key,
std::vector<std::complex<double>>(
static_cast<std::complex<double>*>(buf.ptr),
static_cast<std::complex<double>*>(buf.ptr) + buf.size
) );
else if( buf.format.find("Zg") != std::string::npos )
return attr.setAttribute( key,
std::vector<std::complex<long double>>(
static_cast<std::complex<long double>*>(buf.ptr),
static_cast<std::complex<long double>*>(buf.ptr) + buf.size
) );
else if( buf.format.find("f") != std::string::npos )
return attr.setAttribute( key,
std::vector<float>(
Expand Down
7 changes: 7 additions & 0 deletions src/binding/python/RecordComponent.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include "openPMD/auxiliary/ShareRaw.hpp"
#include "openPMD/binding/python/Numpy.hpp"

#include <complex>
#include <string>
#include <algorithm>
#include <tuple>
Expand Down Expand Up @@ -395,6 +396,12 @@ load_chunk(RecordComponent & r, Offset const & offset, Extent const & extent, st
r.loadChunk<double>(shareRaw((double*) a.mutable_data()), offset, extent);
else if( r.getDatatype() == Datatype::FLOAT )
r.loadChunk<float>(shareRaw((float*) a.mutable_data()), offset, extent);
//else if( r.getDatatype() == Datatype::CLONG_DOUBLE )
// r.loadChunk<std::complex<long double>>(shareRaw((std::complex<long double>*) a.mutable_data()), offset, extent);
else if( r.getDatatype() == Datatype::CDOUBLE )
r.loadChunk<std::complex<double>>(shareRaw((std::complex<double>*) a.mutable_data()), offset, extent);
else if( r.getDatatype() == Datatype::CFLOAT )
r.loadChunk<std::complex<float>>(shareRaw((std::complex<float>*) a.mutable_data()), offset, extent);
else if( r.getDatatype() == Datatype::BOOL )
r.loadChunk<bool>(shareRaw((bool*) a.mutable_data()), offset, extent);
else
Expand Down
77 changes: 41 additions & 36 deletions test/python/unittest/API/APITest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,9 +158,9 @@ def attributeRoundTrip(self, file_ending):
series.set_attribute("single", np.single(1.234))
series.set_attribute("double", np.double(1.234567))
series.set_attribute("longdouble", np.longdouble(1.23456789))
series.set_attribute("csingle", np.complex64([1., 2.]))
series.set_attribute("cdouble", np.complex128([3., 4.]))
series.set_attribute("clongdouble", np.complex256([5., 6.]))
series.set_attribute("csingle", np.complex64(1.+2.j))
series.set_attribute("cdouble", np.complex128(3.+4.j))
#series.set_attribute("clongdouble", np.complex256(5.+6.j))
# array of ...
series.set_attribute("arr_int16", (np.int16(23), np.int16(26), ))
series.set_attribute("arr_int32", (np.int32(34), np.int32(37), ))
Expand All @@ -187,14 +187,14 @@ def attributeRoundTrip(self, file_ending):
series.set_attribute("l_longdouble",
[np.longdouble(7.8e9), np.longdouble(8.2e3)])
series.set_attribute("l_csingle",
[np.complex64([5.6, 7.8]),
np.complex64([5.9, 5.8])])
[np.csingle(5.6+7.8j),
np.csingle(5.9+5.8j)]) # ComplexWarning: Casting complex values to real discards the imaginary part
series.set_attribute("l_cdouble",
[np.complex128([6.7, 6.8]),
np.complex128([7.1, 7.2])])
series.set_attribute("l_clongdouble",
[np.complex256([7.8e9, -6.5e9]),
np.complex256([8.2e3, -9.1e3])])
[np.complex_(6.7+6.8j),
np.complex_(7.1+7.2j)])
#series.set_attribute("l_clongdouble",
# [np.clongfloat(7.8e9-6.5e9j),
# np.clongfloat(8.2e3-9.1e3j)])
# numpy.array of ...
series.set_attribute("nparr_int16",
np.array([234, 567], dtype=np.int16))
Expand All @@ -214,9 +214,9 @@ def attributeRoundTrip(self, file_ending):
series.set_attribute("nparr_cdouble",
np.array([4.5 + 1.1j, 6.7 - 2.2j],
dtype=np.complex128))
series.set_attribute("nparr_clongdouble",
np.array([8.9 + 7.8j, 7.6 + 9.2j],
dtype=np.complex256))
#series.set_attribute("nparr_clongdouble",
# np.array([8.9 + 7.8j, 7.6 + 9.2j],
# dtype=np.complex256))

# c_types
# TODO remove the .value and handle types directly?
Expand Down Expand Up @@ -260,7 +260,8 @@ def attributeRoundTrip(self, file_ending):
1.234567)
self.assertAlmostEqual(series.get_attribute("longdouble"),
1.23456789)
self.assertAlmostEqual(series.get_attribute("csingle"), 1.+2.j)
np.testing.assert_almost_equal(series.get_attribute("csingle"),
np.complex64(1.+2.j))
self.assertAlmostEqual(series.get_attribute("cdouble"),
3.+4.j)
self.assertAlmostEqual(series.get_attribute("clongdouble"),
Expand All @@ -287,13 +288,13 @@ def attributeRoundTrip(self, file_ending):
[np.double(6.7), np.double(7.1)])
self.assertListEqual(series.get_attribute("l_longdouble"),
[np.longdouble(7.8e9), np.longdouble(8.2e3)])
# l_csingle
self.assertListEqual(series.get_attribute("l_cdouble"),
[np.complex128(6.7 + 6.8j),
np.double(7.1 + 7.2j)])
self.assertListEqual(series.get_attribute("l_clongdouble"),
[np.complex256(7.8e9 - 6.5e9j),
np.complex256(8.2e3 - 9.1e3j)])
# TODO: l_csingle
#self.assertListEqual(series.get_attribute("l_cdouble"),
# [np.complex128(6.7 + 6.8j),
# np.double(7.1 + 7.2j)])
#self.assertListEqual(series.get_attribute("l_clongdouble"),
# [np.complex256(7.8e9 - 6.5e9j),
# np.complex256(8.2e3 - 9.1e3j)])

# numpy.array of ...
self.assertListEqual(series.get_attribute("nparr_int16"),
Expand All @@ -314,9 +315,9 @@ def attributeRoundTrip(self, file_ending):
np.testing.assert_almost_equal(
series.get_attribute("nparr_cdouble"),
[4.5 + 1.1j, 6.7 - 2.2j])
np.testing.assert_almost_equal(
series.get_attribute("nparr_clongdouble"),
[8.9 + 7.8j, 7.6 + 9.2j])
#np.testing.assert_almost_equal(
# series.get_attribute("nparr_clongdouble"),
# [8.9 + 7.8j, 7.6 + 9.2j])
# TODO instead of returning lists, return all arrays as np.array?
# self.assertEqual(
# series.get_attribute("nparr_int16").dtype, np.int16)
Expand Down Expand Up @@ -420,13 +421,14 @@ def makeConstantRoundTrip(self, file_ending):
DS(np.dtype("complex128"), extent))
ms["complex128"][SCALAR].make_constant(
np.complex128(1.234567 + 2.345678j))
ms["complex256"][SCALAR].reset_dataset(
DS(np.dtype("complex256"), extent))
ms["complex256"][SCALAR].make_constant(
np.complex256(1.23456789 + 2.34567890j))
#ms["complex256"][SCALAR].reset_dataset(
# DS(np.dtype("complex256"), extent))
#ms["complex256"][SCALAR].make_constant(
# np.complex256(1.23456789 + 2.34567890j))

# flush and close file
del series
#return

# read back
series = io.Series(
Expand Down Expand Up @@ -493,12 +495,15 @@ def makeConstantRoundTrip(self, file_ending):
np.dtype('double'))
self.assertTrue(ms["longdouble"][SCALAR].load_chunk(o, e).dtype
== np.dtype('longdouble'))
self.assertTrue(ms["complex64"][SCALAR].load_chunk(o, e).dtype ==
np.dtype('complex64'))
self.assertTrue(ms["complex128"][SCALAR].load_chunk(o, e).dtype ==
np.dtype('complex128'))
self.assertTrue(ms["complex256"][SCALAR].load_chunk(o, e).dtype
== np.dtype('complex256'))
if file_ending != "json":
print("++++++++++++++++", file_ending)
print(ms["complex64"][SCALAR].load_chunk(o, e).dtype)
self.assertTrue(ms["complex64"][SCALAR].load_chunk(o, e).dtype ==
np.dtype('complex64'))
self.assertTrue(ms["complex128"][SCALAR].load_chunk(o, e).dtype ==
np.dtype('complex128'))
#self.assertTrue(ms["complex256"][SCALAR].load_chunk(o, e).dtype
# == np.dtype('complex256'))

self.assertEqual(ms["int16"][SCALAR].load_chunk(o, e),
np.int16(234))
Expand All @@ -522,8 +527,8 @@ def makeConstantRoundTrip(self, file_ending):
np.complex64(1.234 + 2.345j))
self.assertEqual(ms["complex128"][SCALAR].load_chunk(o, e),
np.complex128(1.234567 + 2.345678j))
self.assertEqual(ms["complex256"][SCALAR].load_chunk(o, e),
np.complex256(1.23456789 + 2.34567890j))
#self.assertEqual(ms["complex256"][SCALAR].load_chunk(o, e),
# np.complex256(1.23456789 + 2.34567890j))

def testConstantRecords(self):
backend_filesupport = {
Expand Down

0 comments on commit 69f91f9

Please sign in to comment.