Skip to content

Commit

Permalink
Convert vector of basic type to numpy array instead of list
Browse files Browse the repository at this point in the history
-Port numpy_proxy class from nda
  • Loading branch information
Wentzell committed May 26, 2020
1 parent 06304b4 commit a6e12d6
Show file tree
Hide file tree
Showing 4 changed files with 260 additions and 27 deletions.
2 changes: 1 addition & 1 deletion c++/cpp2py/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
add_library(cpp2py signal_handler.cpp exceptions.cpp)
add_library(cpp2py signal_handler.cpp exceptions.cpp numpy_proxy.cpp)
add_library(cpp2py::cpp2py ALIAS cpp2py)

target_compile_options(cpp2py PRIVATE -std=c++14 -fPIC)
Expand Down
109 changes: 83 additions & 26 deletions c++/cpp2py/converters/vector.hpp
Original file line number Diff line number Diff line change
@@ -1,49 +1,106 @@
#pragma once
//#include <vector>
//#include <numeric>
#include <vector>
#include <numpy/arrayobject.h>

#include "../numpy_proxy.hpp"

namespace cpp2py {

template <typename T>
static void delete_pycapsule(PyObject *capsule) {
auto *ptr = static_cast<std::unique_ptr<T[]> *>(PyCapsule_GetPointer(capsule, "guard"));
delete ptr;
}

// Convert vector to numpy_proxy, WARNING: Deep Copy
template <typename T> numpy_proxy make_numpy_proxy_from_vector(std::vector<T> const &v) {

auto * data_ptr = new std::unique_ptr<T[]>{new T[v.size()]};
std::copy(begin(v), end(v), data_ptr->get());
auto capsule = PyCapsule_New(data_ptr, "guard", &delete_pycapsule<T>);

return {1, // rank
npy_type<std::remove_const_t<T>>,
(void *)data_ptr->get(),
std::is_const_v<T>,
v_t{static_cast<long>(v.size())}, // extents
v_t{sizeof(T)}, // strides
capsule};
}

// Make a new vector from numpy view
template <typename T>
std::vector<T> make_vector_from_numpy_proxy(numpy_proxy const &p) {
//EXPECTS(p.extents.size() == 1);
//EXPECTS(p.strides == v_t{sizeof(T)});

T * data = static_cast<T *>(p.data);
long size = p.extents[0];

std::vector<T> v(size);
std::copy(data, data + size, begin(v));
return v;
}

// --------------------------------------

template <typename T> struct py_converter<std::vector<T>> {

// --------------------------------------

static PyObject *c2py(std::vector<T> const &v) {
PyObject *list = PyList_New(0);
for (auto const &x : v) {
pyref y = py_converter<T>::c2py(x);
if (y.is_null() or (PyList_Append(list, y) == -1)) {
Py_DECREF(list);
return NULL;
} // error

static PyObject *c2py(std::vector<T> const &v) {

if constexpr (has_npy_type<T>) {
return make_numpy_proxy_from_vector(v).to_python();
}
else{ // Convert to Python List
PyObject *list = PyList_New(0);
for (auto const &x : v) {
pyref y = py_converter<T>::c2py(x);
if (y.is_null() or (PyList_Append(list, y) == -1)) {
Py_DECREF(list);
return NULL;
} // error
}
return list;
}
return list;
}

// --------------------------------------
// --------------------------------------

static bool is_convertible(PyObject *ob, bool raise_exception) {
if (!PySequence_Check(ob)) goto _false;
{
static bool is_convertible(PyObject *ob, bool raise_exception) {
if (PySequence_Check(ob)) {
pyref seq = PySequence_Fast(ob, "expected a sequence");
int len = PySequence_Size(ob);
for (int i = 0; i < len; i++)
if (!py_converter<T>::is_convertible(PySequence_Fast_GET_ITEM((PyObject *)seq, i), raise_exception)) goto _false; //borrowed ref
return true;
} else if (PyArray_Check(ob)) {
PyArrayObject *arr = (PyArrayObject *)(ob);
if (PyArray_TYPE(arr) != npy_type<T>) goto _false;
#ifdef PYTHON_NUMPY_VERSION_LT_17
int rank = arr->nd;
#else
int rank = PyArray_NDIM(arr);
#endif
if (rank != 1) goto _false;
return true;
}
_false:
if (raise_exception) { PyErr_SetString(PyExc_TypeError, "Cannot convert to std::vector"); }
return false;
}

// --------------------------------------

static std::vector<T> py2c(PyObject *ob) {
pyref seq = PySequence_Fast(ob, "expected a sequence");
std::vector<T> res;
int len = PySequence_Size(ob);
for (int i = 0; i < len; i++) res.push_back(py_converter<T>::py2c(PySequence_Fast_GET_ITEM((PyObject *)seq, i))); //borrowed ref
return res;
// --------------------------------------

static std::vector<T> py2c(PyObject *ob) {
if (PySequence_Check(ob)) {
std::vector<T> res;
pyref seq = PySequence_Fast(ob, "expected a sequence");
int len = PySequence_Size(ob);
for (int i = 0; i < len; i++) res.push_back(py_converter<T>::py2c(PySequence_Fast_GET_ITEM((PyObject *)seq, i))); //borrowed ref
return res;
}
//ASSERT(PyArray_Check(ob));
return make_vector_from_numpy_proxy<T>(make_numpy_proxy(ob));
}
};

Expand Down
118 changes: 118 additions & 0 deletions c++/cpp2py/numpy_proxy.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
#include "numpy_proxy.hpp"

namespace cpp2py {

// Make a new view_info
PyObject *numpy_proxy::to_python() {

// Apparently we can not get rid of this
_import_array();

#ifdef PYTHON_NUMPY_VERSION_LT_17
int flags = NPY_BEHAVED & ~NPY_OWNDATA;
#else
int flags = NPY_ARRAY_BEHAVED & ~NPY_ARRAY_OWNDATA;
#endif
// make the array read only
if (is_const) flags &= ~NPY_ARRAY_WRITEABLE;
PyObject *result =
PyArray_NewFromDescr(&PyArray_Type, PyArray_DescrFromType(element_type), rank, extents.data(), strides.data(), data, flags, NULL);
if (not result) return nullptr; // the Python error is set

if (!PyArray_Check(result)) {
PyErr_SetString(PyExc_RuntimeError, "The python object is not a numpy array");
return nullptr;
}

PyArrayObject *arr = (PyArrayObject *)(result);
#ifdef PYTHON_NUMPY_VERSION_LT_17
arr->base = base;
assert(arr->flags == (arr->flags & ~NPY_OWNDATA));
#else
int r = PyArray_SetBaseObject(arr, base);
//EXPECTS(r == 0);
//EXPECTS(PyArray_FLAGS(arr) == (PyArray_FLAGS(arr) & ~NPY_ARRAY_OWNDATA));
#endif
base = nullptr; // ref is stolen by the new object

return result;
}

// ----------------------------------------------------------

// Extract a view_info from python
numpy_proxy make_numpy_proxy(PyObject *obj) {

// Apparently we can not get rid of this
_import_array();

if (obj == NULL) return {};
if (not PyArray_Check(obj)) return {};

numpy_proxy result;

// extract strides and lengths
PyArrayObject *arr = (PyArrayObject *)(obj);

#ifdef PYTHON_NUMPY_VERSION_LT_17
result.rank = arr->nd;
#else
result.rank = PyArray_NDIM(arr);
#endif

result.element_type = PyArray_TYPE(arr);
result.extents.resize(result.rank);
result.strides.resize(result.rank);
result.data = PyArray_DATA(arr);
// base is ignored, stays at nullptr

#ifdef PYTHON_NUMPY_VERSION_LT_17
for (long i = 0; i < result.rank; ++i) {
result.extents[i] = size_t(arr->dimensions[i]);
result.strides[i] = std::ptrdiff_t(arr->strides[i]);
}
#else
for (size_t i = 0; i < result.rank; ++i) {
result.extents[i] = size_t(PyArray_DIMS(arr)[i]);
result.strides[i] = std::ptrdiff_t(PyArray_STRIDES(arr)[i]);
}
#endif

//PRINT(result.rank);
//PRINT(result.element_type);
//PRINT(result.data);

return result;
}

// ----------------------------------------------------------

PyObject *make_numpy_copy(PyObject *obj, int rank, long element_type) {

if (obj == nullptr) return nullptr;

// From obj, we ask the numpy library to make a numpy, and of the correct type.
// This handles automatically the cases where :
// - we have list, or list of list/tuple
// - the numpy type is not the one we want.
// - adjust the dimension if needed
// If obj is an array :
// - if Order is same, don't change it
// - else impose it (may provoque a copy).
// if obj is not array :
// - Order = FortranOrder or SameOrder - > Fortran order otherwise C

int flags = 0; //(ForceCast ? NPY_FORCECAST : 0) ;// do NOT force a copy | (make_copy ? NPY_ENSURECOPY : 0);
//if (!(PyArray_Check(obj) ))
//flags |= ( IndexMapType::traversal_order == indexmaps::mem_layout::c_order(rank) ? NPY_C_CONTIGUOUS : NPY_F_CONTIGUOUS); //impose mem order
#ifdef PYTHON_NUMPY_VERSION_LT_17
flags |= (NPY_C_CONTIGUOUS); //impose mem order
flags |= (NPY_ENSURECOPY);
#else
flags |= (NPY_ARRAY_C_CONTIGUOUS); // impose mem order
flags |= (NPY_ARRAY_ENSURECOPY);
#endif
return PyArray_FromAny(obj, PyArray_DescrFromType(element_type), rank, rank, flags, NULL); // new ref
}

} // namespace nda::python
58 changes: 58 additions & 0 deletions c++/cpp2py/numpy_proxy.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#pragma once
#include <Python.h>
#include <numpy/arrayobject.h>

#include <vector>
#include <complex>

namespace cpp2py {

using v_t = std::vector<long>;

// the basic information for a numpy array
struct numpy_proxy {
int rank = 0;
long element_type = 0;
void *data = nullptr;
bool is_const = false;
v_t extents, strides;
PyObject *base = nullptr; // The ref. counting guard typically

// Returns a new ref (or NULL if failure) with a new numpy.
// If failure, return null with the Python exception set
PyObject *to_python();
};

// From a numpy, extract the info. Better than a constructor, I want to use the aggregate constructor of the struct also.
numpy_proxy make_numpy_proxy(PyObject *);

// Make a copy in Python with the given rank and element_type
// If failure, return null with the Python exception set
PyObject *make_numpy_copy(PyObject *obj, int rank, long elements_type);

//
template <typename T> constexpr long npy_type = -1;
template <typename T> constexpr bool has_npy_type = (npy_type<T> == -1);

#define NPY_CONVERT(C, P) template <> constexpr long npy_type<C> = P;
NPY_CONVERT(bool, NPY_BOOL)
NPY_CONVERT(char, NPY_STRING)
NPY_CONVERT(signed char, NPY_BYTE)
NPY_CONVERT(unsigned char, NPY_UBYTE)
NPY_CONVERT(short, NPY_SHORT)
NPY_CONVERT(unsigned short, NPY_USHORT)
NPY_CONVERT(int, NPY_INT)
NPY_CONVERT(unsigned int, NPY_UINT)
NPY_CONVERT(long, NPY_LONG)
NPY_CONVERT(unsigned long, NPY_ULONG)
NPY_CONVERT(long long, NPY_LONGLONG)
NPY_CONVERT(unsigned long long, NPY_ULONGLONG)
NPY_CONVERT(float, NPY_FLOAT)
NPY_CONVERT(double, NPY_DOUBLE)
NPY_CONVERT(long double, NPY_LONGDOUBLE)
NPY_CONVERT(std::complex<float>, NPY_CFLOAT)
NPY_CONVERT(std::complex<double>, NPY_CDOUBLE)
NPY_CONVERT(std::complex<long double>, NPY_CLONGDOUBLE)
#undef NPY_CONVERT

} // namespace cpp2py

0 comments on commit a6e12d6

Please sign in to comment.