From a6e12d62370966555cb289f97babd9659303c66b Mon Sep 17 00:00:00 2001 From: Nils Wentzell Date: Tue, 26 May 2020 16:32:18 -0400 Subject: [PATCH] Convert vector of basic type to numpy array instead of list -Port numpy_proxy class from nda --- c++/cpp2py/CMakeLists.txt | 2 +- c++/cpp2py/converters/vector.hpp | 109 +++++++++++++++++++++------- c++/cpp2py/numpy_proxy.cpp | 118 +++++++++++++++++++++++++++++++ c++/cpp2py/numpy_proxy.hpp | 58 +++++++++++++++ 4 files changed, 260 insertions(+), 27 deletions(-) create mode 100644 c++/cpp2py/numpy_proxy.cpp create mode 100644 c++/cpp2py/numpy_proxy.hpp diff --git a/c++/cpp2py/CMakeLists.txt b/c++/cpp2py/CMakeLists.txt index 4012ad3..662e03e 100644 --- a/c++/cpp2py/CMakeLists.txt +++ b/c++/cpp2py/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(cpp2py signal_handler.cpp exceptions.cpp) +add_library(cpp2py signal_handler.cpp exceptions.cpp numpy_proxy.cpp) add_library(cpp2py::cpp2py ALIAS cpp2py) target_compile_options(cpp2py PRIVATE -std=c++14 -fPIC) diff --git a/c++/cpp2py/converters/vector.hpp b/c++/cpp2py/converters/vector.hpp index 8b77265..ec97887 100644 --- a/c++/cpp2py/converters/vector.hpp +++ b/c++/cpp2py/converters/vector.hpp @@ -1,49 +1,106 @@ #pragma once -//#include -//#include +#include +#include + +#include "../numpy_proxy.hpp" namespace cpp2py { + template + static void delete_pycapsule(PyObject *capsule) { + auto *ptr = static_cast *>(PyCapsule_GetPointer(capsule, "guard")); + delete ptr; + } + + // Convert vector to numpy_proxy, WARNING: Deep Copy + template numpy_proxy make_numpy_proxy_from_vector(std::vector const &v) { + + auto * data_ptr = new std::unique_ptr{new T[v.size()]}; + std::copy(begin(v), end(v), data_ptr->get()); + auto capsule = PyCapsule_New(data_ptr, "guard", &delete_pycapsule); + + return {1, // rank + npy_type>, + (void *)data_ptr->get(), + std::is_const_v, + v_t{static_cast(v.size())}, // extents + v_t{sizeof(T)}, // strides + capsule}; + } + + // Make a new vector from numpy view + template + std::vector make_vector_from_numpy_proxy(numpy_proxy const &p) { + //EXPECTS(p.extents.size() == 1); + //EXPECTS(p.strides == v_t{sizeof(T)}); + + T * data = static_cast(p.data); + long size = p.extents[0]; + + std::vector v(size); + std::copy(data, data + size, begin(v)); + return v; + } + + // -------------------------------------- + template struct py_converter> { - - // -------------------------------------- - - static PyObject *c2py(std::vector const &v) { - PyObject *list = PyList_New(0); - for (auto const &x : v) { - pyref y = py_converter::c2py(x); - if (y.is_null() or (PyList_Append(list, y) == -1)) { - Py_DECREF(list); - return NULL; - } // error + + static PyObject *c2py(std::vector const &v) { + + if constexpr (has_npy_type) { + return make_numpy_proxy_from_vector(v).to_python(); + } + else{ // Convert to Python List + PyObject *list = PyList_New(0); + for (auto const &x : v) { + pyref y = py_converter::c2py(x); + if (y.is_null() or (PyList_Append(list, y) == -1)) { + Py_DECREF(list); + return NULL; + } // error + } + return list; } - return list; } - // -------------------------------------- + // -------------------------------------- - static bool is_convertible(PyObject *ob, bool raise_exception) { - if (!PySequence_Check(ob)) goto _false; - { + static bool is_convertible(PyObject *ob, bool raise_exception) { + if (PySequence_Check(ob)) { pyref seq = PySequence_Fast(ob, "expected a sequence"); int len = PySequence_Size(ob); for (int i = 0; i < len; i++) if (!py_converter::is_convertible(PySequence_Fast_GET_ITEM((PyObject *)seq, i), raise_exception)) goto _false; //borrowed ref return true; + } else if (PyArray_Check(ob)) { + PyArrayObject *arr = (PyArrayObject *)(ob); + if (PyArray_TYPE(arr) != npy_type) goto _false; +#ifdef PYTHON_NUMPY_VERSION_LT_17 + int rank = arr->nd; +#else + int rank = PyArray_NDIM(arr); +#endif + if (rank != 1) goto _false; + return true; } _false: if (raise_exception) { PyErr_SetString(PyExc_TypeError, "Cannot convert to std::vector"); } return false; } - // -------------------------------------- - - static std::vector py2c(PyObject *ob) { - pyref seq = PySequence_Fast(ob, "expected a sequence"); - std::vector res; - int len = PySequence_Size(ob); - for (int i = 0; i < len; i++) res.push_back(py_converter::py2c(PySequence_Fast_GET_ITEM((PyObject *)seq, i))); //borrowed ref - return res; + // -------------------------------------- + + static std::vector py2c(PyObject *ob) { + if (PySequence_Check(ob)) { + std::vector res; + pyref seq = PySequence_Fast(ob, "expected a sequence"); + int len = PySequence_Size(ob); + for (int i = 0; i < len; i++) res.push_back(py_converter::py2c(PySequence_Fast_GET_ITEM((PyObject *)seq, i))); //borrowed ref + return res; + } + //ASSERT(PyArray_Check(ob)); + return make_vector_from_numpy_proxy(make_numpy_proxy(ob)); } }; diff --git a/c++/cpp2py/numpy_proxy.cpp b/c++/cpp2py/numpy_proxy.cpp new file mode 100644 index 0000000..5d59367 --- /dev/null +++ b/c++/cpp2py/numpy_proxy.cpp @@ -0,0 +1,118 @@ +#include "numpy_proxy.hpp" + +namespace cpp2py { + + // Make a new view_info + PyObject *numpy_proxy::to_python() { + + // Apparently we can not get rid of this + _import_array(); + +#ifdef PYTHON_NUMPY_VERSION_LT_17 + int flags = NPY_BEHAVED & ~NPY_OWNDATA; +#else + int flags = NPY_ARRAY_BEHAVED & ~NPY_ARRAY_OWNDATA; +#endif + // make the array read only + if (is_const) flags &= ~NPY_ARRAY_WRITEABLE; + PyObject *result = + PyArray_NewFromDescr(&PyArray_Type, PyArray_DescrFromType(element_type), rank, extents.data(), strides.data(), data, flags, NULL); + if (not result) return nullptr; // the Python error is set + + if (!PyArray_Check(result)) { + PyErr_SetString(PyExc_RuntimeError, "The python object is not a numpy array"); + return nullptr; + } + + PyArrayObject *arr = (PyArrayObject *)(result); +#ifdef PYTHON_NUMPY_VERSION_LT_17 + arr->base = base; + assert(arr->flags == (arr->flags & ~NPY_OWNDATA)); +#else + int r = PyArray_SetBaseObject(arr, base); + //EXPECTS(r == 0); + //EXPECTS(PyArray_FLAGS(arr) == (PyArray_FLAGS(arr) & ~NPY_ARRAY_OWNDATA)); +#endif + base = nullptr; // ref is stolen by the new object + + return result; + } + + // ---------------------------------------------------------- + + // Extract a view_info from python + numpy_proxy make_numpy_proxy(PyObject *obj) { + + // Apparently we can not get rid of this + _import_array(); + + if (obj == NULL) return {}; + if (not PyArray_Check(obj)) return {}; + + numpy_proxy result; + + // extract strides and lengths + PyArrayObject *arr = (PyArrayObject *)(obj); + +#ifdef PYTHON_NUMPY_VERSION_LT_17 + result.rank = arr->nd; +#else + result.rank = PyArray_NDIM(arr); +#endif + + result.element_type = PyArray_TYPE(arr); + result.extents.resize(result.rank); + result.strides.resize(result.rank); + result.data = PyArray_DATA(arr); + // base is ignored, stays at nullptr + +#ifdef PYTHON_NUMPY_VERSION_LT_17 + for (long i = 0; i < result.rank; ++i) { + result.extents[i] = size_t(arr->dimensions[i]); + result.strides[i] = std::ptrdiff_t(arr->strides[i]); + } +#else + for (size_t i = 0; i < result.rank; ++i) { + result.extents[i] = size_t(PyArray_DIMS(arr)[i]); + result.strides[i] = std::ptrdiff_t(PyArray_STRIDES(arr)[i]); + } +#endif + + //PRINT(result.rank); + //PRINT(result.element_type); + //PRINT(result.data); + + return result; + } + + // ---------------------------------------------------------- + + PyObject *make_numpy_copy(PyObject *obj, int rank, long element_type) { + + if (obj == nullptr) return nullptr; + + // From obj, we ask the numpy library to make a numpy, and of the correct type. + // This handles automatically the cases where : + // - we have list, or list of list/tuple + // - the numpy type is not the one we want. + // - adjust the dimension if needed + // If obj is an array : + // - if Order is same, don't change it + // - else impose it (may provoque a copy). + // if obj is not array : + // - Order = FortranOrder or SameOrder - > Fortran order otherwise C + + int flags = 0; //(ForceCast ? NPY_FORCECAST : 0) ;// do NOT force a copy | (make_copy ? NPY_ENSURECOPY : 0); + //if (!(PyArray_Check(obj) )) + //flags |= ( IndexMapType::traversal_order == indexmaps::mem_layout::c_order(rank) ? NPY_C_CONTIGUOUS : NPY_F_CONTIGUOUS); //impose mem order +#ifdef PYTHON_NUMPY_VERSION_LT_17 + flags |= (NPY_C_CONTIGUOUS); //impose mem order + flags |= (NPY_ENSURECOPY); +#else + flags |= (NPY_ARRAY_C_CONTIGUOUS); // impose mem order + flags |= (NPY_ARRAY_ENSURECOPY); +#endif + return PyArray_FromAny(obj, PyArray_DescrFromType(element_type), rank, rank, flags, NULL); // new ref + } + +} // namespace nda::python diff --git a/c++/cpp2py/numpy_proxy.hpp b/c++/cpp2py/numpy_proxy.hpp new file mode 100644 index 0000000..de2d6f0 --- /dev/null +++ b/c++/cpp2py/numpy_proxy.hpp @@ -0,0 +1,58 @@ +#pragma once +#include +#include + +#include +#include + +namespace cpp2py { + + using v_t = std::vector; + + // the basic information for a numpy array + struct numpy_proxy { + int rank = 0; + long element_type = 0; + void *data = nullptr; + bool is_const = false; + v_t extents, strides; + PyObject *base = nullptr; // The ref. counting guard typically + + // Returns a new ref (or NULL if failure) with a new numpy. + // If failure, return null with the Python exception set + PyObject *to_python(); + }; + + // From a numpy, extract the info. Better than a constructor, I want to use the aggregate constructor of the struct also. + numpy_proxy make_numpy_proxy(PyObject *); + + // Make a copy in Python with the given rank and element_type + // If failure, return null with the Python exception set + PyObject *make_numpy_copy(PyObject *obj, int rank, long elements_type); + + // + template constexpr long npy_type = -1; + template constexpr bool has_npy_type = (npy_type == -1); + +#define NPY_CONVERT(C, P) template <> constexpr long npy_type = P; + NPY_CONVERT(bool, NPY_BOOL) + NPY_CONVERT(char, NPY_STRING) + NPY_CONVERT(signed char, NPY_BYTE) + NPY_CONVERT(unsigned char, NPY_UBYTE) + NPY_CONVERT(short, NPY_SHORT) + NPY_CONVERT(unsigned short, NPY_USHORT) + NPY_CONVERT(int, NPY_INT) + NPY_CONVERT(unsigned int, NPY_UINT) + NPY_CONVERT(long, NPY_LONG) + NPY_CONVERT(unsigned long, NPY_ULONG) + NPY_CONVERT(long long, NPY_LONGLONG) + NPY_CONVERT(unsigned long long, NPY_ULONGLONG) + NPY_CONVERT(float, NPY_FLOAT) + NPY_CONVERT(double, NPY_DOUBLE) + NPY_CONVERT(long double, NPY_LONGDOUBLE) + NPY_CONVERT(std::complex, NPY_CFLOAT) + NPY_CONVERT(std::complex, NPY_CDOUBLE) + NPY_CONVERT(std::complex, NPY_CLONGDOUBLE) +#undef NPY_CONVERT + +} // namespace cpp2py