PROTON-2816/PROTON-2817: Support building python extension with unbun…

python/CMakeLists.txt

@@ -53,6 +53,8 @@ set(py_dist_files
     README.rst
     MANIFEST.in
     ext_build.py
+    ext_build_devtree.py
+    ext_build_unbundled.py
     cproton.h
     cproton_ext.c
     cproton.py
@@ -128,7 +130,6 @@ add_custom_command(OUTPUT .timestamp.copied_pysrc
                    COMMAND ${CMAKE_COMMAND} -E copy ${PN_C_SOURCE_DIR}/core/frame_generators.h src/core
                    COMMAND ${CMAKE_COMMAND} -E copy ${PN_C_SOURCE_DIR}/core/frame_consumers.c  src/core
                    COMMAND ${CMAKE_COMMAND} -E copy ${PN_C_SOURCE_DIR}/core/frame_consumers.h  src/core
-                   COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/VERSION.txt            .
                    COMMAND ${CMAKE_COMMAND} -E touch .timestamp.copied_pysrc
                    DEPENDS generated_c_files ${py_cgen} ${py_csrc} ${py_cinc} ${PROJECT_SOURCE_DIR}/VERSION.txt)
 
@@ -139,28 +140,55 @@ foreach(file IN LISTS py_dist_files pysrc)
   list(APPEND pysrc_files "${CMAKE_CURRENT_BINARY_DIR}/${file}")
 endforeach()
 
-add_custom_target(pysrc_copied ALL DEPENDS .timestamp.copied_pysrc ${pysrc_files})
+add_custom_command(OUTPUT VERSION.txt
+                  COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/VERSION.txt .
+                  DEPENDS ${PROJECT_SOURCE_DIR}/VERSION.txt)
+
+add_custom_target(pysrc_copied DEPENDS ${pysrc_files} VERSION.txt)
+add_custom_target(pypkg_src_copied ALL DEPENDS pysrc_copied .timestamp.copied_pysrc)
 
 add_custom_command(OUTPUT ./tox.ini
                    COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/tox.ini" tox.ini
                    DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/tox.ini")
 
+option(ENABLE_PYTHON_ISOLATED "Enable python building/testing with isolated environments." ON)
+option(BUILD_PYTHON_UNBUNDLED_PKG "Build Python package without bundling qpid-proton-core library" Off)
+
 # Make python source and binary packages if we have prerequisites
+check_python_module("build" BUILD_MODULE_FOUND)
+
+if (ENABLE_PYTHON_ISOLATED)
+  set (pypkgbuildoption "")
+else ()
+  set (pypkgbuildoption "-n")
+endif ()
+
 check_python_module("setuptools" SETUPTOOLS_MODULE_FOUND)
 check_python_module("wheel" WHEEL_MODULE_FOUND)
-check_python_module("build" BUILD_MODULE_FOUND)
 check_python_module("cffi" CFFI_MODULE_FOUND)
-if (BUILD_MODULE_FOUND AND SETUPTOOLS_MODULE_FOUND AND WHEEL_MODULE_FOUND AND CFFI_MODULE_FOUND)
-  add_custom_command(OUTPUT .timestamp.dist
-    DEPENDS pysrc_copied
-    COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.dist
-    COMMAND ${Python_EXECUTABLE} -m build -n
-    COMMAND ${CMAKE_COMMAND} -E touch .timestamp.dist)
+
+if (BUILD_MODULE_FOUND AND
+    (ENABLE_PYTHON_ISOLATED OR (SETUPTOOLS_MODULE_FOUND AND WHEEL_MODULE_FOUND AND CFFI_MODULE_FOUND)))
+  if (BUILD_PYTHON_UNBUNDLED_PKG)
+    add_custom_command(OUTPUT .timestamp.dist
+      DEPENDS pypkg_src_copied
+      COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.dist
+      COMMAND ${CMAKE_COMMAND} -E env
+        QPID_PYTHON_UNBUNDLING=unbundled
+        ${Python_EXECUTABLE} -m build ${pypkgbuildoption}
+      COMMAND ${CMAKE_COMMAND} -E touch .timestamp.dist)
+  else ()
+    add_custom_command(OUTPUT .timestamp.dist
+      DEPENDS pypkg_src_copied
+      COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.dist
+      COMMAND ${Python_EXECUTABLE} -m build ${pypkgbuildoption}
+      COMMAND ${CMAKE_COMMAND} -E touch .timestamp.dist)
+  endif ()
   add_custom_target(pydist ALL DEPENDS .timestamp.dist)
 endif ()
 
 if (BUILD_TESTING)
-  add_custom_target(pytest_cffi ALL DEPENDS .timestamp.cproton_ffi)
+  add_custom_target(pytest_cffi ALL DEPENDS .timestamp.test_env)
 
   # python test: python/tests/proton-test
   set (py_src "${CMAKE_CURRENT_SOURCE_DIR}")
@@ -179,8 +207,7 @@ if (BUILD_TESTING)
     set (python_coverage_options -m coverage run --parallel-mode)
   endif(CMAKE_BUILD_TYPE MATCHES "Coverage")
 
-  option(ENABLE_PYTHON_TEST_VENV "Enable python testing within a separate virtual environment." ON)
-  if (ENABLE_PYTHON_TEST_VENV)
+  if (ENABLE_PYTHON_ISOLATED)
     # Create Python virtual environment to run tests
     set(pytest_venv "${py_bin}/pytest_env")
     # Have to use a conditional here as you can't use generator expressions in OUTPUT or BYPRODUCTS
@@ -193,27 +220,41 @@ if (BUILD_TESTING)
     set(pytest_executable "${pytest_bin}/python${CMAKE_EXECUTABLE_SUFFIX}")
 
     add_custom_command(
-      OUTPUT ${pytest_venv}/env.txt
+      OUTPUT .timestamp.test_env
       COMMAND ${Python_EXECUTABLE} -m venv ${pytest_venv}
       COMMAND ${pytest_executable} -m pip install --disable-pip-version-check cffi setuptools
+      COMMAND ${CMAKE_COMMAND} -E env
+        "CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}"
+        "QPID_PROTON_CORE_TARGET_DIR=$<TARGET_FILE_DIR:qpid-proton-core>"
+        "QPID_PYTHON_UNBUNDLING=devtree"
+        ${pytest_executable} -m pip install -e .
       COMMAND ${pytest_executable} -m pip freeze > ${pytest_venv}/env.txt
+      COMMAND ${CMAKE_COMMAND} -E touch .timestamp.test_env
       BYPRODUCTS ${pytest_executable}
     )
-
   else()
     set(pytest_executable "${Python_EXECUTABLE}")
     set(pytest_venv "${py_bin}")
-    FILE(TOUCH "${pytest_venv}/env.txt")
+    add_custom_command(
+      OUTPUT .timestamp.test_env
+      COMMAND ${CMAKE_COMMAND} -E remove -f ${pytest_venv}/env.txt
+      COMMAND ${CMAKE_COMMAND} -E env
+        "CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}"
+        "QPID_PROTON_CORE_TARGET_DIR=$<TARGET_FILE_DIR:qpid-proton-core>"
+        ${pytest_executable} ext_build_devtree.py
+      COMMAND ${CMAKE_COMMAND} -E touch .timestamp.test_env
+      DEPENDS pysrc_copied qpid-proton-core
+    )
   endif()
 
-  # Create c code for cffi extension
-  add_custom_command(
-    OUTPUT .timestamp.cproton_ffi
-    COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.cproton_ffi
-    COMMAND ${pytest_executable} ext_build.py
-    COMMAND ${CMAKE_COMMAND} -E touch .timestamp.cproton_ffi
-    DEPENDS ${pytest_venv}/env.txt pysrc_copied
-  )
+  # If we are on windows copy the qpid-proton-core dll to the test directory so we can find it easily
+  if (WIN32)
+    add_custom_command(
+      OUTPUT .timestamp.test_env
+      APPEND
+      COMMAND ${CMAKE_COMMAND} -E copy "$<TARGET_FILE:qpid-proton-core>" .
+    )
+  endif()
 
   pn_add_test(
     INTERPRETED

python/MANIFEST.in

@@ -18,6 +18,7 @@
 
 include VERSION.txt
 include ext_build.py
+include ext_build_unbundled.py
 include cproton.h
 include cproton_ext.c
 include cproton.py

python/ext_build_devtree.py

@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+
+from cffi import FFI
+
+bld_tree_top = os.environ.get("CMAKE_BINARY_DIR")
+bld_clibdir = os.environ.get("QPID_PROTON_CORE_TARGET_DIR")
+cdefs = open('cproton.h').read()
+c_code = open('cproton_ext.c').read()
+extra_link_args = [f"-Wl,-rpath,{bld_clibdir}"] if os.name == 'posix' else None
+ffibuilder = FFI()
+ffibuilder.cdef(cdefs)
+ffibuilder.set_source(
+    "cproton_ffi",
+    c_code,
+    include_dirs=[f"{bld_tree_top}/c/include"],
+    library_dirs=[f"{bld_clibdir}"],
+    libraries=["qpid-proton-core"],
+    extra_link_args=extra_link_args,
+)
+
+if __name__ == "__main__":
+    ffibuilder.compile(verbose=True)

python/ext_build_unbundled.py

@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import cffi.pkgconfig
+
+from cffi import FFI
+
+ffibuilder = FFI()
+
+# cdef() expects a single string declaring the C types, functions and
+# globals needed to use the shared object. It must be in valid C syntax
+# with cffi extensions
+cdefs = open('cproton.h').read()
+ffibuilder.cdef(cdefs)
+
+cffi.pkgconfig.flags_from_pkgconfig(['libqpid-proton-core'])
+
+c_code = open('cproton_ext.c').read()
+ffibuilder.set_source_pkgconfig(
+    "cproton_ffi",
+    ['libqpid-proton-core'],
+    c_code,
+)
+
+if __name__ == "__main__":
+    ffibuilder.compile(verbose=True)

python/setup.py

@@ -16,8 +16,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import os
 from setuptools import setup
 
+unbundling = os.environ.get("QPID_PYTHON_UNBUNDLING")
+unbundling = f'_{unbundling}' if unbundling else ''
+
 setup(
-    cffi_modules='ext_build.py:ffibuilder'
+    cffi_modules=f'ext_build{unbundling}.py:ffibuilder'
 )