Skip to content

Commit

Permalink
PROTON-2816/PROTON-2817: Support building python extension with unbun…
Browse files Browse the repository at this point in the history
…dled proton

There are 3 cases:
* Bundled proton library - this should be the default and is what you
  should use with pip etc.
* System proton library - this is what you should use in a distro
  package.
* Use the in tree library built earlier in the build process - this is
  what you usually want for developer debugging, and should be the
  version actually tested by the in tree tests.

This change also add control of the ability to package and test in
isolated virtual python environments. Using these is the default, but
under some circumstances you do not want the build process to construct
a virtual environment by pulling packages from the internet. So we now
have an option to turn this behaviour off and rely on the hosts python
environment.
  • Loading branch information
astitcher committed Apr 19, 2024
1 parent e3ba722 commit a2e9eb6
Show file tree
Hide file tree
Showing 5 changed files with 153 additions and 24 deletions.
87 changes: 64 additions & 23 deletions python/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ set(py_dist_files
README.rst
MANIFEST.in
ext_build.py
ext_build_devtree.py
ext_build_unbundled.py
cproton.h
cproton_ext.c
cproton.py
Expand Down Expand Up @@ -128,7 +130,6 @@ add_custom_command(OUTPUT .timestamp.copied_pysrc
COMMAND ${CMAKE_COMMAND} -E copy ${PN_C_SOURCE_DIR}/core/frame_generators.h src/core
COMMAND ${CMAKE_COMMAND} -E copy ${PN_C_SOURCE_DIR}/core/frame_consumers.c src/core
COMMAND ${CMAKE_COMMAND} -E copy ${PN_C_SOURCE_DIR}/core/frame_consumers.h src/core
COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/VERSION.txt .
COMMAND ${CMAKE_COMMAND} -E touch .timestamp.copied_pysrc
DEPENDS generated_c_files ${py_cgen} ${py_csrc} ${py_cinc} ${PROJECT_SOURCE_DIR}/VERSION.txt)

Expand All @@ -139,28 +140,55 @@ foreach(file IN LISTS py_dist_files pysrc)
list(APPEND pysrc_files "${CMAKE_CURRENT_BINARY_DIR}/${file}")
endforeach()

add_custom_target(pysrc_copied ALL DEPENDS .timestamp.copied_pysrc ${pysrc_files})
add_custom_command(OUTPUT VERSION.txt
COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/VERSION.txt .
DEPENDS ${PROJECT_SOURCE_DIR}/VERSION.txt)

add_custom_target(pysrc_copied DEPENDS ${pysrc_files} VERSION.txt)
add_custom_target(pypkg_src_copied ALL DEPENDS pysrc_copied .timestamp.copied_pysrc)

add_custom_command(OUTPUT ./tox.ini
COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/tox.ini" tox.ini
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/tox.ini")

option(ENABLE_PYTHON_ISOLATED "Enable python building/testing with isolated environments." ON)
option(BUILD_PYTHON_UNBUNDLED_PKG "Build Python package without bundling qpid-proton-core library" Off)

# Make python source and binary packages if we have prerequisites
check_python_module("build" BUILD_MODULE_FOUND)

if (ENABLE_PYTHON_ISOLATED)
set (pypkgbuildoption "")
else ()
set (pypkgbuildoption "-n")
endif ()

check_python_module("setuptools" SETUPTOOLS_MODULE_FOUND)
check_python_module("wheel" WHEEL_MODULE_FOUND)
check_python_module("build" BUILD_MODULE_FOUND)
check_python_module("cffi" CFFI_MODULE_FOUND)
if (BUILD_MODULE_FOUND AND SETUPTOOLS_MODULE_FOUND AND WHEEL_MODULE_FOUND AND CFFI_MODULE_FOUND)
add_custom_command(OUTPUT .timestamp.dist
DEPENDS pysrc_copied
COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.dist
COMMAND ${Python_EXECUTABLE} -m build -n
COMMAND ${CMAKE_COMMAND} -E touch .timestamp.dist)

if (BUILD_MODULE_FOUND AND
(ENABLE_PYTHON_ISOLATED OR (SETUPTOOLS_MODULE_FOUND AND WHEEL_MODULE_FOUND AND CFFI_MODULE_FOUND)))
if (BUILD_PYTHON_UNBUNDLED_PKG)
add_custom_command(OUTPUT .timestamp.dist
DEPENDS pypkg_src_copied
COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.dist
COMMAND ${CMAKE_COMMAND} -E env
QPID_PYTHON_UNBUNDLING=unbundled
${Python_EXECUTABLE} -m build ${pypkgbuildoption}
COMMAND ${CMAKE_COMMAND} -E touch .timestamp.dist)
else ()
add_custom_command(OUTPUT .timestamp.dist
DEPENDS pypkg_src_copied
COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.dist
COMMAND ${Python_EXECUTABLE} -m build ${pypkgbuildoption}
COMMAND ${CMAKE_COMMAND} -E touch .timestamp.dist)
endif ()
add_custom_target(pydist ALL DEPENDS .timestamp.dist)
endif ()

if (BUILD_TESTING)
add_custom_target(pytest_cffi ALL DEPENDS .timestamp.cproton_ffi)
add_custom_target(pytest_cffi ALL DEPENDS .timestamp.test_env)

# python test: python/tests/proton-test
set (py_src "${CMAKE_CURRENT_SOURCE_DIR}")
Expand All @@ -179,8 +207,7 @@ if (BUILD_TESTING)
set (python_coverage_options -m coverage run --parallel-mode)
endif(CMAKE_BUILD_TYPE MATCHES "Coverage")

option(ENABLE_PYTHON_TEST_VENV "Enable python testing within a separate virtual environment." ON)
if (ENABLE_PYTHON_TEST_VENV)
if (ENABLE_PYTHON_ISOLATED)
# Create Python virtual environment to run tests
set(pytest_venv "${py_bin}/pytest_env")
# Have to use a conditional here as you can't use generator expressions in OUTPUT or BYPRODUCTS
Expand All @@ -193,27 +220,41 @@ if (BUILD_TESTING)
set(pytest_executable "${pytest_bin}/python${CMAKE_EXECUTABLE_SUFFIX}")

add_custom_command(
OUTPUT ${pytest_venv}/env.txt
OUTPUT .timestamp.test_env
COMMAND ${Python_EXECUTABLE} -m venv ${pytest_venv}
COMMAND ${pytest_executable} -m pip install --disable-pip-version-check cffi setuptools
COMMAND ${CMAKE_COMMAND} -E env
"CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}"
"QPID_PROTON_CORE_TARGET_DIR=$<TARGET_FILE_DIR:qpid-proton-core>"
"QPID_PYTHON_UNBUNDLING=devtree"
${pytest_executable} -m pip install -e .
COMMAND ${pytest_executable} -m pip freeze > ${pytest_venv}/env.txt
COMMAND ${CMAKE_COMMAND} -E touch .timestamp.test_env
BYPRODUCTS ${pytest_executable}
)

else()
set(pytest_executable "${Python_EXECUTABLE}")
set(pytest_venv "${py_bin}")
FILE(TOUCH "${pytest_venv}/env.txt")
add_custom_command(
OUTPUT .timestamp.test_env
COMMAND ${CMAKE_COMMAND} -E remove -f ${pytest_venv}/env.txt
COMMAND ${CMAKE_COMMAND} -E env
"CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}"
"QPID_PROTON_CORE_TARGET_DIR=$<TARGET_FILE_DIR:qpid-proton-core>"
${pytest_executable} ext_build_devtree.py
COMMAND ${CMAKE_COMMAND} -E touch .timestamp.test_env
DEPENDS pysrc_copied qpid-proton-core
)
endif()

# Create c code for cffi extension
add_custom_command(
OUTPUT .timestamp.cproton_ffi
COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.cproton_ffi
COMMAND ${pytest_executable} ext_build.py
COMMAND ${CMAKE_COMMAND} -E touch .timestamp.cproton_ffi
DEPENDS ${pytest_venv}/env.txt pysrc_copied
)
# If we are on windows copy the qpid-proton-core dll to the test directory so we can find it easily
if (WIN32)
add_custom_command(
OUTPUT .timestamp.test_env
APPEND
COMMAND ${CMAKE_COMMAND} -E copy "$<TARGET_FILE:qpid-proton-core>" .
)
endif()

pn_add_test(
INTERPRETED
Expand Down
1 change: 1 addition & 0 deletions python/MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

include VERSION.txt
include ext_build.py
include ext_build_unbundled.py
include cproton.h
include cproton_ext.c
include cproton.py
Expand Down
41 changes: 41 additions & 0 deletions python/ext_build_devtree.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

import os

from cffi import FFI

bld_tree_top = os.environ.get("CMAKE_BINARY_DIR")
bld_clibdir = os.environ.get("QPID_PROTON_CORE_TARGET_DIR")
cdefs = open('cproton.h').read()
c_code = open('cproton_ext.c').read()
extra_link_args = [f"-Wl,-rpath,{bld_clibdir}"] if os.name == 'posix' else None
ffibuilder = FFI()
ffibuilder.cdef(cdefs)
ffibuilder.set_source(
"cproton_ffi",
c_code,
include_dirs=[f"{bld_tree_top}/c/include"],
library_dirs=[f"{bld_clibdir}"],
libraries=["qpid-proton-core"],
extra_link_args=extra_link_args,
)

if __name__ == "__main__":
ffibuilder.compile(verbose=True)
42 changes: 42 additions & 0 deletions python/ext_build_unbundled.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

import cffi.pkgconfig

from cffi import FFI

ffibuilder = FFI()

# cdef() expects a single string declaring the C types, functions and
# globals needed to use the shared object. It must be in valid C syntax
# with cffi extensions
cdefs = open('cproton.h').read()
ffibuilder.cdef(cdefs)

cffi.pkgconfig.flags_from_pkgconfig(['libqpid-proton-core'])

c_code = open('cproton_ext.c').read()
ffibuilder.set_source_pkgconfig(
"cproton_ffi",
['libqpid-proton-core'],
c_code,
)

if __name__ == "__main__":
ffibuilder.compile(verbose=True)
6 changes: 5 additions & 1 deletion python/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,12 @@
# specific language governing permissions and limitations
# under the License.

import os
from setuptools import setup

unbundling = os.environ.get("QPID_PYTHON_UNBUNDLING")
unbundling = f'_{unbundling}' if unbundling else ''

setup(
cffi_modules='ext_build.py:ffibuilder'
cffi_modules=f'ext_build{unbundling}.py:ffibuilder'
)

0 comments on commit a2e9eb6

Please sign in to comment.