Skip to content

Commit

Permalink
Bulding manylinux wheels using cibuildwheel (#73)
Browse files Browse the repository at this point in the history
  • Loading branch information
mwydmuch authored Oct 11, 2023
1 parent 867f45b commit cec42fc
Show file tree
Hide file tree
Showing 16 changed files with 324 additions and 98 deletions.
55 changes: 0 additions & 55 deletions .github/workflows/build-publish-linux.yml

This file was deleted.

90 changes: 90 additions & 0 deletions .github/workflows/build-publish.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# This workflow will build and (if release) publish Python distributions to PyPI
# For more information see:
# - https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
# - https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
# - Adapted from https://github.com/Farama-Foundation/PettingZoo/blob/e230f4d80a5df3baf9bd905149f6d4e8ce22be31/.github/workflows/build-publish.yml

name: build-publish

on:
workflow_dispatch:
push:
branches: [master]
pull_request:
branches: [master]
release:
types: [published]

jobs:
build_wheels:
name: Build wheels on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-22.04] #, macos-11] - disable macos, as it's not working for a moment

steps:
- uses: actions/checkout@v3

- name: Set up QEMU
if: runner.os == 'Linux'
uses: docker/setup-qemu-action@v2
with:
platforms: all

- name: Build wheels
uses: pypa/[email protected]
env:
# Configure cibuildwheel to build native archs, and some emulated ones
CIBW_ARCHS_LINUX: x86_64 # aarch64 - disable ARM, as it's not working for a moment
#CIBW_ARCHS_MACOS: x86_64 arm64 - macos is disabled for a moment
CIBW_BUILD_VERBOSITY: 3 # Increase verbosity to see what's going on
CIBW_REPAIR_WHEEL_COMMAND_LINUX: > # Print additional info from auditwheel
auditwheel show {wheel} && auditwheel repair -w {dest_dir} {wheel}
- name: Report built wheels
run: |
ls -l ./wheelhouse/*.whl
- name: Upload artifacts
uses: actions/upload-artifact@v3
with:
path: ./wheelhouse/*.whl

build_sdist:
name: Build source distribution
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

- name: Build sdist
run: pipx run build --sdist

- name: Upload sdist
uses: actions/upload-artifact@v3
with:
path: dist/*.tar.gz

upload_pypi:
needs: [build_wheels, build_sdist]
runs-on: ubuntu-latest
environment: pypi
permissions:
id-token: write
if: github.event_name == 'release' && github.event.action == 'published'
steps:
- name: Download all dists
uses: actions/download-artifact@v3
with:
# Unpacks default artifact into dist/
# If `name: artifact` is omitted, the action will create extra parent dir
name: artifact
path: dist

- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
password: ${{ secrets.PYPI_API_TOKEN }}
# To test:
# with:
# repository_url: https://test.pypi.org/legacy/
11 changes: 8 additions & 3 deletions .github/workflows/test-cpp.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,19 @@ jobs:
steps:
- uses: actions/checkout@v3

- uses: actions/setup-python@v4
with:
python-version: 3.9

- name: Install system packages
run: |
sudo apt-get update
sudo apt-get install git cmake capnproto zlib1g-dev g++ build-essential python3-dev python3-pip pkg-config libzip-dev software-properties-common libbz2-dev python3-opengl
sudo apt-get install git cmake capnproto zlib1g-dev build-essential pkg-config libzip-dev software-properties-common libbz2-dev python3-opengl
- name: Build test files
- name: Install pip package and build tests
run: |
python3 -m pip install -e .
cmake .
make -j
make -f tests/Makefile
- name: Run tests
Expand Down
13 changes: 7 additions & 6 deletions .github/workflows/test-python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,24 +10,25 @@ jobs:
runs-on: ubuntu-latest # todo, add more OS systems to see if they work as well
strategy:
matrix:
python-version: ['3.7', '3.8', '3.9', '3.10'] # '3.11'
python-version: ['3.8', '3.9', '3.10'] # '3.11'

steps:
- uses: actions/checkout@v3

- uses: actions/setup-python@v2
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

- name: Install system packages
run: |
sudo apt-get update
sudo apt-get install git cmake capnproto zlib1g-dev g++ build-essential python3-dev python3-pip pkg-config libzip-dev software-properties-common libbz2-dev xvfb python3-opengl
sudo apt-get install git cmake capnproto zlib1g-dev build-essential pkg-config libzip-dev software-properties-common libbz2-dev xvfb python3-opengl
- name: Install pip packages
run: |
python -m pip install --upgrade pip cmake wheel setuptools pytest
pip install -e .
python3 -m pip install --upgrade pip pytest
python3 -m pip install -e .
- name: Run tests
run: xvfb-run -s '-screen 0 1024x768x24' pytest
run: |
xvfb-run -s '-screen 0 1024x768x24' pytest
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -139,3 +139,7 @@ Temporary

# VSCode config
/.vscode/

# cibuildwheel
wheelhouse
tests/test_cibuildwheel/tmp_dockerfiles
48 changes: 32 additions & 16 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,32 @@ if(POLICY CMP0048)
cmake_policy(SET CMP0048 NEW)
endif()

find_package(Git QUIET)
if(LINUX)
set(PYEXT_SUFFIX
".so"
CACHE STRING "Suffix for Python extension modules")
set(DYNLIB_SUFFIX
".so"
CACHE STRING "Suffix for dynamic libraries")
elseif(APPLE)
set(PYEXT_SUFFIX
".so"
CACHE STRING "Suffix for Python extension modules")
set(DYNLIB_SUFFIX
".dylib"
CACHE STRING "Suffix for dynamic libraries")
elseif(WIN32)
set(PYEXT_SUFFIX
".pyd"
CACHE STRING "Suffix for Python extension modules" FORCE)
set(DYNLIB_SUFFIX
".dll"
CACHE STRING "Suffix for dynamic libraries")
else()
message(FATAL_ERROR "Unsupported platform")
endif()

find_package(PythonInterp REQUIRED)
find_package(Git QUIET)

file(READ "${CMAKE_SOURCE_DIR}/retro/VERSION.txt" PROJECT_VERSION)
string(REGEX REPLACE "\n$" "" PROJECT_VERSION "${PROJECT_VERSION}")
Expand Down Expand Up @@ -45,7 +68,9 @@ set(BUILD_PYTHON
mark_as_advanced(BUILD_PYTHON)

if(BUILD_PYTHON)
find_package(PythonLibs REQUIRED)
find_package(Python 3 COMPONENTS Interpreter Development.Module)
else()
find_package(Python 3 COMPONENTS Interpreter)
endif()

if(WIN32 OR BUILD_MANYLINUX)
Expand Down Expand Up @@ -103,16 +128,6 @@ else()
WORKING_DIRECTORY "${LUA_INCLUDE_DIRS}")
endif()

if(NOT WIN32)
set(PYEXT_SUFFIX
".so"
CACHE STRING "Suffix for Python extension modules")
else()
set(PYEXT_SUFFIX
".pyd"
CACHE STRING "Suffix for Python extension modules" FORCE)
endif()

if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
set(STATIC_LDFLAGS "-static-libstdc++ -Wl,--exclude-libs,ALL")
endif()
Expand Down Expand Up @@ -184,7 +199,8 @@ function(add_core platform core_name)
${CMAKE_COMMAND} -E env CFLAGS=${core_cflags} CXXFLAGS=${core_cxxflags}
LDFLAGS=${core_ldflags} $(MAKE) -f ${makefile} CC="${CMAKE_C_COMPILER}"
CXX="${CMAKE_CXX_COMPILER}" fpic=${core_fpic_flags} ${libretro_platform}
COMMAND ${CMAKE_COMMAND} -E copy "${core_name}_libretro*" "${TARGET_PATH}"
COMMAND ${CMAKE_COMMAND} -E copy "${core_name}_libretro${DYNLIB_SUFFIX}"
"${TARGET_PATH}"
WORKING_DIRECTORY "cores/${platform}/${subdir}"
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/retro/cores/${core_name}-version")
unset(core_ldflags)
Expand Down Expand Up @@ -339,7 +355,7 @@ if(CAPN_PROTO_FOUND)
endif()

include_directories(src retro third-party/pybind11/include third-party
third-party/gtest/googletest/include ${PYTHON_INCLUDE_DIRS})
third-party/gtest/googletest/include ${Python_INCLUDE_DIRS})

if(BUILD_PYTHON)
add_library(retro SHARED src/retro.cpp)
Expand All @@ -359,7 +375,7 @@ if(BUILD_PYTHON)
add_definitions(-DMS_WIN64)
endif()

set(PYBIND_LIBS "${PYTHON_LIBRARY}")
set(PYBIND_LIBS "${Python_LIBRARY}")
endif()

target_link_libraries(retro retro-base ${PYBIND_LIBS} ${STATIC_LDFLAGS})
Expand Down
21 changes: 21 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
[build-system]
requires = ["cmake>=3.2.0", "setuptools", "wheel", "build"]

[tool.cibuildwheel]
# We need to build for the following Python versions:
build = "cp{38,39,310,311}-*"

[tool.cibuildwheel.linux]
# Only manylinux is supported (no musl)
build = "cp{38,39,310,311}-manylinux*"

# For manylinux_2_28 we need to install the following dependencies using yum:
before-all = "yum install -y cmake git pkgconf-pkg-config zlib-devel libzip-devel bzip2-devel"

# Only build for x86_64 and aarch64 are officially supported
archs = "x86_64 aarch64"
manylinux-x86_64-image = "manylinux_2_28"
manylinux-aarch64-image = "manylinux_2_28"

[tool.cibuildwheel.macos]
before-all = "brew install pkg-config capnp [email protected] qt5"
Loading

0 comments on commit cec42fc

Please sign in to comment.