Bulding manylinux wheels using cibuildwheel

.github/workflows/build-publish.yml

@@ -0,0 +1,90 @@
+# This workflow will build and (if release) publish Python distributions to PyPI
+# For more information see:
+#   - https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+#   - https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
+#   - Adapted from https://github.com/Farama-Foundation/PettingZoo/blob/e230f4d80a5df3baf9bd905149f6d4e8ce22be31/.github/workflows/build-publish.yml
+
+name: build-publish
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+  release:
+    types: [published]
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-22.04] #, macos-11] - disable macos, as it's not working for a moment
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: all
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+        env:
+          # Configure cibuildwheel to build native archs, and some emulated ones
+          CIBW_ARCHS_LINUX: x86_64 # aarch64 - disable ARM, as it's not working for a moment
+          #CIBW_ARCHS_MACOS: x86_64 arm64 - macos is disabled for a moment
+          CIBW_BUILD_VERBOSITY: 3  # Increase verbosity to see what's going on
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: >  # Print additional info from auditwheel
+            auditwheel show {wheel} && auditwheel repair -w {dest_dir} {wheel}
+
+      - name: Report built wheels
+        run: |
+          ls -l ./wheelhouse/*.whl
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - name: Upload sdist
+        uses: actions/upload-artifact@v3
+        with:
+          path: dist/*.tar.gz
+
+  upload_pypi:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    if: github.event_name == 'release' && github.event.action == 'published'
+    steps:
+      - name: Download all dists
+        uses: actions/download-artifact@v3
+        with:
+          # Unpacks default artifact into dist/
+          # If `name: artifact` is omitted, the action will create extra parent dir
+          name: artifact
+          path: dist
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+        # To test:
+        # with:
+        #   repository_url: https://test.pypi.org/legacy/

.github/workflows/test-cpp.yml

@@ -12,14 +12,19 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
       - name: Install system packages
         run: |
           sudo apt-get update
-          sudo apt-get install git cmake capnproto zlib1g-dev g++ build-essential python3-dev python3-pip pkg-config libzip-dev software-properties-common libbz2-dev python3-opengl
+          sudo apt-get install git cmake capnproto zlib1g-dev build-essential pkg-config libzip-dev software-properties-common libbz2-dev python3-opengl
 
-      - name: Build test files
+      - name: Install pip package and build tests
         run: |
-          python3 -m pip install -e .
+          cmake .
+          make -j
           make -f tests/Makefile
 
       - name: Run tests

.github/workflows/test-python.yml

@@ -10,24 +10,25 @@ jobs:
     runs-on: ubuntu-latest  # todo, add more OS systems to see if they work as well
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']  # '3.11'
+        python-version: ['3.8', '3.9', '3.10']  # '3.11'
 
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install system packages
         run: |
           sudo apt-get update
-          sudo apt-get install git cmake capnproto zlib1g-dev g++ build-essential python3-dev python3-pip pkg-config libzip-dev software-properties-common libbz2-dev xvfb python3-opengl
+          sudo apt-get install git cmake capnproto zlib1g-dev build-essential pkg-config libzip-dev software-properties-common libbz2-dev xvfb python3-opengl
 
       - name: Install pip packages
         run: |
-          python -m pip install --upgrade pip cmake wheel setuptools pytest
-          pip install -e .
+          python3 -m pip install --upgrade pip pytest
+          python3 -m pip install -e .
 
       - name: Run tests
-        run: xvfb-run -s '-screen 0 1024x768x24' pytest
+        run: |
+          xvfb-run -s '-screen 0 1024x768x24' pytest

.gitignore

@@ -139,3 +139,7 @@ Temporary
 
 # VSCode config
 /.vscode/
+
+# cibuildwheel
+wheelhouse
+tests/test_cibuildwheel/tmp_dockerfiles

CMakeLists.txt

@@ -4,9 +4,32 @@ if(POLICY CMP0048)
   cmake_policy(SET CMP0048 NEW)
 endif()
 
-find_package(Git QUIET)
+if(LINUX)
+  set(PYEXT_SUFFIX
+      ".so"
+      CACHE STRING "Suffix for Python extension modules")
+  set(DYNLIB_SUFFIX
+      ".so"
+      CACHE STRING "Suffix for dynamic libraries")
+elseif(APPLE)
+  set(PYEXT_SUFFIX
+      ".so"
+      CACHE STRING "Suffix for Python extension modules")
+  set(DYNLIB_SUFFIX
+      ".dylib"
+      CACHE STRING "Suffix for dynamic libraries")
+elseif(WIN32)
+  set(PYEXT_SUFFIX
+      ".pyd"
+      CACHE STRING "Suffix for Python extension modules" FORCE)
+  set(DYNLIB_SUFFIX
+      ".dll"
+      CACHE STRING "Suffix for dynamic libraries")
+else()
+  message(FATAL_ERROR "Unsupported platform")
+endif()
 
-find_package(PythonInterp REQUIRED)
+find_package(Git QUIET)
 
 file(READ "${CMAKE_SOURCE_DIR}/retro/VERSION" PROJECT_VERSION)
 string(REGEX REPLACE "\n$" "" PROJECT_VERSION "${PROJECT_VERSION}")
@@ -45,7 +68,9 @@ set(BUILD_PYTHON
 mark_as_advanced(BUILD_PYTHON)
 
 if(BUILD_PYTHON)
-  find_package(PythonLibs REQUIRED)
+  find_package(Python 3 COMPONENTS Interpreter Development.Module)
+else()
+  find_package(Python 3 COMPONENTS Interpreter)
 endif()
 
 if(WIN32 OR BUILD_MANYLINUX)
@@ -103,16 +128,6 @@ else()
     WORKING_DIRECTORY "${LUA_INCLUDE_DIRS}")
 endif()
 
-if(NOT WIN32)
-  set(PYEXT_SUFFIX
-      ".so"
-      CACHE STRING "Suffix for Python extension modules")
-else()
-  set(PYEXT_SUFFIX
-      ".pyd"
-      CACHE STRING "Suffix for Python extension modules" FORCE)
-endif()
-
 if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
   set(STATIC_LDFLAGS "-static-libstdc++ -Wl,--exclude-libs,ALL")
 endif()
@@ -184,7 +199,8 @@ function(add_core platform core_name)
       ${CMAKE_COMMAND} -E env CFLAGS=${core_cflags} CXXFLAGS=${core_cxxflags}
       LDFLAGS=${core_ldflags} $(MAKE) -f ${makefile} CC="${CMAKE_C_COMPILER}"
       CXX="${CMAKE_CXX_COMPILER}" fpic=${core_fpic_flags} ${libretro_platform}
-    COMMAND ${CMAKE_COMMAND} -E copy "${core_name}_libretro*" "${TARGET_PATH}"
+    COMMAND ${CMAKE_COMMAND} -E copy "${core_name}_libretro${DYNLIB_SUFFIX}"
+            "${TARGET_PATH}"
     WORKING_DIRECTORY "cores/${platform}/${subdir}"
     DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/retro/cores/${core_name}-version")
   unset(core_ldflags)
@@ -339,7 +355,7 @@ if(CAPN_PROTO_FOUND)
 endif()
 
 include_directories(src retro third-party/pybind11/include third-party
-                    third-party/gtest/googletest/include ${PYTHON_INCLUDE_DIRS})
+                    third-party/gtest/googletest/include ${Python_INCLUDE_DIRS})
 
 if(BUILD_PYTHON)
   add_library(retro SHARED src/retro.cpp)
@@ -359,7 +375,7 @@ if(BUILD_PYTHON)
       add_definitions(-DMS_WIN64)
     endif()
 
-    set(PYBIND_LIBS "${PYTHON_LIBRARY}")
+    set(PYBIND_LIBS "${Python_LIBRARY}")
   endif()
 
   target_link_libraries(retro retro-base ${PYBIND_LIBS} ${STATIC_LDFLAGS})

pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["cmake>=3.2.0", "setuptools", "wheel", "build"]
+
+[tool.cibuildwheel]
+# We need to build for the following Python versions:
+build = "cp{38,39,310,311}-*"
+
+[tool.cibuildwheel.linux]
+# Only manylinux is supported (no musl)
+build = "cp{38,39,310,311}-manylinux*"
+
+# For manylinux_2_28 we need to install the following dependencies using yum:
+before-all = "yum install -y cmake git pkgconf-pkg-config zlib-devel libzip-devel bzip2-devel"
+
+# Only build for x86_64 and aarch64 are officially supported
+archs = "x86_64 aarch64"
+manylinux-x86_64-image = "manylinux_2_28"
+manylinux-aarch64-image = "manylinux_2_28"
+
+[tool.cibuildwheel.macos]
+before-all = "brew install pkg-config capnp [email protected] qt5"