gh-127906: Backport test_cext from the main branch

Lib/test/test_cext/__init__.py

@@ -0,0 +1,109 @@
+# gh-116869: Build a basic C test extension to check that the Python C API
+# does not emit C compiler warnings.
+#
+# The Python C API must be compatible with building
+# with the -Werror=declaration-after-statement compiler flag.
+
+import os.path
+import shlex
+import shutil
+import subprocess
+import unittest
+from test import support
+
+
+SOURCE = os.path.join(os.path.dirname(__file__), 'extension.c')
+SETUP = os.path.join(os.path.dirname(__file__), 'setup.py')
+
+
+# With MSVC on a debug build, the linker fails with: cannot open file
+# 'python311.lib', it should look 'python311_d.lib'.
+@unittest.skipIf(support.MS_WINDOWS and support.Py_DEBUG,
+                 'test fails on Windows debug build')
+# Building and running an extension in clang sanitizing mode is not
+# straightforward
+@support.skip_if_sanitizer('test does not work with analyzing builds',
+                           address=True, memory=True, ub=True, thread=True)
+# the test uses venv+pip: skip if it's not available
+@support.requires_venv_with_pip()
+@support.requires_subprocess()
+@support.requires_resource('cpu')
+class TestExt(unittest.TestCase):
+    # Default build with no options
+    def test_build(self):
+        self.check_build('_test_cext')
+
+    def test_build_c11(self):
+        self.check_build('_test_c11_cext', std='c11')
+
+    @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99")
+    def test_build_c99(self):
+        self.check_build('_test_c99_cext', std='c99')
+
+    def test_build_limited(self):
+        self.check_build('_test_limited_cext', limited=True)
+
+    def test_build_limited_c11(self):
+        self.check_build('_test_limited_c11_cext', limited=True, std='c11')
+
+    def check_build(self, extension_name, std=None, limited=False):
+        venv_dir = 'env'
+        with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe:
+            self._check_build(extension_name, python_exe,
+                              std=std, limited=limited)
+
+    def _check_build(self, extension_name, python_exe, std, limited):
+        pkg_dir = 'pkg'
+        os.mkdir(pkg_dir)
+        shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP)))
+        shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE)))
+
+        def run_cmd(operation, cmd):
+            env = os.environ.copy()
+            if std:
+                env['CPYTHON_TEST_STD'] = std
+            if limited:
+                env['CPYTHON_TEST_LIMITED'] = '1'
+            env['CPYTHON_TEST_EXT_NAME'] = extension_name
+            if support.verbose:
+                print('Run:', ' '.join(map(shlex.quote, cmd)))
+                subprocess.run(cmd, check=True, env=env)
+            else:
+                proc = subprocess.run(cmd,
+                                      env=env,
+                                      stdout=subprocess.PIPE,
+                                      stderr=subprocess.STDOUT,
+                                      text=True)
+                if proc.returncode:
+                    print('Run:', ' '.join(map(shlex.quote, cmd)))
+                    print(proc.stdout, end='')
+                    self.fail(
+                        f"{operation} failed with exit code {proc.returncode}")
+
+        # Build and install the C extension
+        cmd = [python_exe, '-X', 'dev',
+               '-m', 'pip', 'install', '--no-build-isolation',
+               os.path.abspath(pkg_dir)]
+        if support.verbose:
+            cmd.append('-v')
+        run_cmd('Install', cmd)
+
+        # Do a reference run. Until we test that running python
+        # doesn't leak references (gh-94755), run it so one can manually check
+        # -X showrefcount results against this baseline.
+        cmd = [python_exe,
+               '-X', 'dev',
+               '-X', 'showrefcount',
+               '-c', 'pass']
+        run_cmd('Reference run', cmd)
+
+        # Import the C extension
+        cmd = [python_exe,
+               '-X', 'dev',
+               '-X', 'showrefcount',
+               '-c', f"import {extension_name}"]
+        run_cmd('Import', cmd)
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_cext/extension.c

@@ -0,0 +1,89 @@
+// gh-116869: Basic C test extension to check that the Python C API
+// does not emit C compiler warnings.
+
+// Always enable assertions
+#undef NDEBUG
+
+#include "Python.h"
+
+#ifndef MODULE_NAME
+#  error "MODULE_NAME macro must be defined"
+#endif
+
+#define _STR(NAME) #NAME
+#define STR(NAME) _STR(NAME)
+
+PyDoc_STRVAR(_testcext_add_doc,
+"add(x, y)\n"
+"\n"
+"Return the sum of two integers: x + y.");
+
+static PyObject *
+_testcext_add(PyObject *Py_UNUSED(module), PyObject *args)
+{
+    long i, j, res;
+    if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) {
+        return NULL;
+    }
+    res = i + j;
+    return PyLong_FromLong(res);
+}
+
+
+static PyMethodDef _testcext_methods[] = {
+    {"add", _testcext_add, METH_VARARGS, _testcext_add_doc},
+    {NULL, NULL, 0, NULL}  // sentinel
+};
+
+
+static int
+_testcext_exec(
+#ifdef __STDC_VERSION__
+    PyObject *module
+#else
+    PyObject *Py_UNUSED(module)
+#endif
+    )
+{
+#ifdef __STDC_VERSION__
+    if (PyModule_AddIntMacro(module, __STDC_VERSION__) < 0) {
+        return -1;
+    }
+#endif
+
+    // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR()
+    Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int));
+    assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0);
+
+    return 0;
+}
+
+static PyModuleDef_Slot _testcext_slots[] = {
+    {Py_mod_exec, (void*)_testcext_exec},
+    {0, NULL}
+};
+
+
+PyDoc_STRVAR(_testcext_doc, "C test extension.");
+
+static struct PyModuleDef _testcext_module = {
+    PyModuleDef_HEAD_INIT,  // m_base
+    STR(MODULE_NAME),  // m_name
+    _testcext_doc,  // m_doc
+    0,  // m_size
+    _testcext_methods,  // m_methods
+    _testcext_slots,  // m_slots
+    NULL,  // m_traverse
+    NULL,  // m_clear
+    NULL,  // m_free
+};
+
+
+#define _FUNC_NAME(NAME) PyInit_ ## NAME
+#define FUNC_NAME(NAME) _FUNC_NAME(NAME)
+
+PyMODINIT_FUNC
+FUNC_NAME(MODULE_NAME)(void)
+{
+    return PyModuleDef_Init(&_testcext_module);
+}

Lib/test/test_cext/setup.py

@@ -0,0 +1,107 @@
+# gh-91321: Build a basic C test extension to check that the Python C API is
+# compatible with C and does not emit C compiler warnings.
+import os
+import platform
+import shlex
+import sys
+import sysconfig
+from test import support
+
+from setuptools import setup, Extension
+
+
+SOURCE = 'extension.c'
+
+if not support.MS_WINDOWS:
+    # C compiler flags for GCC and clang
+    CFLAGS = [
+        # The purpose of test_cext extension is to check that building a C
+        # extension using the Python C API does not emit C compiler warnings.
+        '-Werror',
+
+        # gh-120593: Check the 'const' qualifier
+        '-Wcast-qual',
+
+        # gh-116869: The Python C API must be compatible with building
+        # with the -Werror=declaration-after-statement compiler flag.
+        '-Werror=declaration-after-statement',
+    ]
+else:
+    # MSVC compiler flags
+    CFLAGS = [
+        # Display warnings level 1 to 4
+        '/W4',
+        # Treat all compiler warnings as compiler errors
+        '/WX',
+    ]
+
+
+def main():
+    std = os.environ.get("CPYTHON_TEST_STD", "")
+    module_name = os.environ["CPYTHON_TEST_EXT_NAME"]
+    limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", ""))
+
+    cflags = list(CFLAGS)
+    cflags.append(f'-DMODULE_NAME={module_name}')
+
+    # Add -std=STD or /std:STD (MSVC) compiler flag
+    if std:
+        if support.MS_WINDOWS:
+            cflags.append(f'/std:{std}')
+        else:
+            cflags.append(f'-std={std}')
+
+    # Remove existing -std or /std options from CC command line.
+    # Python adds -std=c11 option.
+    cmd = (sysconfig.get_config_var('CC') or '')
+    if cmd is not None:
+        if support.MS_WINDOWS:
+            std_prefix = '/std'
+        else:
+            std_prefix = '-std'
+        cmd = shlex.split(cmd)
+        cmd = [arg for arg in cmd if not arg.startswith(std_prefix)]
+        cmd = shlex.join(cmd)
+        # CC env var overrides sysconfig CC variable in setuptools
+        os.environ['CC'] = cmd
+
+    # Define Py_LIMITED_API macro
+    if limited:
+        version = sys.hexversion
+        cflags.append(f'-DPy_LIMITED_API={version:#x}')
+
+    # On Windows, add PCbuild\amd64\ to include and library directories
+    include_dirs = []
+    library_dirs = []
+    if support.MS_WINDOWS:
+        srcdir = sysconfig.get_config_var('srcdir')
+        machine = platform.uname().machine
+        pcbuild = os.path.join(srcdir, 'PCbuild', machine)
+        if os.path.exists(pcbuild):
+            # pyconfig.h is generated in PCbuild\amd64\
+            include_dirs.append(pcbuild)
+            # python313.lib is generated in PCbuild\amd64\
+            library_dirs.append(pcbuild)
+            print(f"Add PCbuild directory: {pcbuild}")
+
+    # Display information to help debugging
+    for env_name in ('CC', 'CFLAGS'):
+        if env_name in os.environ:
+            print(f"{env_name} env var: {os.environ[env_name]!r}")
+        else:
+            print(f"{env_name} env var: <missing>")
+    print(f"extra_compile_args: {cflags!r}")
+
+    ext = Extension(
+        module_name,
+        sources=[SOURCE],
+        extra_compile_args=cflags,
+        include_dirs=include_dirs,
+        library_dirs=library_dirs)
+    setup(name=f'internal_{module_name}',
+          version='0.0',
+          ext_modules=[ext])
+
+
+if __name__ == "__main__":
+    main()

Misc/NEWS.d/next/Tests/2024-12-13-13-16-43.gh-issue-127906.wsZJ29.rst

@@ -0,0 +1 @@
+Backport test_cext from the main branch. Patch by Victor Stinner.