diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index bd347e6448f1a0..5846d784c88ccc 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -845,7 +845,6 @@ which incur interpreter overhead. def quantify(iterable, pred=bool): "Given a predicate that returns True or False, count the True results." - "Count how many times the predicate is True" return sum(map(pred, iterable)) def all_equal(iterable): diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3117d9183761b0..1223f859265ed9 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -291,9 +291,11 @@ can be used to customize buffer creation. PEP 684: A Per-Interpreter GIL ------------------------------ -Sub-interpreters may now be created with a unique GIL per interpreter. +:pep:`684` introduces a per-interpreter :term:`GIL `, +so that sub-interpreters may now be created with a unique GIL per interpreter. This allows Python programs to take full advantage of multiple CPU -cores. +cores. This is currently only available through the C-API, +though a Python API is :pep:`anticipated for 3.13 <554>`. Use the new :c:func:`Py_NewInterpreterFromConfig` function to create an interpreter with its own GIL:: @@ -312,8 +314,6 @@ create an interpreter with its own GIL:: For further examples how to use the C-API for sub-interpreters with a per-interpreter GIL, see :source:`Modules/_xxsubinterpretersmodule.c`. -A Python API is anticipated for 3.13. (See :pep:`554`.) - (Contributed by Eric Snow in :gh:`104210`, etc.) .. _whatsnew312-pep669: @@ -321,13 +321,15 @@ A Python API is anticipated for 3.13. (See :pep:`554`.) PEP 669: Low impact monitoring for CPython ------------------------------------------ -CPython 3.12 now supports the ability to monitor calls, -returns, lines, exceptions and other events using instrumentation. +:pep:`669` defines a new :mod:`API ` for profilers, +debuggers, and other tools to monitor events in CPython. +It covers a wide range of events, including calls, +returns, lines, exceptions, jumps, and more. This means that you only pay for what you use, providing support for near-zero overhead debuggers and coverage tools. - See :mod:`sys.monitoring` for details. +(Contributed by Mark Shannon in :gh:`103083`.) New Features Related to Type Hints ================================== @@ -459,12 +461,12 @@ and others in :gh:`103764`.) Other Language Changes ====================== -* Add :ref:`perf_profiling` through the new - environment variable :envvar:`PYTHONPERFSUPPORT`, - the new command-line option :option:`-X perf <-X>`, +* Add :ref:`support for the perf profiler ` through the new + environment variable :envvar:`PYTHONPERFSUPPORT` + and command-line option :option:`-X perf <-X>`, as well as the new :func:`sys.activate_stack_trampoline`, :func:`sys.deactivate_stack_trampoline`, - and :func:`sys.is_stack_trampoline_active` APIs. + and :func:`sys.is_stack_trampoline_active` functions. (Design by Pablo Galindo. Contributed by Pablo Galindo and Christian Heimes with contributions from Gregory P. Smith [Google] and Mark Shannon in :gh:`96123`.) @@ -473,7 +475,7 @@ Other Language Changes have a new a *filter* argument that allows limiting tar features than may be surprising or dangerous, such as creating files outside the destination directory. - See :ref:`tarfile-extraction-filter` for details. + See :ref:`tarfile extraction filters ` for details. In Python 3.14, the default will switch to ``'data'``. (Contributed by Petr Viktorin in :pep:`706`.) @@ -501,8 +503,8 @@ Other Language Changes * A backslash-character pair that is not a valid escape sequence now generates a :exc:`SyntaxWarning`, instead of :exc:`DeprecationWarning`. For example, ``re.compile("\d+\.\d+")`` now emits a :exc:`SyntaxWarning` - (``"\d"`` is an invalid escape sequence), use raw strings for regular - expression: ``re.compile(r"\d+\.\d+")``. + (``"\d"`` is an invalid escape sequence, use raw strings for regular + expression: ``re.compile(r"\d+\.\d+")``). In a future Python version, :exc:`SyntaxError` will eventually be raised, instead of :exc:`SyntaxWarning`. (Contributed by Victor Stinner in :gh:`98401`.) @@ -531,7 +533,7 @@ Other Language Changes when summing floats or mixed ints and floats. (Contributed by Raymond Hettinger in :gh:`100425`.) -* Exceptions raised in a typeobject's ``__set_name__`` method are no longer +* Exceptions raised in a class or type's ``__set_name__`` method are no longer wrapped by a :exc:`RuntimeError`. Context information is added to the exception as a :pep:`678` note. (Contributed by Irit Katriel in :gh:`77757`.) @@ -567,7 +569,7 @@ asyncio * Added :func:`asyncio.eager_task_factory` and :func:`asyncio.create_eager_task_factory` functions to allow opting an event loop in to eager task execution, making some use-cases 2x to 5x faster. - (Contributed by Jacob Bower & Itamar O in :gh:`102853`, :gh:`104140`, and :gh:`104138`) + (Contributed by Jacob Bower & Itamar Oren in :gh:`102853`, :gh:`104140`, and :gh:`104138`) * On Linux, :mod:`asyncio` uses :class:`asyncio.PidfdChildWatcher` by default if :func:`os.pidfd_open` is available and functional instead of @@ -594,7 +596,7 @@ asyncio (Contributed by Kumar Aditya in :gh:`99388`.) * Add C implementation of :func:`asyncio.current_task` for 4x-6x speedup. - (Contributed by Itamar Ostricher and Pranav Thulasiram Bhat in :gh:`100344`.) + (Contributed by Itamar Oren and Pranav Thulasiram Bhat in :gh:`100344`.) * :func:`asyncio.iscoroutine` now returns ``False`` for generators as :mod:`asyncio` does not support legacy generator-based coroutines. @@ -985,7 +987,7 @@ Optimizations (Contributed by Serhiy Storchaka in :gh:`91524`.) * Speed up :class:`asyncio.Task` creation by deferring expensive string formatting. - (Contributed by Itamar O in :gh:`103793`.) + (Contributed by Itamar Oren in :gh:`103793`.) * The :func:`tokenize.tokenize` and :func:`tokenize.generate_tokens` functions are up to 64% faster as a side effect of the changes required to cover :pep:`701` in @@ -1000,9 +1002,9 @@ Optimizations CPython bytecode changes ======================== -* Remove the :opcode:`LOAD_METHOD` instruction. It has been merged into +* Remove the :opcode:`!LOAD_METHOD` instruction. It has been merged into :opcode:`LOAD_ATTR`. :opcode:`LOAD_ATTR` will now behave like the old - :opcode:`LOAD_METHOD` instruction if the low bit of its oparg is set. + :opcode:`!LOAD_METHOD` instruction if the low bit of its oparg is set. (Contributed by Ken Jin in :gh:`93429`.) * Remove the :opcode:`!JUMP_IF_FALSE_OR_POP` and :opcode:`!JUMP_IF_TRUE_OR_POP` @@ -1838,7 +1840,7 @@ New Features * Added :c:func:`PyCode_AddWatcher` and :c:func:`PyCode_ClearWatcher` APIs to register callbacks to receive notification on creation and destruction of code objects. - (Contributed by Itamar Ostricher in :gh:`91054`.) + (Contributed by Itamar Oren in :gh:`91054`.) * Add :c:func:`PyFrame_GetVar` and :c:func:`PyFrame_GetVarString` functions to get a frame variable by its name. diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index c4e5ba2061cffc..043359bbd03f8a 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -147,15 +147,17 @@ def kill(self): async def _feed_stdin(self, input): debug = self._loop.get_debug() - if input is not None: - self.stdin.write(input) - if debug: - logger.debug( - '%r communicate: feed stdin (%s bytes)', self, len(input)) try: + if input is not None: + self.stdin.write(input) + if debug: + logger.debug( + '%r communicate: feed stdin (%s bytes)', self, len(input)) + await self.stdin.drain() except (BrokenPipeError, ConnectionResetError) as exc: - # communicate() ignores BrokenPipeError and ConnectionResetError + # communicate() ignores BrokenPipeError and ConnectionResetError. + # write() and drain() can raise these exceptions. if debug: logger.debug('%r communicate: stdin got %r', self, exc) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index fba19d39d5c9c2..48d8db3ed423a5 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -71,6 +71,11 @@ def __init__(self): self._reader, self._writer = mp.Pipe(duplex=False) def close(self): + # Please note that we do not take the shutdown lock when + # calling clear() (to avoid deadlocking) so this method can + # only be called safely from the same thread as all calls to + # clear() even if you hold the shutdown lock. Otherwise we + # might try to read from the closed pipe. if not self._closed: self._closed = True self._writer.close() @@ -426,8 +431,12 @@ def wait_result_broken_or_wakeup(self): elif wakeup_reader in ready: is_broken = False - with self.shutdown_lock: - self.thread_wakeup.clear() + # No need to hold the _shutdown_lock here because: + # 1. we're the only thread to use the wakeup reader + # 2. we're also the only thread to call thread_wakeup.close() + # 3. we want to avoid a possible deadlock when both reader and writer + # would block (gh-105829) + self.thread_wakeup.clear() return result_item, is_broken, cause @@ -717,7 +726,10 @@ def __init__(self, max_workers=None, mp_context=None, # as it could result in a deadlock if a worker process dies with the # _result_queue write lock still acquired. # - # _shutdown_lock must be locked to access _ThreadWakeup. + # _shutdown_lock must be locked to access _ThreadWakeup.close() and + # .wakeup(). Care must also be taken to not call clear or close from + # more than one thread since _ThreadWakeup.clear() is not protected by + # the _shutdown_lock self._executor_manager_thread_wakeup = _ThreadWakeup() # Create communication channels for the executor diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 429ef16fdb0e05..dc5a48d500e8d5 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -1,6 +1,7 @@ import os import signal import sys +import textwrap import unittest import warnings from unittest import mock @@ -12,9 +13,14 @@ from test import support from test.support import os_helper -if sys.platform != 'win32': + +MS_WINDOWS = (sys.platform == 'win32') +if MS_WINDOWS: + import msvcrt +else: from asyncio import unix_events + if support.check_sanitizer(address=True): raise unittest.SkipTest("Exposes ASAN flakiness in GitHub CI") @@ -270,26 +276,43 @@ async def send_signal(proc): finally: signal.signal(signal.SIGHUP, old_handler) - def prepare_broken_pipe_test(self): + def test_stdin_broken_pipe(self): # buffer large enough to feed the whole pipe buffer large_data = b'x' * support.PIPE_MAX_SIZE + rfd, wfd = os.pipe() + self.addCleanup(os.close, rfd) + self.addCleanup(os.close, wfd) + if MS_WINDOWS: + handle = msvcrt.get_osfhandle(rfd) + os.set_handle_inheritable(handle, True) + code = textwrap.dedent(f''' + import os, msvcrt + handle = {handle} + fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) + os.read(fd, 1) + ''') + from subprocess import STARTUPINFO + startupinfo = STARTUPINFO() + startupinfo.lpAttributeList = {"handle_list": [handle]} + kwargs = dict(startupinfo=startupinfo) + else: + code = f'import os; fd = {rfd}; os.read(fd, 1)' + kwargs = dict(pass_fds=(rfd,)) + # the program ends before the stdin can be fed proc = self.loop.run_until_complete( asyncio.create_subprocess_exec( - sys.executable, '-c', 'pass', + sys.executable, '-c', code, stdin=subprocess.PIPE, + **kwargs ) ) - return (proc, large_data) - - def test_stdin_broken_pipe(self): - proc, large_data = self.prepare_broken_pipe_test() - async def write_stdin(proc, data): - await asyncio.sleep(0.5) proc.stdin.write(data) + # Only exit the child process once the write buffer is filled + os.write(wfd, b'go') await proc.stdin.drain() coro = write_stdin(proc, large_data) @@ -300,7 +323,16 @@ async def write_stdin(proc, data): self.loop.run_until_complete(proc.wait()) def test_communicate_ignore_broken_pipe(self): - proc, large_data = self.prepare_broken_pipe_test() + # buffer large enough to feed the whole pipe buffer + large_data = b'x' * support.PIPE_MAX_SIZE + + # the program ends before the stdin can be fed + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec( + sys.executable, '-c', 'pass', + stdin=subprocess.PIPE, + ) + ) # communicate() must ignore BrokenPipeError when feeding stdin self.loop.set_exception_handler(lambda loop, msg: None) diff --git a/Lib/test/test_concurrent_futures/test_deadlock.py b/Lib/test/test_concurrent_futures/test_deadlock.py index 1675a55b89eb80..a76e075c3be180 100644 --- a/Lib/test/test_concurrent_futures/test_deadlock.py +++ b/Lib/test/test_concurrent_futures/test_deadlock.py @@ -1,10 +1,13 @@ import contextlib +import queue +import signal import sys import time import unittest +import unittest.mock from pickle import PicklingError from concurrent import futures -from concurrent.futures.process import BrokenProcessPool +from concurrent.futures.process import BrokenProcessPool, _ThreadWakeup from test import support @@ -241,6 +244,73 @@ def test_crash_big_data(self): executor.shutdown(wait=True) + def test_gh105829_should_not_deadlock_if_wakeup_pipe_full(self): + # Issue #105829: The _ExecutorManagerThread wakeup pipe could + # fill up and block. See: https://github.com/python/cpython/issues/105829 + + # Lots of cargo culting while writing this test, apologies if + # something is really stupid... + + self.executor.shutdown(wait=True) + + if not hasattr(signal, 'alarm'): + raise unittest.SkipTest( + "Tested platform does not support the alarm signal") + + def timeout(_signum, _frame): + import faulthandler + faulthandler.dump_traceback() + + raise RuntimeError("timed out while submitting jobs?") + + thread_run = futures.process._ExecutorManagerThread.run + def mock_run(self): + # Delay thread startup so the wakeup pipe can fill up and block + time.sleep(3) + thread_run(self) + + class MockWakeup(_ThreadWakeup): + """Mock wakeup object to force the wakeup to block""" + def __init__(self): + super().__init__() + self._dummy_queue = queue.Queue(maxsize=1) + + def wakeup(self): + self._dummy_queue.put(None, block=True) + super().wakeup() + + def clear(self): + try: + while True: + self._dummy_queue.get_nowait() + except queue.Empty: + super().clear() + + with (unittest.mock.patch.object(futures.process._ExecutorManagerThread, + 'run', mock_run), + unittest.mock.patch('concurrent.futures.process._ThreadWakeup', + MockWakeup)): + with self.executor_type(max_workers=2, + mp_context=self.get_context()) as executor: + self.executor = executor # Allow clean up in fail_on_deadlock + + job_num = 100 + job_data = range(job_num) + + # Need to use sigalarm for timeout detection because + # Executor.submit is not guarded by any timeout (both + # self._work_ids.put(self._queue_count) and + # self._executor_manager_thread_wakeup.wakeup() might + # timeout, maybe more?). In this specific case it was + # the wakeup call that deadlocked on a blocking pipe. + old_handler = signal.signal(signal.SIGALRM, timeout) + try: + signal.alarm(int(self.TIMEOUT)) + self.assertEqual(job_num, len(list(executor.map(int, job_data)))) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old_handler) + create_executor_tests(globals(), ExecutorDeadlockTest, executor_mixins=(ProcessPoolForkMixin, diff --git a/Misc/ACKS b/Misc/ACKS index fd3c68b58a180c..b8a4f10347d74e 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -504,6 +504,7 @@ Daniel Ellis Phil Elson David Ely Victor van den Elzen +Vlad Emelianov Jeff Epler Tom Epperly Gökcen Eraslan @@ -1332,6 +1333,7 @@ Ethan Onstott Ken Jin Ooi Piet van Oostrum Tomas Oppelstrup +Itamar Oren Jason Orendorff Yan "yyyyyyyan" Orestes Bastien Orivel @@ -1342,7 +1344,6 @@ Michele Orrù Tomáš Orsava Oleg Oshmyan Denis Osipov -Itamar Ostricher Denis S. Otkidach Peter Otten Michael Otteneder @@ -1870,6 +1871,7 @@ Steven Troxler Brent Tubbs Anthony Tuininga Erno Tukia +Adam Turner David Turner Stephen Turner Itamar Turner-Trauring @@ -2077,7 +2079,5 @@ Jelle Zijlstra Gennadiy Zlobin Doug Zongker Peter Åstrand -Vlad Emelianov -Andrey Doroschenko (Entries should be added in rough alphabetical order by last names) diff --git a/Misc/NEWS.d/next/Library/2023-08-26-12-35-39.gh-issue-105829.kyYhWI.rst b/Misc/NEWS.d/next/Library/2023-08-26-12-35-39.gh-issue-105829.kyYhWI.rst new file mode 100644 index 00000000000000..eaa2a5a4330e28 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-26-12-35-39.gh-issue-105829.kyYhWI.rst @@ -0,0 +1 @@ +Fix concurrent.futures.ProcessPoolExecutor deadlock diff --git a/Modules/_testcapi/clinic/long.c.h b/Modules/_testcapi/clinic/long.c.h index b77cb51810cb65..e2f7042be12c48 100644 --- a/Modules/_testcapi/clinic/long.c.h +++ b/Modules/_testcapi/clinic/long.c.h @@ -133,23 +133,6 @@ _testcapi_test_long_as_double(PyObject *module, PyObject *Py_UNUSED(ignored)) return _testcapi_test_long_as_double_impl(module); } -PyDoc_STRVAR(_testcapi_test_long_numbits__doc__, -"test_long_numbits($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONG_NUMBITS_METHODDEF \ - {"test_long_numbits", (PyCFunction)_testcapi_test_long_numbits, METH_NOARGS, _testcapi_test_long_numbits__doc__}, - -static PyObject * -_testcapi_test_long_numbits_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_numbits_impl(module); -} - PyDoc_STRVAR(_testcapi_call_long_compact_api__doc__, "call_long_compact_api($module, arg, /)\n" "--\n" @@ -165,4 +148,4 @@ PyDoc_STRVAR(_testcapi_PyLong_AsInt__doc__, #define _TESTCAPI_PYLONG_ASINT_METHODDEF \ {"PyLong_AsInt", (PyCFunction)_testcapi_PyLong_AsInt, METH_O, _testcapi_PyLong_AsInt__doc__}, -/*[clinic end generated code: output=31267ab2dd90aa1d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=de762870526e241d input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index c1d2d42a2c434e..4362f431fc3f4d 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -4,7 +4,6 @@ #include "parts.h" #include "clinic/long.c.h" -#include "pycore_long.h" // _PyLong_Sign() /*[clinic input] module _testcapi @@ -535,57 +534,6 @@ _testcapi_test_long_as_double_impl(PyObject *module) return Py_None; } -/*[clinic input] -_testcapi.test_long_numbits -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_numbits_impl(PyObject *module) -/*[clinic end generated code: output=9eaf8458cb15d7f7 input=265c02d48a13059e]*/ -{ - struct triple { - long input; - size_t nbits; - int sign; - } testcases[] = {{0, 0, 0}, - {1L, 1, 1}, - {-1L, 1, -1}, - {2L, 2, 1}, - {-2L, 2, -1}, - {3L, 2, 1}, - {-3L, 2, -1}, - {4L, 3, 1}, - {-4L, 3, -1}, - {0x7fffL, 15, 1}, /* one Python int digit */ - {-0x7fffL, 15, -1}, - {0xffffL, 16, 1}, - {-0xffffL, 16, -1}, - {0xfffffffL, 28, 1}, - {-0xfffffffL, 28, -1}}; - size_t i; - - for (i = 0; i < Py_ARRAY_LENGTH(testcases); ++i) { - size_t nbits; - int sign; - PyObject *plong; - - plong = PyLong_FromLong(testcases[i].input); - if (plong == NULL) - return NULL; - nbits = _PyLong_NumBits(plong); - sign = _PyLong_Sign(plong); - - Py_DECREF(plong); - if (nbits != testcases[i].nbits) - return raiseTestError("test_long_numbits", - "wrong result for _PyLong_NumBits"); - if (sign != testcases[i].sign) - return raiseTestError("test_long_numbits", - "wrong result for _PyLong_Sign"); - } - Py_RETURN_NONE; -} - /*[clinic input] _testcapi.call_long_compact_api arg: object @@ -631,7 +579,6 @@ static PyMethodDef test_methods[] = { _TESTCAPI_TEST_LONG_AS_SIZE_T_METHODDEF _TESTCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF _TESTCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF - _TESTCAPI_TEST_LONG_NUMBITS_METHODDEF _TESTCAPI_TEST_LONGLONG_API_METHODDEF _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF _TESTCAPI_PYLONG_ASINT_METHODDEF diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index c162dbc65db81a..24abe54814e611 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -4,8 +4,24 @@ // Always enable assertions #undef NDEBUG +// The _testcapi extension tests the public C API: header files in Include/ and +// Include/cpython/ directories. The internal C API must not be tested by +// _testcapi: use _testinternalcapi for that. +// +// _testcapi C files can built with the Py_BUILD_CORE_BUILTIN macro defined if +// one of the Modules/Setup files asks to build _testcapi as "static" +// (gh-109723). +// +// The Visual Studio projects builds _testcapi with Py_BUILD_CORE_MODULE. +#undef Py_BUILD_CORE_MODULE +#undef Py_BUILD_CORE_BUILTIN + #include "Python.h" +#ifdef Py_BUILD_CORE +# error "_testcapi must test the public Python C API, not the internal C API" +#endif + int _PyTestCapi_Init_Vectorcall(PyObject *module); int _PyTestCapi_Init_Heaptype(PyObject *module); int _PyTestCapi_Init_Abstract(PyObject *module); diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index f0be2cfccccc98..5aedf687705707 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -4,10 +4,6 @@ * This only tests basic functionality, not any synchronizing ordering. */ -/* Always enable assertions */ -#undef NDEBUG - -#include "Python.h" #include "parts.h" // We define atomic bitwise operations on these types diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f356fc5a6a016e..e09fd8806d2f64 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5,19 +5,13 @@ * standard Python regression test, via Lib/test/test_capi.py. */ -/* This module tests the public (Include/ and Include/cpython/) C API. - The internal C API must not be used here: use _testinternalcapi for that. - - The Visual Studio projects builds _testcapi with Py_BUILD_CORE_MODULE - macro defined, but only the public C API must be tested here. */ - -#undef Py_BUILD_CORE_MODULE -#undef Py_BUILD_CORE_BUILTIN - -/* Always enable assertions */ -#undef NDEBUG +// Include parts.h first since it takes care of NDEBUG and Py_BUILD_CORE macros +// and including Python.h. +// +// Several parts of this module are broken out into files in _testcapi/. +// Include definitions from there. +#include "_testcapi/parts.h" -#include "Python.h" #include "frameobject.h" // PyFrame_New() #include "marshal.h" // PyMarshal_WriteLongToFile() @@ -29,17 +23,10 @@ # include // W_STOPCODE #endif -#ifdef Py_BUILD_CORE -# error "_testcapi must test the public Python C API, not CPython internal C API" -#endif - #ifdef bool # error "The public headers should not include , see gh-48924" #endif -// Several parts of this module are broken out into files in _testcapi/. -// Include definitions from there. -#include "_testcapi/parts.h" #include "_testcapi/util.h" diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index f97b609f7ad2ef..c6b80fffdec16d 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -22,6 +22,7 @@ #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() +#include "pycore_long.h" // _PyLong_Sign() #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() @@ -1466,6 +1467,66 @@ _testinternalcapi_write_unraisable_exc_impl(PyObject *module, PyObject *exc, } +static PyObject * +raiseTestError(const char* test_name, const char* msg) +{ + PyErr_Format(PyExc_AssertionError, "%s: %s", test_name, msg); + return NULL; +} + + +/*[clinic input] +_testinternalcapi.test_long_numbits +[clinic start generated code]*/ + +static PyObject * +_testinternalcapi_test_long_numbits_impl(PyObject *module) +/*[clinic end generated code: output=745d62d120359434 input=f14ca6f638e44dad]*/ +{ + struct triple { + long input; + size_t nbits; + int sign; + } testcases[] = {{0, 0, 0}, + {1L, 1, 1}, + {-1L, 1, -1}, + {2L, 2, 1}, + {-2L, 2, -1}, + {3L, 2, 1}, + {-3L, 2, -1}, + {4L, 3, 1}, + {-4L, 3, -1}, + {0x7fffL, 15, 1}, /* one Python int digit */ + {-0x7fffL, 15, -1}, + {0xffffL, 16, 1}, + {-0xffffL, 16, -1}, + {0xfffffffL, 28, 1}, + {-0xfffffffL, 28, -1}}; + size_t i; + + for (i = 0; i < Py_ARRAY_LENGTH(testcases); ++i) { + size_t nbits; + int sign; + PyObject *plong; + + plong = PyLong_FromLong(testcases[i].input); + if (plong == NULL) + return NULL; + nbits = _PyLong_NumBits(plong); + sign = _PyLong_Sign(plong); + + Py_DECREF(plong); + if (nbits != testcases[i].nbits) + return raiseTestError("test_long_numbits", + "wrong result for _PyLong_NumBits"); + if (sign != testcases[i].sign) + return raiseTestError("test_long_numbits", + "wrong result for _PyLong_Sign"); + } + Py_RETURN_NONE; +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1521,6 +1582,7 @@ static PyMethodDef module_functions[] = { _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, _TESTINTERNALCAPI_WRITE_UNRAISABLE_EXC_METHODDEF + _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index 38a3579d7dec77..c1b42672e13d53 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -296,4 +296,21 @@ _testinternalcapi_write_unraisable_exc(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=c7156622e80df1ce input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_testinternalcapi_test_long_numbits__doc__, +"test_long_numbits($module, /)\n" +"--\n" +"\n"); + +#define _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF \ + {"test_long_numbits", (PyCFunction)_testinternalcapi_test_long_numbits, METH_NOARGS, _testinternalcapi_test_long_numbits__doc__}, + +static PyObject * +_testinternalcapi_test_long_numbits_impl(PyObject *module); + +static PyObject * +_testinternalcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testinternalcapi_test_long_numbits_impl(module); +} +/*[clinic end generated code: output=59144f59957627bd input=a9049054013a1b77]*/ diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index d206ceb43a268e..6ece70c77fd55c 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -3,18 +3,20 @@ from . import common as _common -# The following C files define the Py_LIMITED_API macro, and so must not be -# built with the Py_BUILD_CORE macro defined. -USE_LIMITED_C_API = frozenset(( +# The following C files must not built with Py_BUILD_CORE. +FILES_WITHOUT_INTERNAL_CAPI = frozenset(( # Modules/ '_testcapimodule.c', '_testclinic_limited.c', 'xxlimited.c', 'xxlimited_35.c', +)) +# C files in the fhe following directories must not be built with +# Py_BUILD_CORE. +DIRS_WITHOUT_INTERNAL_CAPI = frozenset(( # Modules/_testcapi/ - 'heaptype_relative.c', - 'vectorcall_limited.c', + '_testcapi', )) TOOL = 'gcc' @@ -75,7 +77,10 @@ def preprocess(filename, filename = _normpath(filename, cwd) postargs = POST_ARGS - if os.path.basename(filename) not in USE_LIMITED_C_API: + basename = os.path.basename(filename) + dirname = os.path.basename(os.path.dirname(filename)) + if (basename not in FILES_WITHOUT_INTERNAL_CAPI + and dirname not in DIRS_WITHOUT_INTERNAL_CAPI): postargs += ('-DPy_BUILD_CORE=1',) text = _common.preprocess(