From bb2e96f6f4d6c397c4eb5775a09262a207675577 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 3 Oct 2023 14:13:13 +0300 Subject: [PATCH 01/73] gh-109956: Also test typing.NamedTuple with copy.replace() (GH-109957) --- Lib/test/test_copy.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index c66c6eeb00811e..60735ba89a80ee 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -936,14 +936,24 @@ def __replace__(self, **changes): def test_namedtuple(self): from collections import namedtuple - Point = namedtuple('Point', 'x y', defaults=(0,)) - p = Point(11, 22) - self.assertEqual(copy.replace(p), (11, 22)) - self.assertEqual(copy.replace(p, x=1), (1, 22)) - self.assertEqual(copy.replace(p, y=2), (11, 2)) - self.assertEqual(copy.replace(p, x=1, y=2), (1, 2)) - with self.assertRaisesRegex(ValueError, 'unexpected field name'): - copy.replace(p, x=1, error=2) + from typing import NamedTuple + PointFromCall = namedtuple('Point', 'x y', defaults=(0,)) + class PointFromInheritance(PointFromCall): + pass + class PointFromClass(NamedTuple): + x: int + y: int = 0 + for Point in (PointFromCall, PointFromInheritance, PointFromClass): + with self.subTest(Point=Point): + p = Point(11, 22) + self.assertIsInstance(p, Point) + self.assertEqual(copy.replace(p), (11, 22)) + self.assertIsInstance(copy.replace(p), Point) + self.assertEqual(copy.replace(p, x=1), (1, 22)) + self.assertEqual(copy.replace(p, y=2), (11, 2)) + self.assertEqual(copy.replace(p, x=1, y=2), (1, 2)) + with self.assertRaisesRegex(ValueError, 'unexpected field name'): + copy.replace(p, x=1, error=2) def test_dataclass(self): from dataclasses import dataclass From dddc757303c6512915ef79b9029213415f1c6f1b Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 3 Oct 2023 13:54:21 +0100 Subject: [PATCH 02/73] gh-110276: Run `test_str`, not `test_unicode`, as part of the PGO build (#110277) `test_unicode` does not exist --- Lib/test/libregrtest/pgo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/pgo.py b/Lib/test/libregrtest/pgo.py index cabbba73d5eff5..e3a6927be5db1d 100644 --- a/Lib/test/libregrtest/pgo.py +++ b/Lib/test/libregrtest/pgo.py @@ -42,10 +42,10 @@ 'test_set', 'test_sqlite3', 'test_statistics', + 'test_str', 'test_struct', 'test_tabnanny', 'test_time', - 'test_unicode', 'test_xml_etree', 'test_xml_etree_c', ] From f1663a492e14c80c30cb9741fdc36fa221d5e30a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 3 Oct 2023 14:10:03 +0100 Subject: [PATCH 03/73] Bump various dependencies in `Doc/requirements-oldest-sphinx.txt` (#110278) This resolves a Dependabot security alert on the repository for urllib3==2.0.4. --- Doc/requirements-oldest-sphinx.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/requirements-oldest-sphinx.txt b/Doc/requirements-oldest-sphinx.txt index d3ef5bc17650ae..5de739fc10b085 100644 --- a/Doc/requirements-oldest-sphinx.txt +++ b/Doc/requirements-oldest-sphinx.txt @@ -13,16 +13,16 @@ python-docs-theme>=2022.1 # Sphinx 4.2 comes from ``needs_sphinx = '4.2'`` in ``Doc/conf.py``. alabaster==0.7.13 -Babel==2.12.1 +Babel==2.13.0 certifi==2023.7.22 -charset-normalizer==3.2.0 +charset-normalizer==3.3.0 colorama==0.4.6 -docutils==0.16 +docutils==0.17.1 idna==3.4 imagesize==1.4.1 -Jinja2==2.11.3 -MarkupSafe==1.1.1 -packaging==23.1 +Jinja2==3.1.2 +MarkupSafe==2.1.3 +packaging==23.2 Pygments==2.16.1 requests==2.31.0 snowballstemmer==2.2.0 @@ -33,4 +33,4 @@ sphinxcontrib-htmlhelp==2.0.1 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 -urllib3==2.0.4 +urllib3==2.0.6 From 2d4865d775123e8889c7a79fc49b4bf627176c4b Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Tue, 3 Oct 2023 22:09:49 +0800 Subject: [PATCH 04/73] gh-110267: Add tests for pickling and copying PyStructSequence objects (GH-110272) --- Lib/test/test_structseq.py | 75 ++++++++++++++++++- ...-10-03-10-54-09.gh-issue-110267.O-c47G.rst | 2 + 2 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-10-03-10-54-09.gh-issue-110267.O-c47G.rst diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index a9fe193028ebe4..c6c0afaf077acc 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -1,4 +1,6 @@ +import copy import os +import pickle import time import unittest @@ -106,9 +108,78 @@ def __len__(self): self.assertRaises(Exc, time.struct_time, C()) - def test_reduce(self): + def test_pickling(self): t = time.gmtime() - x = t.__reduce__() + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + p = pickle.dumps(t, proto) + t2 = pickle.loads(p) + self.assertEqual(t2.__class__, t.__class__) + self.assertEqual(t2, t) + self.assertEqual(t2.tm_year, t.tm_year) + self.assertEqual(t2.tm_zone, t.tm_zone) + + def test_pickling_with_unnamed_fields(self): + assert os.stat_result.n_unnamed_fields > 0 + + r = os.stat_result(range(os.stat_result.n_sequence_fields), + {'st_atime': 1.0, 'st_atime_ns': 2.0}) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + p = pickle.dumps(r, proto) + r2 = pickle.loads(p) + self.assertEqual(r2.__class__, r.__class__) + self.assertEqual(r2, r) + self.assertEqual(r2.st_mode, r.st_mode) + self.assertEqual(r2.st_atime, r.st_atime) + self.assertEqual(r2.st_atime_ns, r.st_atime_ns) + + def test_copying(self): + n_fields = time.struct_time.n_fields + t = time.struct_time([[i] for i in range(n_fields)]) + + t2 = copy.copy(t) + self.assertEqual(t2.__class__, t.__class__) + self.assertEqual(t2, t) + self.assertEqual(t2.tm_year, t.tm_year) + self.assertEqual(t2.tm_zone, t.tm_zone) + self.assertIs(t2[0], t[0]) + self.assertIs(t2.tm_year, t.tm_year) + + t3 = copy.deepcopy(t) + self.assertEqual(t3.__class__, t.__class__) + self.assertEqual(t3, t) + self.assertEqual(t3.tm_year, t.tm_year) + self.assertEqual(t3.tm_zone, t.tm_zone) + self.assertIsNot(t3[0], t[0]) + self.assertIsNot(t3.tm_year, t.tm_year) + + def test_copying_with_unnamed_fields(self): + assert os.stat_result.n_unnamed_fields > 0 + + n_sequence_fields = os.stat_result.n_sequence_fields + r = os.stat_result([[i] for i in range(n_sequence_fields)], + {'st_atime': [1.0], 'st_atime_ns': [2.0]}) + + r2 = copy.copy(r) + self.assertEqual(r2.__class__, r.__class__) + self.assertEqual(r2, r) + self.assertEqual(r2.st_mode, r.st_mode) + self.assertEqual(r2.st_atime, r.st_atime) + self.assertEqual(r2.st_atime_ns, r.st_atime_ns) + self.assertIs(r2[0], r[0]) + self.assertIs(r2.st_mode, r.st_mode) + self.assertIs(r2.st_atime, r.st_atime) + self.assertIs(r2.st_atime_ns, r.st_atime_ns) + + r3 = copy.deepcopy(r) + self.assertEqual(r3.__class__, r.__class__) + self.assertEqual(r3, r) + self.assertEqual(r3.st_mode, r.st_mode) + self.assertEqual(r3.st_atime, r.st_atime) + self.assertEqual(r3.st_atime_ns, r.st_atime_ns) + self.assertIsNot(r3[0], r[0]) + self.assertIsNot(r3.st_mode, r.st_mode) + self.assertIsNot(r3.st_atime, r.st_atime) + self.assertIsNot(r3.st_atime_ns, r.st_atime_ns) def test_extended_getslice(self): # Test extended slicing by comparing with list slicing. diff --git a/Misc/NEWS.d/next/Tests/2023-10-03-10-54-09.gh-issue-110267.O-c47G.rst b/Misc/NEWS.d/next/Tests/2023-10-03-10-54-09.gh-issue-110267.O-c47G.rst new file mode 100644 index 00000000000000..2bae7715cc3d5b --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-10-03-10-54-09.gh-issue-110267.O-c47G.rst @@ -0,0 +1,2 @@ +Add tests for pickling and copying PyStructSequence objects. +Patched by Xuehai Pan. From 4227bfa8b273207a2b882f7d69c8ac49c3d2b57d Mon Sep 17 00:00:00 2001 From: Lincoln <71312724+Lincoln-developer@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:32:43 +0300 Subject: [PATCH 05/73] Enhanced sqlite3 connection context management documentation with contextlib.closing gh-109234 (#109322) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Enhanced sqlite3 connection context management documentation with contextlib.closing * 📜🤖 Added by blurb_it. * Fixed gitignore spelling error from nitignore to gitignore * Renamed .gitignore to .nitignore * Added generated doctests * Deleted sqlite3 generated files * Removed white-space changes * Removed News entry from the doc * Expanded a note that context manager can be used for connection management using contextlib.closing * Removed repeated contextlib.closing code snippet * Expanded the note around usage of context manageer for sqlite3 connection management * Deleted extra white-spaces * Deleted extra white-space * re-arranged context manager wording * Re-arranged word layout on how to use context manager * Fix whitespace errors * Remove unneeded change in .gitignore * Added suggested changes * Added suggested change redirecting to the contextlib.closing implementation * Added closing keyword * Removed line 2473 --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Erlend E. Aasland --- Doc/library/sqlite3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 7b8c7810165164..aa34bcc9388e1c 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -2437,9 +2437,9 @@ or if :attr:`~Connection.autocommit` is ``True``, the context manager does nothing. .. note:: - The context manager neither implicitly opens a new transaction - nor closes the connection. + nor closes the connection. If you need a closing context manager, consider + using :meth:`contextlib.closing`. .. testcode:: From f5198b09e16bca1886f8245fa88203d07d51ec11 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Oct 2023 09:20:48 -0600 Subject: [PATCH 06/73] gh-109860: Use a New Thread State When Switching Interpreters, When Necessary (gh-110245) In a few places we switch to another interpreter without knowing if it has a thread state associated with the current thread. For the main interpreter there wasn't much of a problem, but for subinterpreters we were *mostly* okay re-using the tstate created with the interpreter (located via PyInterpreterState_ThreadHead()). There was a good chance that tstate wasn't actually in use by another thread. However, there are no guarantees of that. Furthermore, re-using an already used tstate is currently fragile. To address this, now we create a new thread state in each of those places and use it. One consequence of this change is that PyInterpreterState_ThreadHead() may not return NULL (though that won't happen for the main interpreter). --- Include/cpython/pystate.h | 9 ++ Include/internal/pycore_pystate.h | 5 +- Include/internal/pycore_runtime_init.h | 1 + Lib/threading.py | 7 +- Modules/_threadmodule.c | 2 +- Modules/_xxsubinterpretersmodule.c | 111 +++++++++++++++---------- Python/pylifecycle.c | 6 +- Python/pystate.c | 78 +++++++++++++---- 8 files changed, 151 insertions(+), 68 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 7e4c57efc7c00c..af5cc4a2f3bf63 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -92,6 +92,15 @@ struct _ts { /* padding to align to 4 bytes */ unsigned int :24; } _status; +#ifdef Py_BUILD_CORE +# define _PyThreadState_WHENCE_NOTSET -1 +# define _PyThreadState_WHENCE_UNKNOWN 0 +# define _PyThreadState_WHENCE_INTERP 1 +# define _PyThreadState_WHENCE_THREADING 2 +# define _PyThreadState_WHENCE_GILSTATE 3 +# define _PyThreadState_WHENCE_EXEC 4 +#endif + int _whence; int py_recursion_remaining; int py_recursion_limit; diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 6a36dba3708e38..5f8b576e4a69ab 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -48,6 +48,7 @@ _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp) PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IsRunningMain(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_FailIfRunningMain(PyInterpreterState *); static inline const PyConfig * @@ -139,7 +140,9 @@ static inline PyInterpreterState* _PyInterpreterState_GET(void) { // PyThreadState functions -extern PyThreadState * _PyThreadState_New(PyInterpreterState *interp); +extern PyThreadState * _PyThreadState_New( + PyInterpreterState *interp, + int whence); extern void _PyThreadState_Bind(PyThreadState *tstate); extern void _PyThreadState_DeleteExcept(PyThreadState *tstate); diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 2deba02a89f33c..574a3c1a9db66c 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -185,6 +185,7 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyThreadState_INIT \ { \ + ._whence = _PyThreadState_WHENCE_NOTSET, \ .py_recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ .context_ver = 1, \ } diff --git a/Lib/threading.py b/Lib/threading.py index 0edfaf880f711a..41c3a9ff93856f 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1593,8 +1593,11 @@ def _shutdown(): # The main thread isn't finished yet, so its thread state lock can't # have been released. assert tlock is not None - assert tlock.locked() - tlock.release() + if tlock.locked(): + # It should have been released already by + # _PyInterpreterState_SetNotRunningMain(), but there may be + # embedders that aren't calling that yet. + tlock.release() _main_thread._stop() else: # bpo-1596321: _shutdown() must be called in the main thread. diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index ee46b37d92e128..86bd560b92ba6b 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1205,7 +1205,7 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs) if (boot == NULL) { return PyErr_NoMemory(); } - boot->tstate = _PyThreadState_New(interp); + boot->tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_THREADING); if (boot->tstate == NULL) { PyMem_RawFree(boot); if (!PyErr_Occurred()) { diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index e1c7d4ab2fd78f..700282efb8c619 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -242,6 +242,11 @@ _sharedns_apply(_sharedns *shared, PyObject *ns) // of the exception in the calling interpreter. typedef struct _sharedexception { + PyInterpreterState *interp; +#define ERR_NOT_SET 0 +#define ERR_NO_MEMORY 1 +#define ERR_ALREADY_RUNNING 2 + int code; const char *name; const char *msg; } _sharedexception; @@ -263,14 +268,26 @@ _sharedexception_clear(_sharedexception *exc) } static const char * -_sharedexception_bind(PyObject *exc, _sharedexception *sharedexc) +_sharedexception_bind(PyObject *exc, int code, _sharedexception *sharedexc) { + if (sharedexc->interp == NULL) { + sharedexc->interp = PyInterpreterState_Get(); + } + + if (code != ERR_NOT_SET) { + assert(exc == NULL); + assert(code > 0); + sharedexc->code = code; + return NULL; + } + assert(exc != NULL); const char *failure = NULL; PyObject *nameobj = PyUnicode_FromString(Py_TYPE(exc)->tp_name); if (nameobj == NULL) { failure = "unable to format exception type name"; + code = ERR_NO_MEMORY; goto error; } sharedexc->name = _copy_raw_string(nameobj); @@ -281,6 +298,7 @@ _sharedexception_bind(PyObject *exc, _sharedexception *sharedexc) } else { failure = "unable to encode and copy exception type name"; } + code = ERR_NO_MEMORY; goto error; } @@ -288,6 +306,7 @@ _sharedexception_bind(PyObject *exc, _sharedexception *sharedexc) PyObject *msgobj = PyUnicode_FromFormat("%S", exc); if (msgobj == NULL) { failure = "unable to format exception message"; + code = ERR_NO_MEMORY; goto error; } sharedexc->msg = _copy_raw_string(msgobj); @@ -298,6 +317,7 @@ _sharedexception_bind(PyObject *exc, _sharedexception *sharedexc) } else { failure = "unable to encode and copy exception message"; } + code = ERR_NO_MEMORY; goto error; } } @@ -308,7 +328,10 @@ _sharedexception_bind(PyObject *exc, _sharedexception *sharedexc) assert(failure != NULL); PyErr_Clear(); _sharedexception_clear(sharedexc); - *sharedexc = no_exception; + *sharedexc = (_sharedexception){ + .interp = sharedexc->interp, + .code = code, + }; return failure; } @@ -316,6 +339,7 @@ static void _sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass) { if (exc->name != NULL) { + assert(exc->code == ERR_NOT_SET); if (exc->msg != NULL) { PyErr_Format(wrapperclass, "%s: %s", exc->name, exc->msg); } @@ -324,9 +348,19 @@ _sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass) } } else if (exc->msg != NULL) { + assert(exc->code == ERR_NOT_SET); PyErr_SetString(wrapperclass, exc->msg); } + else if (exc->code == ERR_NO_MEMORY) { + PyErr_NoMemory(); + } + else if (exc->code == ERR_ALREADY_RUNNING) { + assert(exc->interp != NULL); + assert(_PyInterpreterState_IsRunningMain(exc->interp)); + _PyInterpreterState_FailIfRunningMain(exc->interp); + } else { + assert(exc->code == ERR_NOT_SET); PyErr_SetNone(wrapperclass); } } @@ -362,9 +396,16 @@ static int _run_script(PyInterpreterState *interp, const char *codestr, _sharedns *shared, _sharedexception *sharedexc) { + int errcode = ERR_NOT_SET; + if (_PyInterpreterState_SetRunningMain(interp) < 0) { - // We skip going through the shared exception. - return -1; + assert(PyErr_Occurred()); + // In the case where we didn't switch interpreters, it would + // be more efficient to leave the exception in place and return + // immediately. However, life is simpler if we don't. + PyErr_Clear(); + errcode = ERR_ALREADY_RUNNING; + goto error; } PyObject *excval = NULL; @@ -403,16 +444,17 @@ _run_script(PyInterpreterState *interp, const char *codestr, error: excval = PyErr_GetRaisedException(); - const char *failure = _sharedexception_bind(excval, sharedexc); + const char *failure = _sharedexception_bind(excval, errcode, sharedexc); if (failure != NULL) { fprintf(stderr, "RunFailedError: script raised an uncaught exception (%s)", failure); - PyErr_Clear(); } Py_XDECREF(excval); + if (errcode != ERR_ALREADY_RUNNING) { + _PyInterpreterState_SetNotRunningMain(interp); + } assert(!PyErr_Occurred()); - _PyInterpreterState_SetNotRunningMain(interp); return -1; } @@ -421,6 +463,7 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, const char *codestr, PyObject *shareables) { module_state *state = get_module_state(mod); + assert(state != NULL); _sharedns *shared = _get_shared_ns(shareables); if (shared == NULL && PyErr_Occurred()) { @@ -429,50 +472,30 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, // Switch to interpreter. PyThreadState *save_tstate = NULL; + PyThreadState *tstate = NULL; if (interp != PyInterpreterState_Get()) { - // XXX gh-109860: Using the "head" thread isn't strictly correct. - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - assert(tstate != NULL); - // Hack (until gh-109860): The interpreter's initial thread state - // is least likely to break. - while(tstate->next != NULL) { - tstate = tstate->next; - } - // We must do this check before switching interpreters, so any - // exception gets raised in the right one. - // XXX gh-109860: Drop this redundant check once we stop - // re-using tstates that might already be in use. - if (_PyInterpreterState_IsRunningMain(interp)) { - PyErr_SetString(PyExc_RuntimeError, - "interpreter already running"); - if (shared != NULL) { - _sharedns_free(shared); - } - return -1; - } + tstate = PyThreadState_New(interp); + tstate->_whence = _PyThreadState_WHENCE_EXEC; // XXX Possible GILState issues? save_tstate = PyThreadState_Swap(tstate); } // Run the script. - _sharedexception exc = {NULL, NULL}; + _sharedexception exc = (_sharedexception){ .interp = interp }; int result = _run_script(interp, codestr, shared, &exc); // Switch back. if (save_tstate != NULL) { + PyThreadState_Clear(tstate); PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); } // Propagate any exception out to the caller. - if (exc.name != NULL) { - assert(state != NULL); + if (result < 0) { + assert(!PyErr_Occurred()); _sharedexception_apply(&exc, state->RunFailedError); - } - else if (result != 0) { - if (!PyErr_Occurred()) { - // We were unable to allocate a shared exception. - PyErr_NoMemory(); - } + assert(PyErr_Occurred()); } if (shared != NULL) { @@ -502,6 +525,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) const PyInterpreterConfig config = isolated ? (PyInterpreterConfig)_PyInterpreterConfig_INIT : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + // XXX Possible GILState issues? PyThreadState *tstate = NULL; PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); @@ -517,6 +541,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } assert(tstate != NULL); + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); PyObject *idobj = PyInterpreterState_GetIDObject(interp); if (idobj == NULL) { @@ -526,6 +551,10 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) PyThreadState_Swap(save_tstate); return NULL; } + + PyThreadState_Clear(tstate); + PyThreadState_Delete(tstate); + _PyInterpreterState_RequireIDRef(interp, 1); return idobj; } @@ -573,14 +602,8 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) } // Destroy the interpreter. - // XXX gh-109860: Using the "head" thread isn't strictly correct. - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - assert(tstate != NULL); - // Hack (until gh-109860): The interpreter's initial thread state - // is least likely to break. - while(tstate->next != NULL) { - tstate = tstate->next; - } + PyThreadState *tstate = PyThreadState_New(interp); + tstate->_whence = _PyThreadState_WHENCE_INTERP; // XXX Possible GILState issues? PyThreadState *save_tstate = PyThreadState_Swap(tstate); Py_EndInterpreter(tstate); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index c0323763f44890..eb10aa3562dfce 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -655,7 +655,8 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return status; } - PyThreadState *tstate = _PyThreadState_New(interp); + PyThreadState *tstate = _PyThreadState_New(interp, + _PyThreadState_WHENCE_INTERP); if (tstate == NULL) { return _PyStatus_ERR("can't make first thread"); } @@ -2050,7 +2051,8 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) return _PyStatus_OK(); } - PyThreadState *tstate = _PyThreadState_New(interp); + PyThreadState *tstate = _PyThreadState_New(interp, + _PyThreadState_WHENCE_INTERP); if (tstate == NULL) { PyInterpreterState_Delete(interp); *tstate_p = NULL; diff --git a/Python/pystate.c b/Python/pystate.c index fe056bf4687026..25a957c31f0134 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1094,9 +1094,7 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) int _PyInterpreterState_SetRunningMain(PyInterpreterState *interp) { - if (interp->threads.main != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "interpreter already running"); + if (_PyInterpreterState_FailIfRunningMain(interp) < 0) { return -1; } PyThreadState *tstate = current_fast_get(&_PyRuntime); @@ -1113,7 +1111,20 @@ _PyInterpreterState_SetRunningMain(PyInterpreterState *interp) void _PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp) { - assert(interp->threads.main == current_fast_get(&_PyRuntime)); + PyThreadState *tstate = interp->threads.main; + assert(tstate == current_fast_get(&_PyRuntime)); + + if (tstate->on_delete != NULL) { + // The threading module was imported for the first time in this + // thread, so it was set as threading._main_thread. (See gh-75698.) + // The thread has finished running the Python program so we mark + // the thread object as finished. + assert(tstate->_whence != _PyThreadState_WHENCE_THREADING); + tstate->on_delete(tstate->on_delete_data); + tstate->on_delete = NULL; + tstate->on_delete_data = NULL; + } + interp->threads.main = NULL; } @@ -1123,6 +1134,17 @@ _PyInterpreterState_IsRunningMain(PyInterpreterState *interp) return (interp->threads.main != NULL); } +int +_PyInterpreterState_FailIfRunningMain(PyInterpreterState *interp) +{ + if (interp->threads.main != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "interpreter already running"); + return -1; + } + return 0; +} + //---------- // accessors @@ -1183,8 +1205,10 @@ _PyInterpreterState_IDDecref(PyInterpreterState *interp) PyThread_release_lock(interp->id_mutex); if (refcount == 0 && interp->requires_idref) { - // XXX Using the "head" thread isn't strictly correct. - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + PyThreadState *tstate = _PyThreadState_New(interp, + _PyThreadState_WHENCE_INTERP); + _PyThreadState_Bind(tstate); + // XXX Possible GILState issues? PyThreadState *save_tstate = _PyThreadState_Swap(runtime, tstate); Py_EndInterpreter(tstate); @@ -1338,7 +1362,14 @@ free_threadstate(PyThreadState *tstate) { // The initial thread state of the interpreter is allocated // as part of the interpreter state so should not be freed. - if (tstate != &tstate->interp->_initial_thread) { + if (tstate == &tstate->interp->_initial_thread) { + // Restore to _PyThreadState_INIT. + tstate = &tstate->interp->_initial_thread; + memcpy(tstate, + &initial._main_interpreter._initial_thread, + sizeof(*tstate)); + } + else { PyMem_RawFree(tstate); } } @@ -1353,7 +1384,7 @@ free_threadstate(PyThreadState *tstate) static void init_threadstate(PyThreadState *tstate, - PyInterpreterState *interp, uint64_t id) + PyInterpreterState *interp, uint64_t id, int whence) { if (tstate->_status.initialized) { Py_FatalError("thread state already initialized"); @@ -1366,6 +1397,10 @@ init_threadstate(PyThreadState *tstate, assert(tstate->next == NULL); assert(tstate->prev == NULL); + assert(tstate->_whence == _PyThreadState_WHENCE_NOTSET); + assert(whence >= 0 && whence <= _PyThreadState_WHENCE_EXEC); + tstate->_whence = whence; + assert(id > 0); tstate->id = id; @@ -1395,8 +1430,6 @@ add_threadstate(PyInterpreterState *interp, PyThreadState *tstate, PyThreadState *next) { assert(interp->threads.head != tstate); - assert((next != NULL && tstate->id != 1) || - (next == NULL && tstate->id == 1)); if (next != NULL) { assert(next->prev == NULL || next->prev == tstate); next->prev = tstate; @@ -1407,7 +1440,7 @@ add_threadstate(PyInterpreterState *interp, PyThreadState *tstate, } static PyThreadState * -new_threadstate(PyInterpreterState *interp) +new_threadstate(PyInterpreterState *interp, int whence) { PyThreadState *tstate; _PyRuntimeState *runtime = interp->runtime; @@ -1430,10 +1463,10 @@ new_threadstate(PyInterpreterState *interp) PyThreadState *old_head = interp->threads.head; if (old_head == NULL) { // It's the interpreter's initial thread state. - assert(id == 1); used_newtstate = 0; tstate = &interp->_initial_thread; } + // XXX Re-use interp->_initial_thread if not in use? else { // Every valid interpreter must have at least one thread. assert(id > 1); @@ -1446,7 +1479,7 @@ new_threadstate(PyInterpreterState *interp) sizeof(*tstate)); } - init_threadstate(tstate, interp, id); + init_threadstate(tstate, interp, id, whence); add_threadstate(interp, tstate, old_head); HEAD_UNLOCK(runtime); @@ -1460,7 +1493,8 @@ new_threadstate(PyInterpreterState *interp) PyThreadState * PyThreadState_New(PyInterpreterState *interp) { - PyThreadState *tstate = new_threadstate(interp); + PyThreadState *tstate = new_threadstate(interp, + _PyThreadState_WHENCE_UNKNOWN); if (tstate) { bind_tstate(tstate); // This makes sure there's a gilstate tstate bound @@ -1474,16 +1508,16 @@ PyThreadState_New(PyInterpreterState *interp) // This must be followed by a call to _PyThreadState_Bind(); PyThreadState * -_PyThreadState_New(PyInterpreterState *interp) +_PyThreadState_New(PyInterpreterState *interp, int whence) { - return new_threadstate(interp); + return new_threadstate(interp, whence); } // We keep this for stable ABI compabibility. PyAPI_FUNC(PyThreadState*) _PyThreadState_Prealloc(PyInterpreterState *interp) { - return _PyThreadState_New(interp); + return _PyThreadState_New(interp, _PyThreadState_WHENCE_UNKNOWN); } // We keep this around for (accidental) stable ABI compatibility. @@ -1580,6 +1614,12 @@ PyThreadState_Clear(PyThreadState *tstate) Py_CLEAR(tstate->context); if (tstate->on_delete != NULL) { + // For the "main" thread of each interpreter, this is meant + // to be done in _PyInterpreterState_SetNotRunningMain(). + // That leaves threads created by the threading module, + // and any threads killed by forking. + // However, we also accommodate "main" threads that still + // don't call _PyInterpreterState_SetNotRunningMain() yet. tstate->on_delete(tstate->on_delete_data); } @@ -2240,7 +2280,9 @@ PyGILState_Ensure(void) int has_gil; if (tcur == NULL) { /* Create a new Python thread state for this thread */ - tcur = new_threadstate(runtime->gilstate.autoInterpreterState); + // XXX Use PyInterpreterState_EnsureThreadState()? + tcur = new_threadstate(runtime->gilstate.autoInterpreterState, + _PyThreadState_WHENCE_GILSTATE); if (tcur == NULL) { Py_FatalError("Couldn't create thread-state for new thread"); } From eeb4e974d074436b968749d51ce922303355a347 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 3 Oct 2023 17:50:10 +0200 Subject: [PATCH 07/73] gh-107073: Mention pythoncapi-compat for PyObject_VisitManagedDict() (#110291) --- Doc/whatsnew/3.13.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 1ef04fa7ae6adc..a99d1141f53d77 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -998,7 +998,9 @@ New Features * Add :c:func:`PyObject_VisitManagedDict` and :c:func:`PyObject_ClearManagedDict` functions which must be called by the traverse and clear functions of a type using - :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag. + :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag. The `pythoncapi-compat project + `__ can be used to get these + functions on Python 3.11 and 3.12. (Contributed by Victor Stinner in :gh:`107073`.) Porting to Python 3.13 From 6ab6040054e5ca2d3eb7833dc8bf4eb0bbaa0aac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 3 Oct 2023 18:43:23 +0200 Subject: [PATCH 08/73] gh-110276: No longer ignore PROFILE_TASK failure silently (#110295) --- Doc/using/configure.rst | 3 +++ Makefile.pre.in | 4 ++-- .../next/Build/2023-10-03-17-55-09.gh-issue-110276.luaKRg.rst | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-10-03-17-55-09.gh-issue-110276.luaKRg.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 83b4c7aa0481e9..eb8f2442d0f0a6 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -518,6 +518,9 @@ also be used to improve performance. .. versionadded:: 3.8 + .. versionchanged:: 3.13 + Task failure is no longer ignored silently. + .. cmdoption:: --with-lto=[full|thin|no|yes] Enable Link Time Optimization (LTO) in any build (disabled by default). diff --git a/Makefile.pre.in b/Makefile.pre.in index cf03c86f18b3c3..97eb767b8fcdeb 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -657,7 +657,7 @@ profile-run-stamp: $(MAKE) profile-gen-stamp # Next, run the profile task to generate the profile information. @ # FIXME: can't run for a cross build - $(LLVM_PROF_FILE) $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true + $(LLVM_PROF_FILE) $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) $(LLVM_PROF_MERGER) # Remove profile generation binary since we are done with it. $(MAKE) clean-retain-profile @@ -706,7 +706,7 @@ profile-bolt-stamp: $(BUILDPYTHON) mv "$${bin}.bolt_inst" "$${bin}"; \ done # Run instrumented binaries to collect data. - $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true + $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) # Merge all the data files together. for bin in $(BOLT_BINARIES); do \ @MERGE_FDATA@ $${bin}.*.fdata > "$${bin}.fdata"; \ diff --git a/Misc/NEWS.d/next/Build/2023-10-03-17-55-09.gh-issue-110276.luaKRg.rst b/Misc/NEWS.d/next/Build/2023-10-03-17-55-09.gh-issue-110276.luaKRg.rst new file mode 100644 index 00000000000000..392203d21ca45d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-10-03-17-55-09.gh-issue-110276.luaKRg.rst @@ -0,0 +1,2 @@ +No longer ignore :envvar:`PROFILE_TASK` failure silently: command used by +Profile Guided Optimization (PGO). Patch by Victor Stinner. From d73501602f863a54c872ce103cd3fa119e38bac9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 3 Oct 2023 18:53:51 +0200 Subject: [PATCH 09/73] gh-108867: Add PyThreadState_GetUnchecked() function (#108870) Add PyThreadState_GetUnchecked() function: similar to PyThreadState_Get(), but don't issue a fatal error if it is NULL. The caller is responsible to check if the result is NULL. Previously, this function was private and known as _PyThreadState_UncheckedGet(). --- Doc/c-api/init.rst | 13 +++++++++++++ Doc/whatsnew/3.13.rst | 7 +++++++ Include/cpython/object.h | 2 +- Include/cpython/pystate.h | 2 +- Include/internal/pycore_pystate.h | 2 +- Include/pystate.h | 2 +- .../2023-09-04-11-47-12.gh-issue-108867.Cr_LKd.rst | 5 +++++ Modules/_testcapimodule.c | 4 ++-- Modules/getpath.c | 3 ++- Python/pystate.c | 2 +- 10 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-09-04-11-47-12.gh-issue-108867.Cr_LKd.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 60f5c81cff572c..d164d1a752e295 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -870,6 +870,19 @@ code, or when embedding the Python interpreter: When the current thread state is ``NULL``, this issues a fatal error (so that the caller needn't check for ``NULL``). + See also :c:func:`PyThreadState_GetUnchecked`. + + +.. c:function:: PyThreadState* PyThreadState_GetUnchecked() + + Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a + fatal error if it is NULL. The caller is responsible to check if the result + is NULL. + + .. versionadded:: 3.13 + In Python 3.5 to 3.12, the function was private and known as + ``_PyThreadState_UncheckedGet()``. + .. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a99d1141f53d77..785deea1c1e48f 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1003,6 +1003,13 @@ New Features functions on Python 3.11 and 3.12. (Contributed by Victor Stinner in :gh:`107073`.) +* Add :c:func:`PyThreadState_GetUnchecked()` function: similar to + :c:func:`PyThreadState_Get()`, but don't kill the process with a fatal error + if it is NULL. The caller is responsible to check if the result is NULL. + Previously, the function was private and known as + ``_PyThreadState_UncheckedGet()``. + (Contributed by Victor Stinner in :gh:`108867`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 3838f19c75a230..ede394d9673d7e 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -425,7 +425,7 @@ PyAPI_FUNC(int) _PyTrash_cond(PyObject *op, destructor dealloc); /* If "cond" is false, then _tstate remains NULL and the deallocator \ * is run normally without involving the trashcan */ \ if (cond) { \ - _tstate = _PyThreadState_UncheckedGet(); \ + _tstate = PyThreadState_GetUnchecked(); \ if (_PyTrash_begin(_tstate, _PyObject_CAST(op))) { \ break; \ } \ diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index af5cc4a2f3bf63..8fd06242bc82e0 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -218,7 +218,7 @@ struct _ts { /* Similar to PyThreadState_Get(), but don't issue a fatal error * if it is NULL. */ -PyAPI_FUNC(PyThreadState *) _PyThreadState_UncheckedGet(void); +PyAPI_FUNC(PyThreadState *) PyThreadState_GetUnchecked(void); // Disable tracing and profiling. diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 5f8b576e4a69ab..a60d949bff1eba 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -93,7 +93,7 @@ PyAPI_FUNC(PyThreadState *) _PyThreadState_GetCurrent(void); The caller must hold the GIL. - See also PyThreadState_Get() and _PyThreadState_UncheckedGet(). */ + See also PyThreadState_Get() and PyThreadState_GetUnchecked(). */ static inline PyThreadState* _PyThreadState_GET(void) { diff --git a/Include/pystate.h b/Include/pystate.h index e6b4de979c87b8..727b8fbfffe0e6 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -56,7 +56,7 @@ PyAPI_FUNC(void) PyThreadState_Delete(PyThreadState *); The caller must hold the GIL. - See also _PyThreadState_UncheckedGet() and _PyThreadState_GET(). */ + See also PyThreadState_GetUnchecked() and _PyThreadState_GET(). */ PyAPI_FUNC(PyThreadState *) PyThreadState_Get(void); // Alias to PyThreadState_Get() diff --git a/Misc/NEWS.d/next/C API/2023-09-04-11-47-12.gh-issue-108867.Cr_LKd.rst b/Misc/NEWS.d/next/C API/2023-09-04-11-47-12.gh-issue-108867.Cr_LKd.rst new file mode 100644 index 00000000000000..2f56466833f6dd --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-09-04-11-47-12.gh-issue-108867.Cr_LKd.rst @@ -0,0 +1,5 @@ +Add :c:func:`PyThreadState_GetUnchecked()` function: similar to +:c:func:`PyThreadState_Get()`, but don't kill the process with a fatal error if +it is NULL. The caller is responsible to check if the result is NULL. +Previously, the function was private and known as +``_PyThreadState_UncheckedGet()``. Patch by Victor Stinner. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 64bcb49d365774..a46d986c18ecd4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2458,8 +2458,8 @@ test_tstate_capi(PyObject *self, PyObject *Py_UNUSED(args)) PyThreadState *tstate2 = PyThreadState_Get(); assert(tstate2 == tstate); - // private _PyThreadState_UncheckedGet() - PyThreadState *tstate3 = _PyThreadState_UncheckedGet(); + // PyThreadState_GetUnchecked() + PyThreadState *tstate3 = PyThreadState_GetUnchecked(); assert(tstate3 == tstate); // PyThreadState_EnterTracing(), PyThreadState_LeaveTracing() diff --git a/Modules/getpath.c b/Modules/getpath.c index 3b926cac0d3f24..6f76a84e78bf62 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -6,6 +6,7 @@ #include "pycore_pathconfig.h" // _PyPathConfig_ReadGlobal() #include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #include "pycore_pymem.h" // _PyMem_RawWcsdup() +#include "pycore_pystate.h" // _PyThreadState_GET() #include "marshal.h" // PyMarshal_ReadObjectFromString #include "osdefs.h" // DELIM @@ -821,7 +822,7 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) return status; } - if (!_PyThreadState_UncheckedGet()) { + if (!_PyThreadState_GET()) { return PyStatus_Error("cannot calculate path configuration without GIL"); } diff --git a/Python/pystate.c b/Python/pystate.c index 25a957c31f0134..ae33259f8df2f6 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1908,7 +1908,7 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) //--------------------------------- PyThreadState * -_PyThreadState_UncheckedGet(void) +PyThreadState_GetUnchecked(void) { return current_fast_get(&_PyRuntime); } From d67edcf0b361c9ee0d29ed719562c58a85304cd0 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 3 Oct 2023 10:13:50 -0700 Subject: [PATCH 10/73] gh-109979: Auto-generate the target for DEOPT_IF() (#110193) In Python/bytecodes.c, you now write ``` DEOPT_IF(condition); ``` The code generator expands this to ``` DEOPT_IF(condition, opcode); ``` where `opcode` is the name of the unspecialized instruction. This works inside macro expansions too. **CAVEAT:** The entire `DEOPT_IF(condition)` statement must be on a single line. If it isn't, the substitution will fail; an error will be printed by the code generator and the C compiler will report some errors. --- Lib/test/test_generated_cases.py | 11 +- Python/bytecodes.c | 352 ++++++++++++------------ Python/executor_cases.c.h | 73 +++-- Python/generated_cases.c.h | 132 ++++----- Tools/cases_generator/analysis.py | 26 +- Tools/cases_generator/generate_cases.py | 4 +- Tools/cases_generator/instructions.py | 16 +- Tools/cases_generator/stacking.py | 16 +- 8 files changed, 315 insertions(+), 315 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index b5eaf824aee706..5971d2e436e4aa 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -239,16 +239,22 @@ def test_overlap(self): def test_predictions_and_eval_breaker(self): input = """ - inst(OP1, (--)) { + inst(OP1, (arg -- rest)) { } inst(OP3, (arg -- res)) { - DEOPT_IF(xxx, OP1); + DEOPT_IF(xxx); CHECK_EVAL_BREAKER(); } + family(OP1, INLINE_CACHE_ENTRIES_OP1) = { OP3 }; """ output = """ TARGET(OP1) { PREDICTED(OP1); + static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); + PyObject *arg; + PyObject *rest; + arg = stack_pointer[-1]; + stack_pointer[-1] = rest; DISPATCH(); } @@ -371,6 +377,7 @@ def test_macro_instruction(self): } TARGET(OP) { + PREDICTED(OP); static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); PyObject *right; PyObject *left; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f7681bd234a43f..035629dda6e7db 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -41,8 +41,6 @@ #include "ceval_macros.h" /* Flow control macros */ -#define DEOPT_IF(cond, instname) ((void)0) -#define ERROR_IF(cond, labelname) ((void)0) #define GO_TO_INSTRUCTION(instname) ((void)0) #define inst(name, ...) case name: @@ -153,13 +151,12 @@ dummy_func( inst(RESUME_CHECK, (--)) { #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + DEOPT_IF(_Py_emscripten_signal_clock == 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ - DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version - != tstate->interp->monitoring_version, RESUME); - DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); + DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version); + DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker)); } inst(INSTRUMENTED_RESUME, (--)) { @@ -325,12 +322,12 @@ dummy_func( } inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { - DEOPT_IF(!PyBool_Check(value), TO_BOOL); + DEOPT_IF(!PyBool_Check(value)); STAT_INC(TO_BOOL, hit); } inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyLong_CheckExact(value)); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -343,7 +340,7 @@ dummy_func( } inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyList_CheckExact(value)); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; DECREF_INPUTS(); @@ -351,13 +348,13 @@ dummy_func( inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); + DEOPT_IF(!Py_IsNone(value)); STAT_INC(TO_BOOL, hit); res = Py_False; } inst(TO_BOOL_STR, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyUnicode_CheckExact(value)); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -373,7 +370,7 @@ dummy_func( inst(TO_BOOL_ALWAYS_TRUE, (unused/1, version/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); + DEOPT_IF(Py_TYPE(value)->tp_version_tag != version); STAT_INC(TO_BOOL, hit); DECREF_INPUTS(); res = Py_True; @@ -397,8 +394,8 @@ dummy_func( }; op(_GUARD_BOTH_INT, (left, right -- left, right)) { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left)); + DEOPT_IF(!PyLong_CheckExact(right)); } op(_BINARY_OP_MULTIPLY_INT, (unused/1, left, right -- res)) { @@ -433,8 +430,8 @@ dummy_func( _GUARD_BOTH_INT + _BINARY_OP_SUBTRACT_INT; op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left)); + DEOPT_IF(!PyFloat_CheckExact(right)); } op(_BINARY_OP_MULTIPLY_FLOAT, (unused/1, left, right -- res)) { @@ -469,8 +466,8 @@ dummy_func( _GUARD_BOTH_FLOAT + _BINARY_OP_SUBTRACT_FLOAT; op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left)); + DEOPT_IF(!PyUnicode_CheckExact(right)); } op(_BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) { @@ -494,7 +491,7 @@ dummy_func( _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; assert(true_next.op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(true_next.op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); + DEOPT_IF(*target_local != left); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -574,13 +571,13 @@ dummy_func( } inst(BINARY_SUBSCR_LIST_INT, (unused/1, list, sub -- res)) { - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyList_CheckExact(list)); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list)); STAT_INC(BINARY_SUBSCR, hit); res = PyList_GET_ITEM(list, index); assert(res != NULL); @@ -590,14 +587,14 @@ dummy_func( } inst(BINARY_SUBSCR_STR_INT, (unused/1, str, sub -- res)) { - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyUnicode_CheckExact(str)); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); @@ -605,13 +602,13 @@ dummy_func( } inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple, sub -- res)) { - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyTuple_CheckExact(tuple)); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple)); STAT_INC(BINARY_SUBSCR, hit); res = PyTuple_GET_ITEM(tuple, index); assert(res != NULL); @@ -621,7 +618,7 @@ dummy_func( } inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) { - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); if (res == NULL) { @@ -636,19 +633,19 @@ dummy_func( } inst(BINARY_SUBSCR_GETITEM, (unused/1, container, sub -- unused)) { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *tp = Py_TYPE(container); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *cached = ht->_spec_cache.getitem; - DEOPT_IF(cached == NULL, BINARY_SUBSCR); + DEOPT_IF(cached == NULL); assert(PyFunction_Check(cached)); PyFunctionObject *getitem = (PyFunctionObject *)cached; uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); + DEOPT_IF(getitem->func_version != cached_version); PyCodeObject *code = (PyCodeObject *)getitem->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(BINARY_SUBSCR, hit); Py_INCREF(getitem); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); @@ -693,14 +690,14 @@ dummy_func( } inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) { - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyList_CheckExact(list)); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list)); STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -712,7 +709,7 @@ dummy_func( } inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) { - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); @@ -1009,11 +1006,10 @@ dummy_func( } inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) { - DEOPT_IF(tstate->interp->eval_frame, SEND); + DEOPT_IF(tstate->interp->eval_frame); PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && - Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(SEND, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; STACK_SHRINK(1); @@ -1194,8 +1190,8 @@ dummy_func( } inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq)); + DEOPT_IF(PyTuple_GET_SIZE(seq) != 2); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); @@ -1204,8 +1200,8 @@ dummy_func( } inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq)); + DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1215,8 +1211,8 @@ dummy_func( } inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + DEOPT_IF(!PyList_CheckExact(seq)); + DEOPT_IF(PyList_GET_SIZE(seq) != oparg); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1412,15 +1408,15 @@ dummy_func( op(_GUARD_GLOBALS_VERSION, (version/1 --)) { PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict)); + DEOPT_IF(dict->ma_keys->dk_version != version); assert(DK_IS_UNICODE(dict->ma_keys)); } op(_GUARD_BUILTINS_VERSION, (version/1 --)) { PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict)); + DEOPT_IF(dict->ma_keys->dk_version != version); assert(DK_IS_UNICODE(dict->ma_keys)); } @@ -1428,7 +1424,7 @@ dummy_func( PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1438,7 +1434,7 @@ dummy_func( PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1763,8 +1759,8 @@ dummy_func( inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- attr, unused if (0))) { assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type); + DEOPT_IF(!PyType_Check(class)); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); @@ -1774,8 +1770,8 @@ dummy_func( inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- attr, self_or_null)) { assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type); + DEOPT_IF(!PyType_Check(class)); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -1864,22 +1860,20 @@ dummy_func( op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version); } op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); } op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1894,15 +1888,15 @@ dummy_func( unused/5; // Skip over rest of cache inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { - DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); + DEOPT_IF(!PyModule_CheckExact(owner)); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); + DEOPT_IF(dict->ma_keys->dk_version != type_version); assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; attr = ep->me_value; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1912,27 +1906,27 @@ dummy_func( inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + DEOPT_IF(_PyDictOrValues_IsValues(dorv)); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, LOAD_ATTR); + DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); uint16_t hint = index; - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); + DEOPT_IF(ep->me_key != name); attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); + DEOPT_IF(ep->me_key != name); attr = ep->me_value; } - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1942,7 +1936,7 @@ dummy_func( op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { char *addr = (char *)owner + index; attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1957,9 +1951,8 @@ dummy_func( inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, null if (oparg & 1))) { - DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, - LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner)); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version); assert(type_version != 0); STAT_INC(LOAD_ATTR, hit); @@ -1972,18 +1965,18 @@ dummy_func( inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused, unused if (0))) { assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(cls->tp_version_tag != type_version); assert(type_version != 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + DEOPT_IF(f->func_version != func_version); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 1); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); @@ -1997,17 +1990,17 @@ dummy_func( inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) { assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(cls->tp_version_tag != type_version); assert(type_version != 0); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + DEOPT_IF(f->func_version != func_version); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); @@ -2025,7 +2018,7 @@ dummy_func( op(_GUARD_DORV_VALUES, (owner -- owner)) { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv)); } op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) { @@ -2052,30 +2045,30 @@ dummy_func( inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); + DEOPT_IF(_PyDictOrValues_IsValues(dorv)); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); PyObject *old_value; uint64_t new_version; if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); + DEOPT_IF(ep->me_key != name); old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); + DEOPT_IF(old_value == NULL); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); + DEOPT_IF(ep->me_key != name); old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); + DEOPT_IF(old_value == NULL); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } @@ -2093,7 +2086,7 @@ dummy_func( op(_GUARD_TYPE_VERSION_STORE, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version); } op(_STORE_ATTR_SLOT, (index/1, value, owner --)) { @@ -2140,8 +2133,8 @@ dummy_func( } inst(COMPARE_OP_FLOAT, (unused/1, left, right -- res)) { - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(left)); + DEOPT_IF(!PyFloat_CheckExact(right)); STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); @@ -2155,10 +2148,10 @@ dummy_func( // Similar to COMPARE_OP_FLOAT inst(COMPARE_OP_INT, (unused/1, left, right -- res)) { - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(left)); + DEOPT_IF(!PyLong_CheckExact(right)); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left)); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right)); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); @@ -2174,8 +2167,8 @@ dummy_func( // Similar to COMPARE_OP_FLOAT, but for ==, != only inst(COMPARE_OP_STR, (unused/1, left, right -- res)) { - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(left)); + DEOPT_IF(!PyUnicode_CheckExact(right)); STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left, right); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); @@ -2492,7 +2485,7 @@ dummy_func( } op(_ITER_CHECK_LIST, (iter -- iter)) { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type); } op(_ITER_JUMP_LIST, (iter -- iter)) { @@ -2548,7 +2541,7 @@ dummy_func( _ITER_NEXT_LIST; op(_ITER_CHECK_TUPLE, (iter -- iter)) { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type); } op(_ITER_JUMP_TUPLE, (iter -- iter)) { @@ -2605,7 +2598,7 @@ dummy_func( op(_ITER_CHECK_RANGE, (iter -- iter)) { _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type); } op(_ITER_JUMP_RANGE, (iter -- iter)) { @@ -2647,10 +2640,10 @@ dummy_func( _ITER_NEXT_RANGE; inst(FOR_ITER_GEN, (unused/1, iter -- iter, unused)) { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + DEOPT_IF(tstate->interp->eval_frame); PyGenObject *gen = (PyGenObject *)iter; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; _PyFrame_StackPush(gen_frame, Py_None); @@ -2790,16 +2783,13 @@ dummy_func( op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); } op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); } op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { @@ -2839,15 +2829,12 @@ dummy_func( assert((oparg & 1) == 0); PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(owner_cls->tp_version_tag != type_version); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); @@ -2858,7 +2845,7 @@ dummy_func( assert((oparg & 1) == 0); PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(owner_cls->tp_version_tag != type_version); assert(owner_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -2869,12 +2856,12 @@ dummy_func( inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, self if (1))) { assert(oparg & 1); PyTypeObject *owner_cls = Py_TYPE(owner); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(owner_cls->tp_version_tag != type_version); Py_ssize_t dictoffset = owner_cls->tp_dictoffset; assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); + DEOPT_IF(dict != NULL); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); @@ -3002,8 +2989,8 @@ dummy_func( } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + DEOPT_IF(null != NULL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type); } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { @@ -3016,22 +3003,22 @@ dummy_func( } op(_CHECK_PEP_523, (--)) { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame); } op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable)); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL)); } op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); + DEOPT_IF(tstate->py_recursion_remaining <= 1); } op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { @@ -3094,24 +3081,24 @@ dummy_func( _PUSH_FRAME; inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame); int argcount = oparg; if (self_or_null != NULL) { args--; argcount++; } - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable)); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version); PyCodeObject *code = (PyCodeObject *)func->func_code; assert(func->func_defaults); assert(PyTuple_CheckExact(func->func_defaults)); int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); assert(defcount <= code->co_argcount); int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount, CALL); - DEOPT_IF(argcount < min_args, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(argcount > code->co_argcount); + DEOPT_IF(argcount < min_args); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); for (int i = 0; i < argcount; i++) { @@ -3130,9 +3117,9 @@ dummy_func( inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); + DEOPT_IF(null != NULL); PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + DEOPT_IF(callable != (PyObject *)&PyType_Type); STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); @@ -3141,8 +3128,8 @@ dummy_func( inst(CALL_STR_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + DEOPT_IF(null != NULL); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type); STAT_INC(CALL, hit); PyObject *arg = args[0]; res = PyObject_Str(arg); @@ -3154,8 +3141,8 @@ dummy_func( inst(CALL_TUPLE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + DEOPT_IF(null != NULL); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type); STAT_INC(CALL, hit); PyObject *arg = args[0]; res = PySequence_Tuple(arg); @@ -3172,15 +3159,15 @@ dummy_func( * 3. Pushes the frame for ``__init__`` to the frame stack * */ _PyCallCache *cache = (_PyCallCache *)next_instr; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(!PyType_Check(callable), CALL); + DEOPT_IF(null != NULL); + DEOPT_IF(!PyType_Check(callable)); PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version)); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + DEOPT_IF(code->co_argcount != oparg+1); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)); STAT_INC(CALL, hit); PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { @@ -3233,9 +3220,9 @@ dummy_func( args--; total_args++; } - DEOPT_IF(!PyType_Check(callable), CALL); + DEOPT_IF(!PyType_Check(callable)); PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + DEOPT_IF(tp->tp_vectorcall == NULL); STAT_INC(CALL, hit); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); /* Free the arguments. */ @@ -3254,9 +3241,9 @@ dummy_func( args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + DEOPT_IF(total_args != 1); + DEOPT_IF(!PyCFunction_CheckExact(callable)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); // This is slower but CPython promises to check all non-vectorcall @@ -3282,8 +3269,8 @@ dummy_func( args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ @@ -3314,9 +3301,8 @@ dummy_func( args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != - (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = @@ -3341,9 +3327,9 @@ dummy_func( args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); + DEOPT_IF(callable != interp->callable_cache.len); STAT_INC(CALL, hit); PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); @@ -3365,9 +3351,9 @@ dummy_func( args--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(total_args != 2); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + DEOPT_IF(callable != interp->callable_cache.isinstance); STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; @@ -3388,9 +3374,9 @@ dummy_func( inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { assert(oparg == 1); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); + DEOPT_IF(callable != interp->callable_cache.list_append); assert(self != NULL); - DEOPT_IF(!PyList_Check(self), CALL); + DEOPT_IF(!PyList_Check(self)); STAT_INC(CALL, hit); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { goto pop_1_error; // Since arg is DECREF'ed already @@ -3411,13 +3397,13 @@ dummy_func( total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(total_args != 2); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + DEOPT_IF(meth->ml_flags != METH_O); PyObject *arg = args[1]; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3442,12 +3428,12 @@ dummy_func( total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, d_type)); STAT_INC(CALL, hit); int nargs = total_args - 1; _PyCFunctionFastWithKeywords cfunc = @@ -3471,13 +3457,13 @@ dummy_func( args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1); PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); + DEOPT_IF(meth->ml_flags != METH_NOARGS); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3502,11 +3488,11 @@ dummy_func( } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + DEOPT_IF(meth->ml_flags != METH_FASTCALL); PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 55a03c9a23a572..662de57553fb47 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -13,8 +13,7 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ - DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version - != tstate->interp->monitoring_version, RESUME); + DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version, RESUME); DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); break; } @@ -236,8 +235,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), _GUARD_BOTH_INT); + DEOPT_IF(!PyLong_CheckExact(right), _GUARD_BOTH_INT); break; } @@ -294,8 +293,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), _GUARD_BOTH_FLOAT); + DEOPT_IF(!PyFloat_CheckExact(right), _GUARD_BOTH_FLOAT); break; } @@ -352,8 +351,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), _GUARD_BOTH_UNICODE); + DEOPT_IF(!PyUnicode_CheckExact(right), _GUARD_BOTH_UNICODE); break; } @@ -1189,8 +1188,8 @@ case _GUARD_GLOBALS_VERSION: { uint16_t version = (uint16_t)operand; PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION); + DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION); assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1198,8 +1197,8 @@ case _GUARD_BUILTINS_VERSION: { uint16_t version = (uint16_t)operand; PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION); + DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION); assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1211,7 +1210,7 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, _LOAD_GLOBAL_MODULE); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1229,7 +1228,7 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, _LOAD_GLOBAL_BUILTINS); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1679,7 +1678,7 @@ uint32_t type_version = (uint32_t)operand; PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION); break; } @@ -1689,9 +1688,7 @@ assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _CHECK_MANAGED_OBJECT_HAS_VALUES); break; } @@ -1703,7 +1700,7 @@ uint16_t index = (uint16_t)operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1722,7 +1719,7 @@ uint16_t index = (uint16_t)operand; char *addr = (char *)owner + index; attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1738,7 +1735,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), _GUARD_DORV_VALUES); break; } @@ -1770,7 +1767,7 @@ uint32_t type_version = (uint32_t)operand; PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION_STORE); break; } @@ -2114,7 +2111,7 @@ case _ITER_CHECK_LIST: { PyObject *iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, _ITER_CHECK_LIST); break; } @@ -2159,7 +2156,7 @@ case _ITER_CHECK_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, _ITER_CHECK_TUPLE); break; } @@ -2205,7 +2202,7 @@ PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, _ITER_CHECK_RANGE); break; } @@ -2300,9 +2297,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT); break; } @@ -2312,8 +2307,7 @@ uint32_t keys_version = (uint32_t)operand; PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION); break; } @@ -2360,8 +2354,8 @@ PyObject *callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + DEOPT_IF(null != NULL, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); break; } @@ -2382,7 +2376,7 @@ } case _CHECK_PEP_523: { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, _CHECK_PEP_523); break; } @@ -2392,11 +2386,11 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)operand; - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), _CHECK_FUNCTION_EXACT_ARGS); break; } @@ -2405,8 +2399,8 @@ callable = stack_pointer[-2 - oparg]; PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), _CHECK_STACK_SPACE); + DEOPT_IF(tstate->py_recursion_remaining <= 1, _CHECK_STACK_SPACE); break; } @@ -2671,8 +2665,7 @@ total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != - (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2701d416648a20..96d9d9db9c7928 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -32,8 +32,7 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ - DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version - != tstate->interp->monitoring_version, RESUME); + DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version, RESUME); DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); DISPATCH(); } @@ -391,8 +390,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_MULTIPLY_INT { @@ -416,8 +415,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_INT { @@ -441,8 +440,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_SUBTRACT_INT { @@ -466,8 +465,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_MULTIPLY_FLOAT { @@ -491,8 +490,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_FLOAT { @@ -516,8 +515,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_SUBTRACT_FLOAT { @@ -541,8 +540,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_UNICODE { @@ -565,15 +564,15 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } // _BINARY_OP_INPLACE_ADD_UNICODE { _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; assert(true_next.op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(true_next.op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); + DEOPT_IF(*target_local != left, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -1294,8 +1293,7 @@ receiver = stack_pointer[-2]; DEOPT_IF(tstate->interp->eval_frame, SEND); PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && - Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); STAT_INC(SEND, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; @@ -1806,8 +1804,8 @@ { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } // _LOAD_GLOBAL_MODULE @@ -1816,7 +1814,7 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1836,16 +1834,16 @@ { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } // _GUARD_BUILTINS_VERSION { uint16_t version = read_u16(&next_instr[2].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } // _LOAD_GLOBAL_BUILTINS @@ -1854,7 +1852,7 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -2403,23 +2401,21 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { uint16_t index = read_u16(&next_instr[3].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -2510,14 +2506,14 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { uint16_t index = read_u16(&next_instr[3].cache); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -2539,8 +2535,7 @@ PyObject *descr = read_obj(&next_instr[5].cache); DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, - LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); STAT_INC(LOAD_ATTR, hit); @@ -2627,13 +2622,13 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _GUARD_DORV_VALUES { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); } // _STORE_ATTR_INSTANCE_VALUE value = stack_pointer[-2]; @@ -2716,7 +2711,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _STORE_ATTR_SLOT value = stack_pointer[-2]; @@ -3297,7 +3292,7 @@ // _ITER_CHECK_LIST iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); } // _ITER_JUMP_LIST { @@ -3339,7 +3334,7 @@ // _ITER_CHECK_TUPLE iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -3382,7 +3377,7 @@ iter = stack_pointer[-1]; { _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } // _ITER_JUMP_RANGE { @@ -3585,23 +3580,20 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&next_instr[3].cache); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -3631,7 +3623,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_NO_DICT { @@ -3664,12 +3656,9 @@ DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); Py_DECREF(owner); @@ -3845,14 +3834,14 @@ _PyInterpreterFrame *new_frame; // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -3868,18 +3857,18 @@ callable = func; { uint32_t func_version = read_u32(&next_instr[1].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); } // _CHECK_STACK_SPACE { PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS args = stack_pointer - oparg; @@ -3939,25 +3928,25 @@ _PyInterpreterFrame *new_frame; // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_FUNCTION_EXACT_ARGS self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { uint32_t func_version = read_u32(&next_instr[1].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); } // _CHECK_STACK_SPACE { PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS args = stack_pointer - oparg; @@ -4328,8 +4317,7 @@ total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != - (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 91dcba8ceee13d..7bbc924e5083f1 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -26,7 +26,7 @@ "co_names": "Use FRAME_CO_NAMES.", } -RE_PREDICTED = r"^\s*(?:GO_TO_INSTRUCTION\(|DEOPT_IF\(.*?,\s*)(\w+)\);\s*(?://.*)?$" +RE_GO_TO_INSTR = r"^\s*GO_TO_INSTRUCTION\((\w+)\);\s*(?://.*)?$" class Analyzer: @@ -187,16 +187,23 @@ def analyze(self) -> None: Raises SystemExit if there is an error. """ self.analyze_macros_and_pseudos() - self.find_predictions() self.map_families() + self.mark_predictions() self.check_families() - def find_predictions(self) -> None: - """Find the instructions that need PREDICTED() labels.""" + def mark_predictions(self) -> None: + """Mark the instructions that need PREDICTED() labels.""" + # Start with family heads + for family in self.families.values(): + if family.name in self.instrs: + self.instrs[family.name].predicted = True + if family.name in self.macro_instrs: + self.macro_instrs[family.name].predicted = True + # Also look for GO_TO_INSTRUCTION() calls for instr in self.instrs.values(): targets: set[str] = set() for line in instr.block_text: - if m := re.match(RE_PREDICTED, line): + if m := re.match(RE_GO_TO_INSTR, line): targets.add(m.group(1)) for target in targets: if target_instr := self.instrs.get(target): @@ -225,11 +232,18 @@ def map_families(self) -> None: ) else: member_instr.family = family - elif not self.macro_instrs.get(member): + if member_mac := self.macro_instrs.get(member): + assert member_mac.family is None, (member, member_mac.family.name) + member_mac.family = family + if not member_instr and not member_mac: self.error( f"Unknown instruction {member!r} referenced in family {family.name!r}", family, ) + # A sanctioned exception: + # This opcode is a member of the family but it doesn't pass the checks. + if mac := self.macro_instrs.get("BINARY_OP_INPLACE_ADD_UNICODE"): + mac.family = self.families.get("BINARY_OP") def check_families(self) -> None: """Check each family: diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 9192d1038ab7d6..01ab83bedb2985 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -781,9 +781,7 @@ def write_instructions( case parsing.Macro(): n_macros += 1 mac = self.macro_instrs[thing.name] - stacking.write_macro_instr( - mac, self.out, self.families.get(mac.name) - ) + stacking.write_macro_instr(mac, self.out) case parsing.Pseudo(): pass case _: diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index 6fbf7d93f42fde..5384bbe81169cc 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -144,7 +144,8 @@ def write_body( out: Formatter, dedent: int, active_caches: list[ActiveCacheEffect], - tier: Tiers = TIER_ONE, + tier: Tiers, + family: parsing.Family | None, ) -> None: """Write the instruction body.""" # Write cache effect variable declarations and initializations @@ -207,6 +208,14 @@ def write_body( ) else: out.write_raw(f"{space}if ({cond}) goto {label};\n") + elif m := re.match(r"(\s*)DEOPT_IF\((.+)\);\s*(?://.*)?$", line): + space, cond = m.groups() + target = family.name if family else self.name + out.write_raw(f"{space}DEOPT_IF({cond}, {target});\n") + elif "DEOPT" in line: + filename = context.owner.filename + lineno = context.owner.tokens[context.begin].line + print(f"{filename}:{lineno}: ERROR: DEOPT_IF() must be all on one line") elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line): out.reset_lineno() space = extra + m.group(1) @@ -244,7 +253,8 @@ def write_body( out: Formatter, dedent: int, active_caches: list[ActiveCacheEffect], - tier: Tiers = TIER_ONE, + tier: Tiers, + family: parsing.Family | None, ) -> None: pass @@ -268,7 +278,9 @@ class MacroInstruction: macro: parsing.Macro parts: MacroParts cache_offset: int + # Set later predicted: bool = False + family: parsing.Family | None = None @dataclasses.dataclass diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 1f9fda66a5f034..bba2db8b059da8 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -351,14 +351,13 @@ def write_single_instr( out, tier, 0, + instr.family, ) except AssertionError as err: raise AssertionError(f"Error writing instruction {instr.name}") from err -def write_macro_instr( - mac: MacroInstruction, out: Formatter, family: Family | None -) -> None: +def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None: parts = [ part for part in mac.parts @@ -368,9 +367,11 @@ def write_macro_instr( with out.block(f"TARGET({mac.name})"): if mac.predicted: out.emit(f"PREDICTED({mac.name});") - out.static_assert_family_size(mac.name, family, mac.cache_offset) + out.static_assert_family_size(mac.name, mac.family, mac.cache_offset) try: - next_instr_is_set = write_components(parts, out, TIER_ONE, mac.cache_offset) + next_instr_is_set = write_components( + parts, out, TIER_ONE, mac.cache_offset, mac.family + ) except AssertionError as err: raise AssertionError(f"Error writing macro {mac.name}") from err if not parts[-1].instr.always_exits: @@ -386,6 +387,7 @@ def write_components( out: Formatter, tier: Tiers, cache_offset: int, + family: Family | None, ) -> bool: managers = get_managers(parts) @@ -454,10 +456,10 @@ def write_components( assert_no_pokes(managers) if len(parts) == 1: - mgr.instr.write_body(out, 0, mgr.active_caches, tier) + mgr.instr.write_body(out, 0, mgr.active_caches, tier, family) else: with out.block(""): - mgr.instr.write_body(out, -4, mgr.active_caches, tier) + mgr.instr.write_body(out, -4, mgr.active_caches, tier, family) if mgr is managers[-1] and not next_instr_is_set and not mgr.instr.always_exits: # Adjust the stack to its final depth, *then* write the From 77e9aae3837d9f0cf87461d023896f2c4aeb282f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 3 Oct 2023 18:38:12 +0100 Subject: [PATCH 11/73] Docs: Avoid the deprecated ``.. cmdoption::`` directive (#110292) --- Doc/library/ast.rst | 14 +-- Doc/library/compileall.rst | 34 +++--- Doc/library/gzip.rst | 10 +- Doc/library/inspect.rst | 2 +- Doc/library/json.rst | 14 +-- Doc/library/pickletools.rst | 10 +- Doc/library/py_compile.rst | 6 +- Doc/library/site.rst | 4 +- Doc/library/tarfile.rst | 20 ++-- Doc/library/timeit.rst | 14 +-- Doc/library/tokenize.rst | 4 +- Doc/library/trace.rst | 30 +++--- Doc/library/unittest.rst | 20 ++-- Doc/library/uuid.rst | 14 +-- Doc/library/zipapp.rst | 12 +-- Doc/library/zipfile.rst | 18 ++-- Doc/using/cmdline.rst | 60 +++++------ Doc/using/configure.rst | 200 ++++++++++++++++++------------------ 18 files changed, 243 insertions(+), 243 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 10b3e2dae472e1..4ebbe0e5471c88 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2483,26 +2483,26 @@ The following options are accepted: .. program:: ast -.. cmdoption:: -h, --help +.. option:: -h, --help Show the help message and exit. -.. cmdoption:: -m - --mode +.. option:: -m + --mode Specify what kind of code must be compiled, like the *mode* argument in :func:`parse`. -.. cmdoption:: --no-type-comments +.. option:: --no-type-comments Don't parse type comments. -.. cmdoption:: -a, --include-attributes +.. option:: -a, --include-attributes Include attributes such as line numbers and column offsets. -.. cmdoption:: -i - --indent +.. option:: -i + --indent Indentation of nodes in AST (number of spaces). diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index b4723b98f67bb2..df1eefab839cc1 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -26,28 +26,28 @@ compile Python sources. .. program:: compileall -.. cmdoption:: directory ... - file ... +.. option:: directory ... + file ... Positional arguments are files to compile or directories that contain source files, traversed recursively. If no argument is given, behave as if the command line was :samp:`-l {}`. -.. cmdoption:: -l +.. option:: -l Do not recurse into subdirectories, only compile source code files directly contained in the named or implied directories. -.. cmdoption:: -f +.. option:: -f Force rebuild even if timestamps are up-to-date. -.. cmdoption:: -q +.. option:: -q Do not print the list of files compiled. If passed once, error messages will still be printed. If passed twice (``-qq``), all output is suppressed. -.. cmdoption:: -d destdir +.. option:: -d destdir Directory prepended to the path to each file being compiled. This will appear in compilation time tracebacks, and is also compiled in to the @@ -55,45 +55,45 @@ compile Python sources. cases where the source file does not exist at the time the byte-code file is executed. -.. cmdoption:: -s strip_prefix -.. cmdoption:: -p prepend_prefix +.. option:: -s strip_prefix +.. option:: -p prepend_prefix Remove (``-s``) or append (``-p``) the given prefix of paths recorded in the ``.pyc`` files. Cannot be combined with ``-d``. -.. cmdoption:: -x regex +.. option:: -x regex regex is used to search the full path to each file considered for compilation, and if the regex produces a match, the file is skipped. -.. cmdoption:: -i list +.. option:: -i list Read the file ``list`` and add each line that it contains to the list of files and directories to compile. If ``list`` is ``-``, read lines from ``stdin``. -.. cmdoption:: -b +.. option:: -b Write the byte-code files to their legacy locations and names, which may overwrite byte-code files created by another version of Python. The default is to write files to their :pep:`3147` locations and names, which allows byte-code files from multiple versions of Python to coexist. -.. cmdoption:: -r +.. option:: -r Control the maximum recursion level for subdirectories. If this is given, then ``-l`` option will not be taken into account. :program:`python -m compileall -r 0` is equivalent to :program:`python -m compileall -l`. -.. cmdoption:: -j N +.. option:: -j N Use *N* workers to compile the files within the given directory. If ``0`` is used, then the result of :func:`os.process_cpu_count()` will be used. -.. cmdoption:: --invalidation-mode [timestamp|checked-hash|unchecked-hash] +.. option:: --invalidation-mode [timestamp|checked-hash|unchecked-hash] Control how the generated byte-code files are invalidated at runtime. The ``timestamp`` value, means that ``.pyc`` files with the source timestamp @@ -106,17 +106,17 @@ compile Python sources. variable is not set, and ``checked-hash`` if the ``SOURCE_DATE_EPOCH`` environment variable is set. -.. cmdoption:: -o level +.. option:: -o level Compile with the given optimization level. May be used multiple times to compile for multiple levels at a time (for example, ``compileall -o 1 -o 2``). -.. cmdoption:: -e dir +.. option:: -e dir Ignore symlinks pointing outside the given directory. -.. cmdoption:: --hardlink-dupes +.. option:: --hardlink-dupes If two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 6a4f2c76ae4e10..f931d0e399c9f2 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -268,23 +268,23 @@ Once executed the :mod:`gzip` module keeps the input file(s). Command line options ^^^^^^^^^^^^^^^^^^^^ -.. cmdoption:: file +.. option:: file If *file* is not specified, read from :data:`sys.stdin`. -.. cmdoption:: --fast +.. option:: --fast Indicates the fastest compression method (less compression). -.. cmdoption:: --best +.. option:: --best Indicates the slowest compression method (best compression). -.. cmdoption:: -d, --decompress +.. option:: -d, --decompress Decompress the given file. -.. cmdoption:: -h, --help +.. option:: -h, --help Show the help message. diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index fe0ed135029f0f..d0c3dd761e4d56 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1655,6 +1655,6 @@ By default, accepts the name of a module and prints the source of that module. A class or function within the module can be printed instead by appended a colon and the qualified name of the target object. -.. cmdoption:: --details +.. option:: --details Print information about the specified object rather than the source code diff --git a/Doc/library/json.rst b/Doc/library/json.rst index b337b5f9960e8e..0ce4b697145cb3 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -714,7 +714,7 @@ specified, :data:`sys.stdin` and :data:`sys.stdout` will be used respectively: Command line options ^^^^^^^^^^^^^^^^^^^^ -.. cmdoption:: infile +.. option:: infile The JSON file to be validated or pretty-printed: @@ -734,36 +734,36 @@ Command line options If *infile* is not specified, read from :data:`sys.stdin`. -.. cmdoption:: outfile +.. option:: outfile Write the output of the *infile* to the given *outfile*. Otherwise, write it to :data:`sys.stdout`. -.. cmdoption:: --sort-keys +.. option:: --sort-keys Sort the output of dictionaries alphabetically by key. .. versionadded:: 3.5 -.. cmdoption:: --no-ensure-ascii +.. option:: --no-ensure-ascii Disable escaping of non-ascii characters, see :func:`json.dumps` for more information. .. versionadded:: 3.9 -.. cmdoption:: --json-lines +.. option:: --json-lines Parse every input line as separate JSON object. .. versionadded:: 3.8 -.. cmdoption:: --indent, --tab, --no-indent, --compact +.. option:: --indent, --tab, --no-indent, --compact Mutually exclusive options for whitespace control. .. versionadded:: 3.9 -.. cmdoption:: -h, --help +.. option:: -h, --help Show the help message. diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 76f5b0cadf975a..41930f8cbe8412 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -53,24 +53,24 @@ Command line options .. program:: pickletools -.. cmdoption:: -a, --annotate +.. option:: -a, --annotate Annotate each line with a short opcode description. -.. cmdoption:: -o, --output= +.. option:: -o, --output= Name of a file where the output should be written. -.. cmdoption:: -l, --indentlevel= +.. option:: -l, --indentlevel= The number of blanks by which to indent a new MARK level. -.. cmdoption:: -m, --memo +.. option:: -m, --memo When multiple objects are disassembled, preserve memo between disassemblies. -.. cmdoption:: -p, --preamble= +.. option:: -p, --preamble= When more than one pickle file are specified, print given preamble before each disassembly. diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 5501db8f87de81..38c416f9ad0305 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -139,13 +139,13 @@ not be compiled. .. program:: python -m py_compile -.. cmdoption:: ... - - +.. option:: ... + - Positional arguments are files to compile. If ``-`` is the only parameter, the list of files is taken from standard input. -.. cmdoption:: -q, --quiet +.. option:: -q, --quiet Suppress errors output. diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 02880c56416615..2dc9fb09d727e2 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -266,11 +266,11 @@ If it is called without arguments, it will print the contents of :data:`USER_BASE` and whether the directory exists, then the same thing for :data:`USER_SITE`, and finally the value of :data:`ENABLE_USER_SITE`. -.. cmdoption:: --user-base +.. option:: --user-base Print the path to the user base directory. -.. cmdoption:: --user-site +.. option:: --user-site Print the path to the user site-packages directory. diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 62d67bc577c7d0..3e5723a66780ca 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -1156,31 +1156,31 @@ For a list of the files in a tar archive, use the :option:`-l` option: Command-line options ~~~~~~~~~~~~~~~~~~~~ -.. cmdoption:: -l - --list +.. option:: -l + --list List files in a tarfile. -.. cmdoption:: -c ... - --create ... +.. option:: -c ... + --create ... Create tarfile from source files. -.. cmdoption:: -e [] - --extract [] +.. option:: -e [] + --extract [] Extract tarfile into the current directory if *output_dir* is not specified. -.. cmdoption:: -t - --test +.. option:: -t + --test Test whether the tarfile is valid or not. -.. cmdoption:: -v, --verbose +.. option:: -v, --verbose Verbose output. -.. cmdoption:: --filter +.. option:: --filter Specifies the *filter* for ``--extract``. See :ref:`tarfile-extraction-filter` for details. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index b3d2a1b9e0600f..60ec135873c414 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -214,36 +214,36 @@ Where the following options are understood: .. program:: timeit -.. cmdoption:: -n N, --number=N +.. option:: -n N, --number=N how many times to execute 'statement' -.. cmdoption:: -r N, --repeat=N +.. option:: -r N, --repeat=N how many times to repeat the timer (default 5) -.. cmdoption:: -s S, --setup=S +.. option:: -s S, --setup=S statement to be executed once initially (default ``pass``) -.. cmdoption:: -p, --process +.. option:: -p, --process measure process time, not wallclock time, using :func:`time.process_time` instead of :func:`time.perf_counter`, which is the default .. versionadded:: 3.3 -.. cmdoption:: -u, --unit=U +.. option:: -u, --unit=U specify a time unit for timer output; can select ``nsec``, ``usec``, ``msec``, or ``sec`` .. versionadded:: 3.5 -.. cmdoption:: -v, --verbose +.. option:: -v, --verbose print raw timing results; repeat for more digits precision -.. cmdoption:: -h, --help +.. option:: -h, --help print a short usage message and exit diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index bffe93006edc7b..92bdb052267a68 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -166,11 +166,11 @@ The following options are accepted: .. program:: tokenize -.. cmdoption:: -h, --help +.. option:: -h, --help show this help message and exit -.. cmdoption:: -e, --exact +.. option:: -e, --exact display token names using the exact type diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index 40cf198f1287d7..e9b59a6d186ba2 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -34,11 +34,11 @@ all Python modules imported during the execution into the current directory. .. program:: trace -.. cmdoption:: --help +.. option:: --help Display usage and exit. -.. cmdoption:: --version +.. option:: --version Display the version of the module and exit. @@ -56,28 +56,28 @@ the :option:`--trace <-t>` and :option:`--count <-c>` options. When .. program:: trace -.. cmdoption:: -c, --count +.. option:: -c, --count Produce a set of annotated listing files upon program completion that shows how many times each statement was executed. See also :option:`--coverdir <-C>`, :option:`--file <-f>` and :option:`--no-report <-R>` below. -.. cmdoption:: -t, --trace +.. option:: -t, --trace Display lines as they are executed. -.. cmdoption:: -l, --listfuncs +.. option:: -l, --listfuncs Display the functions executed by running the program. -.. cmdoption:: -r, --report +.. option:: -r, --report Produce an annotated list from an earlier program run that used the :option:`--count <-c>` and :option:`--file <-f>` option. This does not execute any code. -.. cmdoption:: -T, --trackcalls +.. option:: -T, --trackcalls Display the calling relationships exposed by running the program. @@ -86,33 +86,33 @@ Modifiers .. program:: trace -.. cmdoption:: -f, --file= +.. option:: -f, --file= Name of a file to accumulate counts over several tracing runs. Should be used with the :option:`--count <-c>` option. -.. cmdoption:: -C, --coverdir= +.. option:: -C, --coverdir= Directory where the report files go. The coverage report for ``package.module`` is written to file :file:`{dir}/{package}/{module}.cover`. -.. cmdoption:: -m, --missing +.. option:: -m, --missing When generating annotated listings, mark lines which were not executed with ``>>>>>>``. -.. cmdoption:: -s, --summary +.. option:: -s, --summary When using :option:`--count <-c>` or :option:`--report <-r>`, write a brief summary to stdout for each file processed. -.. cmdoption:: -R, --no-report +.. option:: -R, --no-report Do not generate annotated listings. This is useful if you intend to make several runs with :option:`--count <-c>`, and then produce a single set of annotated listings at the end. -.. cmdoption:: -g, --timing +.. option:: -g, --timing Prefix each line with the time since the program started. Only used while tracing. @@ -124,12 +124,12 @@ These options may be repeated multiple times. .. program:: trace -.. cmdoption:: --ignore-module= +.. option:: --ignore-module= Ignore each of the given module names and its submodules (if it is a package). The argument can be a list of names separated by a comma. -.. cmdoption:: --ignore-dir= +.. option:: --ignore-dir= Ignore all modules and packages in the named directory and subdirectories. The argument can be a list of directories separated by :data:`os.pathsep`. diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 4c28e8fae8b088..21abc583f853a7 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -206,13 +206,13 @@ Command-line options .. program:: unittest -.. cmdoption:: -b, --buffer +.. option:: -b, --buffer The standard output and standard error streams are buffered during the test run. Output during a passing test is discarded. Output is echoed normally on test fail or error and is added to the failure messages. -.. cmdoption:: -c, --catch +.. option:: -c, --catch :kbd:`Control-C` during the test run waits for the current test to end and then reports all the results so far. A second :kbd:`Control-C` raises the normal @@ -220,11 +220,11 @@ Command-line options See `Signal Handling`_ for the functions that provide this functionality. -.. cmdoption:: -f, --failfast +.. option:: -f, --failfast Stop the test run on the first error or failure. -.. cmdoption:: -k +.. option:: -k Only run test methods and classes that match the pattern or substring. This option may be used multiple times, in which case all test cases that @@ -240,11 +240,11 @@ Command-line options For example, ``-k foo`` matches ``foo_tests.SomeTest.test_something``, ``bar_tests.SomeTest.test_foo``, but not ``bar_tests.FooTest.test_something``. -.. cmdoption:: --locals +.. option:: --locals Show local variables in tracebacks. -.. cmdoption:: --durations N +.. option:: --durations N Show the N slowest test cases (N=0 for all). @@ -292,19 +292,19 @@ The ``discover`` sub-command has the following options: .. program:: unittest discover -.. cmdoption:: -v, --verbose +.. option:: -v, --verbose Verbose output -.. cmdoption:: -s, --start-directory directory +.. option:: -s, --start-directory directory Directory to start discovery (``.`` default) -.. cmdoption:: -p, --pattern pattern +.. option:: -p, --pattern pattern Pattern to match test files (``test*.py`` default) -.. cmdoption:: -t, --top-level-directory directory +.. option:: -t, --top-level-directory directory Top level directory of project (defaults to start directory) diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index adf01770656754..e2d231da38fd9a 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -289,25 +289,25 @@ The following options are accepted: .. program:: uuid -.. cmdoption:: -h, --help +.. option:: -h, --help Show the help message and exit. -.. cmdoption:: -u - --uuid +.. option:: -u + --uuid Specify the function name to use to generate the uuid. By default :func:`uuid4` is used. -.. cmdoption:: -n - --namespace +.. option:: -n + --namespace The namespace is a ``UUID``, or ``@ns`` where ``ns`` is a well-known predefined UUID addressed by namespace name. Such as ``@dns``, ``@url``, ``@oid``, and ``@x500``. Only required for :func:`uuid3` / :func:`uuid5` functions. -.. cmdoption:: -N - --name +.. option:: -N + --name The name used as part of generating the uuid. Only required for :func:`uuid3` / :func:`uuid5` functions. diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index 7c01fc102fca07..104afca23a20b4 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -54,7 +54,7 @@ The following options are understood: .. program:: zipapp -.. cmdoption:: -o , --output= +.. option:: -o , --output= Write the output to a file named *output*. If this option is not specified, the output filename will be the same as the input *source*, with the @@ -64,13 +64,13 @@ The following options are understood: An output filename must be specified if the *source* is an archive (and in that case, *output* must not be the same as *source*). -.. cmdoption:: -p , --python= +.. option:: -p , --python= Add a ``#!`` line to the archive specifying *interpreter* as the command to run. Also, on POSIX, make the archive executable. The default is to write no ``#!`` line, and not make the file executable. -.. cmdoption:: -m , --main= +.. option:: -m , --main= Write a ``__main__.py`` file to the archive that executes *mainfn*. The *mainfn* argument should have the form "pkg.mod:fn", where "pkg.mod" is a @@ -79,7 +79,7 @@ The following options are understood: :option:`--main` cannot be specified when copying an archive. -.. cmdoption:: -c, --compress +.. option:: -c, --compress Compress files with the deflate method, reducing the size of the output file. By default, files are stored uncompressed in the archive. @@ -88,13 +88,13 @@ The following options are understood: .. versionadded:: 3.7 -.. cmdoption:: --info +.. option:: --info Display the interpreter embedded in the archive, for diagnostic purposes. In this case, any other options are ignored and SOURCE must be an archive, not a directory. -.. cmdoption:: -h, --help +.. option:: -h, --help Print a short usage message and exit. diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index bd951e4872f113..a77e49a7643826 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -906,27 +906,27 @@ For a list of the files in a ZIP archive, use the :option:`-l` option: Command-line options ~~~~~~~~~~~~~~~~~~~~ -.. cmdoption:: -l - --list +.. option:: -l + --list List files in a zipfile. -.. cmdoption:: -c ... - --create ... +.. option:: -c ... + --create ... Create zipfile from source files. -.. cmdoption:: -e - --extract +.. option:: -e + --extract Extract zipfile into target directory. -.. cmdoption:: -t - --test +.. option:: -t + --test Test whether the zipfile is valid or not. -.. cmdoption:: --metadata-encoding +.. option:: --metadata-encoding Specify encoding of member names for :option:`-l`, :option:`-e` and :option:`-t`. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index bade3ca6650e62..f68a2251f06d4a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -59,7 +59,7 @@ all consecutive arguments will end up in :data:`sys.argv` -- note that the first element, subscript zero (``sys.argv[0]``), is a string reflecting the program's source. -.. cmdoption:: -c +.. option:: -c Execute the Python code in *command*. *command* can be one or more statements separated by newlines, with significant leading whitespace as in @@ -72,7 +72,7 @@ source. .. audit-event:: cpython.run_command command cmdoption-c -.. cmdoption:: -m +.. option:: -m Search :data:`sys.path` for the named module and execute its contents as the :mod:`__main__` module. @@ -188,35 +188,35 @@ automatically enabled, if available on your platform (see Generic options ~~~~~~~~~~~~~~~ -.. cmdoption:: -? - -h - --help +.. option:: -? + -h + --help Print a short description of all command line options and corresponding environment variables and exit. -.. cmdoption:: --help-env +.. option:: --help-env Print a short description of Python-specific environment variables and exit. .. versionadded:: 3.11 -.. cmdoption:: --help-xoptions +.. option:: --help-xoptions Print a description of implementation-specific :option:`-X` options and exit. .. versionadded:: 3.11 -.. cmdoption:: --help-all +.. option:: --help-all Print complete usage information and exit. .. versionadded:: 3.11 -.. cmdoption:: -V - --version +.. option:: -V + --version Print the Python version number and exit. Example output could be: @@ -240,7 +240,7 @@ Generic options Miscellaneous options ~~~~~~~~~~~~~~~~~~~~~ -.. cmdoption:: -b +.. option:: -b Issue a warning when comparing :class:`bytes` or :class:`bytearray` with :class:`str` or :class:`bytes` with :class:`int`. Issue an error when the @@ -249,13 +249,13 @@ Miscellaneous options .. versionchanged:: 3.5 Affects comparisons of :class:`bytes` with :class:`int`. -.. cmdoption:: -B +.. option:: -B If given, Python won't try to write ``.pyc`` files on the import of source modules. See also :envvar:`PYTHONDONTWRITEBYTECODE`. -.. cmdoption:: --check-hash-based-pycs default|always|never +.. option:: --check-hash-based-pycs default|always|never Control the validation behavior of hash-based ``.pyc`` files. See :ref:`pyc-invalidation`. When set to ``default``, checked and unchecked @@ -269,7 +269,7 @@ Miscellaneous options option. -.. cmdoption:: -d +.. option:: -d Turn on parser debugging output (for expert only). See also the :envvar:`PYTHONDEBUG` environment variable. @@ -278,7 +278,7 @@ Miscellaneous options it's ignored. -.. cmdoption:: -E +.. option:: -E Ignore all :envvar:`PYTHON*` environment variables, e.g. :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. @@ -286,7 +286,7 @@ Miscellaneous options See also the :option:`-P` and :option:`-I` (isolated) options. -.. cmdoption:: -i +.. option:: -i When a script is passed as first argument or the :option:`-c` option is used, enter interactive mode after executing the script or the command, even when @@ -297,7 +297,7 @@ Miscellaneous options raises an exception. See also :envvar:`PYTHONINSPECT`. -.. cmdoption:: -I +.. option:: -I Run Python in isolated mode. This also implies :option:`-E`, :option:`-P` and :option:`-s` options. @@ -310,7 +310,7 @@ Miscellaneous options .. versionadded:: 3.4 -.. cmdoption:: -O +.. option:: -O Remove assert statements and any code conditional on the value of :const:`__debug__`. Augment the filename for compiled @@ -321,7 +321,7 @@ Miscellaneous options Modify ``.pyc`` filenames according to :pep:`488`. -.. cmdoption:: -OO +.. option:: -OO Do :option:`-O` and also discard docstrings. Augment the filename for compiled (:term:`bytecode`) files by adding ``.opt-2`` before the @@ -331,7 +331,7 @@ Miscellaneous options Modify ``.pyc`` filenames according to :pep:`488`. -.. cmdoption:: -P +.. option:: -P Don't prepend a potentially unsafe path to :data:`sys.path`: @@ -348,14 +348,14 @@ Miscellaneous options .. versionadded:: 3.11 -.. cmdoption:: -q +.. option:: -q Don't display the copyright and version messages even in interactive mode. .. versionadded:: 3.2 -.. cmdoption:: -R +.. option:: -R Turn on hash randomization. This option only has an effect if the :envvar:`PYTHONHASHSEED` environment variable is set to ``0``, since hash @@ -381,7 +381,7 @@ Miscellaneous options .. versionadded:: 3.2.3 -.. cmdoption:: -s +.. option:: -s Don't add the :data:`user site-packages directory ` to :data:`sys.path`. @@ -391,7 +391,7 @@ Miscellaneous options :pep:`370` -- Per user site-packages directory -.. cmdoption:: -S +.. option:: -S Disable the import of the module :mod:`site` and the site-dependent manipulations of :data:`sys.path` that it entails. Also disable these @@ -399,7 +399,7 @@ Miscellaneous options :func:`site.main` if you want them to be triggered). -.. cmdoption:: -u +.. option:: -u Force the stdout and stderr streams to be unbuffered. This option has no effect on the stdin stream. @@ -410,7 +410,7 @@ Miscellaneous options The text layer of the stdout and stderr streams now is unbuffered. -.. cmdoption:: -v +.. option:: -v Print a message each time a module is initialized, showing the place (filename or built-in module) from which it is loaded. When given twice @@ -425,7 +425,7 @@ Miscellaneous options .. _using-on-warnings: -.. cmdoption:: -W arg +.. option:: -W arg Warning control. Python's warning machinery by default prints warning messages to :data:`sys.stderr`. @@ -484,13 +484,13 @@ Miscellaneous options details. -.. cmdoption:: -x +.. option:: -x Skip the first line of the source, allowing use of non-Unix forms of ``#!cmd``. This is intended for a DOS specific hack only. -.. cmdoption:: -X +.. option:: -X Reserved for various implementation-specific options. CPython currently defines the following possible values: @@ -597,7 +597,7 @@ Miscellaneous options Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ -.. cmdoption:: -J +.. option:: -J Reserved for use by Jython_. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index eb8f2442d0f0a6..5f9e695d10ad44 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -97,7 +97,7 @@ See also the :file:`Misc/SpecialBuilds.txt` in the Python source distribution. General Options --------------- -.. cmdoption:: --enable-loadable-sqlite-extensions +.. option:: --enable-loadable-sqlite-extensions Support loadable extensions in the :mod:`!_sqlite` extension module (default is no) of the :mod:`sqlite3` module. @@ -107,12 +107,12 @@ General Options .. versionadded:: 3.6 -.. cmdoption:: --disable-ipv6 +.. option:: --disable-ipv6 Disable IPv6 support (enabled by default if supported), see the :mod:`socket` module. -.. cmdoption:: --enable-big-digits=[15|30] +.. option:: --enable-big-digits=[15|30] Define the size in bits of Python :class:`int` digits: 15 or 30 bits. @@ -122,7 +122,7 @@ General Options See :data:`sys.int_info.bits_per_digit `. -.. cmdoption:: --with-suffix=SUFFIX +.. option:: --with-suffix=SUFFIX Set the Python executable suffix to *SUFFIX*. @@ -135,7 +135,7 @@ General Options The default suffix on WASM platform is one of ``.js``, ``.html`` or ``.wasm``. -.. cmdoption:: --with-tzpath= +.. option:: --with-tzpath= Select the default time zone search path for :const:`zoneinfo.TZPATH`. See the :ref:`Compile-time configuration @@ -147,7 +147,7 @@ General Options .. versionadded:: 3.9 -.. cmdoption:: --without-decimal-contextvar +.. option:: --without-decimal-contextvar Build the ``_decimal`` extension module using a thread-local context rather than a coroutine-local context (default), see the :mod:`decimal` module. @@ -156,7 +156,7 @@ General Options .. versionadded:: 3.9 -.. cmdoption:: --with-dbmliborder= +.. option:: --with-dbmliborder= Override order to check db backends for the :mod:`dbm` module @@ -166,7 +166,7 @@ General Options * ``gdbm``; * ``bdb``. -.. cmdoption:: --without-c-locale-coercion +.. option:: --without-c-locale-coercion Disable C locale coercion to a UTF-8 based locale (enabled by default). @@ -174,13 +174,13 @@ General Options See :envvar:`PYTHONCOERCECLOCALE` and the :pep:`538`. -.. cmdoption:: --without-freelists +.. option:: --without-freelists Disable all freelists except the empty tuple singleton. .. versionadded:: 3.11 -.. cmdoption:: --with-platlibdir=DIRNAME +.. option:: --with-platlibdir=DIRNAME Python library directory name (default is ``lib``). @@ -190,7 +190,7 @@ General Options .. versionadded:: 3.9 -.. cmdoption:: --with-wheel-pkg-dir=PATH +.. option:: --with-wheel-pkg-dir=PATH Directory of wheel packages used by the :mod:`ensurepip` module (none by default). @@ -202,7 +202,7 @@ General Options .. versionadded:: 3.10 -.. cmdoption:: --with-pkg-config=[check|yes|no] +.. option:: --with-pkg-config=[check|yes|no] Whether configure should use :program:`pkg-config` to detect build dependencies. @@ -213,7 +213,7 @@ General Options .. versionadded:: 3.11 -.. cmdoption:: --enable-pystats +.. option:: --enable-pystats Turn on internal Python performance statistics gathering. @@ -280,7 +280,7 @@ General Options .. versionadded:: 3.11 -.. cmdoption:: --disable-gil +.. option:: --disable-gil Enables **experimental** support for running Python without the :term:`global interpreter lock` (GIL). @@ -289,12 +289,12 @@ General Options .. versionadded:: 3.13 -.. cmdoption:: PKG_CONFIG +.. option:: PKG_CONFIG Path to ``pkg-config`` utility. -.. cmdoption:: PKG_CONFIG_LIBDIR -.. cmdoption:: PKG_CONFIG_PATH +.. option:: PKG_CONFIG_LIBDIR +.. option:: PKG_CONFIG_PATH ``pkg-config`` options. @@ -302,19 +302,19 @@ General Options C compiler options ------------------ -.. cmdoption:: CC +.. option:: CC C compiler command. -.. cmdoption:: CFLAGS +.. option:: CFLAGS C compiler flags. -.. cmdoption:: CPP +.. option:: CPP C preprocessor command. -.. cmdoption:: CPPFLAGS +.. option:: CPPFLAGS C preprocessor flags, e.g. :samp:`-I{include_dir}`. @@ -322,15 +322,15 @@ C compiler options Linker options -------------- -.. cmdoption:: LDFLAGS +.. option:: LDFLAGS Linker flags, e.g. :samp:`-L{library_directory}`. -.. cmdoption:: LIBS +.. option:: LIBS Libraries to pass to the linker, e.g. :samp:`-l{library}`. -.. cmdoption:: MACHDEP +.. option:: MACHDEP Name for machine-dependent library files. @@ -340,80 +340,80 @@ Options for third-party dependencies .. versionadded:: 3.11 -.. cmdoption:: BZIP2_CFLAGS -.. cmdoption:: BZIP2_LIBS +.. option:: BZIP2_CFLAGS +.. option:: BZIP2_LIBS C compiler and linker flags to link Python to ``libbz2``, used by :mod:`bz2` module, overriding ``pkg-config``. -.. cmdoption:: CURSES_CFLAGS -.. cmdoption:: CURSES_LIBS +.. option:: CURSES_CFLAGS +.. option:: CURSES_LIBS C compiler and linker flags for ``libncurses`` or ``libncursesw``, used by :mod:`curses` module, overriding ``pkg-config``. -.. cmdoption:: GDBM_CFLAGS -.. cmdoption:: GDBM_LIBS +.. option:: GDBM_CFLAGS +.. option:: GDBM_LIBS C compiler and linker flags for ``gdbm``. -.. cmdoption:: LIBB2_CFLAGS -.. cmdoption:: LIBB2_LIBS +.. option:: LIBB2_CFLAGS +.. option:: LIBB2_LIBS C compiler and linker flags for ``libb2`` (:ref:`BLAKE2 `), used by :mod:`hashlib` module, overriding ``pkg-config``. -.. cmdoption:: LIBEDIT_CFLAGS -.. cmdoption:: LIBEDIT_LIBS +.. option:: LIBEDIT_CFLAGS +.. option:: LIBEDIT_LIBS C compiler and linker flags for ``libedit``, used by :mod:`readline` module, overriding ``pkg-config``. -.. cmdoption:: LIBFFI_CFLAGS -.. cmdoption:: LIBFFI_LIBS +.. option:: LIBFFI_CFLAGS +.. option:: LIBFFI_LIBS C compiler and linker flags for ``libffi``, used by :mod:`ctypes` module, overriding ``pkg-config``. -.. cmdoption:: LIBLZMA_CFLAGS -.. cmdoption:: LIBLZMA_LIBS +.. option:: LIBLZMA_CFLAGS +.. option:: LIBLZMA_LIBS C compiler and linker flags for ``liblzma``, used by :mod:`lzma` module, overriding ``pkg-config``. -.. cmdoption:: LIBREADLINE_CFLAGS -.. cmdoption:: LIBREADLINE_LIBS +.. option:: LIBREADLINE_CFLAGS +.. option:: LIBREADLINE_LIBS C compiler and linker flags for ``libreadline``, used by :mod:`readline` module, overriding ``pkg-config``. -.. cmdoption:: LIBSQLITE3_CFLAGS -.. cmdoption:: LIBSQLITE3_LIBS +.. option:: LIBSQLITE3_CFLAGS +.. option:: LIBSQLITE3_LIBS C compiler and linker flags for ``libsqlite3``, used by :mod:`sqlite3` module, overriding ``pkg-config``. -.. cmdoption:: LIBUUID_CFLAGS -.. cmdoption:: LIBUUID_LIBS +.. option:: LIBUUID_CFLAGS +.. option:: LIBUUID_LIBS C compiler and linker flags for ``libuuid``, used by :mod:`uuid` module, overriding ``pkg-config``. -.. cmdoption:: PANEL_CFLAGS -.. cmdoption:: PANEL_LIBS +.. option:: PANEL_CFLAGS +.. option:: PANEL_LIBS C compiler and Linker flags for PANEL, overriding ``pkg-config``. C compiler and linker flags for ``libpanel`` or ``libpanelw``, used by :mod:`curses.panel` module, overriding ``pkg-config``. -.. cmdoption:: TCLTK_CFLAGS -.. cmdoption:: TCLTK_LIBS +.. option:: TCLTK_CFLAGS +.. option:: TCLTK_LIBS C compiler and linker flags for TCLTK, overriding ``pkg-config``. -.. cmdoption:: ZLIB_CFLAGS -.. cmdoption:: ZLIB_LIBS +.. option:: ZLIB_CFLAGS +.. option:: ZLIB_LIBS C compiler and linker flags for ``libzlib``, used by :mod:`gzip` module, overriding ``pkg-config``. @@ -422,7 +422,7 @@ Options for third-party dependencies WebAssembly Options ------------------- -.. cmdoption:: --with-emscripten-target=[browser|node] +.. option:: --with-emscripten-target=[browser|node] Set build flavor for ``wasm32-emscripten``. @@ -431,7 +431,7 @@ WebAssembly Options .. versionadded:: 3.11 -.. cmdoption:: --enable-wasm-dynamic-linking +.. option:: --enable-wasm-dynamic-linking Turn on dynamic linking support for WASM. @@ -440,7 +440,7 @@ WebAssembly Options .. versionadded:: 3.11 -.. cmdoption:: --enable-wasm-pthreads +.. option:: --enable-wasm-pthreads Turn on pthreads support for WASM. @@ -450,7 +450,7 @@ WebAssembly Options Install Options --------------- -.. cmdoption:: --prefix=PREFIX +.. option:: --prefix=PREFIX Install architecture-independent files in PREFIX. On Unix, it defaults to :file:`/usr/local`. @@ -460,20 +460,20 @@ Install Options As an example, one can use ``--prefix="$HOME/.local/"`` to install a Python in its home directory. -.. cmdoption:: --exec-prefix=EPREFIX +.. option:: --exec-prefix=EPREFIX Install architecture-dependent files in EPREFIX, defaults to :option:`--prefix`. This value can be retrieved at runtime using :data:`sys.exec_prefix`. -.. cmdoption:: --disable-test-modules +.. option:: --disable-test-modules Don't build nor install test modules, like the :mod:`test` package or the :mod:`!_testcapi` extension module (built and installed by default). .. versionadded:: 3.10 -.. cmdoption:: --with-ensurepip=[upgrade|install|no] +.. option:: --with-ensurepip=[upgrade|install|no] Select the :mod:`ensurepip` command run on Python installation: @@ -492,7 +492,7 @@ Configuring Python using ``--enable-optimizations --with-lto`` (PGO + LTO) is recommended for best performance. The experimental ``--enable-bolt`` flag can also be used to improve performance. -.. cmdoption:: --enable-optimizations +.. option:: --enable-optimizations Enable Profile Guided Optimization (PGO) using :envvar:`PROFILE_TASK` (disabled by default). @@ -521,7 +521,7 @@ also be used to improve performance. .. versionchanged:: 3.13 Task failure is no longer ignored silently. -.. cmdoption:: --with-lto=[full|thin|no|yes] +.. option:: --with-lto=[full|thin|no|yes] Enable Link Time Optimization (LTO) in any build (disabled by default). @@ -536,7 +536,7 @@ also be used to improve performance. .. versionchanged:: 3.12 Use ThinLTO as the default optimization policy on Clang if the compiler accepts the flag. -.. cmdoption:: --enable-bolt +.. option:: --enable-bolt Enable usage of the `BOLT post-link binary optimizer `_ (disabled by @@ -561,32 +561,32 @@ also be used to improve performance. .. versionadded:: 3.12 -.. cmdoption:: BOLT_APPLY_FLAGS +.. option:: BOLT_APPLY_FLAGS Arguments to ``llvm-bolt`` when creating a `BOLT optimized binary `_. .. versionadded:: 3.12 -.. cmdoption:: BOLT_INSTRUMENT_FLAGS +.. option:: BOLT_INSTRUMENT_FLAGS Arguments to ``llvm-bolt`` when instrumenting binaries. .. versionadded:: 3.12 -.. cmdoption:: --with-computed-gotos +.. option:: --with-computed-gotos Enable computed gotos in evaluation loop (enabled by default on supported compilers). -.. cmdoption:: --without-pymalloc +.. option:: --without-pymalloc Disable the specialized Python memory allocator :ref:`pymalloc ` (enabled by default). See also :envvar:`PYTHONMALLOC` environment variable. -.. cmdoption:: --without-doc-strings +.. option:: --without-doc-strings Disable static documentation strings to reduce the memory footprint (enabled by default). Documentation strings defined in Python are not affected. @@ -595,11 +595,11 @@ also be used to improve performance. See the ``PyDoc_STRVAR()`` macro. -.. cmdoption:: --enable-profiling +.. option:: --enable-profiling Enable C-level code profiling with ``gprof`` (disabled by default). -.. cmdoption:: --with-strict-overflow +.. option:: --with-strict-overflow Add ``-fstrict-overflow`` to the C compiler flags (by default we add ``-fno-strict-overflow`` instead). @@ -655,12 +655,12 @@ See also the :ref:`Python Development Mode ` and the Debug options ------------- -.. cmdoption:: --with-pydebug +.. option:: --with-pydebug :ref:`Build Python in debug mode `: define the ``Py_DEBUG`` macro (disabled by default). -.. cmdoption:: --with-trace-refs +.. option:: --with-trace-refs Enable tracing references for debugging purpose (disabled by default). @@ -681,7 +681,7 @@ Debug options .. versionadded:: 3.8 -.. cmdoption:: --with-assertions +.. option:: --with-assertions Build with C assertions enabled (default is no): ``assert(...);`` and ``_PyObject_ASSERT(...);``. @@ -694,11 +694,11 @@ Debug options .. versionadded:: 3.6 -.. cmdoption:: --with-valgrind +.. option:: --with-valgrind Enable Valgrind support (default is no). -.. cmdoption:: --with-dtrace +.. option:: --with-dtrace Enable DTrace support (default is no). @@ -707,19 +707,19 @@ Debug options .. versionadded:: 3.6 -.. cmdoption:: --with-address-sanitizer +.. option:: --with-address-sanitizer Enable AddressSanitizer memory error detector, ``asan`` (default is no). .. versionadded:: 3.6 -.. cmdoption:: --with-memory-sanitizer +.. option:: --with-memory-sanitizer Enable MemorySanitizer allocation error detector, ``msan`` (default is no). .. versionadded:: 3.6 -.. cmdoption:: --with-undefined-behavior-sanitizer +.. option:: --with-undefined-behavior-sanitizer Enable UndefinedBehaviorSanitizer undefined behaviour detector, ``ubsan`` (default is no). @@ -730,11 +730,11 @@ Debug options Linker options -------------- -.. cmdoption:: --enable-shared +.. option:: --enable-shared Enable building a shared Python library: ``libpython`` (default is no). -.. cmdoption:: --without-static-libpython +.. option:: --without-static-libpython Do not build ``libpythonMAJOR.MINOR.a`` and do not install ``python.o`` (built and enabled by default). @@ -745,23 +745,23 @@ Linker options Libraries options ----------------- -.. cmdoption:: --with-libs='lib1 ...' +.. option:: --with-libs='lib1 ...' Link against additional libraries (default is no). -.. cmdoption:: --with-system-expat +.. option:: --with-system-expat Build the :mod:`!pyexpat` module using an installed ``expat`` library (default is no). -.. cmdoption:: --with-system-libmpdec +.. option:: --with-system-libmpdec Build the ``_decimal`` extension module using an installed ``mpdec`` library, see the :mod:`decimal` module (default is no). .. versionadded:: 3.3 -.. cmdoption:: --with-readline=readline|editline +.. option:: --with-readline=readline|editline Designate a backend library for the :mod:`readline` module. @@ -770,7 +770,7 @@ Libraries options .. versionadded:: 3.10 -.. cmdoption:: --without-readline +.. option:: --without-readline Don't build the :mod:`readline` module (built by default). @@ -778,21 +778,21 @@ Libraries options .. versionadded:: 3.10 -.. cmdoption:: --with-libm=STRING +.. option:: --with-libm=STRING Override ``libm`` math library to *STRING* (default is system-dependent). -.. cmdoption:: --with-libc=STRING +.. option:: --with-libc=STRING Override ``libc`` C library to *STRING* (default is system-dependent). -.. cmdoption:: --with-openssl=DIR +.. option:: --with-openssl=DIR Root of the OpenSSL directory. .. versionadded:: 3.7 -.. cmdoption:: --with-openssl-rpath=[no|auto|DIR] +.. option:: --with-openssl-rpath=[no|auto|DIR] Set runtime library directory (rpath) for OpenSSL libraries: @@ -807,7 +807,7 @@ Libraries options Security Options ---------------- -.. cmdoption:: --with-hash-algorithm=[fnv|siphash13|siphash24] +.. option:: --with-hash-algorithm=[fnv|siphash13|siphash24] Select hash algorithm for use in ``Python/pyhash.c``: @@ -820,7 +820,7 @@ Security Options .. versionadded:: 3.11 ``siphash13`` is added and it is the new default. -.. cmdoption:: --with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2 +.. option:: --with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2 Built-in hash modules: @@ -833,7 +833,7 @@ Security Options .. versionadded:: 3.9 -.. cmdoption:: --with-ssl-default-suites=[python|openssl|STRING] +.. option:: --with-ssl-default-suites=[python|openssl|STRING] Override the OpenSSL default cipher suites string: @@ -855,19 +855,19 @@ macOS Options See ``Mac/README.rst``. -.. cmdoption:: --enable-universalsdk -.. cmdoption:: --enable-universalsdk=SDKDIR +.. option:: --enable-universalsdk +.. option:: --enable-universalsdk=SDKDIR Create a universal binary build. *SDKDIR* specifies which macOS SDK should be used to perform the build (default is no). -.. cmdoption:: --enable-framework -.. cmdoption:: --enable-framework=INSTALLDIR +.. option:: --enable-framework +.. option:: --enable-framework=INSTALLDIR Create a Python.framework rather than a traditional Unix install. Optional *INSTALLDIR* specifies the installation path (default is no). -.. cmdoption:: --with-universal-archs=ARCH +.. option:: --with-universal-archs=ARCH Specify the kind of universal binary that should be created. This option is only valid when :option:`--enable-universalsdk` is set. @@ -883,7 +883,7 @@ See ``Mac/README.rst``. * ``intel-64``; * ``all``. -.. cmdoption:: --with-framework-name=FRAMEWORK +.. option:: --with-framework-name=FRAMEWORK Specify the name for the python framework on macOS only valid when :option:`--enable-framework` is set (default: ``Python``). @@ -897,21 +897,21 @@ for another CPU architecture or platform. Cross compiling requires a Python interpreter for the build platform. The version of the build Python must match the version of the cross compiled host Python. -.. cmdoption:: --build=BUILD +.. option:: --build=BUILD configure for building on BUILD, usually guessed by :program:`config.guess`. -.. cmdoption:: --host=HOST +.. option:: --host=HOST cross-compile to build programs to run on HOST (target platform) -.. cmdoption:: --with-build-python=path/to/python +.. option:: --with-build-python=path/to/python path to build ``python`` binary for cross compiling .. versionadded:: 3.11 -.. cmdoption:: CONFIG_SITE=file +.. option:: CONFIG_SITE=file An environment variable that points to a file with configure overrides. @@ -922,7 +922,7 @@ the version of the cross compiled host Python. ac_cv_file__dev_ptmx=yes ac_cv_file__dev_ptc=no -.. cmdoption:: HOSTRUNNER +.. option:: HOSTRUNNER Program to run CPython for the host platform for cross-compilation. From 6bc889aedc11cc8e0b9f57948a3d528ad2685497 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Oct 2023 14:41:02 -0600 Subject: [PATCH 12/73] gh-76785: Print the Traceback from Interpreter.run() (gh-110248) This is a temporary fix. The full fix may involve serializing the traceback in some form. --- Modules/_xxsubinterpretersmodule.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 700282efb8c619..33feae8ee82ff2 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -450,6 +450,10 @@ _run_script(PyInterpreterState *interp, const char *codestr, "RunFailedError: script raised an uncaught exception (%s)", failure); } + // XXX Instead, store the rendered traceback on sharedexc, + // attach it to the exception when applied, + // and teach PyErr_Display() to print it. + PyErr_Display(NULL, excval, NULL); Py_XDECREF(excval); if (errcode != ERR_ALREADY_RUNNING) { _PyInterpreterState_SetNotRunningMain(interp); From 4467d2c3ac728f21af02594df5f151f29422c3a5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Oct 2023 15:18:25 -0600 Subject: [PATCH 13/73] Revert "gh-76785: Print the Traceback from Interpreter.run() (gh-110248)" (gh-110314) This reverts commit 6bc889aedc11cc8e0b9f57948a3d528ad2685497. That commit is causing some buildbot failures. --- Modules/_xxsubinterpretersmodule.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 33feae8ee82ff2..700282efb8c619 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -450,10 +450,6 @@ _run_script(PyInterpreterState *interp, const char *codestr, "RunFailedError: script raised an uncaught exception (%s)", failure); } - // XXX Instead, store the rendered traceback on sharedexc, - // attach it to the exception when applied, - // and teach PyErr_Display() to print it. - PyErr_Display(NULL, excval, NULL); Py_XDECREF(excval); if (errcode != ERR_ALREADY_RUNNING) { _PyInterpreterState_SetNotRunningMain(interp); From a376a72bd92cd7c9930467dd1aba40045fb75e3b Mon Sep 17 00:00:00 2001 From: elfstrom Date: Tue, 3 Oct 2023 23:59:49 +0200 Subject: [PATCH 14/73] gh-109917: Fix test instability in test_concurrent_futures (#110306) The test had an instability issue due to the ordering of the dummy queue operation and the real wakeup pipe operations. Both primitives are thread safe but not done atomically as a single update and may interleave arbitrarily. With the old order of operations this can lead to an incorrect state where the dummy queue is full but the wakeup pipe is empty. By swapping the order in clear() I think this can no longer happen in any possible operation interleaving (famous last words). --- Lib/test/test_concurrent_futures/test_deadlock.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_concurrent_futures/test_deadlock.py b/Lib/test/test_concurrent_futures/test_deadlock.py index af702542081ad9..3c30c4558c0b3e 100644 --- a/Lib/test/test_concurrent_futures/test_deadlock.py +++ b/Lib/test/test_concurrent_futures/test_deadlock.py @@ -286,11 +286,12 @@ def wakeup(self): super().wakeup() def clear(self): + super().clear() try: while True: self._dummy_queue.get_nowait() except queue.Empty: - super().clear() + pass with (unittest.mock.patch.object(futures.process._ExecutorManagerThread, 'run', mock_run), From 625ecbe92eb69d2850c2e6dbe9538e9b1a098baa Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 3 Oct 2023 15:37:21 -0700 Subject: [PATCH 15/73] gh-109979: Unify _GUARD_TYPE_VERSION{,_STORE} (#110301) Now the target for `DEOPT_IF()` is auto-filled, we don't need a separate `_GUARD_TYPE_VERSION_STORE` uop. --- Include/internal/pycore_opcode_metadata.h | 71 ++++++++++------------- Python/abstract_interp_cases.c.h | 4 -- Python/bytecodes.c | 10 +--- Python/executor_cases.c.h | 10 ---- Python/generated_cases.c.h | 4 +- 5 files changed, 36 insertions(+), 63 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 16c1637e496033..431a914d26e685 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -49,37 +49,36 @@ #define _LOAD_ATTR_SLOT 321 #define _GUARD_DORV_VALUES 322 #define _STORE_ATTR_INSTANCE_VALUE 323 -#define _GUARD_TYPE_VERSION_STORE 324 -#define _STORE_ATTR_SLOT 325 -#define _IS_NONE 326 -#define _ITER_CHECK_LIST 327 -#define _ITER_JUMP_LIST 328 -#define _IS_ITER_EXHAUSTED_LIST 329 -#define _ITER_NEXT_LIST 330 -#define _ITER_CHECK_TUPLE 331 -#define _ITER_JUMP_TUPLE 332 -#define _IS_ITER_EXHAUSTED_TUPLE 333 -#define _ITER_NEXT_TUPLE 334 -#define _ITER_CHECK_RANGE 335 -#define _ITER_JUMP_RANGE 336 -#define _IS_ITER_EXHAUSTED_RANGE 337 -#define _ITER_NEXT_RANGE 338 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 339 -#define _GUARD_KEYS_VERSION 340 -#define _LOAD_ATTR_METHOD_WITH_VALUES 341 -#define _LOAD_ATTR_METHOD_NO_DICT 342 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 343 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 344 -#define _CHECK_PEP_523 345 -#define _CHECK_FUNCTION_EXACT_ARGS 346 -#define _CHECK_STACK_SPACE 347 -#define _INIT_CALL_PY_EXACT_ARGS 348 -#define _PUSH_FRAME 349 -#define _POP_JUMP_IF_FALSE 350 -#define _POP_JUMP_IF_TRUE 351 -#define _JUMP_TO_TOP 352 -#define _SAVE_CURRENT_IP 353 -#define _INSERT 354 +#define _STORE_ATTR_SLOT 324 +#define _IS_NONE 325 +#define _ITER_CHECK_LIST 326 +#define _ITER_JUMP_LIST 327 +#define _IS_ITER_EXHAUSTED_LIST 328 +#define _ITER_NEXT_LIST 329 +#define _ITER_CHECK_TUPLE 330 +#define _ITER_JUMP_TUPLE 331 +#define _IS_ITER_EXHAUSTED_TUPLE 332 +#define _ITER_NEXT_TUPLE 333 +#define _ITER_CHECK_RANGE 334 +#define _ITER_JUMP_RANGE 335 +#define _IS_ITER_EXHAUSTED_RANGE 336 +#define _ITER_NEXT_RANGE 337 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 338 +#define _GUARD_KEYS_VERSION 339 +#define _LOAD_ATTR_METHOD_WITH_VALUES 340 +#define _LOAD_ATTR_METHOD_NO_DICT 341 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 343 +#define _CHECK_PEP_523 344 +#define _CHECK_FUNCTION_EXACT_ARGS 345 +#define _CHECK_STACK_SPACE 346 +#define _INIT_CALL_PY_EXACT_ARGS 347 +#define _PUSH_FRAME 348 +#define _POP_JUMP_IF_FALSE 349 +#define _POP_JUMP_IF_TRUE 350 +#define _JUMP_TO_TOP 351 +#define _SAVE_CURRENT_IP 352 +#define _INSERT 353 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -383,8 +382,6 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case STORE_ATTR_WITH_HINT: return 2; - case _GUARD_TYPE_VERSION_STORE: - return 1; case _STORE_ATTR_SLOT: return 2; case STORE_ATTR_SLOT: @@ -941,8 +938,6 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case STORE_ATTR_WITH_HINT: return 0; - case _GUARD_TYPE_VERSION_STORE: - return 1; case _STORE_ATTR_SLOT: return 0; case STORE_ATTR_SLOT: @@ -1413,7 +1408,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC, 0 }, [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, - [_GUARD_TYPE_VERSION_STORE] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, [_STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC, 0 }, [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, @@ -1638,8 +1632,8 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_ATTR] = { .nuops = 1, .uops = { { LOAD_ATTR, 0, 0 } } }, [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, [LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } }, - [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION_STORE, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, - [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION_STORE, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, [COMPARE_OP] = { .nuops = 1, .uops = { { COMPARE_OP, 0, 0 } } }, [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { COMPARE_OP_FLOAT, 0, 0 } } }, [COMPARE_OP_INT] = { .nuops = 1, .uops = { { COMPARE_OP_INT, 0, 0 } } }, @@ -1714,7 +1708,6 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", - [_GUARD_TYPE_VERSION_STORE] = "_GUARD_TYPE_VERSION_STORE", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", [_IS_NONE] = "_IS_NONE", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 61b1db9e5a1543..e98f7642542d8d 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -490,10 +490,6 @@ break; } - case _GUARD_TYPE_VERSION_STORE: { - break; - } - case _STORE_ATTR_SLOT: { STACK_SHRINK(2); break; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 035629dda6e7db..08f80810d5a340 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2038,7 +2038,7 @@ dummy_func( macro(STORE_ATTR_INSTANCE_VALUE) = unused/1 + - _GUARD_TYPE_VERSION_STORE + + _GUARD_TYPE_VERSION + _GUARD_DORV_VALUES + _STORE_ATTR_INSTANCE_VALUE; @@ -2083,12 +2083,6 @@ dummy_func( Py_DECREF(owner); } - op(_GUARD_TYPE_VERSION_STORE, (type_version/2, owner -- owner)) { - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version); - } - op(_STORE_ATTR_SLOT, (index/1, value, owner --)) { char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); @@ -2100,7 +2094,7 @@ dummy_func( macro(STORE_ATTR_SLOT) = unused/1 + - _GUARD_TYPE_VERSION_STORE + + _GUARD_TYPE_VERSION + _STORE_ATTR_SLOT; family(COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP) = { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 662de57553fb47..bbdc508a036657 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1761,16 +1761,6 @@ break; } - case _GUARD_TYPE_VERSION_STORE: { - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)operand; - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION_STORE); - break; - } - case _STORE_ATTR_SLOT: { PyObject *owner; PyObject *value; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 96d9d9db9c7928..c616d03b352fbd 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2616,7 +2616,7 @@ TARGET(STORE_ATTR_INSTANCE_VALUE) { PyObject *owner; PyObject *value; - // _GUARD_TYPE_VERSION_STORE + // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&next_instr[1].cache); @@ -2705,7 +2705,7 @@ TARGET(STORE_ATTR_SLOT) { PyObject *owner; PyObject *value; - // _GUARD_TYPE_VERSION_STORE + // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&next_instr[1].cache); From 269005e78429d6ee518643daa633e9aa7e8b9c33 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Oct 2023 18:36:50 -0600 Subject: [PATCH 16/73] gh-110300: Fix Refleaks in test_interpreters and test__xxinterpchannels (gh-110318) --- Lib/test/support/interpreters.py | 3 ++- Lib/test/test_interpreters.py | 26 +++++++++++++++++++------- Modules/_xxinterpchannelsmodule.c | 11 +++++++++-- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index d2beba31e80283..3b501614bc4b4d 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -215,4 +215,5 @@ def close(self): _channels.close(self._id, send=True) -_channels._register_end_types(SendChannel, RecvChannel) +# XXX This is causing leaks (gh-110318): +#_channels._register_end_types(SendChannel, RecvChannel) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index ffdd8a12769397..f2ef172d26dfc8 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -68,6 +68,17 @@ def run(): class TestBase(unittest.TestCase): + def pipe(self): + def ensure_closed(fd): + try: + os.close(fd) + except OSError: + pass + r, w = os.pipe() + self.addCleanup(lambda: ensure_closed(r)) + self.addCleanup(lambda: ensure_closed(w)) + return r, w + def tearDown(self): clean_up_interpreters() @@ -262,7 +273,7 @@ def test_subinterpreter(self): self.assertFalse(interp.is_running()) def test_finished(self): - r, w = os.pipe() + r, w = self.pipe() interp = interpreters.create() interp.run(f"""if True: import os @@ -299,8 +310,8 @@ def test_bad_id(self): interp.is_running() def test_with_only_background_threads(self): - r_interp, w_interp = os.pipe() - r_thread, w_thread = os.pipe() + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() DONE = b'D' FINISHED = b'F' @@ -425,8 +436,8 @@ def test_still_running(self): self.assertTrue(interp.is_running()) def test_subthreads_still_running(self): - r_interp, w_interp = os.pipe() - r_thread, w_thread = os.pipe() + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() FINISHED = b'F' @@ -532,8 +543,8 @@ def test_bytes_for_script(self): interp.run(b'print("spam")') def test_with_background_threads_still_running(self): - r_interp, w_interp = os.pipe() - r_thread, w_thread = os.pipe() + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() RAN = b'R' DONE = b'D' @@ -822,6 +833,7 @@ def test_list_all(self): after = set(interpreters.list_all_channels()) self.assertEqual(after, created) + @unittest.expectedFailure # See gh-110318: def test_shareable(self): rch, sch = interpreters.create_channel() diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index d5be76f1f0e38e..d762f449c407a3 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -1992,6 +1992,7 @@ _get_current_channel_end_type(int end) return NULL; } } + Py_DECREF(highlevel); if (end == CHANNEL_SEND) { cls = state->send_channel_type; } @@ -2012,6 +2013,7 @@ _channel_end_from_xid(_PyCrossInterpreterData *data) } PyTypeObject *cls = _get_current_channel_end_type(cid->end); if (cls == NULL) { + Py_DECREF(cid); return NULL; } PyObject *obj = PyObject_CallOneArg((PyObject *)cls, (PyObject *)cid); @@ -2027,7 +2029,9 @@ _channel_end_shared(PyThreadState *tstate, PyObject *obj, if (cidobj == NULL) { return -1; } - if (_channelid_shared(tstate, cidobj, data) < 0) { + int res = _channelid_shared(tstate, cidobj, data); + Py_DECREF(cidobj); + if (res < 0) { return -1; } data->new_object = _channel_end_from_xid; @@ -2464,7 +2468,10 @@ channel__channel_id(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } PyTypeObject *cls = state->ChannelIDType; - assert(get_module_from_owned_type(cls) == self); + + PyObject *mod = get_module_from_owned_type(cls); + assert(mod == self); + Py_DECREF(mod); return _channelid_new(self, cls, args, kwds); } From f02f26e29366513b097578fbc6b25e02d0eba7c0 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 3 Oct 2023 20:46:38 -0700 Subject: [PATCH 17/73] gh-85984: Document change in return type of tty functions (#110028) --- Doc/library/tty.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/tty.rst b/Doc/library/tty.rst index fc7f98c7931fa5..a4777772e1fc6c 100644 --- a/Doc/library/tty.rst +++ b/Doc/library/tty.rst @@ -43,6 +43,9 @@ The :mod:`tty` module defines the following functions: :func:`termios.tcsetattr`. The return value of :func:`termios.tcgetattr` is saved before setting *fd* to raw mode; this value is returned. + .. versionchanged:: 3.12 + The return value is now the original tty attributes, instead of None. + .. function:: setcbreak(fd, when=termios.TCSAFLUSH) @@ -51,6 +54,9 @@ The :mod:`tty` module defines the following functions: :func:`termios.tcsetattr`. The return value of :func:`termios.tcgetattr` is saved before setting *fd* to cbreak mode; this value is returned. + .. versionchanged:: 3.12 + The return value is now the original tty attributes, instead of None. + .. seealso:: From bfe7e72522565f828f43c2591fea84a7981ee048 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 4 Oct 2023 06:09:43 +0100 Subject: [PATCH 18/73] gh-109653: Defer importing `warnings` in several modules (#110286) --- Lib/argparse.py | 5 +++-- Lib/calendar.py | 2 +- Lib/getpass.py | 2 +- Lib/shutil.py | 2 +- Lib/tarfile.py | 2 +- .../Library/2023-10-03-15-17-03.gh-issue-109653.9DYOMD.rst | 3 +++ 6 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-03-15-17-03.gh-issue-109653.9DYOMD.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index dfc98695f64e0a..a32884db80d1ea 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -89,8 +89,6 @@ import re as _re import sys as _sys -import warnings - from gettext import gettext as _, ngettext SUPPRESS = '==SUPPRESS==' @@ -910,6 +908,7 @@ def __init__(self, # parser.add_argument('-f', action=BooleanOptionalAction, type=int) for field_name in ('type', 'choices', 'metavar'): if locals()[field_name] is not _deprecated_default: + import warnings warnings._deprecated( field_name, "{name!r} is deprecated as of Python 3.12 and will be " @@ -1700,6 +1699,7 @@ def _remove_action(self, action): self._group_actions.remove(action) def add_argument_group(self, *args, **kwargs): + import warnings warnings.warn( "Nesting argument groups is deprecated.", category=DeprecationWarning, @@ -1728,6 +1728,7 @@ def _remove_action(self, action): self._group_actions.remove(action) def add_mutually_exclusive_group(self, *args, **kwargs): + import warnings warnings.warn( "Nesting mutually exclusive groups is deprecated.", category=DeprecationWarning, diff --git a/Lib/calendar.py b/Lib/calendar.py index 2a4deb70a0111f..03469d8ac96bcd 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -10,7 +10,6 @@ from enum import IntEnum, global_enum import locale as _locale from itertools import repeat -import warnings __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", "firstweekday", "isleap", "leapdays", "weekday", "monthrange", @@ -44,6 +43,7 @@ def __str__(self): def __getattr__(name): if name in ('January', 'February'): + import warnings warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead", DeprecationWarning, stacklevel=2) if name == 'January': diff --git a/Lib/getpass.py b/Lib/getpass.py index 6970d8adfbab36..8b42c0a536b4c4 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -18,7 +18,6 @@ import io import os import sys -import warnings __all__ = ["getpass","getuser","GetPassWarning"] @@ -118,6 +117,7 @@ def win_getpass(prompt='Password: ', stream=None): def fallback_getpass(prompt='Password: ', stream=None): + import warnings warnings.warn("Can not control echo on the terminal.", GetPassWarning, stacklevel=2) if not stream: diff --git a/Lib/shutil.py b/Lib/shutil.py index a278b74fab2ddb..0fed0117a63234 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -10,7 +10,6 @@ import fnmatch import collections import errno -import warnings try: import zlib @@ -723,6 +722,7 @@ def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): """ if onerror is not None: + import warnings warnings.warn("onerror argument is deprecated, use onexc instead", DeprecationWarning, stacklevel=2) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 726f9f50ba2e72..ec32f9ba49b03f 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -46,7 +46,6 @@ import struct import copy import re -import warnings try: import pwd @@ -2219,6 +2218,7 @@ def _get_filter_function(self, filter): if filter is None: filter = self.extraction_filter if filter is None: + import warnings warnings.warn( 'Python 3.14 will, by default, filter extracted tar ' + 'archives and reject files or modify their metadata. ' diff --git a/Misc/NEWS.d/next/Library/2023-10-03-15-17-03.gh-issue-109653.9DYOMD.rst b/Misc/NEWS.d/next/Library/2023-10-03-15-17-03.gh-issue-109653.9DYOMD.rst new file mode 100644 index 00000000000000..92e5a1cada9a69 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-03-15-17-03.gh-issue-109653.9DYOMD.rst @@ -0,0 +1,3 @@ +Slightly improve the import time of several standard-library modules by +deferring imports of :mod:`warnings` within those modules. Patch by Alex +Waygood. From 5b9a3fd6a0ce3c347463e6192a59c15f5fcb0043 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 4 Oct 2023 09:20:14 +0300 Subject: [PATCH 19/73] gh-110273: dataclasses.replace() now raise TypeError for all invalid arguments (GH-110274) dataclasses.replace() now raises TypeError instead of ValueError if specify keyword argument for a field declared with init=False or miss keyword argument for required InitVar field. --- Lib/dataclasses.py | 10 +++++----- Lib/test/test_dataclasses/__init__.py | 12 ++++++------ .../2023-10-03-14-07-05.gh-issue-110273.QaDUmS.rst | 3 +++ 3 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-03-14-07-05.gh-issue-110273.QaDUmS.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 84f8d68ce092a4..31dc6f8abce91a 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1567,15 +1567,15 @@ def _replace(obj, /, **changes): if not f.init: # Error if this field is specified in changes. if f.name in changes: - raise ValueError(f'field {f.name} is declared with ' - 'init=False, it cannot be specified with ' - 'replace()') + raise TypeError(f'field {f.name} is declared with ' + f'init=False, it cannot be specified with ' + f'replace()') continue if f.name not in changes: if f._field_type is _FIELD_INITVAR and f.default is MISSING: - raise ValueError(f"InitVar {f.name!r} " - 'must be specified with replace()') + raise TypeError(f"InitVar {f.name!r} " + f'must be specified with replace()') changes[f.name] = getattr(obj, f.name) # Create the new object, which calls __init__() and diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 7c07dfc77de208..f629d7bb53959b 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -3965,9 +3965,9 @@ class C: self.assertEqual((c1.x, c1.y, c1.z, c1.t), (3, 2, 10, 100)) - with self.assertRaisesRegex(ValueError, 'init=False'): + with self.assertRaisesRegex(TypeError, 'init=False'): replace(c, x=3, z=20, t=50) - with self.assertRaisesRegex(ValueError, 'init=False'): + with self.assertRaisesRegex(TypeError, 'init=False'): replace(c, z=20) replace(c, x=3, z=20, t=50) @@ -4020,10 +4020,10 @@ class C: self.assertEqual((c1.x, c1.y), (5, 10)) # Trying to replace y is an error. - with self.assertRaisesRegex(ValueError, 'init=False'): + with self.assertRaisesRegex(TypeError, 'init=False'): replace(c, x=2, y=30) - with self.assertRaisesRegex(ValueError, 'init=False'): + with self.assertRaisesRegex(TypeError, 'init=False'): replace(c, y=30) def test_classvar(self): @@ -4056,8 +4056,8 @@ def __post_init__(self, y): c = C(1, 10) self.assertEqual(c.x, 10) - with self.assertRaisesRegex(ValueError, r"InitVar 'y' must be " - "specified with replace()"): + with self.assertRaisesRegex(TypeError, r"InitVar 'y' must be " + r"specified with replace\(\)"): replace(c, x=3) c = replace(c, x=3, y=5) self.assertEqual(c.x, 15) diff --git a/Misc/NEWS.d/next/Library/2023-10-03-14-07-05.gh-issue-110273.QaDUmS.rst b/Misc/NEWS.d/next/Library/2023-10-03-14-07-05.gh-issue-110273.QaDUmS.rst new file mode 100644 index 00000000000000..98d87da6295ee5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-03-14-07-05.gh-issue-110273.QaDUmS.rst @@ -0,0 +1,3 @@ +:func:`dataclasses.replace` now raises TypeError instead of ValueError if +specify keyword argument for a field declared with init=False or miss +keyword argument for required InitVar field. From 1465386720cd532a378a5cc1e6de9d96dd8fcc81 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 4 Oct 2023 09:42:12 +0300 Subject: [PATCH 20/73] gh-110171: `libregrtest` always sets `random.seed` (#110172) --- Lib/test/libregrtest/cmdline.py | 13 +++++++------ Lib/test/libregrtest/main.py | 13 +++++++------ Lib/test/libregrtest/setup.py | 3 +-- Lib/test/test_regrtest.py | 8 ++++++-- .../2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst | 3 +++ 5 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 8562a48446b4a7..7cfe904cb5a535 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -27,8 +27,10 @@ Additional option details: -r randomizes test execution order. You can use --randseed=int to provide an -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. +int seed value for the randomizer. The randseed value will be used +to set seeds for all random usages in tests +(including randomizing the tests order if -r is set). +By default we always set random seed, but do not randomize test order. -s On the first invocation of regrtest using -s, the first test file found or the first test file given on the command line is run, and the name of @@ -229,6 +231,9 @@ def _create_parser(): more_details) group.add_argument('-p', '--python', metavar='PYTHON', help='Command to run Python test subprocesses with.') + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a global random seed') group = parser.add_argument_group('Verbosity') group.add_argument('-v', '--verbose', action='count', @@ -249,10 +254,6 @@ def _create_parser(): group = parser.add_argument_group('Selecting tests') group.add_argument('-r', '--randomize', action='store_true', help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') group.add_argument('-f', '--fromfile', metavar='FILE', help='read names of tests to run from a file.' + more_details) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index af5fb0f464f5b5..60179ec7708c1c 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -129,7 +129,11 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Randomize self.randomize: bool = ns.randomize - self.random_seed: int | None = ns.random_seed + self.random_seed: int | None = ( + ns.random_seed + if ns.random_seed is not None + else random.getrandbits(32) + ) if 'SOURCE_DATE_EPOCH' in os.environ: self.randomize = False self.random_seed = None @@ -214,10 +218,8 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList print(f"Cannot find starting test: {self.starting_test}") sys.exit(1) + random.seed(self.random_seed) if self.randomize: - if self.random_seed is None: - self.random_seed = random.randrange(100_000_000) - random.seed(self.random_seed) random.shuffle(selected) return (tuple(selected), tests) @@ -439,8 +441,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: or tests or self.cmdline_args)): display_header(self.use_resources, self.python_cmd) - if self.randomize: - print("Using random seed", self.random_seed) + print("Using random seed", self.random_seed) runtests = self.create_run_tests(selected) self.first_runtests = runtests diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index f0d8d7ebaa2fdb..cb410da5acb4c3 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -126,5 +126,4 @@ def setup_tests(runtests: RunTests): if runtests.gc_threshold is not None: gc.set_threshold(runtests.gc_threshold) - if runtests.randomize: - random.seed(runtests.random_seed) + random.seed(runtests.random_seed) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 38071341006092..ba23b3666924c4 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -392,7 +392,7 @@ def check_ci_mode(self, args, use_resources, rerun=True): self.assertEqual(regrtest.num_workers, -1) self.assertEqual(regrtest.want_rerun, rerun) self.assertTrue(regrtest.randomize) - self.assertIsNone(regrtest.random_seed) + self.assertIsInstance(regrtest.random_seed, int) self.assertTrue(regrtest.fail_env_changed) self.assertTrue(regrtest.fail_rerun) self.assertTrue(regrtest.print_slowest) @@ -663,7 +663,7 @@ def list_regex(line_format, tests): def parse_random_seed(self, output): match = self.regex_search(r'Using random seed ([0-9]+)', output) randseed = int(match.group(1)) - self.assertTrue(0 <= randseed <= 100_000_000, randseed) + self.assertTrue(0 <= randseed, randseed) return randseed def run_command(self, args, input=None, exitcode=0, **kw): @@ -950,6 +950,10 @@ def test_random(self): test_random2 = int(match.group(1)) self.assertEqual(test_random2, test_random) + # check that random.seed is used by default + output = self.run_tests(test, exitcode=EXITCODE_NO_TESTS_RAN) + self.assertIsInstance(self.parse_random_seed(output), int) + def test_fromfile(self): # test --fromfile tests = [self.create_test() for index in range(5)] diff --git a/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst b/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst new file mode 100644 index 00000000000000..9b41b033bc7f2b --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst @@ -0,0 +1,3 @@ +``libregrtest`` now always sets and shows ``random.seed``, +so tests are more reproducible. Use ``--randseed`` flag +to pass the explicit random seed for tests. From 1de9406f9136e3952b849487f0151be3c669a3ea Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Oct 2023 10:57:35 +0200 Subject: [PATCH 21/73] gh-110166: Fix gdb CFunctionFullTests on ppc64le clang build (#110331) CFunctionFullTests now also runs "bt" command before "py-bt-full", similar to CFunctionTests which also runs "bt" command before "py-bt". So test_gdb can skip the test if patterns like "?? ()" are found in the gdb output. --- Lib/test/test_gdb/test_cfunction_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_gdb/test_cfunction_full.py b/Lib/test/test_gdb/test_cfunction_full.py index 3e90cb1d392f49..572cbdab5d77c0 100644 --- a/Lib/test/test_gdb/test_cfunction_full.py +++ b/Lib/test/test_gdb/test_cfunction_full.py @@ -18,7 +18,7 @@ def check(self, func_name, cmd): gdb_output = self.get_stack_trace( cmd, breakpoint=func_name, - cmds_after_breakpoint=['py-bt-full'], + cmds_after_breakpoint=['bt', 'py-bt-full'], # bpo-45207: Ignore 'Function "meth_varargs" not # defined.' message in stderr. ignore_stderr=True, From 1337765225d7d593169205672e004f97e15237ec Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Oct 2023 11:19:08 +0200 Subject: [PATCH 22/73] gh-110335: asyncio test_unix_events cleans multiprocessing (#110336) test_unix_events tests using the multiprocessing module now call multiprocessing.util._cleanup_tests(). --- Lib/test/test_asyncio/test_unix_events.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 7322be597ae2d2..d2c8cba6acfa31 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -4,6 +4,7 @@ import errno import io import multiprocessing +from multiprocessing.util import _cleanup_tests as multiprocessing_cleanup_tests import os import pathlib import signal @@ -15,6 +16,7 @@ import unittest from unittest import mock import warnings + from test import support from test.support import os_helper from test.support import socket_helper @@ -1903,6 +1905,8 @@ async def test_fork_not_share_event_loop(self): @hashlib_helper.requires_hashdigest('md5') def test_fork_signal_handling(self): + self.addCleanup(multiprocessing_cleanup_tests) + # Sending signal to the forked process should not affect the parent # process ctx = multiprocessing.get_context('fork') @@ -1947,6 +1951,8 @@ async def func(): @hashlib_helper.requires_hashdigest('md5') def test_fork_asyncio_run(self): + self.addCleanup(multiprocessing_cleanup_tests) + ctx = multiprocessing.get_context('fork') manager = ctx.Manager() self.addCleanup(manager.shutdown) @@ -1964,6 +1970,8 @@ async def child_main(): @hashlib_helper.requires_hashdigest('md5') def test_fork_asyncio_subprocess(self): + self.addCleanup(multiprocessing_cleanup_tests) + ctx = multiprocessing.get_context('fork') manager = ctx.Manager() self.addCleanup(manager.shutdown) From efd8c7a7c97aa411098a4bee0ef1d428337deebb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Oct 2023 11:39:50 +0200 Subject: [PATCH 23/73] gh-109276: regrtest: shorter list of resources (#110326) --- Lib/test/libregrtest/cmdline.py | 14 +--------- Lib/test/libregrtest/utils.py | 46 ++++++++++++++++++++++++++++++--- Lib/test/test_regrtest.py | 20 ++++++++++++++ 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 7cfe904cb5a535..dd4cd335bef7e3 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -3,6 +3,7 @@ import shlex import sys from test.support import os_helper +from .utils import ALL_RESOURCES, RESOURCE_NAMES USAGE = """\ @@ -132,19 +133,6 @@ """ -ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'walltime') - -# Other resources excluded from --use=all: -# -# - extralagefile (ex: test_zipfile64): really too slow to be enabled -# "by default" -# - tzdata: while needed to validate fully test_datetime, it makes -# test_datetime too slow (15-20 min on some buildbots) and so is disabled by -# default (see bpo-30822). -RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') - - class Namespace(argparse.Namespace): def __init__(self, **kwargs) -> None: self.ci = False diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 86fb820a23f535..ea2086cd71b173 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -33,6 +33,19 @@ EXIT_TIMEOUT = 120.0 +ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'walltime') + +# Other resources excluded from --use=all: +# +# - extralagefile (ex: test_zipfile64): really too slow to be enabled +# "by default" +# - tzdata: while needed to validate fully test_datetime, it makes +# test_datetime too slow (15-20 min on some buildbots) and so is disabled by +# default (see bpo-30822). +RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') + + # Types for types hints StrPath = str TestName = str @@ -535,6 +548,30 @@ def is_cross_compiled(): return ('_PYTHON_HOST_PLATFORM' in os.environ) +def format_resources(use_resources: tuple[str, ...]): + use_resources = set(use_resources) + all_resources = set(ALL_RESOURCES) + + # Express resources relative to "all" + relative_all = ['all'] + for name in sorted(all_resources - use_resources): + relative_all.append(f'-{name}') + for name in sorted(use_resources - all_resources): + relative_all.append(f'{name}') + all_text = ','.join(relative_all) + all_text = f"resources: {all_text}" + + # List of enabled resources + text = ','.join(sorted(use_resources)) + text = f"resources ({len(use_resources)}): {text}" + + # Pick the shortest string (prefer relative to all if lengths are equal) + if len(all_text) <= len(text): + return all_text + else: + return text + + def display_header(use_resources: tuple[str, ...], python_cmd: tuple[str, ...] | None): # Print basic platform information @@ -550,14 +587,15 @@ def display_header(use_resources: tuple[str, ...], if process_cpu_count and process_cpu_count != cpu_count: cpu_count = f"{process_cpu_count} (process) / {cpu_count} (system)" print("== CPU count:", cpu_count) - print("== encodings: locale=%s, FS=%s" + print("== encodings: locale=%s FS=%s" % (locale.getencoding(), sys.getfilesystemencoding())) if use_resources: - print(f"== resources ({len(use_resources)}): " - f"{', '.join(sorted(use_resources))}") + text = format_resources(use_resources) + print(f"== {text}") else: - print("== resources: (all disabled, use -u option)") + print("== resources: all test resources are disabled, " + "use -u option to unskip tests") cross_compile = is_cross_compiled() if cross_compile: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ba23b3666924c4..f24d23e5c2f731 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -2080,6 +2080,26 @@ def test_get_signal_name(self): ): self.assertEqual(utils.get_signal_name(exitcode), expected, exitcode) + def test_format_resources(self): + format_resources = utils.format_resources + ALL_RESOURCES = utils.ALL_RESOURCES + self.assertEqual( + format_resources(("network",)), + 'resources (1): network') + self.assertEqual( + format_resources(("audio", "decimal", "network")), + 'resources (3): audio,decimal,network') + self.assertEqual( + format_resources(ALL_RESOURCES), + 'resources: all') + self.assertEqual( + format_resources(tuple(name for name in ALL_RESOURCES + if name != "cpu")), + 'resources: all,-cpu') + self.assertEqual( + format_resources((*ALL_RESOURCES, "tzdata")), + 'resources: all,tzdata') + if __name__ == '__main__': unittest.main() From e9f2352b7b7503519790ee6f51c2e298cf390e75 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 4 Oct 2023 13:51:45 +0300 Subject: [PATCH 24/73] gh-110332: Remove mentions of `random.WichmannHill` from `test_zlib` (#110334) --- Lib/test/test_zlib.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 9a099adc74f4b4..1dc8b91a453f92 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -512,18 +512,7 @@ def test_odd_flush(self): # Try 17K of data # generate random data stream - try: - # In 2.3 and later, WichmannHill is the RNG of the bug report - gen = random.WichmannHill() - except AttributeError: - try: - # 2.2 called it Random - gen = random.Random() - except AttributeError: - # others might simply have a single RNG - gen = random - gen.seed(1) - data = gen.randbytes(17 * 1024) + data = random.randbytes(17 * 1024) # compress, sync-flush, and decompress first = co.compress(data) From 254e30c487908a52a7545cea205aeaef5fbfeea4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 4 Oct 2023 14:16:44 +0300 Subject: [PATCH 25/73] gh-109151: Enable readline in the sqlite3 CLI (GH-109152) --- Lib/sqlite3/__main__.py | 4 ++++ .../Library/2023-09-08-19-44-01.gh-issue-109151.GkzkQu.rst | 1 + 2 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-09-08-19-44-01.gh-issue-109151.GkzkQu.rst diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index 3b59763375c147..b93b84384a0925 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -116,6 +116,10 @@ def main(*args): else: # No SQL provided; start the REPL. console = SqliteInteractiveConsole(con) + try: + import readline + except ImportError: + pass console.interact(banner, exitmsg="") finally: con.close() diff --git a/Misc/NEWS.d/next/Library/2023-09-08-19-44-01.gh-issue-109151.GkzkQu.rst b/Misc/NEWS.d/next/Library/2023-09-08-19-44-01.gh-issue-109151.GkzkQu.rst new file mode 100644 index 00000000000000..78b4e882baba96 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-08-19-44-01.gh-issue-109151.GkzkQu.rst @@ -0,0 +1 @@ +Enable ``readline`` editing features in the :ref:`sqlite3 command-line interface ` (``python -m sqlite3``). From 43baddc2b9557e06ca4f3427c0717bc3688ed3e4 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 4 Oct 2023 15:42:17 +0300 Subject: [PATCH 26/73] gh-110260: Check for PyList_SetItem() errors in termios module (GH-110261) --- Modules/termios.c | 49 +++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/Modules/termios.c b/Modules/termios.c index c779a757e4fa9b..6f07c93bcdb6b5 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -120,7 +120,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) v = PyBytes_FromStringAndSize(&ch, 1); if (v == NULL) goto err; - PyList_SetItem(cc, i, v); + PyList_SET_ITEM(cc, i, v); } /* Convert the MIN and TIME slots to integer. On some systems, the @@ -128,29 +128,44 @@ termios_tcgetattr_impl(PyObject *module, int fd) only do this in noncanonical input mode. */ if ((mode.c_lflag & ICANON) == 0) { v = PyLong_FromLong((long)mode.c_cc[VMIN]); - if (v == NULL) + if (v == NULL) { + goto err; + } + if (PyList_SetItem(cc, VMIN, v) < 0) { goto err; - PyList_SetItem(cc, VMIN, v); + } v = PyLong_FromLong((long)mode.c_cc[VTIME]); - if (v == NULL) + if (v == NULL) { + goto err; + } + if (PyList_SetItem(cc, VTIME, v) < 0) { goto err; - PyList_SetItem(cc, VTIME, v); + } } - if (!(v = PyList_New(7))) - goto err; - - PyList_SetItem(v, 0, PyLong_FromLong((long)mode.c_iflag)); - PyList_SetItem(v, 1, PyLong_FromLong((long)mode.c_oflag)); - PyList_SetItem(v, 2, PyLong_FromLong((long)mode.c_cflag)); - PyList_SetItem(v, 3, PyLong_FromLong((long)mode.c_lflag)); - PyList_SetItem(v, 4, PyLong_FromLong((long)ispeed)); - PyList_SetItem(v, 5, PyLong_FromLong((long)ospeed)); - if (PyErr_Occurred()) { - Py_DECREF(v); + if (!(v = PyList_New(7))) { goto err; } - PyList_SetItem(v, 6, cc); + +#define ADD_LONG_ITEM(index, val) \ + do { \ + PyObject *l = PyLong_FromLong((long)val); \ + if (l == NULL) { \ + Py_DECREF(v); \ + goto err; \ + } \ + PyList_SET_ITEM(v, index, l); \ + } while (0) + + ADD_LONG_ITEM(0, mode.c_iflag); + ADD_LONG_ITEM(1, mode.c_oflag); + ADD_LONG_ITEM(2, mode.c_cflag); + ADD_LONG_ITEM(3, mode.c_lflag); + ADD_LONG_ITEM(4, ispeed); + ADD_LONG_ITEM(5, ospeed); +#undef ADD_LONG_ITEM + + PyList_SET_ITEM(v, 6, cc); return v; err: Py_DECREF(cc); From d8c00d2a607242932359b995e4637c222fcb2284 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 4 Oct 2023 07:11:32 -0600 Subject: [PATCH 27/73] Lint: Remove files that no longer fail to parse or with F811 (#110356) --- Lib/test/.ruff.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/test/.ruff.toml b/Lib/test/.ruff.toml index b1e6424e785408..f4e68eba14068d 100644 --- a/Lib/test/.ruff.toml +++ b/Lib/test/.ruff.toml @@ -8,9 +8,6 @@ extend-exclude = [ # Failed to lint "encoded_modules/module_iso_8859_1.py", "encoded_modules/module_koi8_r.py", - # Failed to parse - "support/socket_helper.py", - "test_fstring.py", # TODO Fix: F811 Redefinition of unused name "test__opcode.py", "test_buffer.py", @@ -26,7 +23,6 @@ extend-exclude = [ "test_keywordonlyarg.py", "test_pkg.py", "test_subclassinit.py", - "test_unittest/testmock/testpatch.py", "test_yield_from.py", "time_hashlib.py", # Pending https://github.com/python/cpython/pull/109139 From 7c149a76b2bf4c66bb7c8650ffb71acce12f5ea2 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 4 Oct 2023 08:08:02 -0700 Subject: [PATCH 28/73] gh-104909: Split more LOAD_ATTR specializations (GH-110317) * Split LOAD_ATTR_MODULE * Split LOAD_ATTR_WITH_HINT * Split _GUARD_TYPE_VERSION out of the latter * Split LOAD_ATTR_CLASS * Split LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES * Fix indent of DEOPT_IF in macros * Split LOAD_ATTR_METHOD_LAZY_DICT * Split LOAD_ATTR_NONDESCRIPTOR_NO_DICT * Fix omission of _CHECK_ATTR_METHOD_LAZY_DICT --- Include/internal/pycore_opcode_metadata.h | 146 ++++++--- Python/abstract_interp_cases.c.h | 56 ++++ Python/bytecodes.c | 101 +++++-- Python/executor_cases.c.h | 161 ++++++++++ Python/generated_cases.c.h | 351 ++++++++++++---------- Tools/cases_generator/instructions.py | 2 + 6 files changed, 598 insertions(+), 219 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 431a914d26e685..8ef398c5db09f6 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -46,39 +46,49 @@ #define _GUARD_TYPE_VERSION 318 #define _CHECK_MANAGED_OBJECT_HAS_VALUES 319 #define _LOAD_ATTR_INSTANCE_VALUE 320 -#define _LOAD_ATTR_SLOT 321 -#define _GUARD_DORV_VALUES 322 -#define _STORE_ATTR_INSTANCE_VALUE 323 -#define _STORE_ATTR_SLOT 324 -#define _IS_NONE 325 -#define _ITER_CHECK_LIST 326 -#define _ITER_JUMP_LIST 327 -#define _IS_ITER_EXHAUSTED_LIST 328 -#define _ITER_NEXT_LIST 329 -#define _ITER_CHECK_TUPLE 330 -#define _ITER_JUMP_TUPLE 331 -#define _IS_ITER_EXHAUSTED_TUPLE 332 -#define _ITER_NEXT_TUPLE 333 -#define _ITER_CHECK_RANGE 334 -#define _ITER_JUMP_RANGE 335 -#define _IS_ITER_EXHAUSTED_RANGE 336 -#define _ITER_NEXT_RANGE 337 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 338 -#define _GUARD_KEYS_VERSION 339 -#define _LOAD_ATTR_METHOD_WITH_VALUES 340 -#define _LOAD_ATTR_METHOD_NO_DICT 341 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 343 -#define _CHECK_PEP_523 344 -#define _CHECK_FUNCTION_EXACT_ARGS 345 -#define _CHECK_STACK_SPACE 346 -#define _INIT_CALL_PY_EXACT_ARGS 347 -#define _PUSH_FRAME 348 -#define _POP_JUMP_IF_FALSE 349 -#define _POP_JUMP_IF_TRUE 350 -#define _JUMP_TO_TOP 351 -#define _SAVE_CURRENT_IP 352 -#define _INSERT 353 +#define _CHECK_ATTR_MODULE 321 +#define _LOAD_ATTR_MODULE 322 +#define _CHECK_ATTR_WITH_HINT 323 +#define _LOAD_ATTR_WITH_HINT 324 +#define _LOAD_ATTR_SLOT 325 +#define _CHECK_ATTR_CLASS 326 +#define _LOAD_ATTR_CLASS 327 +#define _GUARD_DORV_VALUES 328 +#define _STORE_ATTR_INSTANCE_VALUE 329 +#define _STORE_ATTR_SLOT 330 +#define _IS_NONE 331 +#define _ITER_CHECK_LIST 332 +#define _ITER_JUMP_LIST 333 +#define _IS_ITER_EXHAUSTED_LIST 334 +#define _ITER_NEXT_LIST 335 +#define _ITER_CHECK_TUPLE 336 +#define _ITER_JUMP_TUPLE 337 +#define _IS_ITER_EXHAUSTED_TUPLE 338 +#define _ITER_NEXT_TUPLE 339 +#define _ITER_CHECK_RANGE 340 +#define _ITER_JUMP_RANGE 341 +#define _IS_ITER_EXHAUSTED_RANGE 342 +#define _ITER_NEXT_RANGE 343 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 344 +#define _GUARD_KEYS_VERSION 345 +#define _LOAD_ATTR_METHOD_WITH_VALUES 346 +#define _LOAD_ATTR_METHOD_NO_DICT 347 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 348 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 349 +#define _CHECK_ATTR_METHOD_LAZY_DICT 350 +#define _LOAD_ATTR_METHOD_LAZY_DICT 351 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 352 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 353 +#define _CHECK_PEP_523 354 +#define _CHECK_FUNCTION_EXACT_ARGS 355 +#define _CHECK_STACK_SPACE 356 +#define _INIT_CALL_PY_EXACT_ARGS 357 +#define _PUSH_FRAME 358 +#define _POP_JUMP_IF_FALSE 359 +#define _POP_JUMP_IF_TRUE 360 +#define _JUMP_TO_TOP 361 +#define _SAVE_CURRENT_IP 362 +#define _INSERT 363 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -360,14 +370,26 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case LOAD_ATTR_INSTANCE_VALUE: return 1; + case _CHECK_ATTR_MODULE: + return 1; + case _LOAD_ATTR_MODULE: + return 1; case LOAD_ATTR_MODULE: return 1; + case _CHECK_ATTR_WITH_HINT: + return 1; + case _LOAD_ATTR_WITH_HINT: + return 1; case LOAD_ATTR_WITH_HINT: return 1; case _LOAD_ATTR_SLOT: return 1; case LOAD_ATTR_SLOT: return 1; + case _CHECK_ATTR_CLASS: + return 1; + case _LOAD_ATTR_CLASS: + return 1; case LOAD_ATTR_CLASS: return 1; case LOAD_ATTR_PROPERTY: @@ -506,10 +528,18 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case LOAD_ATTR_METHOD_NO_DICT: return 1; + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + return 1; case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: return 1; + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + return 1; case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: return 1; + case _CHECK_ATTR_METHOD_LAZY_DICT: + return 1; + case _LOAD_ATTR_METHOD_LAZY_DICT: + return 1; case LOAD_ATTR_METHOD_LAZY_DICT: return 1; case INSTRUMENTED_CALL: @@ -916,16 +946,28 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_INSTANCE_VALUE: return (oparg & 1 ? 1 : 0) + 1; + case _CHECK_ATTR_MODULE: + return 1; + case _LOAD_ATTR_MODULE: + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_MODULE: + return (oparg & 1 ? 1 : 0) + 1; + case _CHECK_ATTR_WITH_HINT: + return 1; + case _LOAD_ATTR_WITH_HINT: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_WITH_HINT: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case _LOAD_ATTR_SLOT: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_SLOT: return (oparg & 1 ? 1 : 0) + 1; - case LOAD_ATTR_CLASS: + case _CHECK_ATTR_CLASS: + return 1; + case _LOAD_ATTR_CLASS: return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_ATTR_CLASS: + return (oparg & 1 ? 1 : 0) + 1; case LOAD_ATTR_PROPERTY: return 1; case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: @@ -1062,10 +1104,18 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 2; case LOAD_ATTR_METHOD_NO_DICT: return 2; + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + return 1; case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: return 1; + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + return 1; case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: return 1; + case _CHECK_ATTR_METHOD_LAZY_DICT: + return 1; + case _LOAD_ATTR_METHOD_LAZY_DICT: + return 2; case LOAD_ATTR_METHOD_LAZY_DICT: return 2; case INSTRUMENTED_CALL: @@ -1397,10 +1447,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_CHECK_MANAGED_OBJECT_HAS_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_CHECK_ATTR_MODULE] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, + [_LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_CHECK_ATTR_WITH_HINT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, [_LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_CHECK_ATTR_CLASS] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, + [_LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, @@ -1470,8 +1526,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [_LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_CHECK_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, @@ -1631,7 +1691,10 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { LOAD_ATTR, 0, 0 } } }, [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, + [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, [LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } }, + [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, [COMPARE_OP] = { .nuops = 1, .uops = { { COMPARE_OP, 0, 0 } } }, @@ -1653,6 +1716,9 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { PUSH_EXC_INFO, 0, 0 } } }, [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 9, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SET_IP, 7, 3 }, { _SAVE_CURRENT_IP, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 7, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SET_IP, 7, 3 }, { _SAVE_CURRENT_IP, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_TYPE_1] = { .nuops = 1, .uops = { { CALL_TYPE_1, 0, 0 } } }, @@ -1705,7 +1771,13 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", + [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", + [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT", + [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", + [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", @@ -1726,6 +1798,10 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", + [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", [_CHECK_PEP_523] = "_CHECK_PEP_523", diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index e98f7642542d8d..04fe07fad39937 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -474,6 +474,28 @@ break; } + case _CHECK_ATTR_MODULE: { + break; + } + + case _LOAD_ATTR_MODULE: { + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + + case _CHECK_ATTR_WITH_HINT: { + break; + } + + case _LOAD_ATTR_WITH_HINT: { + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + case _LOAD_ATTR_SLOT: { STACK_GROW(((oparg & 1) ? 1 : 0)); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); @@ -481,6 +503,17 @@ break; } + case _CHECK_ATTR_CLASS: { + break; + } + + case _LOAD_ATTR_CLASS: { + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + case _GUARD_DORV_VALUES: { break; } @@ -670,6 +703,29 @@ break; } + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(0)), true); + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(0)), true); + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT: { + break; + } + + case _LOAD_ATTR_METHOD_LAZY_DICT: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { break; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 08f80810d5a340..a96c5dd75e93b4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1887,11 +1887,15 @@ dummy_func( _LOAD_ATTR_INSTANCE_VALUE + unused/5; // Skip over rest of cache - inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { + op(_CHECK_ATTR_MODULE, (type_version/2, owner -- owner)) { DEOPT_IF(!PyModule_CheckExact(owner)); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); DEOPT_IF(dict->ma_keys->dk_version != type_version); + } + + op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; @@ -1903,19 +1907,26 @@ dummy_func( DECREF_INPUTS(); } - inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + macro(LOAD_ATTR_MODULE) = + unused/1 + + _CHECK_ATTR_MODULE + + _LOAD_ATTR_MODULE + + unused/5; + + op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(_PyDictOrValues_IsValues(dorv)); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - uint16_t hint = index; + } + + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name); @@ -1933,6 +1944,13 @@ dummy_func( DECREF_INPUTS(); } + macro(LOAD_ATTR_WITH_HINT) = + unused/1 + + _GUARD_TYPE_VERSION + + _CHECK_ATTR_WITH_HINT + + _LOAD_ATTR_WITH_HINT + + unused/5; + op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { char *addr = (char *)owner + index; attr = *(PyObject **)addr; @@ -1949,20 +1967,27 @@ dummy_func( _LOAD_ATTR_SLOT + // NOTE: This action may also deopt unused/5; - inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, null if (oparg & 1))) { - + op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { DEOPT_IF(!PyType_Check(owner)); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version); assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version); + + } + op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); null = NULL; - attr = descr; - assert(attr != NULL); - Py_INCREF(attr); DECREF_INPUTS(); } + macro(LOAD_ATTR_CLASS) = + unused/1 + + _CHECK_ATTR_CLASS + + unused/2 + + _LOAD_ATTR_CLASS; + inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused, unused if (0))) { assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame); @@ -2819,43 +2844,46 @@ dummy_func( unused/2 + _LOAD_ATTR_METHOD_NO_DICT; - inst(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, owner -- attr, unused if (0))) { + op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr, unused if (0))) { assert((oparg & 1) == 0); - PyTypeObject *owner_cls = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(owner_cls->tp_version_tag != type_version); - assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); attr = Py_NewRef(descr); } - inst(LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, unused if (0))) { + macro(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) = + unused/1 + + _GUARD_TYPE_VERSION + + _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + + _GUARD_KEYS_VERSION + + _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + + op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr, unused if (0))) { assert((oparg & 1) == 0); - PyTypeObject *owner_cls = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(owner_cls->tp_version_tag != type_version); - assert(owner_cls->tp_dictoffset == 0); + assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); attr = Py_NewRef(descr); } - inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, self if (1))) { - assert(oparg & 1); - PyTypeObject *owner_cls = Py_TYPE(owner); - DEOPT_IF(owner_cls->tp_version_tag != type_version); - Py_ssize_t dictoffset = owner_cls->tp_dictoffset; + macro(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) = + unused/1 + + _GUARD_TYPE_VERSION + + unused/2 + + _LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + + op(_CHECK_ATTR_METHOD_LAZY_DICT, (owner -- owner)) { + Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL); + } + + op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) { + assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); @@ -2863,6 +2891,13 @@ dummy_func( self = owner; } + macro(LOAD_ATTR_METHOD_LAZY_DICT) = + unused/1 + + _GUARD_TYPE_VERSION + + _CHECK_ATTR_METHOD_LAZY_DICT + + unused/2 + + _LOAD_ATTR_METHOD_LAZY_DICT; + inst(INSTRUMENTED_CALL, ( -- )) { int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bbdc508a036657..b3ccf8d4363661 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1711,6 +1711,82 @@ break; } + case _CHECK_ATTR_MODULE: { + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)operand; + DEOPT_IF(!PyModule_CheckExact(owner), _CHECK_ATTR_MODULE); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict != NULL); + DEOPT_IF(dict->ma_keys->dk_version != type_version, _CHECK_ATTR_MODULE); + break; + } + + case _LOAD_ATTR_MODULE: { + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)operand; + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; + attr = ep->me_value; + DEOPT_IF(attr == NULL, _LOAD_ATTR_MODULE); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + break; + } + + case _CHECK_ATTR_WITH_HINT: { + PyObject *owner; + owner = stack_pointer[-1]; + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), _CHECK_ATTR_WITH_HINT); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(dict == NULL, _CHECK_ATTR_WITH_HINT); + assert(PyDict_CheckExact((PyObject *)dict)); + break; + } + + case _LOAD_ATTR_WITH_HINT: { + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + owner = stack_pointer[-1]; + uint16_t hint = (uint16_t)operand; + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, _LOAD_ATTR_WITH_HINT); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); + attr = ep->me_value; + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); + attr = ep->me_value; + } + DEOPT_IF(attr == NULL, _LOAD_ATTR_WITH_HINT); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + break; + } + case _LOAD_ATTR_SLOT: { PyObject *owner; PyObject *attr; @@ -1730,6 +1806,33 @@ break; } + case _CHECK_ATTR_CLASS: { + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)operand; + DEOPT_IF(!PyType_Check(owner), _CHECK_ATTR_CLASS); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, _CHECK_ATTR_CLASS); + break; + } + + case _LOAD_ATTR_CLASS: { + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)operand; + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + null = NULL; + Py_DECREF(owner); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + break; + } + case _GUARD_DORV_VALUES: { PyObject *owner; owner = stack_pointer[-1]; @@ -2339,6 +2442,64 @@ break; } + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { + PyObject *owner; + PyObject *attr; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)operand; + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); + stack_pointer[-1] = attr; + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { + PyObject *owner; + PyObject *attr; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)operand; + assert((oparg & 1) == 0); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); + stack_pointer[-1] = attr; + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT: { + PyObject *owner; + owner = stack_pointer[-1]; + Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; + assert(dictoffset > 0); + PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, _CHECK_ATTR_METHOD_LAZY_DICT); + break; + } + + case _LOAD_ATTR_METHOD_LAZY_DICT: { + PyObject *owner; + PyObject *attr; + PyObject *self; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)operand; + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; + STACK_GROW(1); + stack_pointer[-2] = attr; + stack_pointer[-1] = self; + break; + } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { PyObject *null; PyObject *callable; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c616d03b352fbd..51e3937e55a93c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -390,8 +390,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_MULTIPLY_INT { @@ -415,8 +415,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_INT { @@ -440,8 +440,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_SUBTRACT_INT { @@ -465,8 +465,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_MULTIPLY_FLOAT { @@ -490,8 +490,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_FLOAT { @@ -515,8 +515,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_SUBTRACT_FLOAT { @@ -540,8 +540,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_UNICODE { @@ -564,15 +564,15 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } // _BINARY_OP_INPLACE_ADD_UNICODE { _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; assert(true_next.op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(true_next.op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); + DEOPT_IF(*target_local != left, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -1804,8 +1804,8 @@ { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } // _LOAD_GLOBAL_MODULE @@ -1814,7 +1814,7 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1834,16 +1834,16 @@ { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } // _GUARD_BUILTINS_VERSION { uint16_t version = read_u16(&next_instr[2].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } // _LOAD_GLOBAL_BUILTINS @@ -1852,7 +1852,7 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -2401,21 +2401,21 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { uint16_t index = read_u16(&next_instr[3].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -2432,22 +2432,29 @@ PyObject *owner; PyObject *attr; PyObject *null = NULL; + // _CHECK_ATTR_MODULE owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t index = read_u16(&next_instr[3].cache); - DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + { + uint32_t type_version = read_u32(&next_instr[1].cache); + DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict != NULL); + DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_MODULE + { + uint16_t index = read_u16(&next_instr[3].cache); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; + attr = ep->me_value; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } @@ -2459,36 +2466,46 @@ PyObject *owner; PyObject *attr; PyObject *null = NULL; + // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t index = read_u16(&next_instr[3].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - uint16_t hint = index; - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; + { + uint32_t type_version = read_u32(&next_instr[1].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; + // _CHECK_ATTR_WITH_HINT + { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(dict == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&next_instr[3].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; + } + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); } - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } @@ -2506,14 +2523,14 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { uint16_t index = read_u16(&next_instr[3].cache); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -2530,20 +2547,23 @@ PyObject *owner; PyObject *attr; PyObject *null = NULL; + // _CHECK_ATTR_CLASS owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - PyObject *descr = read_obj(&next_instr[5].cache); - - DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); - assert(type_version != 0); - - STAT_INC(LOAD_ATTR, hit); - null = NULL; - attr = descr; - assert(attr != NULL); - Py_INCREF(attr); - Py_DECREF(owner); + { + uint32_t type_version = read_u32(&next_instr[1].cache); + DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&next_instr[5].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + null = NULL; + Py_DECREF(owner); + } STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } @@ -2622,13 +2642,13 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _GUARD_DORV_VALUES { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); } // _STORE_ATTR_INSTANCE_VALUE value = stack_pointer[-2]; @@ -2711,7 +2731,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _STORE_ATTR_SLOT value = stack_pointer[-2]; @@ -3292,7 +3312,7 @@ // _ITER_CHECK_LIST iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); } // _ITER_JUMP_LIST { @@ -3334,7 +3354,7 @@ // _ITER_CHECK_TUPLE iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -3377,7 +3397,7 @@ iter = stack_pointer[-1]; { _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } // _ITER_JUMP_RANGE { @@ -3580,20 +3600,20 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&next_instr[3].cache); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -3623,7 +3643,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_NO_DICT { @@ -3646,23 +3666,36 @@ TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { PyObject *owner; PyObject *attr; + // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint32_t keys_version = read_u32(&next_instr[3].cache); - PyObject *descr = read_obj(&next_instr[5].cache); - assert((oparg & 1) == 0); - PyTypeObject *owner_cls = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + { + uint32_t type_version = read_u32(&next_instr[1].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&next_instr[3].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&next_instr[5].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); + } stack_pointer[-1] = attr; next_instr += 9; DISPATCH(); @@ -3671,18 +3704,24 @@ TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { PyObject *owner; PyObject *attr; + // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - PyObject *descr = read_obj(&next_instr[5].cache); - assert((oparg & 1) == 0); - PyTypeObject *owner_cls = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(owner_cls->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + { + uint32_t type_version = read_u32(&next_instr[1].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&next_instr[5].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); + } stack_pointer[-1] = attr; next_instr += 9; DISPATCH(); @@ -3692,22 +3731,32 @@ PyObject *owner; PyObject *attr; PyObject *self; + // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - PyObject *descr = read_obj(&next_instr[5].cache); - assert(oparg & 1); - PyTypeObject *owner_cls = Py_TYPE(owner); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); - Py_ssize_t dictoffset = owner_cls->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; + { + uint32_t type_version = read_u32(&next_instr[1].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; + assert(dictoffset > 0); + PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&next_instr[5].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; + } STACK_GROW(1); stack_pointer[-2] = attr; stack_pointer[-1] = self; @@ -3834,14 +3883,14 @@ _PyInterpreterFrame *new_frame; // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -3857,18 +3906,18 @@ callable = func; { uint32_t func_version = read_u32(&next_instr[1].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); } // _CHECK_STACK_SPACE { PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS args = stack_pointer - oparg; @@ -3928,25 +3977,25 @@ _PyInterpreterFrame *new_frame; // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_FUNCTION_EXACT_ARGS self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { uint32_t func_version = read_u32(&next_instr[1].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); } // _CHECK_STACK_SPACE { PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS args = stack_pointer - oparg; diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index 5384bbe81169cc..bd7b7dfbaa8f70 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -210,12 +210,14 @@ def write_body( out.write_raw(f"{space}if ({cond}) goto {label};\n") elif m := re.match(r"(\s*)DEOPT_IF\((.+)\);\s*(?://.*)?$", line): space, cond = m.groups() + space = extra + space target = family.name if family else self.name out.write_raw(f"{space}DEOPT_IF({cond}, {target});\n") elif "DEOPT" in line: filename = context.owner.filename lineno = context.owner.tokens[context.begin].line print(f"{filename}:{lineno}: ERROR: DEOPT_IF() must be all on one line") + out.write_raw(extra + line) elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line): out.reset_lineno() space = extra + m.group(1) From bf4bc36069ef1ed4be4be2ae70404f78bff056d9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 4 Oct 2023 16:09:48 +0100 Subject: [PATCH 29/73] GH-109369: Merge all eval-breaker flags and monitoring version into one word. (GH-109846) --- Include/cpython/code.h | 2 +- Include/internal/pycore_ceval.h | 33 +++ Include/internal/pycore_ceval_state.h | 19 +- Include/internal/pycore_interp.h | 3 +- ...-09-26-03-46-55.gh-issue-109369.OJbxbF.rst | 2 + Modules/gcmodule.c | 7 +- Modules/signalmodule.c | 5 +- Python/bytecodes.c | 22 +- Python/ceval_gil.c | 251 +++++------------- Python/ceval_macros.h | 2 +- Python/executor_cases.c.h | 7 +- Python/generated_cases.c.h | 22 +- Python/instrumentation.c | 47 +++- 13 files changed, 188 insertions(+), 234 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-26-03-46-55.gh-issue-109369.OJbxbF.rst diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 45b09a1265df80..cf715c55a2b3b8 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -167,7 +167,7 @@ typedef struct { PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ _PyExecutorArray *co_executors; /* executors from optimizer */ \ _PyCoCached *_co_cached; /* cached co_* attributes */ \ - uint64_t _co_instrumentation_version; /* current instrumentation version */ \ + uintptr_t _co_instrumentation_version; /* current instrumentation version */ \ _PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ /* Scratch space for extra data relating to the code object. \ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 23d0fa399d7e6f..48fee697324f8f 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -193,6 +193,39 @@ int _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int a void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); +#define _PY_GIL_DROP_REQUEST_BIT 0 +#define _PY_SIGNALS_PENDING_BIT 1 +#define _PY_CALLS_TO_DO_BIT 2 +#define _PY_ASYNC_EXCEPTION_BIT 3 +#define _PY_GC_SCHEDULED_BIT 4 + +/* Reserve a few bits for future use */ +#define _PY_EVAL_EVENTS_BITS 8 +#define _PY_EVAL_EVENTS_MASK ((1 << _PY_EVAL_EVENTS_BITS)-1) + +static inline void +_Py_set_eval_breaker_bit(PyInterpreterState *interp, uint32_t bit, uint32_t set) +{ + assert(set == 0 || set == 1); + uintptr_t to_set = set << bit; + uintptr_t mask = ((uintptr_t)1) << bit; + uintptr_t old = _Py_atomic_load_uintptr(&interp->ceval.eval_breaker); + if ((old & mask) == to_set) { + return; + } + uintptr_t new; + do { + new = (old & ~mask) | to_set; + } while (!_Py_atomic_compare_exchange_uintptr(&interp->ceval.eval_breaker, &old, new)); +} + +static inline bool +_Py_eval_breaker_bit_is_set(PyInterpreterState *interp, int32_t bit) +{ + return _Py_atomic_load_uintptr_relaxed(&interp->ceval.eval_breaker) & (((uintptr_t)1) << bit); +} + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index d0af5b542233e0..47971fbf2b4bfe 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -17,11 +17,7 @@ struct _pending_calls { int busy; PyThread_type_lock lock; /* Request for running pending calls. */ - _Py_atomic_int calls_to_do; - /* Request for looking at the `async_exc` field of the current - thread state. - Guarded by the GIL. */ - int async_exc; + int32_t calls_to_do; #define NPENDINGCALLS 32 struct _pending_call { _Py_pending_call_func func; @@ -62,11 +58,6 @@ struct _ceval_runtime_state { int _not_used; #endif } perf; - /* Request for checking signals. It is shared by all interpreters (see - bpo-40513). Any thread of any interpreter can receive a signal, but only - the main thread of the main interpreter can handle signals: see - _Py_ThreadCanHandleSignals(). */ - _Py_atomic_int signals_pending; /* Pending calls to be made only on the main thread. */ struct _pending_calls pending_mainthread; }; @@ -87,14 +78,12 @@ struct _ceval_state { * the fast path in the eval loop. * It is by far the hottest field in this struct and * should be placed at the beginning. */ - _Py_atomic_int eval_breaker; - /* Request for dropping the GIL */ - _Py_atomic_int gil_drop_request; + uintptr_t eval_breaker; + /* Avoid false sharing */ + int64_t padding[7]; int recursion_limit; struct _gil_runtime_state *gil; int own_gil; - /* The GC is ready to be executed */ - _Py_atomic_int gc_scheduled; struct _pending_calls pending; }; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index ebf02281a7a2a6..21d1ee38b59e6b 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -67,8 +67,7 @@ struct _is { int _initialized; int finalizing; - uint64_t monitoring_version; - uint64_t last_restart_version; + uintptr_t last_restart_version; struct pythreads { uint64_t next_unique_id; /* The linked list of threads, newest first. */ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-26-03-46-55.gh-issue-109369.OJbxbF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-26-03-46-55.gh-issue-109369.OJbxbF.rst new file mode 100644 index 00000000000000..ca1f0f1bd44a8c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-26-03-46-55.gh-issue-109369.OJbxbF.rst @@ -0,0 +1,2 @@ +The internal eval_breaker and supporting flags, plus the monitoring version +have been merged into a single atomic integer to speed up checks. diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 632cabdf4bcfbd..592d527f0bd6a2 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -24,6 +24,7 @@ */ #include "Python.h" +#include "pycore_ceval.h" // _Py_set_eval_breaker_bit() #include "pycore_context.h" #include "pycore_dict.h" // _PyDict_MaybeUntrack() #include "pycore_initconfig.h" @@ -2274,11 +2275,7 @@ _Py_ScheduleGC(PyInterpreterState *interp) if (gcstate->collecting == 1) { return; } - struct _ceval_state *ceval = &interp->ceval; - if (!_Py_atomic_load_relaxed(&ceval->gc_scheduled)) { - _Py_atomic_store_relaxed(&ceval->gc_scheduled, 1); - _Py_atomic_store_relaxed(&ceval->eval_breaker, 1); - } + _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1); } void diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 8d6556727b3a5a..ac3457003b0cb6 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1767,9 +1767,8 @@ PyErr_CheckSignals(void) Python code to ensure signals are handled. Checking for the GC here allows long running native code to clean cycles created using the C-API even if it doesn't run the evaluation loop */ - struct _ceval_state *interp_ceval_state = &tstate->interp->ceval; - if (_Py_atomic_load_relaxed(&interp_ceval_state->gc_scheduled)) { - _Py_atomic_store_relaxed(&interp_ceval_state->gc_scheduled, 0); + if (_Py_eval_breaker_bit_is_set(tstate->interp, _PY_GC_SCHEDULED_BIT)) { + _Py_set_eval_breaker_bit(tstate->interp, _PY_GC_SCHEDULED_BIT, 0); _Py_RunGC(tstate); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a96c5dd75e93b4..6482252eec9fb9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -136,7 +136,12 @@ dummy_func( inst(RESUME, (--)) { TIER_ONE_ONLY assert(frame == tstate->current_frame); - if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((code_version & 255) == 0); + if (code_version != global_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); ERROR_IF(err, error); next_instr--; @@ -154,17 +159,16 @@ dummy_func( DEOPT_IF(_Py_emscripten_signal_clock == 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif - /* Possibly combine these two checks */ - DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version); - DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker)); + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version); } inst(INSTRUMENTED_RESUME, (--)) { - /* Possible performance enhancement: - * We need to check the eval breaker anyway, can we - * combine the instrument verison check and the eval breaker test? - */ - if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { goto error; } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index ba16f5eb9bfe74..6b4ec8eed03aab 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -57,113 +57,62 @@ #define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL) #endif -/* This can set eval_breaker to 0 even though gil_drop_request became - 1. We believe this is all right because the eval loop will release - the GIL eventually anyway. */ +/* bpo-40010: eval_breaker should be recomputed if there + is a pending signal: signal received by another thread which cannot + handle signals. + Similarly, we set CALLS_TO_DO and ASYNC_EXCEPTION to match the thread. +*/ static inline void -COMPUTE_EVAL_BREAKER(PyInterpreterState *interp, - struct _ceval_runtime_state *ceval, - struct _ceval_state *ceval2) +update_eval_breaker_from_thread(PyInterpreterState *interp, PyThreadState *tstate) { - _Py_atomic_store_relaxed(&ceval2->eval_breaker, - _Py_atomic_load_relaxed_int32(&ceval2->gil_drop_request) - | (_Py_atomic_load_relaxed_int32(&ceval->signals_pending) - && _Py_ThreadCanHandleSignals(interp)) - | (_Py_atomic_load_relaxed_int32(&ceval2->pending.calls_to_do)) - | (_Py_IsMainThread() && _Py_IsMainInterpreter(interp) - &&_Py_atomic_load_relaxed_int32(&ceval->pending_mainthread.calls_to_do)) - | ceval2->pending.async_exc - | _Py_atomic_load_relaxed_int32(&ceval2->gc_scheduled)); -} + if (tstate == NULL) { + return; + } + if (_Py_IsMainThread()) { + int32_t calls_to_do = _Py_atomic_load_int32_relaxed( + &_PyRuntime.ceval.pending_mainthread.calls_to_do); + if (calls_to_do) { + _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 1); + } + if (_Py_ThreadCanHandleSignals(interp)) { + if (_Py_atomic_load(&_PyRuntime.signals.is_tripped)) { + _Py_set_eval_breaker_bit(interp, _PY_SIGNALS_PENDING_BIT, 1); + } + } + } + if (tstate->async_exc != NULL) { + _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 1); + } +} static inline void SET_GIL_DROP_REQUEST(PyInterpreterState *interp) { - struct _ceval_state *ceval2 = &interp->ceval; - _Py_atomic_store_relaxed(&ceval2->gil_drop_request, 1); - _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1); + _Py_set_eval_breaker_bit(interp, _PY_GIL_DROP_REQUEST_BIT, 1); } static inline void RESET_GIL_DROP_REQUEST(PyInterpreterState *interp) { - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - _Py_atomic_store_relaxed(&ceval2->gil_drop_request, 0); - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); + _Py_set_eval_breaker_bit(interp, _PY_GIL_DROP_REQUEST_BIT, 0); } static inline void -SIGNAL_PENDING_CALLS(struct _pending_calls *pending, PyInterpreterState *interp) +SIGNAL_PENDING_CALLS(PyInterpreterState *interp) { - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - _Py_atomic_store_relaxed(&pending->calls_to_do, 1); - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); + _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 1); } static inline void UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp) { - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - if (_Py_IsMainThread() && _Py_IsMainInterpreter(interp)) { - _Py_atomic_store_relaxed(&ceval->pending_mainthread.calls_to_do, 0); - } - _Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 0); - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); + _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 0); } - -static inline void -SIGNAL_PENDING_SIGNALS(PyInterpreterState *interp, int force) -{ - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - _Py_atomic_store_relaxed(&ceval->signals_pending, 1); - if (force) { - _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1); - } - else { - /* eval_breaker is not set to 1 if thread_can_handle_signals() is false */ - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); - } -} - - -static inline void -UNSIGNAL_PENDING_SIGNALS(PyInterpreterState *interp) -{ - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - _Py_atomic_store_relaxed(&ceval->signals_pending, 0); - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); -} - - -static inline void -SIGNAL_ASYNC_EXC(PyInterpreterState *interp) -{ - struct _ceval_state *ceval2 = &interp->ceval; - ceval2->pending.async_exc = 1; - _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1); -} - - -static inline void -UNSIGNAL_ASYNC_EXC(PyInterpreterState *interp) -{ - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - ceval2->pending.async_exc = 0; - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); -} - - /* * Implementation of the Global Interpreter Lock (GIL). */ @@ -271,8 +220,9 @@ static void recreate_gil(struct _gil_runtime_state *gil) #endif static void -drop_gil(struct _ceval_state *ceval, PyThreadState *tstate) +drop_gil(PyInterpreterState *interp, PyThreadState *tstate) { + struct _ceval_state *ceval = &interp->ceval; /* If tstate is NULL, the caller is indicating that we're releasing the GIL for the last time in this thread. This is particularly relevant when the current thread state is finalizing or its @@ -310,7 +260,7 @@ drop_gil(struct _ceval_state *ceval, PyThreadState *tstate) the GIL, and that's the only time we might delete the interpreter, so checking tstate first prevents the crash. See https://github.com/python/cpython/issues/104341. */ - if (tstate != NULL && _Py_atomic_load_relaxed(&ceval->gil_drop_request)) { + if (tstate != NULL && _Py_eval_breaker_bit_is_set(interp, _PY_GIL_DROP_REQUEST_BIT)) { MUTEX_LOCK(gil->switch_mutex); /* Not switched yet => wait */ if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate) @@ -356,8 +306,7 @@ take_gil(PyThreadState *tstate) assert(_PyThreadState_CheckConsistency(tstate)); PyInterpreterState *interp = tstate->interp; - struct _ceval_state *ceval = &interp->ceval; - struct _gil_runtime_state *gil = ceval->gil; + struct _gil_runtime_state *gil = interp->ceval.gil; /* Check that _PyEval_InitThreads() was called to create the lock */ assert(gil_created(gil)); @@ -431,27 +380,13 @@ take_gil(PyThreadState *tstate) in take_gil() while the main thread called wait_for_thread_shutdown() from Py_Finalize(). */ MUTEX_UNLOCK(gil->mutex); - drop_gil(ceval, tstate); + drop_gil(interp, tstate); PyThread_exit_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); - if (_Py_atomic_load_relaxed(&ceval->gil_drop_request)) { - RESET_GIL_DROP_REQUEST(interp); - } - else { - /* bpo-40010: eval_breaker should be recomputed to be set to 1 if there - is a pending signal: signal received by another thread which cannot - handle signals. - - Note: RESET_GIL_DROP_REQUEST() calls COMPUTE_EVAL_BREAKER(). */ - COMPUTE_EVAL_BREAKER(interp, &_PyRuntime.ceval, ceval); - } - - /* Don't access tstate if the thread must exit */ - if (tstate->async_exc != NULL) { - _PyEval_SignalAsyncExc(tstate->interp); - } + RESET_GIL_DROP_REQUEST(interp); + update_eval_breaker_from_thread(interp, tstate); MUTEX_UNLOCK(gil->mutex); @@ -611,8 +546,7 @@ PyEval_ReleaseLock(void) /* This function must succeed when the current thread state is NULL. We therefore avoid PyThreadState_Get() which dumps a fatal error in debug mode. */ - struct _ceval_state *ceval = &tstate->interp->ceval; - drop_gil(ceval, tstate); + drop_gil(tstate->interp, tstate); } void @@ -628,8 +562,7 @@ _PyEval_ReleaseLock(PyInterpreterState *interp, PyThreadState *tstate) /* If tstate is NULL then we do not expect the current thread to acquire the GIL ever again. */ assert(tstate == NULL || tstate->interp == interp); - struct _ceval_state *ceval = &interp->ceval; - drop_gil(ceval, tstate); + drop_gil(interp, tstate); } void @@ -653,8 +586,7 @@ PyEval_ReleaseThread(PyThreadState *tstate) if (new_tstate != tstate) { Py_FatalError("wrong thread state"); } - struct _ceval_state *ceval = &tstate->interp->ceval; - drop_gil(ceval, tstate); + drop_gil(tstate->interp, tstate); } #ifdef HAVE_FORK @@ -691,7 +623,7 @@ _PyEval_ReInitThreads(PyThreadState *tstate) void _PyEval_SignalAsyncExc(PyInterpreterState *interp) { - SIGNAL_ASYNC_EXC(interp); + _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 1); } PyThreadState * @@ -700,9 +632,8 @@ PyEval_SaveThread(void) PyThreadState *tstate = _PyThreadState_SwapNoGIL(NULL); _Py_EnsureTstateNotNULL(tstate); - struct _ceval_state *ceval = &tstate->interp->ceval; - assert(gil_created(ceval->gil)); - drop_gil(ceval, tstate); + assert(gil_created(tstate->interp->ceval.gil)); + drop_gil(tstate->interp, tstate); return tstate; } @@ -742,22 +673,9 @@ PyEval_RestoreThread(PyThreadState *tstate) void _PyEval_SignalReceived(PyInterpreterState *interp) { -#ifdef MS_WINDOWS - // bpo-42296: On Windows, _PyEval_SignalReceived() is called from a signal - // handler which can run in a thread different than the Python thread, in - // which case _Py_ThreadCanHandleSignals() is wrong. Ignore - // _Py_ThreadCanHandleSignals() and always set eval_breaker to 1. - // - // The next eval_frame_handle_pending() call will call - // _Py_ThreadCanHandleSignals() to recompute eval_breaker. - int force = 1; -#else - int force = 0; -#endif - /* bpo-30703: Function called when the C signal handler of Python gets a - signal. We cannot queue a callback using _PyEval_AddPendingCall() since - that function is not async-signal-safe. */ - SIGNAL_PENDING_SIGNALS(interp, force); + if (_Py_ThreadCanHandleSignals(interp)) { + _Py_set_eval_breaker_bit(interp, _PY_SIGNALS_PENDING_BIT, 1); + } } /* Push one item onto the queue while holding the lock. */ @@ -773,6 +691,8 @@ _push_pending_call(struct _pending_calls *pending, pending->calls[i].func = func; pending->calls[i].arg = arg; pending->last = j; + assert(pending->calls_to_do < NPENDINGCALLS); + pending->calls_to_do++; return 0; } @@ -800,6 +720,8 @@ _pop_pending_call(struct _pending_calls *pending, if (i >= 0) { pending->calls[i] = (struct _pending_call){0}; pending->first = (i + 1) % NPENDINGCALLS; + assert(pending->calls_to_do > 0); + pending->calls_to_do--; } } @@ -829,7 +751,7 @@ _PyEval_AddPendingCall(PyInterpreterState *interp, PyThread_release_lock(pending->lock); /* signal main loop */ - SIGNAL_PENDING_CALLS(pending, interp); + SIGNAL_PENDING_CALLS(interp); return result; } @@ -846,33 +768,18 @@ static int handle_signals(PyThreadState *tstate) { assert(_PyThreadState_CheckConsistency(tstate)); + _Py_set_eval_breaker_bit(tstate->interp, _PY_SIGNALS_PENDING_BIT, 0); if (!_Py_ThreadCanHandleSignals(tstate->interp)) { return 0; } - - UNSIGNAL_PENDING_SIGNALS(tstate->interp); if (_PyErr_CheckSignalsTstate(tstate) < 0) { /* On failure, re-schedule a call to handle_signals(). */ - SIGNAL_PENDING_SIGNALS(tstate->interp, 0); + _Py_set_eval_breaker_bit(tstate->interp, _PY_SIGNALS_PENDING_BIT, 1); return -1; } return 0; } -static inline int -maybe_has_pending_calls(PyInterpreterState *interp) -{ - struct _pending_calls *pending = &interp->ceval.pending; - if (_Py_atomic_load_relaxed_int32(&pending->calls_to_do)) { - return 1; - } - if (!_Py_IsMainThread() || !_Py_IsMainInterpreter(interp)) { - return 0; - } - pending = &_PyRuntime.ceval.pending_mainthread; - return _Py_atomic_load_relaxed_int32(&pending->calls_to_do); -} - static int _make_pending_calls(struct _pending_calls *pending) { @@ -930,7 +837,7 @@ make_pending_calls(PyInterpreterState *interp) if (_make_pending_calls(pending) != 0) { pending->busy = 0; /* There might not be more calls to make, but we play it safe. */ - SIGNAL_PENDING_CALLS(pending, interp); + SIGNAL_PENDING_CALLS(interp); return -1; } @@ -938,7 +845,7 @@ make_pending_calls(PyInterpreterState *interp) if (_make_pending_calls(pending_main) != 0) { pending->busy = 0; /* There might not be more calls to make, but we play it safe. */ - SIGNAL_PENDING_CALLS(pending_main, interp); + SIGNAL_PENDING_CALLS(interp); return -1; } } @@ -1083,38 +990,35 @@ _PyEval_FiniState(struct _ceval_state *ceval) int _Py_HandlePending(PyThreadState *tstate) { - _PyRuntimeState * const runtime = &_PyRuntime; - struct _ceval_runtime_state *ceval = &runtime->ceval; - struct _ceval_state *interp_ceval_state = &tstate->interp->ceval; + PyInterpreterState *interp = tstate->interp; /* Pending signals */ - if (_Py_atomic_load_relaxed_int32(&ceval->signals_pending)) { + if (_Py_eval_breaker_bit_is_set(interp, _PY_SIGNALS_PENDING_BIT)) { if (handle_signals(tstate) != 0) { return -1; } } /* Pending calls */ - if (maybe_has_pending_calls(tstate->interp)) { - if (make_pending_calls(tstate->interp) != 0) { + if (_Py_eval_breaker_bit_is_set(interp, _PY_CALLS_TO_DO_BIT)) { + if (make_pending_calls(interp) != 0) { return -1; } } /* GC scheduled to run */ - if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->gc_scheduled)) { - _Py_atomic_store_relaxed(&interp_ceval_state->gc_scheduled, 0); - COMPUTE_EVAL_BREAKER(tstate->interp, ceval, interp_ceval_state); + if (_Py_eval_breaker_bit_is_set(interp, _PY_GC_SCHEDULED_BIT)) { + _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 0); _Py_RunGC(tstate); } /* GIL drop request */ - if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->gil_drop_request)) { + if (_Py_eval_breaker_bit_is_set(interp, _PY_GIL_DROP_REQUEST_BIT)) { /* Give another thread a chance */ if (_PyThreadState_SwapNoGIL(NULL) != tstate) { Py_FatalError("tstate mix-up"); } - drop_gil(interp_ceval_state, tstate); + drop_gil(interp, tstate); /* Other threads may run now */ @@ -1126,27 +1030,16 @@ _Py_HandlePending(PyThreadState *tstate) } /* Check for asynchronous exception. */ - if (tstate->async_exc != NULL) { - PyObject *exc = tstate->async_exc; - tstate->async_exc = NULL; - UNSIGNAL_ASYNC_EXC(tstate->interp); - _PyErr_SetNone(tstate, exc); - Py_DECREF(exc); - return -1; + if (_Py_eval_breaker_bit_is_set(interp, _PY_ASYNC_EXCEPTION_BIT)) { + _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 0); + if (tstate->async_exc != NULL) { + PyObject *exc = tstate->async_exc; + tstate->async_exc = NULL; + _PyErr_SetNone(tstate, exc); + Py_DECREF(exc); + return -1; + } } - - - // It is possible that some of the conditions that trigger the eval breaker - // are called in a different thread than the Python thread. An example of - // this is bpo-42296: On Windows, _PyEval_SignalReceived() can be called in - // a different thread than the Python thread, in which case - // _Py_ThreadCanHandleSignals() is wrong. Recompute eval_breaker in the - // current Python thread with the correct _Py_ThreadCanHandleSignals() - // value. It prevents to interrupt the eval loop at every instruction if - // the current Python thread cannot handle signals (if - // _Py_ThreadCanHandleSignals() is false). - COMPUTE_EVAL_BREAKER(tstate->interp, ceval, interp_ceval_state); - return 0; } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 012750df387c1c..872e0a2b7f92ca 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -116,7 +116,7 @@ #define CHECK_EVAL_BREAKER() \ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \ - if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker)) { \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & _PY_EVAL_EVENTS_MASK) { \ if (_Py_HandlePending(tstate) != 0) { \ goto error; \ } \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b3ccf8d4363661..c6eeee4925ef1f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -12,9 +12,10 @@ DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif - /* Possibly combine these two checks */ - DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version, RESUME); - DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 51e3937e55a93c..221d4774beee7f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -12,7 +12,12 @@ static_assert(0 == 0, "incorrect cache size"); TIER_ONE_ONLY assert(frame == tstate->current_frame); - if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((code_version & 255) == 0); + if (code_version != global_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); if (err) goto error; next_instr--; @@ -31,18 +36,17 @@ DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif - /* Possibly combine these two checks */ - DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version, RESUME); - DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); DISPATCH(); } TARGET(INSTRUMENTED_RESUME) { - /* Possible performance enhancement: - * We need to check the eval breaker anyway, can we - * combine the instrument verison check and the eval breaker test? - */ - if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { goto error; } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 0b974f6133ce7d..eee1908e503e43 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -4,6 +4,7 @@ #include "pycore_bitutils.h" // _Py_popcount32 #include "pycore_call.h" +#include "pycore_ceval.h" // _PY_EVAL_EVENTS_BITS #include "pycore_code.h" // _PyCode_Clear_Executors() #include "pycore_frame.h" #include "pycore_interp.h" @@ -895,10 +896,27 @@ static inline int most_significant_bit(uint8_t bits) { return MOST_SIGNIFICANT_BITS[bits]; } +static uint32_t +global_version(PyInterpreterState *interp) +{ + return interp->ceval.eval_breaker & ~_PY_EVAL_EVENTS_MASK; +} + +static void +set_global_version(PyInterpreterState *interp, uint32_t version) +{ + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + uintptr_t old = _Py_atomic_load_uintptr(&interp->ceval.eval_breaker); + intptr_t new; + do { + new = (old & _PY_EVAL_EVENTS_MASK) | version; + } while (!_Py_atomic_compare_exchange_uintptr(&interp->ceval.eval_breaker, &old, new)); +} + static bool is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) { - return interp->monitoring_version == code->_co_instrumentation_version; + return global_version(interp) == code->_co_instrumentation_version; } #ifndef NDEBUG @@ -1556,7 +1574,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { if (is_version_up_to_date(code, interp)) { assert( - interp->monitoring_version == 0 || + (interp->ceval.eval_breaker & ~_PY_EVAL_EVENTS_MASK) == 0 || instrumentation_cross_checks(interp, code) ); return 0; @@ -1594,7 +1612,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) assert(monitors_are_empty(monitors_and(new_events, removed_events))); } code->_co_monitoring->active_monitors = active_events; - code->_co_instrumentation_version = interp->monitoring_version; + code->_co_instrumentation_version = global_version(interp); if (monitors_are_empty(new_events) && monitors_are_empty(removed_events)) { #ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); @@ -1761,6 +1779,10 @@ check_tool(PyInterpreterState *interp, int tool_id) return 0; } +/* We share the eval-breaker with flags, so the monitoring + * version goes in the top 24 bits */ +#define MONITORING_VERSION_INCREMENT (1 << _PY_EVAL_EVENTS_BITS) + int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { @@ -1775,7 +1797,12 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) return 0; } set_events(&interp->monitors, tool_id, events); - interp->monitoring_version++; + uint32_t new_version = global_version(interp) + MONITORING_VERSION_INCREMENT; + if (new_version == 0) { + PyErr_Format(PyExc_OverflowError, "events set too many times"); + return -1; + } + set_global_version(interp, new_version); return instrument_all_executing_code_objects(interp); } @@ -1803,7 +1830,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent set_local_events(local, tool_id, events); if (is_version_up_to_date(code, interp)) { /* Force instrumentation update */ - code->_co_instrumentation_version = UINT64_MAX; + code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT; } if (_Py_Instrument(code, interp)) { return -1; @@ -2086,8 +2113,14 @@ monitoring_restart_events_impl(PyObject *module) * last restart version < current version */ PyInterpreterState *interp = _PyInterpreterState_GET(); - interp->last_restart_version = interp->monitoring_version + 1; - interp->monitoring_version = interp->last_restart_version + 1; + uint32_t restart_version = global_version(interp) + MONITORING_VERSION_INCREMENT; + uint32_t new_version = restart_version + MONITORING_VERSION_INCREMENT; + if (new_version <= MONITORING_VERSION_INCREMENT) { + PyErr_Format(PyExc_OverflowError, "events set too many times"); + return NULL; + } + interp->last_restart_version = restart_version; + set_global_version(interp, new_version); if (instrument_all_executing_code_objects(interp)) { return NULL; } From 9561648f4a5d8486b67ee4bbe24a239b2a93212c Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Thu, 5 Oct 2023 00:44:17 +0800 Subject: [PATCH 30/73] gh-110235: Raise TypeError for duplicate/unknown fields in PyStructSequence constructor (GH-110258) --- Lib/test/test_structseq.py | 60 +++++++++++++++++++ ...-10-03-06-19-10.gh-issue-110235.uec5AG.rst | 2 + Objects/structseq.c | 29 ++++++--- 3 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-10-03-06-19-10.gh-issue-110235.uec5AG.rst diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index c6c0afaf077acc..2ef1316e08fb8b 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -1,6 +1,7 @@ import copy import os import pickle +import re import time import unittest @@ -91,10 +92,69 @@ def test_constructor(self): self.assertRaises(TypeError, t, "123") self.assertRaises(TypeError, t, "123", dict={}) self.assertRaises(TypeError, t, "123456789", dict=None) + self.assertRaises(TypeError, t, seq="123456789", dict={}) + + self.assertEqual(t("123456789"), tuple("123456789")) + self.assertEqual(t("123456789", {}), tuple("123456789")) + self.assertEqual(t("123456789", dict={}), tuple("123456789")) + self.assertEqual(t(sequence="123456789", dict={}), tuple("123456789")) + + self.assertEqual(t("1234567890"), tuple("123456789")) + self.assertEqual(t("1234567890").tm_zone, "0") + self.assertEqual(t("123456789", {"tm_zone": "some zone"}), tuple("123456789")) + self.assertEqual(t("123456789", {"tm_zone": "some zone"}).tm_zone, "some zone") s = "123456789" self.assertEqual("".join(t(s)), s) + def test_constructor_with_duplicate_fields(self): + t = time.struct_time + + error_message = re.escape("got duplicate or unexpected field name(s)") + with self.assertRaisesRegex(TypeError, error_message): + t("1234567890", dict={"tm_zone": "some zone"}) + with self.assertRaisesRegex(TypeError, error_message): + t("1234567890", dict={"tm_zone": "some zone", "tm_mon": 1}) + with self.assertRaisesRegex(TypeError, error_message): + t("1234567890", dict={"error": 0, "tm_zone": "some zone"}) + with self.assertRaisesRegex(TypeError, error_message): + t("1234567890", dict={"error": 0, "tm_zone": "some zone", "tm_mon": 1}) + + def test_constructor_with_duplicate_unnamed_fields(self): + assert os.stat_result.n_unnamed_fields > 0 + n_visible_fields = os.stat_result.n_sequence_fields + + r = os.stat_result(range(n_visible_fields), {'st_atime': -1.0}) + self.assertEqual(r.st_atime, -1.0) + self.assertEqual(r, tuple(range(n_visible_fields))) + + r = os.stat_result((*range(n_visible_fields), -1.0)) + self.assertEqual(r.st_atime, -1.0) + self.assertEqual(r, tuple(range(n_visible_fields))) + + with self.assertRaisesRegex(TypeError, + re.escape("got duplicate or unexpected field name(s)")): + os.stat_result((*range(n_visible_fields), -1.0), {'st_atime': -1.0}) + + def test_constructor_with_unknown_fields(self): + t = time.struct_time + + error_message = re.escape("got duplicate or unexpected field name(s)") + with self.assertRaisesRegex(TypeError, error_message): + t("123456789", dict={"tm_year": 0}) + with self.assertRaisesRegex(TypeError, error_message): + t("123456789", dict={"tm_year": 0, "tm_mon": 1}) + with self.assertRaisesRegex(TypeError, error_message): + t("123456789", dict={"tm_zone": "some zone", "tm_mon": 1}) + with self.assertRaisesRegex(TypeError, error_message): + t("123456789", dict={"tm_zone": "some zone", "error": 0}) + with self.assertRaisesRegex(TypeError, error_message): + t("123456789", dict={"error": 0, "tm_zone": "some zone", "tm_mon": 1}) + with self.assertRaisesRegex(TypeError, error_message): + t("123456789", dict={"error": 0}) + with self.assertRaisesRegex(TypeError, error_message): + t("123456789", dict={"tm_zone": "some zone", "error": 0}) + def test_eviltuple(self): class Exc(Exception): pass diff --git a/Misc/NEWS.d/next/C API/2023-10-03-06-19-10.gh-issue-110235.uec5AG.rst b/Misc/NEWS.d/next/C API/2023-10-03-06-19-10.gh-issue-110235.uec5AG.rst new file mode 100644 index 00000000000000..ff26f25fe71d61 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-10-03-06-19-10.gh-issue-110235.uec5AG.rst @@ -0,0 +1,2 @@ +Raise :exc:`TypeError` for duplicate/unknown fields in ``PyStructSequence`` constructor. +Patched by Xuehai Pan. diff --git a/Objects/structseq.c b/Objects/structseq.c index 0ca622edc2ba37..2c98288039c58c 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -216,19 +216,34 @@ structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict) res->ob_item[i] = Py_NewRef(v); } Py_DECREF(arg); - for (; i < max_len; ++i) { - PyObject *ob = NULL; - if (dict != NULL) { - const char *name = type->tp_members[i-n_unnamed_fields].name; + if (dict != NULL && PyDict_GET_SIZE(dict) > 0) { + Py_ssize_t n_found_keys = 0; + for (i = len; i < max_len; ++i) { + PyObject *ob = NULL; + const char *name = type->tp_members[i - n_unnamed_fields].name; if (PyDict_GetItemStringRef(dict, name, &ob) < 0) { Py_DECREF(res); return NULL; } + if (ob == NULL) { + ob = Py_NewRef(Py_None); + } + else { + ++n_found_keys; + } + res->ob_item[i] = ob; + } + if (PyDict_GET_SIZE(dict) > n_found_keys) { + PyErr_Format(PyExc_TypeError, + "%.500s() got duplicate or unexpected field name(s)", + type->tp_name); + Py_DECREF(res); + return NULL; } - if (ob == NULL) { - ob = Py_NewRef(Py_None); + } else { + for (i = len; i < max_len; ++i) { + res->ob_item[i] = Py_NewRef(Py_None); } - res->ob_item[i] = ob; } _PyObject_GC_TRACK(res); From 3bbe3b7c822091caac90c00ee937848bc4de80eb Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Thu, 5 Oct 2023 00:47:41 +0800 Subject: [PATCH 31/73] gh-110222: Add support of PyStructSequence in copy.replace() (GH-110223) --- Lib/test/test_structseq.py | 78 +++++++++++++++++++ ...-10-02-15-07-28.gh-issue-110222.zl_oHh.rst | 2 + Objects/structseq.c | 76 +++++++++++++++++- 3 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-02-15-07-28.gh-issue-110222.zl_oHh.rst diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index 2ef1316e08fb8b..6aec63e2603412 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -264,6 +264,84 @@ def test_match_args_with_unnamed_fields(self): self.assertEqual(os.stat_result.n_unnamed_fields, 3) self.assertEqual(os.stat_result.__match_args__, expected_args) + def test_copy_replace_all_fields_visible(self): + assert os.times_result.n_unnamed_fields == 0 + assert os.times_result.n_sequence_fields == os.times_result.n_fields + + t = os.times() + + # visible fields + self.assertEqual(copy.replace(t), t) + self.assertIsInstance(copy.replace(t), os.times_result) + self.assertEqual(copy.replace(t, user=1.5), (1.5, *t[1:])) + self.assertEqual(copy.replace(t, system=2.5), (t[0], 2.5, *t[2:])) + self.assertEqual(copy.replace(t, user=1.5, system=2.5), (1.5, 2.5, *t[2:])) + + # unknown fields + with self.assertRaisesRegex(TypeError, 'unexpected field name'): + copy.replace(t, error=-1) + with self.assertRaisesRegex(TypeError, 'unexpected field name'): + copy.replace(t, user=1, error=-1) + + def test_copy_replace_with_invisible_fields(self): + assert time.struct_time.n_unnamed_fields == 0 + assert time.struct_time.n_sequence_fields < time.struct_time.n_fields + + t = time.gmtime(0) + + # visible fields + t2 = copy.replace(t) + self.assertEqual(t2, (1970, 1, 1, 0, 0, 0, 3, 1, 0)) + self.assertIsInstance(t2, time.struct_time) + t3 = copy.replace(t, tm_year=2000) + self.assertEqual(t3, (2000, 1, 1, 0, 0, 0, 3, 1, 0)) + self.assertEqual(t3.tm_year, 2000) + t4 = copy.replace(t, tm_mon=2) + self.assertEqual(t4, (1970, 2, 1, 0, 0, 0, 3, 1, 0)) + self.assertEqual(t4.tm_mon, 2) + t5 = copy.replace(t, tm_year=2000, tm_mon=2) + self.assertEqual(t5, (2000, 2, 1, 0, 0, 0, 3, 1, 0)) + self.assertEqual(t5.tm_year, 2000) + self.assertEqual(t5.tm_mon, 2) + + # named invisible fields + self.assertTrue(hasattr(t, 'tm_zone'), f"{t} has no attribute 'tm_zone'") + with self.assertRaisesRegex(AttributeError, 'readonly attribute'): + t.tm_zone = 'some other zone' + self.assertEqual(t2.tm_zone, t.tm_zone) + self.assertEqual(t3.tm_zone, t.tm_zone) + self.assertEqual(t4.tm_zone, t.tm_zone) + t6 = copy.replace(t, tm_zone='some other zone') + self.assertEqual(t, t6) + self.assertEqual(t6.tm_zone, 'some other zone') + t7 = copy.replace(t, tm_year=2000, tm_zone='some other zone') + self.assertEqual(t7, (2000, 1, 1, 0, 0, 0, 3, 1, 0)) + self.assertEqual(t7.tm_year, 2000) + self.assertEqual(t7.tm_zone, 'some other zone') + + # unknown fields + with self.assertRaisesRegex(TypeError, 'unexpected field name'): + copy.replace(t, error=2) + with self.assertRaisesRegex(TypeError, 'unexpected field name'): + copy.replace(t, tm_year=2000, error=2) + with self.assertRaisesRegex(TypeError, 'unexpected field name'): + copy.replace(t, tm_zone='some other zone', error=2) + + def test_copy_replace_with_unnamed_fields(self): + assert os.stat_result.n_unnamed_fields > 0 + + r = os.stat_result(range(os.stat_result.n_sequence_fields)) + + error_message = re.escape('__replace__() is not supported') + with self.assertRaisesRegex(TypeError, error_message): + copy.replace(r) + with self.assertRaisesRegex(TypeError, error_message): + copy.replace(r, st_mode=1) + with self.assertRaisesRegex(TypeError, error_message): + copy.replace(r, error=2) + with self.assertRaisesRegex(TypeError, error_message): + copy.replace(r, st_mode=1, error=2) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-10-02-15-07-28.gh-issue-110222.zl_oHh.rst b/Misc/NEWS.d/next/Library/2023-10-02-15-07-28.gh-issue-110222.zl_oHh.rst new file mode 100644 index 00000000000000..fd2ecdf6269cf3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-02-15-07-28.gh-issue-110222.zl_oHh.rst @@ -0,0 +1,2 @@ +Add support of struct sequence objects in :func:`copy.replace`. +Patched by Xuehai Pan. diff --git a/Objects/structseq.c b/Objects/structseq.c index 2c98288039c58c..e4a4b45a8db626 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -8,6 +8,7 @@ */ #include "Python.h" +#include "pycore_dict.h" // _PyDict_Pop() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -380,9 +381,82 @@ structseq_reduce(PyStructSequence* self, PyObject *Py_UNUSED(ignored)) return NULL; } + +static PyObject * +structseq_replace(PyStructSequence *self, PyObject *args, PyObject *kwargs) +{ + PyStructSequence *result = NULL; + Py_ssize_t n_fields, n_unnamed_fields, i; + + if (!_PyArg_NoPositional("__replace__", args)) { + return NULL; + } + + n_fields = REAL_SIZE(self); + if (n_fields < 0) { + return NULL; + } + n_unnamed_fields = UNNAMED_FIELDS(self); + if (n_unnamed_fields < 0) { + return NULL; + } + if (n_unnamed_fields > 0) { + PyErr_Format(PyExc_TypeError, + "__replace__() is not supported for %.500s " + "because it has unnamed field(s)", + Py_TYPE(self)->tp_name); + return NULL; + } + + result = (PyStructSequence *) PyStructSequence_New(Py_TYPE(self)); + if (!result) { + return NULL; + } + + if (kwargs != NULL) { + // We do not support types with unnamed fields, so we can iterate over + // i >= n_visible_fields case without slicing with (i - n_unnamed_fields). + for (i = 0; i < n_fields; ++i) { + PyObject *key = PyUnicode_FromString(Py_TYPE(self)->tp_members[i].name); + if (!key) { + goto error; + } + PyObject *ob = _PyDict_Pop(kwargs, key, self->ob_item[i]); + Py_DECREF(key); + if (!ob) { + goto error; + } + result->ob_item[i] = ob; + } + // Check if there are any unexpected fields. + if (PyDict_GET_SIZE(kwargs) > 0) { + PyObject *names = PyDict_Keys(kwargs); + if (names) { + PyErr_Format(PyExc_TypeError, "Got unexpected field name(s): %R", names); + Py_DECREF(names); + } + goto error; + } + } + else + { + // Just create a copy of the original. + for (i = 0; i < n_fields; ++i) { + result->ob_item[i] = Py_NewRef(self->ob_item[i]); + } + } + + return (PyObject *)result; + +error: + Py_DECREF(result); + return NULL; +} + static PyMethodDef structseq_methods[] = { {"__reduce__", (PyCFunction)structseq_reduce, METH_NOARGS, NULL}, - {NULL, NULL} + {"__replace__", _PyCFunction_CAST(structseq_replace), METH_VARARGS | METH_KEYWORDS, NULL}, + {NULL, NULL} // sentinel }; static Py_ssize_t From ee317f7dddd71f2093f9e4b8431f84cbd636a8a2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Oct 2023 11:57:03 -0600 Subject: [PATCH 32/73] gh-76785: Print the Traceback from Interpreter.run() (gh-110322) This is a temporary solution. The full fix may involve serializing the traceback in some form. (FYI, I merged this yesterday and the reverted it due to buildbot failures. See gh-110248.) --- Modules/_xxsubinterpretersmodule.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 700282efb8c619..bca16ac8a62eca 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -450,7 +450,13 @@ _run_script(PyInterpreterState *interp, const char *codestr, "RunFailedError: script raised an uncaught exception (%s)", failure); } - Py_XDECREF(excval); + if (excval != NULL) { + // XXX Instead, store the rendered traceback on sharedexc, + // attach it to the exception when applied, + // and teach PyErr_Display() to print it. + PyErr_Display(NULL, excval, NULL); + Py_DECREF(excval); + } if (errcode != ERR_ALREADY_RUNNING) { _PyInterpreterState_SetNotRunningMain(interp); } From f7860295b16a402621e209871c8eaeeea16f464e Mon Sep 17 00:00:00 2001 From: "P. L. Lim" <2090236+pllim@users.noreply.github.com> Date: Wed, 4 Oct 2023 15:56:11 -0400 Subject: [PATCH 33/73] Add back bltin-boolean-values ref tag (#110371) To avoid breaking downstream intersphinx via numpydoc --- Doc/library/stdtypes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index a351559a84f1ce..f45fd561d2bad3 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -804,6 +804,7 @@ number, :class:`float`, or :class:`complex`:: hash_value = -2 return hash_value +.. _bltin-boolean-values: .. _typebool: Boolean Type - :class:`bool` From e561e9805854980a61967d07869b4ec4205b32c8 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 4 Oct 2023 17:52:28 -0400 Subject: [PATCH 34/73] GH-109329: Add tier 2 stats (GH-109913) --- Include/cpython/pystats.h | 28 +- Include/internal/pycore_code.h | 15 + Python/bytecodes.c | 3 +- Python/executor.c | 25 +- Python/executor_cases.c.h | 1 + Python/generated_cases.c.h | 2 +- Python/optimizer.c | 13 +- Python/specialize.c | 54 +++- Tools/scripts/summarize_stats.py | 470 +++++++++++++++++++++++-------- 9 files changed, 483 insertions(+), 128 deletions(-) diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 150e16faa96ca1..056406e6b5d992 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -86,10 +86,6 @@ typedef struct _object_stats { uint64_t type_cache_dunder_hits; uint64_t type_cache_dunder_misses; uint64_t type_cache_collisions; - uint64_t optimization_attempts; - uint64_t optimization_traces_created; - uint64_t optimization_traces_executed; - uint64_t optimization_uops_executed; /* Temporary value used during GC */ uint64_t object_visits; } ObjectStats; @@ -100,10 +96,34 @@ typedef struct _gc_stats { uint64_t objects_collected; } GCStats; +typedef struct _uop_stats { + uint64_t execution_count; +} UOpStats; + +#define _Py_UOP_HIST_SIZE 32 + +typedef struct _optimization_stats { + uint64_t attempts; + uint64_t traces_created; + uint64_t traces_executed; + uint64_t uops_executed; + uint64_t trace_stack_overflow; + uint64_t trace_stack_underflow; + uint64_t trace_too_long; + uint64_t inner_loop; + uint64_t recursive_call; + UOpStats opcode[512]; + uint64_t unsupported_opcode[256]; + uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; + uint64_t trace_run_length_hist[_Py_UOP_HIST_SIZE]; + uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; +} OptimizationStats; + typedef struct _stats { OpcodeStats opcode_stats[256]; CallStats call_stats; ObjectStats object_stats; + OptimizationStats optimization_stats; GCStats *gc_stats; } PyStats; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index a77fa11baf8413..d31d8363d771ca 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -282,6 +282,17 @@ extern int _PyStaticCode_Init(PyCodeObject *co); #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \ do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0) #define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) +#define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) +#define UOP_EXE_INC(opname) do { if (_Py_stats) _Py_stats->optimization_stats.opcode[opname].execution_count++; } while (0) +#define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0) +#define OPT_HIST(length, name) \ + do { \ + if (_Py_stats) { \ + int bucket = _Py_bit_length(length >= 1 ? length - 1 : 0); \ + bucket = (bucket >= _Py_UOP_HIST_SIZE) ? _Py_UOP_HIST_SIZE - 1 : bucket; \ + _Py_stats->optimization_stats.name[bucket]++; \ + } \ + } while (0) // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -296,6 +307,10 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define EVAL_CALL_STAT_INC(name) ((void)0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) ((void)0) #define GC_STAT_ADD(gen, name, n) ((void)0) +#define OPT_STAT_INC(name) ((void)0) +#define UOP_EXE_INC(opname) ((void)0) +#define OPT_UNSUPPORTED_OPCODE(opname) ((void)0) +#define OPT_HIST(length, name) ((void)0) #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6482252eec9fb9..9b733ce4a8c14b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2278,7 +2278,7 @@ dummy_func( // Double-check that the opcode isn't instrumented or something: here->op.code == JUMP_BACKWARD) { - OBJECT_STAT_INC(optimization_attempts); + OPT_STAT_INC(attempts); int optimized = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); ERROR_IF(optimized < 0, error); if (optimized) { @@ -3965,6 +3965,7 @@ dummy_func( frame->prev_instr--; // Back up to just before destination _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); return frame; } diff --git a/Python/executor.c b/Python/executor.c index ac9104223da8ff..6da3af08822fd0 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -2,6 +2,7 @@ #include "opcode.h" +#include "pycore_bitutils.h" #include "pycore_call.h" #include "pycore_ceval.h" #include "pycore_dict.h" @@ -30,6 +31,16 @@ goto deoptimize; \ } +#ifdef Py_STATS +// Disable these macros that apply to Tier 1 stats when we are in Tier 2 +#undef STAT_INC +#define STAT_INC(opname, name) ((void)0) +#undef STAT_DEC +#define STAT_DEC(opname, name) ((void)0) +#undef CALL_STAT_INC +#define CALL_STAT_INC(name) ((void)0) +#endif + #undef ENABLE_SPECIALIZATION #define ENABLE_SPECIALIZATION 0 @@ -62,12 +73,15 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject CHECK_EVAL_BREAKER(); - OBJECT_STAT_INC(optimization_traces_executed); + OPT_STAT_INC(traces_executed); _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; int pc = 0; int opcode; int oparg; uint64_t operand; +#ifdef Py_STATS + uint64_t trace_uop_execution_counter = 0; +#endif for (;;) { opcode = self->trace[pc].opcode; @@ -81,7 +95,12 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject operand, (int)(stack_pointer - _PyFrame_Stackbase(frame))); pc++; - OBJECT_STAT_INC(optimization_uops_executed); + OPT_STAT_INC(uops_executed); + UOP_EXE_INC(opcode); +#ifdef Py_STATS + trace_uop_execution_counter++; +#endif + switch (opcode) { #include "executor_cases.c.h" @@ -114,6 +133,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject // On ERROR_IF we return NULL as the frame. // The caller recovers the frame from tstate->current_frame. DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); return NULL; @@ -122,6 +142,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject // On DEOPT_IF we just repeat the last instruction. // This presumes nothing was popped from the stack (nor pushed). DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); frame->prev_instr--; // Back up to just before destination _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c6eeee4925ef1f..e2f4f9805b79fa 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3289,6 +3289,7 @@ frame->prev_instr--; // Back up to just before destination _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); return frame; break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 221d4774beee7f..eac136846b169f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2988,7 +2988,7 @@ // Double-check that the opcode isn't instrumented or something: here->op.code == JUMP_BACKWARD) { - OBJECT_STAT_INC(optimization_attempts); + OPT_STAT_INC(attempts); int optimized = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); if (optimized < 0) goto error; if (optimized) { diff --git a/Python/optimizer.c b/Python/optimizer.c index fbdbf7291784c4..f8796eb24073d2 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -461,6 +461,7 @@ translate_bytecode_to_trace( if (trace_length + (n) > max_length) { \ DPRINTF(2, "No room for %s (need %d, got %d)\n", \ (opname), (n), max_length - trace_length); \ + OPT_STAT_INC(trace_too_long); \ goto done; \ } \ reserved = (n); // Keep ADD_TO_TRACE / ADD_TO_STUB honest @@ -472,6 +473,7 @@ translate_bytecode_to_trace( #define TRACE_STACK_PUSH() \ if (trace_stack_depth >= TRACE_STACK_SIZE) { \ DPRINTF(2, "Trace stack overflow\n"); \ + OPT_STAT_INC(trace_stack_overflow); \ ADD_TO_TRACE(_SET_IP, 0, 0); \ goto done; \ } \ @@ -572,6 +574,7 @@ translate_bytecode_to_trace( ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0); } else { + OPT_STAT_INC(inner_loop); DPRINTF(2, "JUMP_BACKWARD not to top ends trace\n"); } goto done; @@ -638,7 +641,9 @@ translate_bytecode_to_trace( // LOAD_CONST + _POP_FRAME. if (trace_stack_depth == 0) { DPRINTF(2, "Trace stack underflow\n"); - goto done;} + OPT_STAT_INC(trace_stack_underflow); + goto done; + } } uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM for (int i = 0; i < nuops; i++) { @@ -713,6 +718,7 @@ translate_bytecode_to_trace( PyUnicode_AsUTF8(new_code->co_qualname), PyUnicode_AsUTF8(new_code->co_filename), new_code->co_firstlineno); + OPT_STAT_INC(recursive_call); ADD_TO_TRACE(_SET_IP, 0, 0); goto done; } @@ -744,6 +750,7 @@ translate_bytecode_to_trace( break; } DPRINTF(2, "Unsupported opcode %s\n", uop_name(opcode)); + OPT_UNSUPPORTED_OPCODE(opcode); goto done; // Break out of loop } // End default @@ -891,7 +898,8 @@ uop_optimize( // Error or nothing translated return trace_length; } - OBJECT_STAT_INC(optimization_traces_created); + OPT_HIST(trace_length, trace_length_hist); + OPT_STAT_INC(traces_created); char *uop_optimize = Py_GETENV("PYTHONUOPSOPTIMIZE"); if (uop_optimize != NULL && *uop_optimize > '0') { trace_length = _Py_uop_analyze_and_optimize(code, trace, trace_length, curr_stackentries); @@ -901,6 +909,7 @@ uop_optimize( if (executor == NULL) { return -1; } + OPT_HIST(trace_length, optimized_trace_length_hist); executor->base.execute = _PyUopExecute; memcpy(executor->trace, trace, trace_length * sizeof(_PyUOpInstruction)); *exec_ptr = (_PyExecutorObject *)executor; diff --git a/Python/specialize.c b/Python/specialize.c index d9b748cad78f4f..ff732eb30ca1e0 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -199,10 +199,6 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits); fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses); - fprintf(out, "Optimization attempts: %" PRIu64 "\n", stats->optimization_attempts); - fprintf(out, "Optimization traces created: %" PRIu64 "\n", stats->optimization_traces_created); - fprintf(out, "Optimization traces executed: %" PRIu64 "\n", stats->optimization_traces_executed); - fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->optimization_uops_executed); } static void @@ -215,6 +211,55 @@ print_gc_stats(FILE *out, GCStats *stats) } } +static void +print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE]) +{ + for (int i = 0; i < _Py_UOP_HIST_SIZE; i++) { + fprintf(out, "%s[%" PRIu64"]: %" PRIu64 "\n", name, (uint64_t)1 << i, hist[i]); + } +} + +static void +print_optimization_stats(FILE *out, OptimizationStats *stats) +{ + fprintf(out, "Optimization attempts: %" PRIu64 "\n", stats->attempts); + fprintf(out, "Optimization traces created: %" PRIu64 "\n", stats->traces_created); + fprintf(out, "Optimization traces executed: %" PRIu64 "\n", stats->traces_executed); + fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->uops_executed); + fprintf(out, "Optimization trace stack overflow: %" PRIu64 "\n", stats->trace_stack_overflow); + fprintf(out, "Optimization trace stack underflow: %" PRIu64 "\n", stats->trace_stack_underflow); + fprintf(out, "Optimization trace too long: %" PRIu64 "\n", stats->trace_too_long); + fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); + fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); + + print_histogram(out, "Trace length", stats->trace_length_hist); + print_histogram(out, "Trace run length", stats->trace_run_length_hist); + print_histogram(out, "Optimized trace length", stats->optimized_trace_length_hist); + + const char* const* names; + for (int i = 0; i < 512; i++) { + if (i < 256) { + names = _PyOpcode_OpName; + } else { + names = _PyOpcode_uop_name; + } + if (stats->opcode[i].execution_count) { + fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", names[i], stats->opcode[i].execution_count); + } + } + + for (int i = 0; i < 256; i++) { + if (stats->unsupported_opcode[i]) { + fprintf( + out, + "unsupported_opcode[%s].count : %" PRIu64 "\n", + _PyOpcode_OpName[i], + stats->unsupported_opcode[i] + ); + } + } +} + static void print_stats(FILE *out, PyStats *stats) { @@ -222,6 +267,7 @@ print_stats(FILE *out, PyStats *stats) print_call_stats(out, &stats->call_stats); print_object_stats(out, &stats->object_stats); print_gc_stats(out, stats->gc_stats); + print_optimization_stats(out, &stats->optimization_stats); } void diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 3b2bdd8015be4a..b9cc2f94080b88 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -13,6 +13,7 @@ from datetime import date import itertools import sys +import re if os.name == "nt": DEFAULT_DIR = "c:\\temp\\py_stats\\" @@ -21,6 +22,7 @@ TOTAL = "specialization.hit", "specialization.miss", "execution_count" + def format_ratio(num, den): """ Format a ratio as a percentage. When the denominator is 0, returns the empty @@ -31,6 +33,7 @@ def format_ratio(num, den): else: return f"{num/den:.01%}" + def percentage_to_float(s): """ Converts a percentage string to a float. The empty string is returned as 0.0 @@ -41,6 +44,7 @@ def percentage_to_float(s): assert s[-1] == "%" return float(s[:-1]) + def join_rows(a_rows, b_rows): """ Joins two tables together, side-by-side, where the first column in each is a @@ -79,40 +83,53 @@ def join_rows(a_rows, b_rows): keys = list(a_data.keys()) + [k for k in b_data.keys() if k not in a_data] return [(k, *a_data.get(k, default), *b_data.get(k, default)) for k in keys] + def calculate_specialization_stats(family_stats, total): rows = [] for key in sorted(family_stats): if key.startswith("specialization.failure_kinds"): continue if key in ("specialization.hit", "specialization.miss"): - label = key[len("specialization."):] + label = key[len("specialization.") :] elif key == "execution_count": continue - elif key in ("specialization.success", "specialization.failure", "specializable"): + elif key in ( + "specialization.success", + "specialization.failure", + "specializable", + ): continue elif key.startswith("pair"): continue else: label = key - rows.append((f"{label:>12}", f"{family_stats[key]:>12}", format_ratio(family_stats[key], total))) + rows.append( + ( + f"{label:>12}", + f"{family_stats[key]:>12}", + format_ratio(family_stats[key], total), + ) + ) return rows + def calculate_specialization_success_failure(family_stats): total_attempts = 0 - for key in ("specialization.success", "specialization.failure"): + for key in ("specialization.success", "specialization.failure"): total_attempts += family_stats.get(key, 0) rows = [] if total_attempts: - for key in ("specialization.success", "specialization.failure"): - label = key[len("specialization."):] + for key in ("specialization.success", "specialization.failure"): + label = key[len("specialization.") :] label = label[0].upper() + label[1:] val = family_stats.get(key, 0) rows.append((label, val, format_ratio(val, total_attempts))) return rows + def calculate_specialization_failure_kinds(name, family_stats, defines): total_failures = family_stats.get("specialization.failure", 0) - failure_kinds = [ 0 ] * 40 + failure_kinds = [0] * 40 for key in family_stats: if not key.startswith("specialization.failure_kind"): continue @@ -125,9 +142,16 @@ def calculate_specialization_failure_kinds(name, family_stats, defines): for value, index in failures: if not value: continue - rows.append((kind_to_text(index, defines, name), value, format_ratio(value, total_failures))) + rows.append( + ( + kind_to_text(index, defines, name), + value, + format_ratio(value, total_failures), + ) + ) return rows + def print_specialization_stats(name, family_stats, defines): if "specializable" not in family_stats: return @@ -144,7 +168,10 @@ def print_specialization_stats(name, family_stats, defines): rows = calculate_specialization_failure_kinds(name, family_stats, defines) emit_table(("Failure kind", "Count:", "Ratio:"), rows) -def print_comparative_specialization_stats(name, base_family_stats, head_family_stats, defines): + +def print_comparative_specialization_stats( + name, base_family_stats, head_family_stats, defines +): if "specializable" not in base_family_stats: return @@ -157,21 +184,34 @@ def print_comparative_specialization_stats(name, base_family_stats, head_family_ head_rows = calculate_specialization_stats(head_family_stats, head_total) emit_table( ("Kind", "Base Count", "Base Ratio", "Head Count", "Head Ratio"), - join_rows(base_rows, head_rows) + join_rows(base_rows, head_rows), ) base_rows = calculate_specialization_success_failure(base_family_stats) head_rows = calculate_specialization_success_failure(head_family_stats) rows = join_rows(base_rows, head_rows) if rows: print_title("Specialization attempts", 4) - emit_table(("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), rows) - base_rows = calculate_specialization_failure_kinds(name, base_family_stats, defines) - head_rows = calculate_specialization_failure_kinds(name, head_family_stats, defines) emit_table( - ("Failure kind", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), - join_rows(base_rows, head_rows) + ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), rows + ) + base_rows = calculate_specialization_failure_kinds( + name, base_family_stats, defines + ) + head_rows = calculate_specialization_failure_kinds( + name, head_family_stats, defines + ) + emit_table( + ( + "Failure kind", + "Base Count:", + "Base Ratio:", + "Head Count:", + "Head Ratio:", + ), + join_rows(base_rows, head_rows), ) + def gather_stats(input): # Note the output of this function must be JSON-serializable @@ -179,7 +219,9 @@ def gather_stats(input): with open(input, "r") as fd: stats = json.load(fd) - stats["_stats_defines"] = {int(k): v for k, v in stats["_stats_defines"].items()} + stats["_stats_defines"] = { + int(k): v for k, v in stats["_stats_defines"].items() + } stats["_defines"] = {int(k): v for k, v in stats["_defines"].items()} return stats @@ -191,18 +233,20 @@ def gather_stats(input): try: key, value = line.split(":") except ValueError: - print(f"Unparsable line: '{line.strip()}' in {filename}", file=sys.stderr) + print( + f"Unparsable line: '{line.strip()}' in {filename}", + file=sys.stderr, + ) continue key = key.strip() value = int(value) stats[key] += value - stats['__nfiles__'] += 1 + stats["__nfiles__"] += 1 import opcode stats["_specialized_instructions"] = [ - op for op in opcode._specialized_opmap.keys() - if "__" not in op + op for op in opcode._specialized_opmap.keys() if "__" not in op ] stats["_stats_defines"] = get_stats_defines() stats["_defines"] = get_defines() @@ -211,15 +255,17 @@ def gather_stats(input): else: raise ValueError(f"{input:r} is not a file or directory path") -def extract_opcode_stats(stats): + +def extract_opcode_stats(stats, prefix): opcode_stats = collections.defaultdict(dict) for key, value in stats.items(): - if not key.startswith("opcode"): + if not key.startswith(prefix): continue - name, _, rest = key[7:].partition("]") + name, _, rest = key[len(prefix) + 1 :].partition("]") opcode_stats[name][rest.strip(".")] = value return opcode_stats + def parse_kinds(spec_src, prefix="SPEC_FAIL"): defines = collections.defaultdict(list) start = "#define " + prefix + "_" @@ -227,14 +273,16 @@ def parse_kinds(spec_src, prefix="SPEC_FAIL"): line = line.strip() if not line.startswith(start): continue - line = line[len(start):] + line = line[len(start) :] name, val = line.split() defines[int(val.strip())].append(name.strip()) return defines + def pretty(defname): return defname.replace("_", " ").lower() + def kind_to_text(kind, defines, opname): if kind <= 8: return pretty(defines[kind][0]) @@ -248,9 +296,10 @@ def kind_to_text(kind, defines, opname): opname = "SUBSCR" for name in defines[kind]: if name.startswith(opname): - return pretty(name[len(opname)+1:]) + return pretty(name[len(opname) + 1 :]) return "kind " + str(kind) + def categorized_counts(opcode_stats, specialized_instructions): basic = 0 specialized = 0 @@ -258,7 +307,7 @@ def categorized_counts(opcode_stats, specialized_instructions): for name, opcode_stat in opcode_stats.items(): if "execution_count" not in opcode_stat: continue - count = opcode_stat['execution_count'] + count = opcode_stat["execution_count"] if "specializable" in opcode_stat: not_specialized += count elif name in specialized_instructions: @@ -269,12 +318,13 @@ def categorized_counts(opcode_stats, specialized_instructions): basic += count return basic, not_specialized, specialized + def print_title(name, level=2): - print("#"*level, name) + print("#" * level, name) print() -class Section: +class Section: def __init__(self, title, level=2, summary=None): self.title = title self.level = level @@ -295,12 +345,14 @@ def __exit__(*args): print("") print() + def to_str(x): if isinstance(x, int): return format(x, ",d") else: return str(x) + def emit_table(header, rows): width = len(header) header_line = "|" @@ -320,11 +372,28 @@ def emit_table(header, rows): print("|", " | ".join(to_str(i) for i in row), "|") print() + +def emit_histogram(title, stats, key, total): + rows = [] + for k, v in stats.items(): + if k.startswith(key): + entry = int(re.match(r".+\[([0-9]+)\]", k).groups()[0]) + rows.append((f"<= {entry}", int(v), format_ratio(int(v), total))) + # Don't include larger buckets with 0 entries + for j in range(len(rows) - 1, -1, -1): + if rows[j][1] != 0: + break + rows = rows[: j + 1] + + print(f"**{title}**\n") + emit_table(("Range", "Count:", "Ratio:"), rows) + + def calculate_execution_counts(opcode_stats, total): counts = [] for name, opcode_stat in opcode_stats.items(): if "execution_count" in opcode_stat: - count = opcode_stat['execution_count'] + count = opcode_stat["execution_count"] miss = 0 if "specializable" not in opcode_stat: miss = opcode_stat.get("specialization.miss") @@ -332,53 +401,61 @@ def calculate_execution_counts(opcode_stats, total): counts.sort(reverse=True) cumulative = 0 rows = [] - for (count, name, miss) in counts: + for count, name, miss in counts: cumulative += count if miss: miss = format_ratio(miss, count) else: miss = "" - rows.append((name, count, format_ratio(count, total), - format_ratio(cumulative, total), miss)) + rows.append( + ( + name, + count, + format_ratio(count, total), + format_ratio(cumulative, total), + miss, + ) + ) return rows + def emit_execution_counts(opcode_stats, total): with Section("Execution counts", summary="execution counts for all instructions"): rows = calculate_execution_counts(opcode_stats, total) - emit_table( - ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), - rows - ) + emit_table(("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), rows) + + +def _emit_comparative_execution_counts(base_rows, head_rows): + base_data = {x[0]: x[1:] for x in base_rows} + head_data = {x[0]: x[1:] for x in head_rows} + opcodes = base_data.keys() | head_data.keys() + + rows = [] + default = [0, "0.0%", "0.0%", 0] + for opcode in opcodes: + base_entry = base_data.get(opcode, default) + head_entry = head_data.get(opcode, default) + if base_entry[0] == 0: + change = 1 + else: + change = (head_entry[0] - base_entry[0]) / base_entry[0] + rows.append((opcode, base_entry[0], head_entry[0], f"{change:0.1%}")) + + rows.sort(key=lambda x: abs(percentage_to_float(x[-1])), reverse=True) + + emit_table(("Name", "Base Count:", "Head Count:", "Change:"), rows) + def emit_comparative_execution_counts( - base_opcode_stats, base_total, head_opcode_stats, head_total + base_opcode_stats, base_total, head_opcode_stats, head_total, level=2 ): - with Section("Execution counts", summary="execution counts for all instructions"): + with Section( + "Execution counts", summary="execution counts for all instructions", level=level + ): base_rows = calculate_execution_counts(base_opcode_stats, base_total) head_rows = calculate_execution_counts(head_opcode_stats, head_total) - base_data = dict((x[0], x[1:]) for x in base_rows) - head_data = dict((x[0], x[1:]) for x in head_rows) - opcodes = set(base_data.keys()) | set(head_data.keys()) - - rows = [] - default = [0, "0.0%", "0.0%", 0] - for opcode in opcodes: - base_entry = base_data.get(opcode, default) - head_entry = head_data.get(opcode, default) - if base_entry[0] == 0: - change = 1 - else: - change = (head_entry[0] - base_entry[0]) / base_entry[0] - rows.append( - (opcode, base_entry[0], head_entry[0], - f"{100*change:0.1f}%")) - - rows.sort(key=lambda x: -abs(percentage_to_float(x[-1]))) + _emit_comparative_execution_counts(base_rows, head_rows) - emit_table( - ("Name", "Base Count:", "Head Count:", "Change:"), - rows - ) def get_defines(): spec_path = os.path.join(os.path.dirname(__file__), "../../Python/specialize.c") @@ -386,12 +463,16 @@ def get_defines(): defines = parse_kinds(spec_src) return defines + def emit_specialization_stats(opcode_stats, defines): with Section("Specialization stats", summary="specialization stats by family"): for name, opcode_stat in opcode_stats.items(): print_specialization_stats(name, opcode_stat, defines) -def emit_comparative_specialization_stats(base_opcode_stats, head_opcode_stats, defines): + +def emit_comparative_specialization_stats( + base_opcode_stats, head_opcode_stats, defines +): with Section("Specialization stats", summary="specialization stats by family"): opcodes = set(base_opcode_stats.keys()) & set(head_opcode_stats.keys()) for opcode in opcodes: @@ -399,6 +480,7 @@ def emit_comparative_specialization_stats(base_opcode_stats, head_opcode_stats, opcode, base_opcode_stats[opcode], head_opcode_stats[opcode], defines ) + def calculate_specialization_effectiveness( opcode_stats, total, specialized_instructions ): @@ -411,11 +493,17 @@ def calculate_specialization_effectiveness( ("Specialized", specialized, format_ratio(specialized, total)), ] + def emit_specialization_overview(opcode_stats, total, specialized_instructions): with Section("Specialization effectiveness"): - rows = calculate_specialization_effectiveness(opcode_stats, total, specialized_instructions) + rows = calculate_specialization_effectiveness( + opcode_stats, total, specialized_instructions + ) emit_table(("Instructions", "Count:", "Ratio:"), rows) - for title, field in (("Deferred", "specialization.deferred"), ("Misses", "specialization.miss")): + for title, field in ( + ("Deferred", "specialization.deferred"), + ("Misses", "specialization.miss"), + ): total = 0 counts = [] for name, opcode_stat in opcode_stats.items(): @@ -428,11 +516,19 @@ def emit_specialization_overview(opcode_stats, total, specialized_instructions): counts.sort(reverse=True) if total: with Section(f"{title} by instruction", 3): - rows = [ (name, count, format_ratio(count, total)) for (count, name) in counts[:10] ] + rows = [ + (name, count, format_ratio(count, total)) + for (count, name) in counts[:10] + ] emit_table(("Name", "Count:", "Ratio:"), rows) + def emit_comparative_specialization_overview( - base_opcode_stats, base_total, head_opcode_stats, head_total, specialized_instructions + base_opcode_stats, + base_total, + head_opcode_stats, + head_total, + specialized_instructions, ): with Section("Specialization effectiveness"): base_rows = calculate_specialization_effectiveness( @@ -442,16 +538,26 @@ def emit_comparative_specialization_overview( head_opcode_stats, head_total, specialized_instructions ) emit_table( - ("Instructions", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), - join_rows(base_rows, head_rows) + ( + "Instructions", + "Base Count:", + "Base Ratio:", + "Head Count:", + "Head Ratio:", + ), + join_rows(base_rows, head_rows), ) + def get_stats_defines(): - stats_path = os.path.join(os.path.dirname(__file__), "../../Include/cpython/pystats.h") + stats_path = os.path.join( + os.path.dirname(__file__), "../../Include/cpython/pystats.h" + ) with open(stats_path) as stats_src: defines = parse_kinds(stats_src, prefix="EVAL_CALL") return defines + def calculate_call_stats(stats, defines): total = 0 for key, value in stats.items(): @@ -463,7 +569,7 @@ def calculate_call_stats(stats, defines): rows.append((key, value, format_ratio(value, total))) elif key.startswith("Calls "): name, index = key[:-1].split("[") - index = int(index) + index = int(index) label = name + " (" + pretty(defines[index][0]) + ")" rows.append((label, value, format_ratio(value, total))) for key, value in stats.items(): @@ -471,11 +577,13 @@ def calculate_call_stats(stats, defines): rows.append((key, value, format_ratio(value, total))) return rows + def emit_call_stats(stats, defines): with Section("Call stats", summary="Inlined calls and frame stats"): rows = calculate_call_stats(stats, defines) emit_table(("", "Count:", "Ratio:"), rows) + def emit_comparative_call_stats(base_stats, head_stats, defines): with Section("Call stats", summary="Inlined calls and frame stats"): base_rows = calculate_call_stats(base_stats, defines) @@ -483,15 +591,21 @@ def emit_comparative_call_stats(base_stats, head_stats, defines): rows = join_rows(base_rows, head_rows) rows.sort(key=lambda x: -percentage_to_float(x[-1])) emit_table( - ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), - rows + ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), rows ) + def calculate_object_stats(stats): total_materializations = stats.get("Object new values") - total_allocations = stats.get("Object allocations") + stats.get("Object allocations from freelist") - total_increfs = stats.get("Object interpreter increfs") + stats.get("Object increfs") - total_decrefs = stats.get("Object interpreter decrefs") + stats.get("Object decrefs") + total_allocations = stats.get("Object allocations") + stats.get( + "Object allocations from freelist" + ) + total_increfs = stats.get("Object interpreter increfs") + stats.get( + "Object increfs" + ) + total_decrefs = stats.get("Object interpreter decrefs") + stats.get( + "Object decrefs" + ) rows = [] for key, value in stats.items(): if key.startswith("Object"): @@ -499,9 +613,9 @@ def calculate_object_stats(stats): ratio = format_ratio(value, total_materializations) elif "allocations" in key: ratio = format_ratio(value, total_allocations) - elif "increfs" in key: + elif "increfs" in key: ratio = format_ratio(value, total_increfs) - elif "decrefs" in key: + elif "decrefs" in key: ratio = format_ratio(value, total_decrefs) else: ratio = "" @@ -510,6 +624,7 @@ def calculate_object_stats(stats): rows.append((label, value, ratio)) return rows + def calculate_gc_stats(stats): gc_stats = [] for key, value in stats.items(): @@ -526,40 +641,58 @@ def calculate_gc_stats(stats): for (i, gen) in enumerate(gc_stats) ] + def emit_object_stats(stats): with Section("Object stats", summary="allocations, frees and dict materializatons"): rows = calculate_object_stats(stats) - emit_table(("", "Count:", "Ratio:"), rows) + emit_table(("", "Count:", "Ratio:"), rows) + def emit_comparative_object_stats(base_stats, head_stats): with Section("Object stats", summary="allocations, frees and dict materializatons"): base_rows = calculate_object_stats(base_stats) head_rows = calculate_object_stats(head_stats) - emit_table(("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), join_rows(base_rows, head_rows)) + emit_table( + ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), + join_rows(base_rows, head_rows), + ) + def emit_gc_stats(stats): with Section("GC stats", summary="GC collections and effectiveness"): rows = calculate_gc_stats(stats) - emit_table(("Generation:", "Collections:", "Objects collected:", "Object visits:"), rows) + emit_table( + ("Generation:", "Collections:", "Objects collected:", "Object visits:"), + rows, + ) + def emit_comparative_gc_stats(base_stats, head_stats): with Section("GC stats", summary="GC collections and effectiveness"): base_rows = calculate_gc_stats(base_stats) head_rows = calculate_gc_stats(head_stats) emit_table( - ("Generation:", - "Base collections:", "Head collections:", - "Base objects collected:", "Head objects collected:", - "Base object visits:", "Head object visits:"), - join_rows(base_rows, head_rows)) + ( + "Generation:", + "Base collections:", + "Head collections:", + "Base objects collected:", + "Head objects collected:", + "Base object visits:", + "Head object visits:", + ), + join_rows(base_rows, head_rows), + ) + def get_total(opcode_stats): total = 0 for opcode_stat in opcode_stats.values(): if "execution_count" in opcode_stat: - total += opcode_stat['execution_count'] + total += opcode_stat["execution_count"] return total + def emit_pair_counts(opcode_stats, total): pair_counts = [] for name_i, opcode_stat in opcode_stats.items(): @@ -572,15 +705,22 @@ def emit_pair_counts(opcode_stats, total): pair_counts.sort(reverse=True) cumulative = 0 rows = [] - for (count, pair) in itertools.islice(pair_counts, 100): + for count, pair in itertools.islice(pair_counts, 100): name_i, name_j = pair cumulative += count - rows.append((f"{name_i} {name_j}", count, format_ratio(count, total), - format_ratio(cumulative, total))) - emit_table(("Pair", "Count:", "Self:", "Cumulative:"), - rows - ) - with Section("Predecessor/Successor Pairs", summary="Top 5 predecessors and successors of each opcode"): + rows.append( + ( + f"{name_i} {name_j}", + count, + format_ratio(count, total), + format_ratio(cumulative, total), + ) + ) + emit_table(("Pair", "Count:", "Self:", "Cumulative:"), rows) + with Section( + "Predecessor/Successor Pairs", + summary="Top 5 predecessors and successors of each opcode", + ): predecessors = collections.defaultdict(collections.Counter) successors = collections.defaultdict(collections.Counter) total_predecessors = collections.Counter() @@ -598,38 +738,135 @@ def emit_pair_counts(opcode_stats, total): continue pred_rows = succ_rows = () if total1: - pred_rows = [(pred, count, f"{count/total1:.1%}") - for (pred, count) in predecessors[name].most_common(5)] + pred_rows = [ + (pred, count, f"{count/total1:.1%}") + for (pred, count) in predecessors[name].most_common(5) + ] if total2: - succ_rows = [(succ, count, f"{count/total2:.1%}") - for (succ, count) in successors[name].most_common(5)] + succ_rows = [ + (succ, count, f"{count/total2:.1%}") + for (succ, count) in successors[name].most_common(5) + ] with Section(name, 3, f"Successors and predecessors for {name}"): - emit_table(("Predecessors", "Count:", "Percentage:"), - pred_rows - ) - emit_table(("Successors", "Count:", "Percentage:"), - succ_rows - ) + emit_table(("Predecessors", "Count:", "Percentage:"), pred_rows) + emit_table(("Successors", "Count:", "Percentage:"), succ_rows) + + +def calculate_optimization_stats(stats): + attempts = stats["Optimization attempts"] + created = stats["Optimization traces created"] + executed = stats["Optimization traces executed"] + uops = stats["Optimization uops executed"] + trace_stack_overflow = stats["Optimization trace stack overflow"] + trace_stack_underflow = stats["Optimization trace stack underflow"] + trace_too_long = stats["Optimization trace too long"] + inner_loop = stats["Optimization inner loop"] + recursive_call = stats["Optimization recursive call"] + + return [ + ("Optimization attempts", attempts, ""), + ("Traces created", created, format_ratio(created, attempts)), + ("Traces executed", executed, ""), + ("Uops executed", uops, int(uops / (executed or 1))), + ("Trace stack overflow", trace_stack_overflow, ""), + ("Trace stack underflow", trace_stack_underflow, ""), + ("Trace too long", trace_too_long, ""), + ("Inner loop found", inner_loop, ""), + ("Recursive call", recursive_call, ""), + ] + + +def calculate_uop_execution_counts(opcode_stats): + total = 0 + counts = [] + for name, opcode_stat in opcode_stats.items(): + if "execution_count" in opcode_stat: + count = opcode_stat["execution_count"] + counts.append((count, name)) + total += count + counts.sort(reverse=True) + cumulative = 0 + rows = [] + for count, name in counts: + cumulative += count + rows.append( + (name, count, format_ratio(count, total), format_ratio(cumulative, total)) + ) + return rows + + +def emit_optimization_stats(stats): + if "Optimization attempts" not in stats: + return + + uop_stats = extract_opcode_stats(stats, "uops") + + with Section( + "Optimization (Tier 2) stats", summary="statistics about the Tier 2 optimizer" + ): + with Section("Overall stats", level=3): + rows = calculate_optimization_stats(stats) + emit_table(("", "Count:", "Ratio:"), rows) + + emit_histogram( + "Trace length histogram", + stats, + "Trace length", + stats["Optimization traces created"], + ) + emit_histogram( + "Optimized trace length histogram", + stats, + "Optimized trace length", + stats["Optimization traces created"], + ) + emit_histogram( + "Trace run length histogram", + stats, + "Trace run length", + stats["Optimization traces executed"], + ) + + with Section("Uop stats", level=3): + rows = calculate_uop_execution_counts(uop_stats) + emit_table(("Uop", "Count:", "Self:", "Cumulative:"), rows) + + with Section("Unsupported opcodes", level=3): + unsupported_opcodes = extract_opcode_stats(stats, "unsupported_opcode") + data = [] + for opcode, entry in unsupported_opcodes.items(): + data.append((entry["count"], opcode)) + data.sort(reverse=True) + rows = [(x[1], x[0]) for x in data] + emit_table(("Opcode", "Count"), rows) + + +def emit_comparative_optimization_stats(base_stats, head_stats): + print("## Comparative optimization stats not implemented\n\n") + def output_single_stats(stats): - opcode_stats = extract_opcode_stats(stats) + opcode_stats = extract_opcode_stats(stats, "opcode") total = get_total(opcode_stats) emit_execution_counts(opcode_stats, total) emit_pair_counts(opcode_stats, total) emit_specialization_stats(opcode_stats, stats["_defines"]) - emit_specialization_overview(opcode_stats, total, stats["_specialized_instructions"]) + emit_specialization_overview( + opcode_stats, total, stats["_specialized_instructions"] + ) emit_call_stats(stats, stats["_stats_defines"]) emit_object_stats(stats) emit_gc_stats(stats) + emit_optimization_stats(stats) with Section("Meta stats", summary="Meta statistics"): - emit_table(("", "Count:"), [('Number of data files', stats['__nfiles__'])]) + emit_table(("", "Count:"), [("Number of data files", stats["__nfiles__"])]) def output_comparative_stats(base_stats, head_stats): - base_opcode_stats = extract_opcode_stats(base_stats) + base_opcode_stats = extract_opcode_stats(base_stats, "opcode") base_total = get_total(base_opcode_stats) - head_opcode_stats = extract_opcode_stats(head_stats) + head_opcode_stats = extract_opcode_stats(head_stats, "opcode") head_total = get_total(head_opcode_stats) emit_comparative_execution_counts( @@ -639,12 +876,17 @@ def output_comparative_stats(base_stats, head_stats): base_opcode_stats, head_opcode_stats, head_stats["_defines"] ) emit_comparative_specialization_overview( - base_opcode_stats, base_total, head_opcode_stats, head_total, - head_stats["_specialized_instructions"] + base_opcode_stats, + base_total, + head_opcode_stats, + head_total, + head_stats["_specialized_instructions"], ) emit_comparative_call_stats(base_stats, head_stats, head_stats["_stats_defines"]) emit_comparative_object_stats(base_stats, head_stats) emit_comparative_gc_stats(base_stats, head_stats) + emit_comparative_optimization_stats(base_stats, head_stats) + def output_stats(inputs, json_output=None): if len(inputs) == 1: @@ -654,9 +896,7 @@ def output_stats(inputs, json_output=None): output_single_stats(stats) elif len(inputs) == 2: if json_output is not None: - raise ValueError( - "Can not output to JSON when there are multiple inputs" - ) + raise ValueError("Can not output to JSON when there are multiple inputs") base_stats = gather_stats(inputs[0]) head_stats = gather_stats(inputs[1]) @@ -665,6 +905,7 @@ def output_stats(inputs, json_output=None): print("---") print("Stats gathered on:", date.today()) + def main(): parser = argparse.ArgumentParser(description="Summarize pystats results") @@ -680,14 +921,14 @@ def main(): If one source is provided, its stats are printed. If two sources are provided, comparative stats are printed. Default is {DEFAULT_DIR}. - """ + """, ) parser.add_argument( "--json-output", nargs="?", type=argparse.FileType("w"), - help="Output complete raw results to the given JSON file." + help="Output complete raw results to the given JSON file.", ) args = parser.parse_args() @@ -697,5 +938,6 @@ def main(): output_stats(args.inputs, json_output=args.json_output) + if __name__ == "__main__": main() From 80dc39e1dc2abc809f448cba5d2c5b9c1c631e11 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Oct 2023 16:35:27 -0600 Subject: [PATCH 35/73] gh-110310: Add a Per-Interpreter XID Registry for Heap Types (gh-110311) We do the following: * add a per-interpreter XID registry (PyInterpreterState.xidregistry) * put heap types there (keep static types in _PyRuntimeState.xidregistry) * clear the registries during interpreter/runtime finalization * avoid duplicate entries in the registry (when _PyCrossInterpreterData_RegisterClass() is called more than once for a type) * use Py_TYPE() instead of PyObject_Type() in _PyCrossInterpreterData_Lookup() The per-interpreter registry helps preserve isolation between interpreters. This is important when heap types are registered, which is something we haven't been doing yet but I will likely do soon. --- Include/internal/pycore_interp.h | 44 ++++++--- Include/internal/pycore_runtime.h | 5 +- Python/pystate.c | 157 ++++++++++++++++++++++-------- 3 files changed, 148 insertions(+), 58 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 21d1ee38b59e6b..523dfdc21deda4 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -39,6 +39,32 @@ struct _Py_long_state { int max_str_digits; }; + +/* cross-interpreter data registry */ + +/* For now we use a global registry of shareable classes. An + alternative would be to add a tp_* slot for a class's + crossinterpdatafunc. It would be simpler and more efficient. */ + +struct _xidregitem; + +struct _xidregitem { + struct _xidregitem *prev; + struct _xidregitem *next; + /* This can be a dangling pointer, but only if weakref is set. */ + PyTypeObject *cls; + /* This is NULL for builtin types. */ + PyObject *weakref; + size_t refcount; + crossinterpdatafunc getdata; +}; + +struct _xidregistry { + PyThread_type_lock mutex; + struct _xidregitem *head; +}; + + /* interpreter state */ /* PyInterpreterState holds the global state for one of the runtime's @@ -149,6 +175,9 @@ struct _is { Py_ssize_t co_extra_user_count; freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS]; + // XXX Remove this field once we have a tp_* slot. + struct _xidregistry xidregistry; + #ifdef HAVE_FORK PyObject *before_forkers; PyObject *after_forkers_parent; @@ -238,21 +267,6 @@ _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tst } -/* cross-interpreter data registry */ - -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ - -struct _xidregitem; - -struct _xidregitem { - struct _xidregitem *prev; - struct _xidregitem *next; - PyObject *cls; // weakref to a PyTypeObject - crossinterpdatafunc getdata; -}; - extern PyInterpreterState* _PyInterpreterState_LookUpID(int64_t); extern int _PyInterpreterState_IDInitref(PyInterpreterState *); diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index cc3a3420befa3d..1dc243e46e8ef8 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -201,10 +201,7 @@ typedef struct pyruntimestate { tools. */ // XXX Remove this field once we have a tp_* slot. - struct _xidregistry { - PyThread_type_lock mutex; - struct _xidregitem *head; - } xidregistry; + struct _xidregistry xidregistry; struct _pymem_allocators allocators; struct _obmalloc_global_state obmalloc; diff --git a/Python/pystate.c b/Python/pystate.c index ae33259f8df2f6..068740828e9c25 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -495,6 +495,8 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) return _PyStatus_OK(); } +static void _xidregistry_clear(struct _xidregistry *); + void _PyRuntimeState_Fini(_PyRuntimeState *runtime) { @@ -503,6 +505,8 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) assert(runtime->object_state.interpreter_leaks == 0); #endif + _xidregistry_clear(&runtime->xidregistry); + if (gilstate_tss_initialized(runtime)) { gilstate_tss_fini(runtime); } @@ -548,6 +552,11 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) for (int i = 0; i < NUMLOCKS; i++) { reinit_err += _PyThread_at_fork_reinit(lockptrs[i]); } + /* PyOS_AfterFork_Child(), which calls this function, later calls + _PyInterpreterState_DeleteExceptMain(), so we only need to update + the main interpreter here. */ + assert(runtime->interpreters.main != NULL); + runtime->interpreters.main->xidregistry.mutex = runtime->xidregistry.mutex; PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -709,6 +718,10 @@ init_interpreter(PyInterpreterState *interp, interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); } interp->f_opcode_trace_set = false; + + assert(runtime->xidregistry.mutex != NULL); + interp->xidregistry.mutex = runtime->xidregistry.mutex; + interp->_initialized = 1; return _PyStatus_OK(); } @@ -930,6 +943,10 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); + _xidregistry_clear(&interp->xidregistry); + /* The lock is owned by the runtime, so we don't free it here. */ + interp->xidregistry.mutex = NULL; + if (tstate->interp == interp) { /* We are now safe to fix tstate->_status.cleared. */ // XXX Do this (much) earlier? @@ -2613,23 +2630,27 @@ _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data) crossinterpdatafunc. It would be simpler and more efficient. */ static int -_xidregistry_add_type(struct _xidregistry *xidregistry, PyTypeObject *cls, - crossinterpdatafunc getdata) +_xidregistry_add_type(struct _xidregistry *xidregistry, + PyTypeObject *cls, crossinterpdatafunc getdata) { - // Note that we effectively replace already registered classes - // rather than failing. struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); if (newhead == NULL) { return -1; } - // XXX Assign a callback to clear the entry from the registry? - newhead->cls = PyWeakref_NewRef((PyObject *)cls, NULL); - if (newhead->cls == NULL) { - PyMem_RawFree(newhead); - return -1; + *newhead = (struct _xidregitem){ + // We do not keep a reference, to avoid keeping the class alive. + .cls = cls, + .refcount = 1, + .getdata = getdata, + }; + if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { + // XXX Assign a callback to clear the entry from the registry? + newhead->weakref = PyWeakref_NewRef((PyObject *)cls, NULL); + if (newhead->weakref == NULL) { + PyMem_RawFree(newhead); + return -1; + } } - newhead->getdata = getdata; - newhead->prev = NULL; newhead->next = xidregistry->head; if (newhead->next != NULL) { newhead->next->prev = newhead; @@ -2654,39 +2675,77 @@ _xidregistry_remove_entry(struct _xidregistry *xidregistry, if (next != NULL) { next->prev = entry->prev; } - Py_DECREF(entry->cls); + Py_XDECREF(entry->weakref); PyMem_RawFree(entry); return next; } +static void +_xidregistry_clear(struct _xidregistry *xidregistry) +{ + struct _xidregitem *cur = xidregistry->head; + xidregistry->head = NULL; + while (cur != NULL) { + struct _xidregitem *next = cur->next; + Py_XDECREF(cur->weakref); + PyMem_RawFree(cur); + cur = next; + } +} + static struct _xidregitem * _xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) { struct _xidregitem *cur = xidregistry->head; while (cur != NULL) { - PyObject *registered = _PyWeakref_GET_REF(cur->cls); - if (registered == NULL) { - // The weakly ref'ed object was freed. - cur = _xidregistry_remove_entry(xidregistry, cur); - } - else { - assert(PyType_Check(registered)); - if (registered == (PyObject *)cls) { - Py_DECREF(registered); - return cur; + if (cur->weakref != NULL) { + // cur is/was a heap type. + PyObject *registered = _PyWeakref_GET_REF(cur->weakref); + if (registered == NULL) { + // The weakly ref'ed object was freed. + cur = _xidregistry_remove_entry(xidregistry, cur); + continue; } + assert(PyType_Check(registered)); + assert(cur->cls == (PyTypeObject *)registered); + assert(cur->cls->tp_flags & Py_TPFLAGS_HEAPTYPE); Py_DECREF(registered); - cur = cur->next; } + if (cur->cls == cls) { + return cur; + } + cur = cur->next; } return NULL; } +static inline struct _xidregistry * +_get_xidregistry(PyInterpreterState *interp, PyTypeObject *cls) +{ + struct _xidregistry *xidregistry = &interp->runtime->xidregistry; + if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { + assert(interp->xidregistry.mutex == xidregistry->mutex); + xidregistry = &interp->xidregistry; + } + return xidregistry; +} + static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry); +static inline void +_ensure_builtins_xid(PyInterpreterState *interp, struct _xidregistry *xidregistry) +{ + if (xidregistry != &interp->xidregistry) { + assert(xidregistry == &interp->runtime->xidregistry); + if (xidregistry->head == NULL) { + _register_builtins_for_crossinterpreter_data(xidregistry); + } + } +} + int _PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, - crossinterpdatafunc getdata) + crossinterpdatafunc getdata) { if (!PyType_Check(cls)) { PyErr_Format(PyExc_ValueError, "only classes may be registered"); @@ -2697,12 +2756,23 @@ _PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, return -1; } - struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ; + int res = 0; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - if (xidregistry->head == NULL) { - _register_builtins_for_crossinterpreter_data(xidregistry); + + _ensure_builtins_xid(interp, xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + if (matched != NULL) { + assert(matched->getdata == getdata); + matched->refcount += 1; + goto finally; } - int res = _xidregistry_add_type(xidregistry, cls, getdata); + + res = _xidregistry_add_type(xidregistry, cls, getdata); + +finally: PyThread_release_lock(xidregistry->mutex); return res; } @@ -2711,13 +2781,20 @@ int _PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) { int res = 0; - struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); if (matched != NULL) { - (void)_xidregistry_remove_entry(xidregistry, matched); + assert(matched->refcount > 0); + matched->refcount -= 1; + if (matched->refcount == 0) { + (void)_xidregistry_remove_entry(xidregistry, matched); + } res = 1; } + PyThread_release_lock(xidregistry->mutex); return res; } @@ -2730,17 +2807,19 @@ _PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *obj) { - struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ; - PyObject *cls = PyObject_Type(obj); + PyTypeObject *cls = Py_TYPE(obj); + + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - if (xidregistry->head == NULL) { - _register_builtins_for_crossinterpreter_data(xidregistry); - } - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, - (PyTypeObject *)cls); - Py_DECREF(cls); + + _ensure_builtins_xid(interp, xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + crossinterpdatafunc func = matched != NULL ? matched->getdata : NULL; + PyThread_release_lock(xidregistry->mutex); - return matched != NULL ? matched->getdata : NULL; + return func; } /* cross-interpreter data for builtin types */ From cf6f23b0e3cdef33f23967cf954a2ca4d1fa6528 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 4 Oct 2023 22:50:29 +0000 Subject: [PATCH 36/73] gh-88402: Add new sysconfig variables on Windows (GH-110049) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Filipe Laíns --- Include/internal/pycore_importdl.h | 57 +++++++++++ Lib/sysconfig.py | 18 ++-- Lib/test/test_sysconfig.py | 24 ++++- ...3-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst | 2 + Modules/Setup | 1 + Modules/Setup.bootstrap.in | 1 + Modules/_sysconfig.c | 98 +++++++++++++++++++ Modules/clinic/_sysconfig.c.h | 22 +++++ PC/config.c | 2 + PCbuild/pythoncore.vcxproj | 3 +- PCbuild/pythoncore.vcxproj.filters | 9 +- Python/dynload_hpux.c | 2 +- Python/dynload_shlib.c | 2 +- Python/dynload_stub.c | 2 +- Python/dynload_win.c | 22 +---- Python/import.c | 2 +- Python/importdl.c | 2 +- Python/importdl.h | 29 ------ Python/stdlib_module_names.h | 1 + Tools/c-analyzer/cpython/ignored.tsv | 2 +- 20 files changed, 231 insertions(+), 70 deletions(-) create mode 100644 Include/internal/pycore_importdl.h create mode 100644 Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst create mode 100644 Modules/_sysconfig.c create mode 100644 Modules/clinic/_sysconfig.c.h delete mode 100644 Python/importdl.h diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h new file mode 100644 index 00000000000000..dee64241c763f3 --- /dev/null +++ b/Include/internal/pycore_importdl.h @@ -0,0 +1,57 @@ +#ifndef Py_INTERNAL_IMPORTDL_H +#define Py_INTERNAL_IMPORTDL_H + +#include "patchlevel.h" // PY_MAJOR_VERSION + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +extern const char *_PyImport_DynLoadFiletab[]; + +extern PyObject *_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *); + +typedef PyObject *(*PyModInitFunction)(void); + +/* Max length of module suffix searched for -- accommodates "module.slb" */ +#define MAXSUFFIXSIZE 12 + +#ifdef MS_WINDOWS +#include +typedef FARPROC dl_funcptr; + +#ifdef _DEBUG +# define PYD_DEBUG_SUFFIX "_d" +#else +# define PYD_DEBUG_SUFFIX "" +#endif + +#ifdef Py_NOGIL +# define PYD_THREADING_TAG "t" +#else +# define PYD_THREADING_TAG "" +#endif + +#ifdef PYD_PLATFORM_TAG +# define PYD_SOABI "cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG "-" PYD_PLATFORM_TAG +#else +# define PYD_SOABI "cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG +#endif + +#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX "." PYD_SOABI ".pyd" +#define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" + +#else +typedef void (*dl_funcptr)(void); +#endif + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_IMPORTDL_H */ diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index a8b5c5f7dfba5b..edfe451a86bfd0 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -544,16 +544,20 @@ def _init_posix(vars): def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories - import _imp + import _winapi + import _sysconfig vars['LIBDEST'] = get_path('stdlib') vars['BINLIBDEST'] = get_path('platstdlib') vars['INCLUDEPY'] = get_path('include') - try: - # GH-99201: _imp.extension_suffixes may be empty when - # HAVE_DYNAMIC_LOADING is not set. In this case, don't set EXT_SUFFIX. - vars['EXT_SUFFIX'] = _imp.extension_suffixes()[0] - except IndexError: - pass + + # Add EXT_SUFFIX, SOABI, and Py_NOGIL + vars.update(_sysconfig.config_vars()) + + vars['LIBDIR'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs')) + if hasattr(sys, 'dllhandle'): + dllhandle = _winapi.GetModuleFileName(sys.dllhandle) + vars['LIBRARY'] = os.path.basename(_safe_realpath(dllhandle)) + vars['LDLIBRARY'] = vars['LIBRARY'] vars['EXE'] = '.exe' vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index b6dbf3d52cb4c3..a077ac5349fdc6 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -17,7 +17,9 @@ get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, _expand_vars, _get_preferred_schemes, _main) +import _imp import _osx_support +import _sysconfig HAS_USER_BASE = sysconfig._HAS_USER_BASE @@ -394,6 +396,24 @@ def test_ldshared_value(self): self.assertIn(ldflags, ldshared) + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") + def test_soabi(self): + soabi = sysconfig.get_config_var('SOABI') + self.assertIn(soabi, _imp.extension_suffixes()[0]) + + def test_library(self): + library = sysconfig.get_config_var('LIBRARY') + ldlibrary = sysconfig.get_config_var('LDLIBRARY') + major, minor = sys.version_info[:2] + if sys.platform == 'win32': + self.assertTrue(library.startswith(f'python{major}{minor}')) + self.assertTrue(library.endswith('.dll')) + self.assertEqual(library, ldlibrary) + else: + self.assertTrue(library.startswith(f'libpython{major}.{minor}')) + self.assertTrue(library.endswith('.a')) + self.assertTrue(ldlibrary.startswith(f'libpython{major}.{minor}')) + @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") @requires_subprocess() def test_platform_in_subprocess(self): @@ -472,10 +492,8 @@ def test_srcdir_independent_of_cwd(self): @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None, 'EXT_SUFFIX required for this test') + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") def test_EXT_SUFFIX_in_vars(self): - import _imp - if not _imp.extension_suffixes(): - self.skipTest("stub loader has no suffixes") vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) diff --git a/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst b/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst new file mode 100644 index 00000000000000..80ec65081c0dc8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst @@ -0,0 +1,2 @@ +Add new variables to :py:meth:`sysconfig.get_config_vars` on Windows: +``LIBRARY``, ``LDLIBRARY``, ``LIBDIR``, ``SOABI``, and ``Py_NOGIL``. diff --git a/Modules/Setup b/Modules/Setup index 8676f9ddce4841..1367f0ef4fa54a 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -155,6 +155,7 @@ PYTHONPATH=$(COREPYTHONPATH) #math mathmodule.c #mmap mmapmodule.c #select selectmodule.c +#_sysconfig _sysconfig.c # XML #_elementtree _elementtree.c diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 8ef0f203a82a8e..cd12c1bd0df8f9 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -19,6 +19,7 @@ errno errnomodule.c _io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c itertools itertoolsmodule.c _sre _sre/sre.c +_sysconfig _sysconfig.c _thread _threadmodule.c time timemodule.c _typing _typingmodule.c diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c new file mode 100644 index 00000000000000..6f1cc16b58467d --- /dev/null +++ b/Modules/_sysconfig.c @@ -0,0 +1,98 @@ +// _sysconfig provides data for the Python sysconfig module + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" + +#include "pycore_importdl.h" // _PyImport_DynLoadFiletab +#include "pycore_long.h" // _PyLong_GetZero, _PyLong_GetOne + + +/*[clinic input] +module _sysconfig +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=0a7c02d3e212ac97]*/ + +#include "clinic/_sysconfig.c.h" + +#ifdef MS_WINDOWS +static int +add_string_value(PyObject *dict, const char *key, const char *str_value) +{ + PyObject *value = PyUnicode_FromString(str_value); + if (value == NULL) { + return -1; + } + int err = PyDict_SetItemString(dict, key, value); + Py_DECREF(value); + return err; +} +#endif + +/*[clinic input] +_sysconfig.config_vars + +Returns a dictionary containing build variables intended to be exposed by sysconfig. +[clinic start generated code]*/ + +static PyObject * +_sysconfig_config_vars_impl(PyObject *module) +/*[clinic end generated code: output=9c41cdee63ea9487 input=391ff42f3af57d01]*/ +{ + PyObject *config = PyDict_New(); + if (config == NULL) { + return NULL; + } + +#ifdef MS_WINDOWS + if (add_string_value(config, "EXT_SUFFIX", PYD_TAGGED_SUFFIX) < 0) { + Py_DECREF(config); + return NULL; + } + if (add_string_value(config, "SOABI", PYD_SOABI) < 0) { + Py_DECREF(config); + return NULL; + } +#endif + +#ifdef Py_NOGIL + PyObject *py_nogil = _PyLong_GetOne(); +#else + PyObject *py_nogil = _PyLong_GetZero(); +#endif + if (PyDict_SetItemString(config, "Py_NOGIL", py_nogil) < 0) { + Py_DECREF(config); + return NULL; + } + + return config; +} + +PyDoc_STRVAR(sysconfig__doc__, +"A helper for the sysconfig module."); + +static struct PyMethodDef sysconfig_methods[] = { + _SYSCONFIG_CONFIG_VARS_METHODDEF + {NULL, NULL} +}; + +static PyModuleDef_Slot sysconfig_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef sysconfig_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_sysconfig", + .m_doc = sysconfig__doc__, + .m_methods = sysconfig_methods, + .m_slots = sysconfig_slots, +}; + +PyMODINIT_FUNC +PyInit__sysconfig(void) +{ + return PyModuleDef_Init(&sysconfig_module); +} diff --git a/Modules/clinic/_sysconfig.c.h b/Modules/clinic/_sysconfig.c.h new file mode 100644 index 00000000000000..eb3d396298bb21 --- /dev/null +++ b/Modules/clinic/_sysconfig.c.h @@ -0,0 +1,22 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_sysconfig_config_vars__doc__, +"config_vars($module, /)\n" +"--\n" +"\n" +"Returns a dictionary containing build variables intended to be exposed by sysconfig."); + +#define _SYSCONFIG_CONFIG_VARS_METHODDEF \ + {"config_vars", (PyCFunction)_sysconfig_config_vars, METH_NOARGS, _sysconfig_config_vars__doc__}, + +static PyObject * +_sysconfig_config_vars_impl(PyObject *module); + +static PyObject * +_sysconfig_config_vars(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _sysconfig_config_vars_impl(module); +} +/*[clinic end generated code: output=25d395cf02eced1f input=a9049054013a1b77]*/ diff --git a/PC/config.c b/PC/config.c index 88f69758aac764..da2bde640961e0 100644 --- a/PC/config.c +++ b/PC/config.c @@ -22,6 +22,7 @@ extern PyObject* PyInit__sha1(void); extern PyObject* PyInit__sha2(void); extern PyObject* PyInit__sha3(void); extern PyObject* PyInit__statistics(void); +extern PyObject* PyInit__sysconfig(void); extern PyObject* PyInit__typing(void); extern PyObject* PyInit__blake2(void); extern PyObject* PyInit_time(void); @@ -102,6 +103,7 @@ struct _inittab _PyImport_Inittab[] = { {"_sha2", PyInit__sha2}, {"_sha3", PyInit__sha3}, {"_blake2", PyInit__blake2}, + {"_sysconfig", PyInit__sysconfig}, {"time", PyInit_time}, {"_thread", PyInit__thread}, {"_tokenize", PyInit__tokenize}, diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 1ec106777db56d..43a79fd5938486 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -240,6 +240,7 @@ + @@ -367,7 +368,6 @@ - @@ -438,6 +438,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index f381120c9b035a..59159ed609968b 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -300,9 +300,6 @@ PC - - Python - Python @@ -633,6 +630,9 @@ Include\internal + + Include\internal + Include\internal @@ -959,6 +959,9 @@ Modules + + Modules + Modules diff --git a/Python/dynload_hpux.c b/Python/dynload_hpux.c index a53373038ed859..1c44722ff9a2d0 100644 --- a/Python/dynload_hpux.c +++ b/Python/dynload_hpux.c @@ -5,7 +5,7 @@ #include #include "Python.h" -#include "importdl.h" +#include "pycore_importdl.h" #if defined(__hp9000s300) #define FUNCNAME_PATTERN "_%.20s_%.200s" diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 6761bba457983b..5a37a83805ba78 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -4,7 +4,7 @@ #include "Python.h" #include "pycore_interp.h" // _PyInterpreterState.dlopenflags #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "importdl.h" +#include "pycore_importdl.h" #include #include diff --git a/Python/dynload_stub.c b/Python/dynload_stub.c index 59160483caa448..11f7e5f643f79e 100644 --- a/Python/dynload_stub.c +++ b/Python/dynload_stub.c @@ -3,7 +3,7 @@ not present. */ #include "Python.h" -#include "importdl.h" +#include "pycore_importdl.h" const char *_PyImport_DynLoadFiletab[] = {NULL}; diff --git a/Python/dynload_win.c b/Python/dynload_win.c index fcb3cb744047ce..a0ac31c80a5f6e 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -5,30 +5,10 @@ #include "pycore_fileutils.h" // _Py_add_relfile() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "importdl.h" // dl_funcptr +#include "pycore_importdl.h" // dl_funcptr #include "patchlevel.h" // PY_MAJOR_VERSION #include -#ifdef _DEBUG -#define PYD_DEBUG_SUFFIX "_d" -#else -#define PYD_DEBUG_SUFFIX "" -#endif - -#ifdef Py_NOGIL -# define PYD_THREADING_TAG "t" -#else -# define PYD_THREADING_TAG "" -#endif - -#ifdef PYD_PLATFORM_TAG -#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG "-" PYD_PLATFORM_TAG ".pyd" -#else -#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG ".pyd" -#endif - -#define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" - const char *_PyImport_DynLoadFiletab[] = { PYD_TAGGED_SUFFIX, PYD_UNTAGGED_SUFFIX, diff --git a/Python/import.c b/Python/import.c index 5636968ed9e63b..cafdd834502224 100644 --- a/Python/import.c +++ b/Python/import.c @@ -17,7 +17,7 @@ #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "marshal.h" // PyMarshal_ReadObjectFromString() -#include "importdl.h" // _PyImport_DynLoadFiletab +#include "pycore_importdl.h" // _PyImport_DynLoadFiletab #include "pydtrace.h" // PyDTrace_IMPORT_FIND_LOAD_START_ENABLED() #include // bool diff --git a/Python/importdl.c b/Python/importdl.c index 9ab0a5ad33aaac..7dfd301d77efb4 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -15,7 +15,7 @@ */ #ifdef HAVE_DYNAMIC_LOADING -#include "importdl.h" +#include "pycore_importdl.h" #ifdef MS_WINDOWS extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, diff --git a/Python/importdl.h b/Python/importdl.h deleted file mode 100644 index 9171adc2770689..00000000000000 --- a/Python/importdl.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef Py_IMPORTDL_H -#define Py_IMPORTDL_H - -#ifdef __cplusplus -extern "C" { -#endif - - -extern const char *_PyImport_DynLoadFiletab[]; - -extern PyObject *_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *); - -typedef PyObject *(*PyModInitFunction)(void); - -/* Max length of module suffix searched for -- accommodates "module.slb" */ -#define MAXSUFFIXSIZE 12 - -#ifdef MS_WINDOWS -#include -typedef FARPROC dl_funcptr; -#else -typedef void (*dl_funcptr)(void); -#endif - - -#ifdef __cplusplus -} -#endif -#endif /* !Py_IMPORTDL_H */ diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 13b1764f0886d1..701bfc35cc8182 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -77,6 +77,7 @@ static const char* _Py_stdlib_module_names[] = { "_strptime", "_struct", "_symtable", +"_sysconfig", "_thread", "_threading_local", "_tkinter", diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index c6c69a3e222f07..f9911643332b5e 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -607,6 +607,7 @@ Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput AST_LITERAL_EVAL_INITIALIZED # XXX Fix the analyzer. ## forward/extern references +Include/internal/pycore_importdl.h - _PyImport_DynLoadFiletab - Include/py_curses.h - PyCurses_API - Include/pydecimal.h - _decimal_api - Modules/_blake2/blake2module.c - blake2b_type_spec - @@ -668,7 +669,6 @@ Objects/object.c - _PyLineIterator - Objects/object.c - _PyPositionsIterator - Python/perf_trampoline.c - _Py_trampoline_func_start - Python/perf_trampoline.c - _Py_trampoline_func_end - -Python/importdl.h - _PyImport_DynLoadFiletab - Modules/expat/xmlrole.c - prolog0 - Modules/expat/xmlrole.c - prolog1 - Modules/expat/xmlrole.c - prolog2 - From 313aa861ce23e83ca64284d97c1dac234c9def7c Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 5 Oct 2023 15:18:17 +1100 Subject: [PATCH 37/73] Remove duplicate word. (#110376) --- Doc/library/sys.monitoring.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 7b02b95fd766a7..5dcbdaf8e5d0e4 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -147,7 +147,7 @@ by another event: * C_RAISE * C_RETURN -The ``C_RETURN`` and ``C_RAISE`` events are are controlled by the ``CALL`` +The ``C_RETURN`` and ``C_RAISE`` events are controlled by the ``CALL`` event. ``C_RETURN`` and ``C_RAISE`` events will only be seen if the corresponding ``CALL`` event is being monitored. From 6592976061a6580fee2ade3564f6497eb685ab67 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Oct 2023 08:24:43 +0200 Subject: [PATCH 38/73] gh-110367: Enhance regrtest -jN --verbose3 (#110368) When using worker processes (-jN) with --verbose3 option, regrtest can now display the worker output even if a worker process does crash. Previously, sys.stdout and sys.stderr were replaced and so the worker output was lost on a crash. --- Lib/test/libregrtest/run_workers.py | 15 ++++++-- Lib/test/test_faulthandler.py | 2 +- Lib/test/test_regrtest.py | 34 +++++++++++++++++++ ...-10-04-18-27-47.gh-issue-110367.Nnq1I7.rst | 4 +++ 4 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-10-04-18-27-47.gh-issue-110367.Nnq1I7.rst diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 6eb32e59635865..106f9730832e54 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -262,6 +262,9 @@ def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> Ru kwargs = {} if match_tests: kwargs['match_tests'] = match_tests + if self.runtests.output_on_failure: + kwargs['verbose'] = True + kwargs['output_on_failure'] = False return self.runtests.copy( tests=tests, json_file=json_file, @@ -562,8 +565,16 @@ def _process_result(self, item: QueueOutput) -> TestResult: self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) - if mp_result.worker_stdout: - print(mp_result.worker_stdout, flush=True) + # Display worker stdout + if not self.runtests.output_on_failure: + show_stdout = True + else: + # --verbose3 ignores stdout on success + show_stdout = (result.state != State.PASSED) + if show_stdout: + stdout = mp_result.worker_stdout + if stdout: + print(stdout, flush=True) return result diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 3c1e8c150ae711..4dce7a7dd8c8df 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -35,7 +35,7 @@ def expected_traceback(lineno1, lineno2, header, min_count=1): return '^' + regex + '$' def skip_segfault_on_android(test): - # Issue #32138: Raising SIGSEGV on Android may not cause a crash. + # gh-76319: Raising SIGSEGV on Android may not cause a crash. return unittest.skipIf(is_android, 'raising SIGSEGV on Android is unreliable')(test) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index f24d23e5c2f731..66463fd2bbea22 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -42,6 +42,8 @@ EXITCODE_RERUN_FAIL = 5 EXITCODE_INTERRUPTED = 130 +MS_WINDOWS = (sys.platform == 'win32') + TEST_INTERRUPTED = textwrap.dedent(""" from signal import SIGINT, raise_signal try: @@ -2036,6 +2038,38 @@ def test_add_python_opts(self): with self.subTest(opt=opt): self.check_add_python_opts(opt) + # gh-76319: Raising SIGSEGV on Android may not cause a crash. + @unittest.skipIf(support.is_android, + 'raising SIGSEGV on Android is unreliable') + def test_worker_output_on_failure(self): + try: + from faulthandler import _sigsegv + except ImportError: + self.skipTest("need faulthandler._sigsegv") + + code = textwrap.dedent(r""" + import faulthandler + import unittest + from test import support + + class CrashTests(unittest.TestCase): + def test_crash(self): + print("just before crash!", flush=True) + + with support.SuppressCrashReport(): + faulthandler._sigsegv(True) + """) + testname = self.create_test(code=code) + + output = self.run_tests("-j1", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + stats=0, parallel=True) + if not MS_WINDOWS: + exitcode = -int(signal.SIGSEGV) + self.assertIn(f"Exit code {exitcode} (SIGSEGV)", output) + self.check_line(output, "just before crash!", full=True, regex=False) + class TestUtils(unittest.TestCase): def test_format_duration(self): diff --git a/Misc/NEWS.d/next/Tests/2023-10-04-18-27-47.gh-issue-110367.Nnq1I7.rst b/Misc/NEWS.d/next/Tests/2023-10-04-18-27-47.gh-issue-110367.Nnq1I7.rst new file mode 100644 index 00000000000000..a1a6a09da509ef --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-10-04-18-27-47.gh-issue-110367.Nnq1I7.rst @@ -0,0 +1,4 @@ +regrtest: When using worker processes (-jN) with --verbose3 option, regrtest +can now display the worker output even if a worker process does crash. +Previously, sys.stdout and sys.stderr were replaced and so the worker output +was lost on a crash. Patch by Victor Stinner. From 2bbbab212fb10b3aeaded188fb5d6c001fb4bf74 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 5 Oct 2023 14:02:52 +0300 Subject: [PATCH 39/73] gh-110365: Fix error overwrite in `termios.tcsetattr` (#110366) Co-authored-by: Erlend E. Aasland --- ...-10-04-18-56-29.gh-issue-110365.LCxiau.rst | 2 + Modules/termios.c | 39 ++++++++++++------- 2 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-04-18-56-29.gh-issue-110365.LCxiau.rst diff --git a/Misc/NEWS.d/next/Library/2023-10-04-18-56-29.gh-issue-110365.LCxiau.rst b/Misc/NEWS.d/next/Library/2023-10-04-18-56-29.gh-issue-110365.LCxiau.rst new file mode 100644 index 00000000000000..a1ac39b60296a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-04-18-56-29.gh-issue-110365.LCxiau.rst @@ -0,0 +1,2 @@ +Fix :func:`termios.tcsetattr` bug that was overwritting existing errors +during parsing integers from ``term`` list. diff --git a/Modules/termios.c b/Modules/termios.c index 6f07c93bcdb6b5..9fc2673ce0e788 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -212,17 +212,25 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) return PyErr_SetFromErrno(state->TermiosError); } - mode.c_iflag = (tcflag_t) PyLong_AsLong(PyList_GetItem(term, 0)); - mode.c_oflag = (tcflag_t) PyLong_AsLong(PyList_GetItem(term, 1)); - mode.c_cflag = (tcflag_t) PyLong_AsLong(PyList_GetItem(term, 2)); - mode.c_lflag = (tcflag_t) PyLong_AsLong(PyList_GetItem(term, 3)); - speed_t ispeed = (speed_t) PyLong_AsLong(PyList_GetItem(term, 4)); - speed_t ospeed = (speed_t) PyLong_AsLong(PyList_GetItem(term, 5)); - PyObject *cc = PyList_GetItem(term, 6); - if (PyErr_Occurred()) { - return NULL; - } - + speed_t ispeed, ospeed; +#define SET_FROM_LIST(TYPE, VAR, LIST, N) do { \ + PyObject *item = PyList_GET_ITEM(LIST, N); \ + long num = PyLong_AsLong(item); \ + if (num == -1 && PyErr_Occurred()) { \ + return NULL; \ + } \ + VAR = (TYPE)num; \ +} while (0) + + SET_FROM_LIST(tcflag_t, mode.c_iflag, term, 0); + SET_FROM_LIST(tcflag_t, mode.c_oflag, term, 1); + SET_FROM_LIST(tcflag_t, mode.c_cflag, term, 2); + SET_FROM_LIST(tcflag_t, mode.c_lflag, term, 3); + SET_FROM_LIST(speed_t, ispeed, term, 4); + SET_FROM_LIST(speed_t, ospeed, term, 5); +#undef SET_FROM_LIST + + PyObject *cc = PyList_GET_ITEM(term, 6); if (!PyList_Check(cc) || PyList_Size(cc) != NCCS) { PyErr_Format(PyExc_TypeError, "tcsetattr: attributes[6] must be %d element list", @@ -237,8 +245,13 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) if (PyBytes_Check(v) && PyBytes_Size(v) == 1) mode.c_cc[i] = (cc_t) * PyBytes_AsString(v); - else if (PyLong_Check(v)) - mode.c_cc[i] = (cc_t) PyLong_AsLong(v); + else if (PyLong_Check(v)) { + long num = PyLong_AsLong(v); + if (num == -1 && PyErr_Occurred()) { + return NULL; + } + mode.c_cc[i] = (cc_t)num; + } else { PyErr_SetString(PyExc_TypeError, "tcsetattr: elements of attributes must be characters or integers"); From af29282fce117cb10f00907fd46d56c2fa6142f5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Oct 2023 14:42:36 +0200 Subject: [PATCH 40/73] gh-110367: Fix regrtest test_worker_output_on_failure() on ASAN build (#110387) Set ASAN_OPTIONS="handle_segv=0" env var to run the test. --- Lib/test/support/__init__.py | 8 ++++++++ Lib/test/test_faulthandler.py | 6 +----- Lib/test/test_regrtest.py | 15 +++++++++++---- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 38d5012ba46c08..900b9c96d08a64 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -436,6 +436,14 @@ def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False): return unittest.skipIf(skip, reason) +def set_sanitizer_env_var(env, option): + for name in ('ASAN_OPTIONS', 'MSAN_OPTIONS', 'UBSAN_OPTIONS'): + if name in env: + env[name] += f':{option}' + else: + env[name] = option + + def system_must_validate_cert(f): """Skip the test on TLS certificate validation failures.""" @functools.wraps(f) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 4dce7a7dd8c8df..0b8299a32b03c0 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -67,11 +67,7 @@ def get_output(self, code, filename=None, fd=None): # Sanitizers must not handle SIGSEGV (ex: for test_enable_fd()) option = 'handle_segv=0' - for name in ('ASAN_OPTIONS', 'MSAN_OPTIONS', 'UBSAN_OPTIONS'): - if name in env: - env[name] += f':{option}' - else: - env[name] = option + support.set_sanitizer_env_var(env, option) with support.SuppressCrashReport(): process = script_helper.spawn_python('-c', code, diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 66463fd2bbea22..de2c4317e71439 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -674,7 +674,7 @@ def run_command(self, args, input=None, exitcode=0, **kw): if 'stderr' not in kw: kw['stderr'] = subprocess.STDOUT proc = subprocess.run(args, - universal_newlines=True, + text=True, input=input, stdout=subprocess.PIPE, **kw) @@ -756,8 +756,8 @@ def check_output(self, output): self.check_executed_tests(output, self.tests, randomize=True, stats=len(self.tests)) - def run_tests(self, args): - output = self.run_python(args) + def run_tests(self, args, env=None): + output = self.run_python(args, env=env) self.check_output(output) def test_script_regrtest(self): @@ -2061,7 +2061,14 @@ def test_crash(self): """) testname = self.create_test(code=code) - output = self.run_tests("-j1", testname, exitcode=EXITCODE_BAD_TEST) + # Sanitizers must not handle SIGSEGV (ex: for test_enable_fd()) + env = dict(os.environ) + option = 'handle_segv=0' + support.set_sanitizer_env_var(env, option) + + output = self.run_tests("-j1", testname, + exitcode=EXITCODE_BAD_TEST, + env=env) self.check_executed_tests(output, testname, failed=[testname], stats=0, parallel=True) From cc389ef627b2a486ab89d9a11245bef48224efb1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 5 Oct 2023 14:26:44 +0100 Subject: [PATCH 41/73] gh-110259: Fix f-strings with multiline expressions and format specs (#110271) Signed-off-by: Pablo Galindo --- Lib/ast.py | 11 ++- Lib/test/test_tokenize.py | 97 +++++++++++++++++++ Lib/test/test_unparse.py | 3 +- ...-10-03-11-43-48.gh-issue-110259.ka93x5.rst | 3 + Parser/tokenizer.c | 24 +++-- 5 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-03-11-43-48.gh-issue-110259.ka93x5.rst diff --git a/Lib/ast.py b/Lib/ast.py index 1f54309c8450d8..f7888d18859ae4 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1270,13 +1270,15 @@ def visit_JoinedStr(self, node): quote_type = quote_types[0] self.write(f"{quote_type}{value}{quote_type}") - def _write_fstring_inner(self, node): + def _write_fstring_inner(self, node, scape_newlines=False): if isinstance(node, JoinedStr): # for both the f-string itself, and format_spec for value in node.values: - self._write_fstring_inner(value) + self._write_fstring_inner(value, scape_newlines=scape_newlines) elif isinstance(node, Constant) and isinstance(node.value, str): value = node.value.replace("{", "{{").replace("}", "}}") + if scape_newlines: + value = value.replace("\n", "\\n") self.write(value) elif isinstance(node, FormattedValue): self.visit_FormattedValue(node) @@ -1299,7 +1301,10 @@ def unparse_inner(inner): self.write(f"!{chr(node.conversion)}") if node.format_spec: self.write(":") - self._write_fstring_inner(node.format_spec) + self._write_fstring_inner( + node.format_spec, + scape_newlines=True + ) def visit_Name(self, node): self.write(node.id) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 94fb6d933de114..9369560788719f 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -566,6 +566,55 @@ def test_string(self): OP '=' (3, 0) (3, 1) OP '}' (3, 1) (3, 2) FSTRING_END "'''" (3, 2) (3, 5) + """) + self.check_tokenize("""\ +f'''__{ + x:a +}__'''""", """\ + FSTRING_START "f'''" (1, 0) (1, 4) + FSTRING_MIDDLE '__' (1, 4) (1, 6) + OP '{' (1, 6) (1, 7) + NL '\\n' (1, 7) (1, 8) + NAME 'x' (2, 4) (2, 5) + OP ':' (2, 5) (2, 6) + FSTRING_MIDDLE 'a\\n' (2, 6) (3, 0) + OP '}' (3, 0) (3, 1) + FSTRING_MIDDLE '__' (3, 1) (3, 3) + FSTRING_END "'''" (3, 3) (3, 6) + """) + self.check_tokenize("""\ +f'''__{ + x:a + b + c + d +}__'''""", """\ + FSTRING_START "f'''" (1, 0) (1, 4) + FSTRING_MIDDLE '__' (1, 4) (1, 6) + OP '{' (1, 6) (1, 7) + NL '\\n' (1, 7) (1, 8) + NAME 'x' (2, 4) (2, 5) + OP ':' (2, 5) (2, 6) + FSTRING_MIDDLE 'a\\n b\\n c\\n d\\n' (2, 6) (6, 0) + OP '}' (6, 0) (6, 1) + FSTRING_MIDDLE '__' (6, 1) (6, 3) + FSTRING_END "'''" (6, 3) (6, 6) + """) + self.check_tokenize("""\ +f'__{ + x:d +}__'""", """\ + FSTRING_START "f'" (1, 0) (1, 2) + FSTRING_MIDDLE '__' (1, 2) (1, 4) + OP '{' (1, 4) (1, 5) + NL '\\n' (1, 5) (1, 6) + NAME 'x' (2, 4) (2, 5) + OP ':' (2, 5) (2, 6) + FSTRING_MIDDLE 'd' (2, 6) (2, 7) + NL '\\n' (2, 7) (2, 8) + OP '}' (3, 0) (3, 1) + FSTRING_MIDDLE '__' (3, 1) (3, 3) + FSTRING_END "'" (3, 3) (3, 4) """) def test_function(self): @@ -2277,6 +2326,54 @@ def test_string(self): FSTRING_START \'f"\' (1, 0) (1, 2) FSTRING_MIDDLE 'hola\\\\\\\\\\\\r\\\\ndfgf' (1, 2) (1, 16) FSTRING_END \'"\' (1, 16) (1, 17) + """) + + self.check_tokenize("""\ +f'''__{ + x:a +}__'''""", """\ + FSTRING_START "f'''" (1, 0) (1, 4) + FSTRING_MIDDLE '__' (1, 4) (1, 6) + LBRACE '{' (1, 6) (1, 7) + NAME 'x' (2, 4) (2, 5) + COLON ':' (2, 5) (2, 6) + FSTRING_MIDDLE 'a\\n' (2, 6) (3, 0) + RBRACE '}' (3, 0) (3, 1) + FSTRING_MIDDLE '__' (3, 1) (3, 3) + FSTRING_END "'''" (3, 3) (3, 6) + """) + + self.check_tokenize("""\ +f'''__{ + x:a + b + c + d +}__'''""", """\ + FSTRING_START "f'''" (1, 0) (1, 4) + FSTRING_MIDDLE '__' (1, 4) (1, 6) + LBRACE '{' (1, 6) (1, 7) + NAME 'x' (2, 4) (2, 5) + COLON ':' (2, 5) (2, 6) + FSTRING_MIDDLE 'a\\n b\\n c\\n d\\n' (2, 6) (6, 0) + RBRACE '}' (6, 0) (6, 1) + FSTRING_MIDDLE '__' (6, 1) (6, 3) + FSTRING_END "'''" (6, 3) (6, 6) + """) + + self.check_tokenize("""\ +f'__{ + x:d +}__'""", """\ + FSTRING_START "f'" (1, 0) (1, 2) + FSTRING_MIDDLE '__' (1, 2) (1, 4) + LBRACE '{' (1, 4) (1, 5) + NAME 'x' (2, 4) (2, 5) + COLON ':' (2, 5) (2, 6) + FSTRING_MIDDLE 'd' (2, 6) (2, 7) + RBRACE '}' (3, 0) (3, 1) + FSTRING_MIDDLE '__' (3, 1) (3, 3) + FSTRING_END "'" (3, 3) (3, 4) """) def test_function(self): diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index bdf7b0588bee67..6f698a8d891815 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -730,7 +730,8 @@ class DirectoryTestCase(ASTTestCase): test_directories = (lib_dir, lib_dir / "test") run_always_files = {"test_grammar.py", "test_syntax.py", "test_compile.py", "test_ast.py", "test_asdl_parser.py", "test_fstring.py", - "test_patma.py", "test_type_alias.py", "test_type_params.py"} + "test_patma.py", "test_type_alias.py", "test_type_params.py", + "test_tokenize.py"} _files_to_test = None diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-03-11-43-48.gh-issue-110259.ka93x5.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-03-11-43-48.gh-issue-110259.ka93x5.rst new file mode 100644 index 00000000000000..55c743d0e4917e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-03-11-43-48.gh-issue-110259.ka93x5.rst @@ -0,0 +1,3 @@ +Correctly identify the format spec in f-strings (with single or triple +quotes) that have multiple lines in the expression part and include a +formatting spec. Patch by Pablo Galindo diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 41d0d16a471dd6..5e3816f59af35d 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -2690,11 +2690,28 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct if (tok->done == E_ERROR) { return MAKE_TOKEN(ERRORTOKEN); } - if (c == EOF || (current_tok->f_string_quote_size == 1 && c == '\n')) { + int in_format_spec = ( + current_tok->last_expr_end != -1 + && + INSIDE_FSTRING_EXPR(current_tok) + ); + + if (c == EOF || (current_tok->f_string_quote_size == 1 && c == '\n')) { if (tok->decoding_erred) { return MAKE_TOKEN(ERRORTOKEN); } + // If we are in a format spec and we found a newline, + // it means that the format spec ends here and we should + // return to the regular mode. + if (in_format_spec && c == '\n') { + tok_backup(tok, c); + TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; + p_start = tok->start; + p_end = tok->cur; + return MAKE_TOKEN(FSTRING_MIDDLE); + } + assert(tok->multi_line_start != NULL); // shift the tok_state's location into // the start of string, and report the error @@ -2726,11 +2743,6 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct end_quote_size = 0; } - int in_format_spec = ( - current_tok->last_expr_end != -1 - && - INSIDE_FSTRING_EXPR(current_tok) - ); if (c == '{') { int peek = tok_nextc(tok); if (peek != '{' || in_format_spec) { From 2cb62c6437fa07e08b4778f7ab9baa5f16ac01f2 Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Thu, 5 Oct 2023 22:08:42 +0800 Subject: [PATCH 42/73] gh-110309: Prune empty constant in format specs (#110320) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Lib/test/test_fstring.py | 48 +++++++++++++++++++ ...-10-03-23-26-18.gh-issue-110309.Y8nDOF.rst | 1 + Parser/action_helpers.c | 40 ++++++++++++---- 3 files changed, 79 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-03-23-26-18.gh-issue-110309.Y8nDOF.rst diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 4f05a149a901b2..dd8c2dd628ee13 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -514,6 +514,54 @@ def test_ast_fstring_empty_format_spec(self): self.assertEqual(type(format_spec), ast.JoinedStr) self.assertEqual(len(format_spec.values), 0) + def test_ast_fstring_format_spec(self): + expr = "f'{1:{name}}'" + + mod = ast.parse(expr) + self.assertEqual(type(mod), ast.Module) + self.assertEqual(len(mod.body), 1) + + fstring = mod.body[0].value + self.assertEqual(type(fstring), ast.JoinedStr) + self.assertEqual(len(fstring.values), 1) + + fv = fstring.values[0] + self.assertEqual(type(fv), ast.FormattedValue) + + format_spec = fv.format_spec + self.assertEqual(type(format_spec), ast.JoinedStr) + self.assertEqual(len(format_spec.values), 1) + + format_spec_value = format_spec.values[0] + self.assertEqual(type(format_spec_value), ast.FormattedValue) + self.assertEqual(format_spec_value.value.id, 'name') + + expr = "f'{1:{name1}{name2}}'" + + mod = ast.parse(expr) + self.assertEqual(type(mod), ast.Module) + self.assertEqual(len(mod.body), 1) + + fstring = mod.body[0].value + self.assertEqual(type(fstring), ast.JoinedStr) + self.assertEqual(len(fstring.values), 1) + + fv = fstring.values[0] + self.assertEqual(type(fv), ast.FormattedValue) + + format_spec = fv.format_spec + self.assertEqual(type(format_spec), ast.JoinedStr) + self.assertEqual(len(format_spec.values), 2) + + format_spec_value = format_spec.values[0] + self.assertEqual(type(format_spec_value), ast.FormattedValue) + self.assertEqual(format_spec_value.value.id, 'name1') + + format_spec_value = format_spec.values[1] + self.assertEqual(type(format_spec_value), ast.FormattedValue) + self.assertEqual(format_spec_value.value.id, 'name2') + + def test_docstring(self): def f(): f'''Not a docstring''' diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-03-23-26-18.gh-issue-110309.Y8nDOF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-03-23-26-18.gh-issue-110309.Y8nDOF.rst new file mode 100644 index 00000000000000..830428730391df --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-03-23-26-18.gh-issue-110309.Y8nDOF.rst @@ -0,0 +1 @@ +Remove unnecessary empty constant nodes in the ast of f-string specs. diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 36e0750220a30d..b8713a329d4ef6 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -998,18 +998,38 @@ _PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, in return NULL; } - // This is needed to keep compatibility with 3.11, where an empty format spec is parsed - // as an *empty* JoinedStr node, instead of having an empty constant in it. - if (asdl_seq_LEN(spec) == 1) { - expr_ty e = asdl_seq_GET(spec, 0); - if (e->kind == Constant_kind - && PyUnicode_Check(e->v.Constant.value) - && PyUnicode_GetLength(e->v.Constant.value) == 0) { - spec = _Py_asdl_expr_seq_new(0, arena); + // This is needed to keep compatibility with 3.11, where an empty format + // spec is parsed as an *empty* JoinedStr node, instead of having an empty + // constant in it. + Py_ssize_t n_items = asdl_seq_LEN(spec); + Py_ssize_t non_empty_count = 0; + for (Py_ssize_t i = 0; i < n_items; i++) { + expr_ty item = asdl_seq_GET(spec, i); + non_empty_count += !(item->kind == Constant_kind && + PyUnicode_CheckExact(item->v.Constant.value) && + PyUnicode_GET_LENGTH(item->v.Constant.value) == 0); + } + if (non_empty_count != n_items) { + asdl_expr_seq *resized_spec = + _Py_asdl_expr_seq_new(non_empty_count, p->arena); + if (resized_spec == NULL) { + return NULL; + } + Py_ssize_t j = 0; + for (Py_ssize_t i = 0; i < n_items; i++) { + expr_ty item = asdl_seq_GET(spec, i); + if (item->kind == Constant_kind && + PyUnicode_CheckExact(item->v.Constant.value) && + PyUnicode_GET_LENGTH(item->v.Constant.value) == 0) { + continue; + } + asdl_seq_SET(resized_spec, j++, item); } + assert(j == non_empty_count); + spec = resized_spec; } - - expr_ty res = _PyAST_JoinedStr(spec, lineno, col_offset, end_lineno, end_col_offset, p->arena); + expr_ty res = _PyAST_JoinedStr(spec, lineno, col_offset, end_lineno, + end_col_offset, p->arena); if (!res) { return NULL; } From d33aa18f15de482a01988aabc75907328e1f9c9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EB=B0=95=EB=AC=B8=EC=8B=9D?= Date: Thu, 5 Oct 2023 23:49:07 +0900 Subject: [PATCH 43/73] gh-82367: Use `FindFirstFile` Win32 API in `ntpath.realpath()` (GH-110298) * Use `FindFirstFile` Win32 API to fix a bug where `ntpath.realpath()` breaks out of traversing a series of paths where a (handled) `ERROR_ACCESS_DENIED` or `ERROR_SHARING_VIOLATION` occurs. * Update docs to reflect that `ntpath.realpath()` eliminates MS-DOS style names. --- Doc/library/os.path.rst | 3 +- Doc/whatsnew/3.13.rst | 3 ++ Lib/ntpath.py | 16 ++++--- Lib/test/test_ntpath.py | 43 +++++++++++++++++++ Misc/ACKS | 1 + ...3-10-03-12-30-59.gh-issue-82367.nxwfMx.rst | 2 + Modules/clinic/posixmodule.c.h | 40 ++++++++++++++++- Modules/posixmodule.c | 32 ++++++++++++++ 8 files changed, 132 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-10-03-12-30-59.gh-issue-82367.nxwfMx.rst diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 6f9e0853bc8947..95933f56d50542 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -377,7 +377,8 @@ the :mod:`glob` module.) Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path (if they are supported by the operating - system). + system). On Windows, this function will also resolve MS-DOS (also called 8.3) + style names such as ``C:\\PROGRA~1`` to ``C:\\Program Files``. If a path doesn't exist or a symlink loop is encountered, and *strict* is ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 785deea1c1e48f..7a62963203e164 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -97,6 +97,9 @@ Other Language Changes if supported. (Contributed by Victor Stinner in :gh:`109649`.) +* :func:`os.path.realpath` now resolves MS-DOS style file names even if + the file is not accessible. + (Contributed by Moonsik Park in :gh:`82367`.) New Modules =========== diff --git a/Lib/ntpath.py b/Lib/ntpath.py index df3402d46c9cc6..3061a4a5ef4c56 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -23,7 +23,6 @@ import genericpath from genericpath import * - __all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", "getatime","getctime", "islink","exists","lexists","isdir","isfile", @@ -601,7 +600,7 @@ def abspath(path): return _abspath_fallback(path) try: - from nt import _getfinalpathname, readlink as _nt_readlink + from nt import _findfirstfile, _getfinalpathname, readlink as _nt_readlink except ImportError: # realpath is a no-op on systems without _getfinalpathname support. realpath = abspath @@ -688,10 +687,15 @@ def _getfinalpathname_nonstrict(path): except OSError: # If we fail to readlink(), let's keep traversing pass - path, name = split(path) - # TODO (bpo-38186): Request the real file name from the directory - # entry using FindFirstFileW. For now, we will return the path - # as best we have it + # If we get these errors, try to get the real name of the file without accessing it. + if ex.winerror in (1, 5, 32, 50, 87, 1920, 1921): + try: + name = _findfirstfile(path) + path, _ = split(path) + except OSError: + path, name = split(path) + else: + path, name = split(path) if path and not name: return path + tail tail = join(name, tail) if tail else name diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index d91dcdfb0c5fac..3e710d1c6dabe4 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -2,6 +2,7 @@ import ntpath import os import string +import subprocess import sys import unittest import warnings @@ -637,6 +638,48 @@ def test_realpath_cwd(self): with os_helper.change_cwd(test_dir_short): self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + def test_realpath_permission(self): + # Test whether python can resolve the real filename of a + # shortened file name even if it does not have permission to access it. + ABSTFN = ntpath.realpath(os_helper.TESTFN) + + os_helper.unlink(ABSTFN) + os_helper.rmtree(ABSTFN) + os.mkdir(ABSTFN) + self.addCleanup(os_helper.rmtree, ABSTFN) + + test_file = ntpath.join(ABSTFN, "LongFileName123.txt") + test_file_short = ntpath.join(ABSTFN, "LONGFI~1.TXT") + + with open(test_file, "wb") as f: + f.write(b"content") + # Automatic generation of short names may be disabled on + # NTFS volumes for the sake of performance. + # They're not supported at all on ReFS and exFAT. + subprocess.run( + # Try to set the short name manually. + ['fsutil.exe', 'file', 'setShortName', test_file, 'LONGFI~1.TXT'], + creationflags=subprocess.DETACHED_PROCESS + ) + + try: + self.assertPathEqual(test_file, ntpath.realpath(test_file_short)) + except AssertionError: + raise unittest.SkipTest('the filesystem seems to lack support for short filenames') + + # Deny the right to [S]YNCHRONIZE on the file to + # force nt._getfinalpathname to fail with ERROR_ACCESS_DENIED. + p = subprocess.run( + ['icacls.exe', test_file, '/deny', '*S-1-5-32-545:(S)'], + creationflags=subprocess.DETACHED_PROCESS + ) + + if p.returncode: + raise unittest.SkipTest('failed to deny access to the test file') + + self.assertPathEqual(test_file, ntpath.realpath(test_file_short)) + def test_expandvars(self): with os_helper.EnvironmentVarGuard() as env: env.clear() diff --git a/Misc/ACKS b/Misc/ACKS index ccdfae66832f0e..94cb1965676f48 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1373,6 +1373,7 @@ Peter Parente Alexandre Parenteau Dan Parisien HyeSoo Park +Moonsik Park William Park Claude Paroz Heikki Partanen diff --git a/Misc/NEWS.d/next/Windows/2023-10-03-12-30-59.gh-issue-82367.nxwfMx.rst b/Misc/NEWS.d/next/Windows/2023-10-03-12-30-59.gh-issue-82367.nxwfMx.rst new file mode 100644 index 00000000000000..613ca075044b51 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-10-03-12-30-59.gh-issue-82367.nxwfMx.rst @@ -0,0 +1,2 @@ +:func:`os.path.realpath` now resolves MS-DOS style file names even if +the file is not accessible. Patch by Moonsik Park. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index fc39ab72bf2a51..0238d3a2f23149 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1848,6 +1848,40 @@ os__getfinalpathname(PyObject *module, PyObject *arg) #if defined(MS_WINDOWS) +PyDoc_STRVAR(os__findfirstfile__doc__, +"_findfirstfile($module, path, /)\n" +"--\n" +"\n" +"A function to get the real file name without accessing the file in Windows."); + +#define OS__FINDFIRSTFILE_METHODDEF \ + {"_findfirstfile", (PyCFunction)os__findfirstfile, METH_O, os__findfirstfile__doc__}, + +static PyObject * +os__findfirstfile_impl(PyObject *module, path_t *path); + +static PyObject * +os__findfirstfile(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + path_t path = PATH_T_INITIALIZE("_findfirstfile", "path", 0, 0); + + if (!path_converter(arg, &path)) { + goto exit; + } + return_value = os__findfirstfile_impl(module, &path); + +exit: + /* Cleanup for path */ + path_cleanup(&path); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + PyDoc_STRVAR(os__getvolumepathname__doc__, "_getvolumepathname($module, /, path)\n" "--\n" @@ -11451,6 +11485,10 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS__GETFINALPATHNAME_METHODDEF #endif /* !defined(OS__GETFINALPATHNAME_METHODDEF) */ +#ifndef OS__FINDFIRSTFILE_METHODDEF + #define OS__FINDFIRSTFILE_METHODDEF +#endif /* !defined(OS__FINDFIRSTFILE_METHODDEF) */ + #ifndef OS__GETVOLUMEPATHNAME_METHODDEF #define OS__GETVOLUMEPATHNAME_METHODDEF #endif /* !defined(OS__GETVOLUMEPATHNAME_METHODDEF) */ @@ -11986,4 +12024,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=8b60de6ddb925bc3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a36904281a8a7507 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d3c0aa6f3c5382..2c32a45a53277f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4809,6 +4809,37 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) return result; } +/*[clinic input] +os._findfirstfile + path: path_t + / +A function to get the real file name without accessing the file in Windows. +[clinic start generated code]*/ + +static PyObject * +os__findfirstfile_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=106dd3f0779c83dd input=0734dff70f60e1a8]*/ +{ + PyObject *result; + HANDLE hFindFile; + WIN32_FIND_DATAW wFileData; + WCHAR *wRealFileName; + + Py_BEGIN_ALLOW_THREADS + hFindFile = FindFirstFileW(path->wide, &wFileData); + Py_END_ALLOW_THREADS + + if (hFindFile == INVALID_HANDLE_VALUE) { + path_error(path); + return NULL; + } + + wRealFileName = wFileData.cFileName; + result = PyUnicode_FromWideChar(wRealFileName, wcslen(wRealFileName)); + FindClose(hFindFile); + return result; +} + /*[clinic input] os._getvolumepathname @@ -15961,6 +15992,7 @@ static PyMethodDef posix_methods[] = { OS__GETFULLPATHNAME_METHODDEF OS__GETDISKUSAGE_METHODDEF OS__GETFINALPATHNAME_METHODDEF + OS__FINDFIRSTFILE_METHODDEF OS__GETVOLUMEPATHNAME_METHODDEF OS__PATH_SPLITROOT_METHODDEF OS__PATH_NORMPATH_METHODDEF From 1328fa31fe9c72748fc6fd11d017c82aafd48a49 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Oct 2023 17:02:48 +0200 Subject: [PATCH 44/73] gh-110393: Remove watchdog with hardcoded timeout (#110400) test_builtin and test_socketserver no longer use signal.alarm() to implement a watchdog with a hardcoded timeout (2 and 60 seconds). Python test runner regrtest has two watchdogs: faulthandler and timeout on running worker processes. Tests using short hardcoded timeout can fail on slowest buildbots just because the timeout is too short. --- Lib/test/test_builtin.py | 2 -- Lib/test/test_socketserver.py | 7 ------- 2 files changed, 9 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 33cb248ff6e82b..b7966f8f03875b 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2203,8 +2203,6 @@ def _run_child(self, child, terminal_input): if pid == 0: # Child try: - # Make sure we don't get stuck if there's a problem - signal.alarm(2) os.close(r) with open(w, "w") as wpipe: child(wpipe) diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index c81d559cde315d..0f62f9eb200e42 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -32,11 +32,6 @@ HAVE_FORKING = test.support.has_fork_support requires_forking = unittest.skipUnless(HAVE_FORKING, 'requires forking') -def signal_alarm(n): - """Call signal.alarm when it exists (i.e. not on Windows).""" - if hasattr(signal, 'alarm'): - signal.alarm(n) - # Remember real select() to avoid interferences with mocking _real_select = select.select @@ -68,12 +63,10 @@ class SocketServerTest(unittest.TestCase): """Test all socket servers.""" def setUp(self): - signal_alarm(60) # Kill deadlocks after 60 seconds. self.port_seed = 0 self.test_files = [] def tearDown(self): - signal_alarm(0) # Didn't deadlock. reap_children() for fn in self.test_files: From 9eb2489266c4c1f115b8f72c0728db737cc8a815 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 5 Oct 2023 11:12:06 -0400 Subject: [PATCH 45/73] gh-109329: Add stat for "trace too short" (GH-110402) --- Include/cpython/pystats.h | 1 + Python/optimizer.c | 1 + Python/specialize.c | 1 + Tools/scripts/summarize_stats.py | 2 ++ 4 files changed, 5 insertions(+) diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 056406e6b5d992..4988caa803723d 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -110,6 +110,7 @@ typedef struct _optimization_stats { uint64_t trace_stack_overflow; uint64_t trace_stack_underflow; uint64_t trace_too_long; + uint64_t trace_too_short; uint64_t inner_loop; uint64_t recursive_call; UOpStats opcode[512]; diff --git a/Python/optimizer.c b/Python/optimizer.c index f8796eb24073d2..65b9638be25e98 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -798,6 +798,7 @@ translate_bytecode_to_trace( return trace_length; } else { + OPT_STAT_INC(trace_too_short); DPRINTF(4, "No trace for %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), diff --git a/Python/specialize.c b/Python/specialize.c index ff732eb30ca1e0..49633b103b3815 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -229,6 +229,7 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "Optimization trace stack overflow: %" PRIu64 "\n", stats->trace_stack_overflow); fprintf(out, "Optimization trace stack underflow: %" PRIu64 "\n", stats->trace_stack_underflow); fprintf(out, "Optimization trace too long: %" PRIu64 "\n", stats->trace_too_long); + fprintf(out, "Optimization trace too short: %" PRIu64 "\n", stats->trace_too_short); fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index b9cc2f94080b88..bdca51df3dac53 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -760,6 +760,7 @@ def calculate_optimization_stats(stats): trace_stack_overflow = stats["Optimization trace stack overflow"] trace_stack_underflow = stats["Optimization trace stack underflow"] trace_too_long = stats["Optimization trace too long"] + trace_too_short = stats["Optimiztion trace too short"] inner_loop = stats["Optimization inner loop"] recursive_call = stats["Optimization recursive call"] @@ -771,6 +772,7 @@ def calculate_optimization_stats(stats): ("Trace stack overflow", trace_stack_overflow, ""), ("Trace stack underflow", trace_stack_underflow, ""), ("Trace too long", trace_too_long, ""), + ("Trace too short", trace_too_short, ""), ("Inner loop found", inner_loop, ""), ("Recursive call", recursive_call, ""), ] From 6e97a9647ae028facb392d12fc24973503693bd6 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 5 Oct 2023 15:46:33 +0000 Subject: [PATCH 46/73] gh-109549: Add new states to PyThreadState to support PEP 703 (gh-109915) This adds a new field 'state' to PyThreadState that can take on one of three values: _Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, or _Py_THREAD_GC. The "attached" and "detached" states correspond closely to acquiring and releasing the GIL. The "gc" state is current unused, but will be used to implement stop-the-world GC for --disable-gil builds in the near future. --- Include/cpython/pystate.h | 4 + Include/internal/pycore_ceval.h | 1 - Include/internal/pycore_pystate.h | 42 ++++++++++ Python/ceval_gil.c | 47 +++-------- Python/pylifecycle.c | 14 ++-- Python/pystate.c | 125 +++++++++++++++++++----------- 6 files changed, 141 insertions(+), 92 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 8fd06242bc82e0..40102f8855090e 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -102,6 +102,10 @@ struct _ts { #endif int _whence; + /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_GC). + See Include/internal/pycore_pystate.h for more details. */ + int state; + int py_recursion_remaining; int py_recursion_limit; diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 48fee697324f8f..d3ea3a898bb425 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -121,7 +121,6 @@ extern void _PyEval_FiniGIL(PyInterpreterState *interp); extern void _PyEval_AcquireLock(PyThreadState *tstate); extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *); -extern PyThreadState * _PyThreadState_SwapNoGIL(PyThreadState *); extern void _PyEval_DeactivateOpCache(void); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index a60d949bff1eba..7135b1e966feb5 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -11,6 +11,33 @@ extern "C" { #include "pycore_runtime.h" // _PyRuntime +// Values for PyThreadState.state. A thread must be in the "attached" state +// before calling most Python APIs. If the GIL is enabled, then "attached" +// implies that the thread holds the GIL and "detached" implies that the +// thread does not hold the GIL (or is in the process of releasing it). In +// `--disable-gil` builds, multiple threads may be "attached" to the same +// interpreter at the same time. Only the "bound" thread may perform the +// transitions between "attached" and "detached" on its own PyThreadState. +// +// The "gc" state is used to implement stop-the-world pauses, such as for +// cyclic garbage collection. It is only used in `--disable-gil` builds. It is +// similar to the "detached" state, but only the thread performing a +// stop-the-world pause may transition threads between the "detached" and "gc" +// states. A thread trying to "attach" from the "gc" state will block until +// it is transitioned back to "detached" when the stop-the-world pause is +// complete. +// +// State transition diagram: +// +// (bound thread) (stop-the-world thread) +// [attached] <-> [detached] <-> [gc] +// +// See `_PyThreadState_Attach()` and `_PyThreadState_Detach()`. +#define _Py_THREAD_DETACHED 0 +#define _Py_THREAD_ATTACHED 1 +#define _Py_THREAD_GC 2 + + /* Check if the current thread is the main thread. Use _Py_IsMainInterpreter() to check if it's the main interpreter. */ static inline int @@ -104,6 +131,21 @@ _PyThreadState_GET(void) #endif } +// Attaches the current thread to the interpreter. +// +// This may block while acquiring the GIL (if the GIL is enabled) or while +// waiting for a stop-the-world pause (if the GIL is disabled). +// +// High-level code should generally call PyEval_RestoreThread() instead, which +// calls this function. +void _PyThreadState_Attach(PyThreadState *tstate); + +// Detaches the current thread from the interpreter. +// +// High-level code should generally call PyEval_SaveThread() instead, which +// calls this function. +void _PyThreadState_Detach(PyThreadState *tstate); + static inline void _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 6b4ec8eed03aab..f237e384333e45 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -462,24 +462,22 @@ PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil) { assert(tstate->interp->ceval.gil == NULL); - int locked; if (!own_gil) { /* The interpreter will share the main interpreter's instead. */ PyInterpreterState *main_interp = _PyInterpreterState_Main(); assert(tstate->interp != main_interp); struct _gil_runtime_state *gil = main_interp->ceval.gil; init_shared_gil(tstate->interp, gil); - locked = current_thread_holds_gil(gil, tstate); + assert(!current_thread_holds_gil(gil, tstate)); } else { PyThread_init_thread(); init_own_gil(tstate->interp, &tstate->interp->_gil); - locked = 0; - } - if (!locked) { - take_gil(tstate); } + // Lock the GIL and mark the current thread as attached. + _PyThreadState_Attach(tstate); + return _PyStatus_OK(); } @@ -569,24 +567,14 @@ void PyEval_AcquireThread(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); - - take_gil(tstate); - - if (_PyThreadState_SwapNoGIL(tstate) != NULL) { - Py_FatalError("non-NULL old thread state"); - } + _PyThreadState_Attach(tstate); } void PyEval_ReleaseThread(PyThreadState *tstate) { assert(_PyThreadState_CheckConsistency(tstate)); - - PyThreadState *new_tstate = _PyThreadState_SwapNoGIL(NULL); - if (new_tstate != tstate) { - Py_FatalError("wrong thread state"); - } - drop_gil(tstate->interp, tstate); + _PyThreadState_Detach(tstate); } #ifdef HAVE_FORK @@ -629,11 +617,8 @@ _PyEval_SignalAsyncExc(PyInterpreterState *interp) PyThreadState * PyEval_SaveThread(void) { - PyThreadState *tstate = _PyThreadState_SwapNoGIL(NULL); - _Py_EnsureTstateNotNULL(tstate); - - assert(gil_created(tstate->interp->ceval.gil)); - drop_gil(tstate->interp, tstate); + PyThreadState *tstate = _PyThreadState_GET(); + _PyThreadState_Detach(tstate); return tstate; } @@ -641,10 +626,7 @@ void PyEval_RestoreThread(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); - - take_gil(tstate); - - _PyThreadState_SwapNoGIL(tstate); + _PyThreadState_Attach(tstate); } @@ -1015,18 +997,11 @@ _Py_HandlePending(PyThreadState *tstate) /* GIL drop request */ if (_Py_eval_breaker_bit_is_set(interp, _PY_GIL_DROP_REQUEST_BIT)) { /* Give another thread a chance */ - if (_PyThreadState_SwapNoGIL(NULL) != tstate) { - Py_FatalError("tstate mix-up"); - } - drop_gil(interp, tstate); + _PyThreadState_Detach(tstate); /* Other threads may run now */ - take_gil(tstate); - - if (_PyThreadState_SwapNoGIL(tstate) != NULL) { - Py_FatalError("orphan tstate"); - } + _PyThreadState_Attach(tstate); } /* Check for asynchronous exception. */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index eb10aa3562dfce..14033162377489 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -661,8 +661,6 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return _PyStatus_ERR("can't make first thread"); } _PyThreadState_Bind(tstate); - // XXX For now we do this before the GIL is created. - (void) _PyThreadState_SwapNoGIL(tstate); status = init_interp_create_gil(tstate, config.gil); if (_PyStatus_EXCEPTION(status)) { @@ -2060,8 +2058,7 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) } _PyThreadState_Bind(tstate); - // XXX For now we do this before the GIL is created. - PyThreadState *save_tstate = _PyThreadState_SwapNoGIL(tstate); + PyThreadState *save_tstate = _PyThreadState_GET(); int has_gil = 0; /* From this point until the init_interp_create_gil() call, @@ -2073,7 +2070,7 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) const PyConfig *src_config; if (save_tstate != NULL) { // XXX Might new_interpreter() have been called without the GIL held? - _PyEval_ReleaseLock(save_tstate->interp, save_tstate); + _PyThreadState_Detach(save_tstate); src_config = _PyInterpreterState_GetConfig(save_tstate->interp); } else @@ -2120,12 +2117,11 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) *tstate_p = NULL; /* Oops, it didn't work. Undo it all. */ - PyErr_PrintEx(0); if (has_gil) { - PyThreadState_Swap(save_tstate); + _PyThreadState_Detach(tstate); } - else { - _PyThreadState_SwapNoGIL(save_tstate); + if (save_tstate != NULL) { + _PyThreadState_Attach(save_tstate); } PyThreadState_Clear(tstate); PyThreadState_Delete(tstate); diff --git a/Python/pystate.c b/Python/pystate.c index 068740828e9c25..a024ae7e3806a6 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -998,6 +998,7 @@ _PyInterpreterState_Clear(PyThreadState *tstate) static inline void tstate_deactivate(PyThreadState *tstate); +static void tstate_set_detached(PyThreadState *tstate); static void zapthreads(PyInterpreterState *interp); void @@ -1011,9 +1012,7 @@ PyInterpreterState_Delete(PyInterpreterState *interp) PyThreadState *tcur = current_fast_get(runtime); if (tcur != NULL && interp == tcur->interp) { /* Unset current thread. After this, many C API calls become crashy. */ - current_fast_clear(runtime); - tstate_deactivate(tcur); - _PyEval_ReleaseLock(interp, NULL); + _PyThreadState_Detach(tcur); } zapthreads(interp); @@ -1651,6 +1650,7 @@ static void tstate_delete_common(PyThreadState *tstate) { assert(tstate->_status.cleared && !tstate->_status.finalized); + assert(tstate->state != _Py_THREAD_ATTACHED); PyInterpreterState *interp = tstate->interp; if (interp == NULL) { @@ -1711,6 +1711,7 @@ void _PyThreadState_DeleteCurrent(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); + tstate_set_detached(tstate); tstate_delete_common(tstate); current_fast_clear(tstate->interp->runtime); _PyEval_ReleaseLock(tstate->interp, NULL); @@ -1867,6 +1868,79 @@ tstate_deactivate(PyThreadState *tstate) // It will still be used in PyGILState_Ensure(). } +static int +tstate_try_attach(PyThreadState *tstate) +{ +#ifdef Py_NOGIL + int expected = _Py_THREAD_DETACHED; + if (_Py_atomic_compare_exchange_int( + &tstate->state, + &expected, + _Py_THREAD_ATTACHED)) { + return 1; + } + return 0; +#else + assert(tstate->state == _Py_THREAD_DETACHED); + tstate->state = _Py_THREAD_ATTACHED; + return 1; +#endif +} + +static void +tstate_set_detached(PyThreadState *tstate) +{ + assert(tstate->state == _Py_THREAD_ATTACHED); +#ifdef Py_NOGIL + _Py_atomic_store_int(&tstate->state, _Py_THREAD_DETACHED); +#else + tstate->state = _Py_THREAD_DETACHED; +#endif +} + +void +_PyThreadState_Attach(PyThreadState *tstate) +{ +#if defined(Py_DEBUG) + // This is called from PyEval_RestoreThread(). Similar + // to it, we need to ensure errno doesn't change. + int err = errno; +#endif + + _Py_EnsureTstateNotNULL(tstate); + if (current_fast_get(&_PyRuntime) != NULL) { + Py_FatalError("non-NULL old thread state"); + } + + _PyEval_AcquireLock(tstate); + + // XXX assert(tstate_is_alive(tstate)); + current_fast_set(&_PyRuntime, tstate); + tstate_activate(tstate); + + if (!tstate_try_attach(tstate)) { + // TODO: Once stop-the-world GC is implemented for --disable-gil builds + // this will need to wait until the GC completes. For now, this case + // should never happen. + Py_FatalError("thread attach failed"); + } + +#if defined(Py_DEBUG) + errno = err; +#endif +} + +void +_PyThreadState_Detach(PyThreadState *tstate) +{ + // XXX assert(tstate_is_alive(tstate) && tstate_is_bound(tstate)); + assert(tstate->state == _Py_THREAD_ATTACHED); + assert(tstate == current_fast_get(&_PyRuntime)); + tstate_set_detached(tstate); + tstate_deactivate(tstate); + current_fast_clear(&_PyRuntime); + _PyEval_ReleaseLock(tstate->interp, tstate); +} //---------- // other API @@ -1939,56 +2013,15 @@ PyThreadState_Get(void) return tstate; } - -static void -_swap_thread_states(_PyRuntimeState *runtime, - PyThreadState *oldts, PyThreadState *newts) -{ - // XXX Do this only if oldts != NULL? - current_fast_clear(runtime); - - if (oldts != NULL) { - // XXX assert(tstate_is_alive(oldts) && tstate_is_bound(oldts)); - tstate_deactivate(oldts); - } - - if (newts != NULL) { - // XXX assert(tstate_is_alive(newts)); - assert(tstate_is_bound(newts)); - current_fast_set(runtime, newts); - tstate_activate(newts); - } -} - -PyThreadState * -_PyThreadState_SwapNoGIL(PyThreadState *newts) -{ -#if defined(Py_DEBUG) - /* This can be called from PyEval_RestoreThread(). Similar - to it, we need to ensure errno doesn't change. - */ - int err = errno; -#endif - - PyThreadState *oldts = current_fast_get(&_PyRuntime); - _swap_thread_states(&_PyRuntime, oldts, newts); - -#if defined(Py_DEBUG) - errno = err; -#endif - return oldts; -} - PyThreadState * _PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) { PyThreadState *oldts = current_fast_get(runtime); if (oldts != NULL) { - _PyEval_ReleaseLock(oldts->interp, oldts); + _PyThreadState_Detach(oldts); } - _swap_thread_states(runtime, oldts, newts); if (newts != NULL) { - _PyEval_AcquireLock(newts); + _PyThreadState_Attach(newts); } return oldts; } From e37d4557c3de0476e76ca4b8a1cc8d2566b86c79 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Oct 2023 18:31:02 +0200 Subject: [PATCH 47/73] gh-110391: socket NetworkConnectionAttributesTest always declare cli (#110401) NetworkConnectionAttributesTest of test_socket now always declare the 'cli' attribute, so clientTearDown() cannot fail with AttributeError. --- Lib/test/test_socket.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 99c4c5cbc4902d..09605f7e774dda 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -5356,6 +5356,7 @@ def test_create_connection_timeout(self): class NetworkConnectionAttributesTest(SocketTCPTest, ThreadableTest): + cli = None def __init__(self, methodName='runTest'): SocketTCPTest.__init__(self, methodName=methodName) @@ -5365,7 +5366,8 @@ def clientSetUp(self): self.source_port = socket_helper.find_unused_port() def clientTearDown(self): - self.cli.close() + if self.cli is not None: + self.cli.close() self.cli = None ThreadableTest.clientTearDown(self) From a973bf0f97e55ace9eab100f9eb95d7eedcb28ac Mon Sep 17 00:00:00 2001 From: Towster15 <105541074+Towster15@users.noreply.github.com> Date: Thu, 5 Oct 2023 18:01:35 +0100 Subject: [PATCH 48/73] gh-110383 TimeIt Docs Spelling Fix (#110407) Make 0.2 second plural --- Doc/library/timeit.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 60ec135873c414..616f8365b80f6c 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -151,7 +151,7 @@ The module defines three convenience functions and a public class: so that the total time >= 0.2 second, returning the eventual (number of loops, time taken for that number of loops). It calls :meth:`.timeit` with increasing numbers from the sequence 1, 2, 5, - 10, 20, 50, ... until the time taken is at least 0.2 second. + 10, 20, 50, ... until the time taken is at least 0.2 seconds. If *callback* is given and is not ``None``, it will be called after each trial with two arguments: ``callback(number, time_taken)``. From a13620685f68957c965fca89343a0e91f95f1bab Mon Sep 17 00:00:00 2001 From: Harmen Stoppels Date: Thu, 5 Oct 2023 19:27:19 +0200 Subject: [PATCH 49/73] Fix env var typo in perf profiling docs (#110404) Fix typo in docs --- Doc/howto/perf_profiling.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index 61812c19ae6ca9..af7b67d2042950 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -162,8 +162,7 @@ the :option:`!-X` option takes precedence over the environment variable. Example, using the environment variable:: - $ PYTHONPERFSUPPORT=1 - $ python script.py + $ PYTHONPERFSUPPORT=1 python script.py $ perf report -g -i perf.data Example, using the :option:`!-X` option:: From 1f3af03f83fead5cd86d54e1b2f47fc5866e635c Mon Sep 17 00:00:00 2001 From: AN Long Date: Fri, 6 Oct 2023 01:52:26 +0800 Subject: [PATCH 50/73] gh-110147: test_msvcrt: run console I/O tests in new processes (#110268) --- Lib/test/test_msvcrt.py | 49 +++++++++++++++----------- PC/_testconsole.c | 19 ----------- PC/clinic/_testconsole.c.h | 70 +------------------------------------- 3 files changed, 31 insertions(+), 107 deletions(-) diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py index 81ec13026014e6..600c4446fd5cd4 100644 --- a/Lib/test/test_msvcrt.py +++ b/Lib/test/test_msvcrt.py @@ -1,17 +1,17 @@ import os +import subprocess import sys import unittest +from textwrap import dedent -from test.support import os_helper +from test.support import os_helper, requires_resource from test.support.os_helper import TESTFN, TESTFN_ASCII if sys.platform != "win32": raise unittest.SkipTest("windows related tests") import _winapi -import msvcrt; - -from _testconsole import write_input, flush_console_input_buffer +import msvcrt class TestFileOperations(unittest.TestCase): @@ -61,34 +61,45 @@ def test_get_osfhandle(self): class TestConsoleIO(unittest.TestCase): + # CREATE_NEW_CONSOLE creates a "popup" window. + @requires_resource('gui') + def run_in_separated_process(self, code): + # Run test in a seprated process to avoid stdin conflicts. + # See: gh-110147 + cmd = [sys.executable, '-c', code] + subprocess.run(cmd, check=True, capture_output=True, + creationflags=subprocess.CREATE_NEW_CONSOLE) + def test_kbhit(self): - h = msvcrt.get_osfhandle(sys.stdin.fileno()) - flush_console_input_buffer(h) - self.assertEqual(msvcrt.kbhit(), 0) + code = dedent(''' + import msvcrt + assert msvcrt.kbhit() == 0 + ''') + self.run_in_separated_process(code) def test_getch(self): msvcrt.ungetch(b'c') self.assertEqual(msvcrt.getch(), b'c') - def test_getwch(self): - with open('CONIN$', 'rb', buffering=0) as stdin: - h = msvcrt.get_osfhandle(stdin.fileno()) - flush_console_input_buffer(h) + def check_getwch(self, funcname): + code = dedent(f''' + import msvcrt + from _testconsole import write_input + with open("CONIN$", "rb", buffering=0) as stdin: + write_input(stdin, {ascii(c_encoded)}) + assert msvcrt.{funcname}() == "{c}" + ''') + self.run_in_separated_process(code) - write_input(stdin, c_encoded) - self.assertEqual(msvcrt.getwch(), c) + def test_getwch(self): + self.check_getwch('getwch') def test_getche(self): msvcrt.ungetch(b'c') self.assertEqual(msvcrt.getche(), b'c') def test_getwche(self): - with open('CONIN$', 'rb', buffering=0) as stdin: - h = msvcrt.get_osfhandle(stdin.fileno()) - flush_console_input_buffer(h) - - write_input(stdin, c_encoded) - self.assertEqual(msvcrt.getwche(), c) + self.check_getwch('getwche') def test_putch(self): msvcrt.putch(b'c') diff --git a/PC/_testconsole.c b/PC/_testconsole.c index 5e5a771b96bfec..1dc0d230c4d7c3 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -133,31 +133,12 @@ _testconsole_read_output_impl(PyObject *module, PyObject *file) Py_RETURN_NONE; } -/*[clinic input] -_testconsole.flush_console_input_buffer - handle: HANDLE - -Flushes the console input buffer. - -All input records currently in the input buffer are discarded. -[clinic start generated code]*/ - -static PyObject * -_testconsole_flush_console_input_buffer_impl(PyObject *module, void *handle) -/*[clinic end generated code: output=1f923a81331465ce input=be8203ae84a288f5]*/ -/*[clinic end generated code:]*/ -{ - FlushConsoleInputBuffer(handle); - - Py_RETURN_NONE; -} #include "clinic\_testconsole.c.h" PyMethodDef testconsole_methods[] = { _TESTCONSOLE_WRITE_INPUT_METHODDEF _TESTCONSOLE_READ_OUTPUT_METHODDEF - _TESTCONSOLE_FLUSH_CONSOLE_INPUT_BUFFER_METHODDEF {NULL, NULL} }; diff --git a/PC/clinic/_testconsole.c.h b/PC/clinic/_testconsole.c.h index b76588909782ea..99cd302ff34698 100644 --- a/PC/clinic/_testconsole.c.h +++ b/PC/clinic/_testconsole.c.h @@ -132,70 +132,6 @@ _testconsole_read_output(PyObject *module, PyObject *const *args, Py_ssize_t nar #endif /* defined(MS_WINDOWS) */ -#if defined(MS_WINDOWS) - -PyDoc_STRVAR(_testconsole_flush_console_input_buffer__doc__, -"flush_console_input_buffer($module, /, handle)\n" -"--\n" -"\n" -"Flushes the console input buffer.\n" -"\n" -"All input records currently in the input buffer are discarded."); - -#define _TESTCONSOLE_FLUSH_CONSOLE_INPUT_BUFFER_METHODDEF \ - {"flush_console_input_buffer", _PyCFunction_CAST(_testconsole_flush_console_input_buffer), METH_FASTCALL|METH_KEYWORDS, _testconsole_flush_console_input_buffer__doc__}, - -static PyObject * -_testconsole_flush_console_input_buffer_impl(PyObject *module, void *handle); - -static PyObject * -_testconsole_flush_console_input_buffer(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(handle), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"handle", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "flush_console_input_buffer", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - void *handle; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - handle = PyLong_AsVoidPtr(args[0]); - if (!handle && PyErr_Occurred()) { - goto exit; - } - return_value = _testconsole_flush_console_input_buffer_impl(module, handle); - -exit: - return return_value; -} - -#endif /* defined(MS_WINDOWS) */ - #ifndef _TESTCONSOLE_WRITE_INPUT_METHODDEF #define _TESTCONSOLE_WRITE_INPUT_METHODDEF #endif /* !defined(_TESTCONSOLE_WRITE_INPUT_METHODDEF) */ @@ -203,8 +139,4 @@ _testconsole_flush_console_input_buffer(PyObject *module, PyObject *const *args, #ifndef _TESTCONSOLE_READ_OUTPUT_METHODDEF #define _TESTCONSOLE_READ_OUTPUT_METHODDEF #endif /* !defined(_TESTCONSOLE_READ_OUTPUT_METHODDEF) */ - -#ifndef _TESTCONSOLE_FLUSH_CONSOLE_INPUT_BUFFER_METHODDEF - #define _TESTCONSOLE_FLUSH_CONSOLE_INPUT_BUFFER_METHODDEF -#endif /* !defined(_TESTCONSOLE_FLUSH_CONSOLE_INPUT_BUFFER_METHODDEF) */ -/*[clinic end generated code: output=5d488564f2500dd9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f59fe72cd4e73704 input=a9049054013a1b77]*/ From 318f5df27109ff8d2519edefa771920a0ec62b92 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Oct 2023 20:53:03 +0200 Subject: [PATCH 51/73] gh-110167: Fix test_socket deadlock in doCleanups() (#110416) Fix a deadlock in test_socket when server fails with a timeout but the client is still running in its thread. Don't hold a lock to call cleanup functions in doCleanups(). One of the cleanup function waits until the client completes, whereas the client could deadlock if it called addCleanup() in such situation. doCleanups() is called when the server completed, but the client can still be running in its thread especially if the server failed with a timeout. Don't put a lock on doCleanups() to prevent deadlock between addCleanup() called in the client and doCleanups() waiting for self.done.wait of ThreadableTest._setUp(). --- Lib/test/test_socket.py | 12 +++++++----- .../2023-10-05-19-33-49.gh-issue-110167.mIdj3v.rst | 5 +++++ 2 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-10-05-19-33-49.gh-issue-110167.mIdj3v.rst diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 09605f7e774dda..0d1c1867a25c40 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -218,8 +218,13 @@ def setUp(self): class ThreadSafeCleanupTestCase: """Subclass of unittest.TestCase with thread-safe cleanup methods. - This subclass protects the addCleanup() and doCleanups() methods - with a recursive lock. + This subclass protects the addCleanup() method with a recursive lock. + + doCleanups() is called when the server completed, but the client can still + be running in its thread especially if the server failed with a timeout. + Don't put a lock on doCleanups() to prevent deadlock between addCleanup() + called in the client and doCleanups() waiting for self.done.wait of + ThreadableTest._setUp() (gh-110167) """ def __init__(self, *args, **kwargs): @@ -230,9 +235,6 @@ def addCleanup(self, *args, **kwargs): with self._cleanup_lock: return super().addCleanup(*args, **kwargs) - def doCleanups(self, *args, **kwargs): - with self._cleanup_lock: - return super().doCleanups(*args, **kwargs) class SocketCANTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Tests/2023-10-05-19-33-49.gh-issue-110167.mIdj3v.rst b/Misc/NEWS.d/next/Tests/2023-10-05-19-33-49.gh-issue-110167.mIdj3v.rst new file mode 100644 index 00000000000000..d0cbbf9c3788bb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-10-05-19-33-49.gh-issue-110167.mIdj3v.rst @@ -0,0 +1,5 @@ +Fix a deadlock in test_socket when server fails with a timeout but the +client is still running in its thread. Don't hold a lock to call cleanup +functions in doCleanups(). One of the cleanup function waits until the +client completes, whereas the client could deadlock if it called +addCleanup() in such situation. Patch by Victor Stinner. From 0db2f1475e6539e1954e1f8bd53e005c3ecd6a26 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Oct 2023 20:54:27 +0200 Subject: [PATCH 52/73] gh-110167: Increase support.LOOPBACK_TIMEOUT to 10 seconds (#110413) Increase support.LOOPBACK_TIMEOUT from 5 to 10 seconds. Also increase the timeout depending on the --timeout option. For example, for a test timeout of 40 minutes (ARM Raspbian 3.x), use LOOPBACK_TIMEOUT of 20 seconds instead of 5 seconds before. --- Lib/test/libregrtest/setup.py | 2 ++ Lib/test/support/__init__.py | 8 +------- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index cb410da5acb4c3..793347f60ad93c 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -111,6 +111,8 @@ def setup_tests(runtests: RunTests): timeout = runtests.timeout if timeout is not None: # For a slow buildbot worker, increase SHORT_TIMEOUT and LONG_TIMEOUT + support.LOOPBACK_TIMEOUT = max(support.LOOPBACK_TIMEOUT, timeout / 120) + # don't increase INTERNET_TIMEOUT support.SHORT_TIMEOUT = max(support.SHORT_TIMEOUT, timeout / 40) support.LONG_TIMEOUT = max(support.LONG_TIMEOUT, timeout / 4) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 900b9c96d08a64..f3270d66bd0852 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -75,13 +75,7 @@ # # The timeout should be long enough for connect(), recv() and send() methods # of socket.socket. -LOOPBACK_TIMEOUT = 5.0 -if sys.platform == 'win32' and ' 32 bit (ARM)' in sys.version: - # bpo-37553: test_socket.SendfileUsingSendTest is taking longer than 2 - # seconds on Windows ARM32 buildbot - LOOPBACK_TIMEOUT = 10 -elif sys.platform == 'vxworks': - LOOPBACK_TIMEOUT = 10 +LOOPBACK_TIMEOUT = 10.0 # Timeout in seconds for network requests going to the internet. The timeout is # short enough to prevent a test to wait for too long if the internet request From 5eae8dc2cb832af6ae1ee340fb0194107fe3bd6e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Oct 2023 21:32:06 +0200 Subject: [PATCH 53/73] gh-109840: Fix multiprocessing test_waitfor_timeout() (#110428) Don't measure the CI performance: don't fail if cond.wait_for() takes longer than 1 second on a slow CI. --- Lib/test/_test_multiprocessing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 39666dd331db0b..d3e713594b0f4b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1651,12 +1651,12 @@ def test_waitfor(self): def _test_waitfor_timeout_f(cls, cond, state, success, sem): sem.release() with cond: - expected = 0.1 + expected = 0.100 dt = time.monotonic() result = cond.wait_for(lambda : state.value==4, timeout=expected) dt = time.monotonic() - dt # borrow logic in assertTimeout() from test/lock_tests.py - if not result and expected * 0.6 < dt < expected * 10.0: + if not result and expected * 0.6 <= dt: success.value = True @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') @@ -1675,7 +1675,7 @@ def test_waitfor_timeout(self): # Only increment 3 times, so state == 4 is never reached. for i in range(3): - time.sleep(0.01) + time.sleep(0.010) with cond: state.value += 1 cond.notify() From 3c2f3215cc81ec8470208176dd32d2aef84faa13 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 5 Oct 2023 19:51:17 +0000 Subject: [PATCH 54/73] gh-110119: Fix test_importlib `--disable-gil` Windows test failures (#110422) Use "t" in the expected tag for `--disable-gil` builds in test_tagged_suffix. --- Lib/test/test_importlib/test_windows.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index a7586b3d95dc14..a60a4c4cebcf1a 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -4,6 +4,7 @@ import os import re import sys +import sysconfig import unittest from test.support import import_helper from contextlib import contextmanager @@ -111,8 +112,10 @@ def test_module_not_found(self): class WindowsExtensionSuffixTests: def test_tagged_suffix(self): suffixes = self.machinery.EXTENSION_SUFFIXES - expected_tag = ".cp{0.major}{0.minor}-{1}.pyd".format(sys.version_info, - re.sub('[^a-zA-Z0-9]', '_', get_platform())) + abi_flags = "t" if sysconfig.get_config_var("Py_NOGIL") else "" + ver = sys.version_info + platform = re.sub('[^a-zA-Z0-9]', '_', get_platform()) + expected_tag = f".cp{ver.major}{ver.minor}{abi_flags}-{platform}.pyd" try: untagged_i = suffixes.index(".pyd") except ValueError: From d257479c2f6cbf3b69ed90062f00635832e4bf91 Mon Sep 17 00:00:00 2001 From: Bradley Reynolds Date: Thu, 5 Oct 2023 14:55:44 -0500 Subject: [PATCH 55/73] gh-110383: Swap 'the all' -> 'all the' in socket docs (#110434) --- Doc/library/socket.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 83957c87990440..7fb47ce2b84593 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -2100,7 +2100,7 @@ The next two examples are identical to the above two, but support both IPv4 and IPv6. The server side will listen to the first address family available (it should listen to both instead). On most of IPv6-ready systems, IPv6 will take precedence and the server may not accept IPv4 traffic. The client side will try -to connect to the all addresses returned as a result of the name resolution, and +to connect to all the addresses returned as a result of the name resolution, and sends traffic to the first one connected successfully. :: # Echo server program From fb6c4ed2bbb2a867d5f0b9a94656e4714be5d9c2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Oct 2023 22:26:37 +0200 Subject: [PATCH 56/73] gh-110429: Fix race condition in "make regen-all" (#110433) "make regen-pegen" now creates a temporary file called "parser.c.new" instead of "parser.new.c". Previously, if "make clinic" was run in parallel with "make regen-all", clinic may try but fail to open "parser.new.c" if the temporay file was removed in the meanwhile. --- Makefile.pre.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 97eb767b8fcdeb..f1f5c8557e7ffc 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1399,8 +1399,8 @@ regen-pegen: PYTHONPATH=$(srcdir)/Tools/peg_generator $(PYTHON_FOR_REGEN) -m pegen -q c \ $(srcdir)/Grammar/python.gram \ $(srcdir)/Grammar/Tokens \ - -o $(srcdir)/Parser/parser.new.c - $(UPDATE_FILE) $(srcdir)/Parser/parser.c $(srcdir)/Parser/parser.new.c + -o $(srcdir)/Parser/parser.c.new + $(UPDATE_FILE) $(srcdir)/Parser/parser.c $(srcdir)/Parser/parser.c.new .PHONY: regen-ast regen-ast: From aaf297c048694cd9652790f8b74e69f7ddadfbde Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Oct 2023 23:40:49 +0200 Subject: [PATCH 57/73] gh-109888: Fix test_os _kill_with_event() on Windows (#110421) Replace os.kill() with proc.kill() which catchs PermissionError. Rewrite _kill_with_event(): * Use subprocess context manager ("with proc:"). * Use sleeping_retry() to wait until the child process is ready. * Replace SIGINT with proc.kill() on error. * Replace 10 seconds with SHORT_TIMEOUT to wait until the process is ready. * Replace 0.5 seconds with SHORT_TIMEOUT to wait for the process exit. --- Lib/test/test_os.py | 50 ++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index c1a78a70c09441..669e27c0473af0 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2566,30 +2566,34 @@ def _kill_with_event(self, event, name): tagname = "test_os_%s" % uuid.uuid1() m = mmap.mmap(-1, 1, tagname) m[0] = 0 + # Run a script which has console control handling enabled. - proc = subprocess.Popen([sys.executable, - os.path.join(os.path.dirname(__file__), - "win_console_handler.py"), tagname], - creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) - # Let the interpreter startup before we send signals. See #3137. - count, max = 0, 100 - while count < max and proc.poll() is None: - if m[0] == 1: - break - time.sleep(0.1) - count += 1 - else: - # Forcefully kill the process if we weren't able to signal it. - os.kill(proc.pid, signal.SIGINT) - self.fail("Subprocess didn't finish initialization") - os.kill(proc.pid, event) - # proc.send_signal(event) could also be done here. - # Allow time for the signal to be passed and the process to exit. - time.sleep(0.5) - if not proc.poll(): - # Forcefully kill the process if we weren't able to signal it. - os.kill(proc.pid, signal.SIGINT) - self.fail("subprocess did not stop on {}".format(name)) + script = os.path.join(os.path.dirname(__file__), + "win_console_handler.py") + cmd = [sys.executable, script, tagname] + proc = subprocess.Popen(cmd, + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) + + with proc: + # Let the interpreter startup before we send signals. See #3137. + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if proc.poll() is None: + break + else: + # Forcefully kill the process if we weren't able to signal it. + proc.kill() + self.fail("Subprocess didn't finish initialization") + + os.kill(proc.pid, event) + + try: + # proc.send_signal(event) could also be done here. + # Allow time for the signal to be passed and the process to exit. + proc.wait(timeout=support.SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + # Forcefully kill the process if we weren't able to signal it. + proc.kill() + self.fail("subprocess did not stop on {}".format(name)) @unittest.skip("subprocesses aren't inheriting Ctrl+C property") @support.requires_subprocess() From bb057b337008626139d97269485b1dddf70ae427 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Oct 2023 23:59:35 +0200 Subject: [PATCH 58/73] gh-85283: Add PySys_AuditTuple() function (#108965) sys.audit() now has assertions to check that the event argument is not NULL and that the format argument does not use the "N" format. Add tests on PySys_AuditTuple(). --- Doc/c-api/sys.rst | 21 +++++-- Doc/whatsnew/3.13.rst | 4 ++ Include/cpython/sysmodule.h | 6 +- Lib/test/test_embed.py | 3 + ...3-09-06-00-14-49.gh-issue-85283.GKY0Cc.rst | 3 + Programs/_testembed.c | 59 ++++++++++++++++++- Python/sysmodule.c | 22 ++++++- 7 files changed, 109 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-09-06-00-14-49.gh-issue-85283.GKY0Cc.rst diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index aed625c5f6cdae..e3c54b075114ff 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -291,19 +291,24 @@ accessible to C code. They all work with the current interpreter thread's Raise an auditing event with any active hooks. Return zero for success and non-zero with an exception set on failure. + The *event* string argument must not be *NULL*. + If any hooks have been added, *format* and other arguments will be used to construct a tuple to pass. Apart from ``N``, the same format characters as used in :c:func:`Py_BuildValue` are available. If the built value is not - a tuple, it will be added into a single-element tuple. (The ``N`` format - option consumes a reference, but since there is no way to know whether - arguments to this function will be consumed, using it may cause reference - leaks.) + a tuple, it will be added into a single-element tuple. + + The ``N`` format option must not be used. It consumes a reference, but since + there is no way to know whether arguments to this function will be consumed, + using it may cause reference leaks. Note that ``#`` format characters should always be treated as :c:type:`Py_ssize_t`, regardless of whether ``PY_SSIZE_T_CLEAN`` was defined. :func:`sys.audit` performs the same function from Python code. + See also :c:func:`PySys_AuditTuple`. + .. versionadded:: 3.8 .. versionchanged:: 3.8.2 @@ -312,6 +317,14 @@ accessible to C code. They all work with the current interpreter thread's unavoidable deprecation warning was raised. +.. c:function:: int PySys_AuditTuple(const char *event, PyObject *args) + + Similar to :c:func:`PySys_Audit`, but pass arguments as a Python object. + *args* must be a :class:`tuple`. To pass no arguments, *args* can be *NULL*. + + .. versionadded:: 3.13 + + .. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) Append the callable *hook* to the list of active auditing hooks. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 7a62963203e164..d5987ae31ce68d 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1013,6 +1013,10 @@ New Features ``_PyThreadState_UncheckedGet()``. (Contributed by Victor Stinner in :gh:`108867`.) +* Add :c:func:`PySys_AuditTuple` function: similar to :c:func:`PySys_Audit`, + but pass event arguments as a Python :class:`tuple` object. + (Contributed by Victor Stinner in :gh:`85283`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/cpython/sysmodule.h b/Include/cpython/sysmodule.h index e028fb7ecd5230..36c4f89432067b 100644 --- a/Include/cpython/sysmodule.h +++ b/Include/cpython/sysmodule.h @@ -6,6 +6,10 @@ typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); PyAPI_FUNC(int) PySys_Audit( const char *event, - const char *argFormat, + const char *format, ...); PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*); + +PyAPI_FUNC(int) PySys_AuditTuple( + const char *event, + PyObject *args); diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index dc476ef83c2519..46c9b03c4178e9 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1716,6 +1716,9 @@ def test_open_code_hook(self): def test_audit(self): self.run_embedded_interpreter("test_audit") + def test_audit_tuple(self): + self.run_embedded_interpreter("test_audit_tuple") + def test_audit_subinterpreter(self): self.run_embedded_interpreter("test_audit_subinterpreter") diff --git a/Misc/NEWS.d/next/C API/2023-09-06-00-14-49.gh-issue-85283.GKY0Cc.rst b/Misc/NEWS.d/next/C API/2023-09-06-00-14-49.gh-issue-85283.GKY0Cc.rst new file mode 100644 index 00000000000000..811551cba73b3a --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-09-06-00-14-49.gh-issue-85283.GKY0Cc.rst @@ -0,0 +1,3 @@ +Add :c:func:`PySys_AuditTuple` function: similar to :c:func:`PySys_Audit`, +but pass event arguments as a Python :class:`tuple` object. Patch by Victor +Stinner. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index bc991020d0fa77..e66c51818227c4 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1278,11 +1278,16 @@ static int _test_audit(Py_ssize_t setValue) printf("Set event failed"); return 4; } + if (PyErr_Occurred()) { + printf("Exception raised"); + return 5; + } if (sawSet != 42) { printf("Failed to see *userData change\n"); - return 5; + return 6; } + return 0; } @@ -1296,6 +1301,57 @@ static int test_audit(void) return result; } +static int test_audit_tuple(void) +{ +#define ASSERT(TEST, EXITCODE) \ + if (!(TEST)) { \ + printf("ERROR test failed at %s:%i\n", __FILE__, __LINE__); \ + return (EXITCODE); \ + } + + Py_ssize_t sawSet = 0; + + // we need at least one hook, otherwise code checking for + // PySys_AuditTuple() is skipped. + PySys_AddAuditHook(_audit_hook, &sawSet); + _testembed_Py_InitializeFromConfig(); + + ASSERT(!PyErr_Occurred(), 0); + + // pass Python tuple object + PyObject *tuple = Py_BuildValue("(i)", 444); + if (tuple == NULL) { + goto error; + } + ASSERT(PySys_AuditTuple("_testembed.set", tuple) == 0, 10); + ASSERT(!PyErr_Occurred(), 11); + ASSERT(sawSet == 444, 12); + Py_DECREF(tuple); + + // pass Python int object + PyObject *int_arg = PyLong_FromLong(555); + if (int_arg == NULL) { + goto error; + } + ASSERT(PySys_AuditTuple("_testembed.set", int_arg) == -1, 20); + ASSERT(PyErr_ExceptionMatches(PyExc_TypeError), 21); + PyErr_Clear(); + Py_DECREF(int_arg); + + // NULL is accepted and means "no arguments" + ASSERT(PySys_AuditTuple("_testembed.test_audit_tuple", NULL) == 0, 30); + ASSERT(!PyErr_Occurred(), 31); + + Py_Finalize(); + return 0; + +error: + PyErr_Print(); + return 1; + +#undef ASSERT +} + static volatile int _audit_subinterpreter_interpreter_count = 0; static int _audit_subinterpreter_hook(const char *event, PyObject *args, void *userdata) @@ -2140,6 +2196,7 @@ static struct TestCase TestCases[] = { // Audit {"test_open_code_hook", test_open_code_hook}, {"test_audit", test_audit}, + {"test_audit_tuple", test_audit_tuple}, {"test_audit_subinterpreter", test_audit_subinterpreter}, {"test_audit_run_command", test_audit_run_command}, {"test_audit_run_file", test_audit_run_file}, diff --git a/Python/sysmodule.c b/Python/sysmodule.c index b00301765e1890..a7ce07d28ae7df 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -191,9 +191,7 @@ static int sys_audit_tstate(PyThreadState *ts, const char *event, const char *argFormat, va_list vargs) { - /* N format is inappropriate, because you do not know - whether the reference is consumed by the call. - Assert rather than exception for perf reasons */ + assert(event != NULL); assert(!argFormat || !strchr(argFormat, 'N')); if (!ts) { @@ -338,6 +336,21 @@ PySys_Audit(const char *event, const char *argFormat, ...) return res; } +int +PySys_AuditTuple(const char *event, PyObject *args) +{ + if (args == NULL) { + return PySys_Audit(event, NULL); + } + + if (!PyTuple_Check(args)) { + PyErr_Format(PyExc_TypeError, "args must be tuple, got %s", + Py_TYPE(args)->tp_name); + return -1; + } + return PySys_Audit(event, "O", args); +} + /* We expose this function primarily for our own cleanup during * finalization. In general, it should not need to be called, * and as such the function is not exported. @@ -509,6 +522,9 @@ sys_audit(PyObject *self, PyObject *const *args, Py_ssize_t argc) return NULL; } + assert(args[0] != NULL); + assert(PyUnicode_Check(args[0])); + if (!should_audit(tstate->interp)) { Py_RETURN_NONE; } From 3c0f65ebce10d5327e07245f7cf2beb96b18c970 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 5 Oct 2023 15:05:29 -0700 Subject: [PATCH 59/73] gh-109287: fix overrides in cases generator (#110419) --- Lib/test/test_generated_cases.py | 35 +++++++++++++++++++++++++ Tools/cases_generator/analysis.py | 11 ++------ Tools/cases_generator/generate_cases.py | 17 ------------ Tools/cases_generator/instructions.py | 5 ---- 4 files changed, 37 insertions(+), 31 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 5971d2e436e4aa..790e6b1b1b91e6 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -598,6 +598,41 @@ def test_macro_push_push(self): """ self.run_cases_test(input, output) + def test_override_inst(self): + input = """ + inst(OP, (--)) { + spam(); + } + override inst(OP, (--)) { + ham(); + } + """ + output = """ + TARGET(OP) { + ham(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_override_op(self): + input = """ + op(OP, (--)) { + spam(); + } + macro(M) = OP; + override op(OP, (--)) { + ham(); + } + """ + output = """ + TARGET(M) { + ham(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + if __name__ == "__main__": unittest.main() diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 7bbc924e5083f1..b2fa0205bea342 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -12,7 +12,6 @@ InstructionOrCacheEffect, MacroInstruction, MacroParts, - OverriddenInstructionPlaceHolder, PseudoInstruction, ) import parsing @@ -66,7 +65,6 @@ def note(self, msg: str, node: parsing.Node) -> None: parsing.InstDef | parsing.Macro | parsing.Pseudo - | OverriddenInstructionPlaceHolder ] instrs: dict[str, Instruction] # Includes ops macros: dict[str, parsing.Macro] @@ -141,7 +139,7 @@ def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: match thing: case parsing.InstDef(name=name): macro: parsing.Macro | None = None - if thing.kind == "inst": + if thing.kind == "inst" and not thing.override: macro = parsing.Macro(name, [parsing.OpName(name)]) if name in self.instrs: if not thing.override: @@ -150,12 +148,7 @@ def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: f"previous definition @ {self.instrs[name].inst.context}", thing_first_token, ) - placeholder = OverriddenInstructionPlaceHolder(name=name) - self.everything[instrs_idx[name]] = placeholder - if macro is not None: - self.warning( - f"Overriding desugared {macro.name} may not work", thing - ) + self.everything[instrs_idx[name]] = thing if name not in self.instrs and thing.override: raise psr.make_syntax_error( f"Definition of '{name}' @ {thing.context} is supposed to be " diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 01ab83bedb2985..dbb16418c0cb24 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -26,7 +26,6 @@ MacroInstruction, MacroParts, PseudoInstruction, - OverriddenInstructionPlaceHolder, TIER_ONE, TIER_TWO, ) @@ -208,8 +207,6 @@ def write_stack_effect_functions(self) -> None: popped_data: list[tuple[AnyInstruction, str]] = [] pushed_data: list[tuple[AnyInstruction, str]] = [] for thing in self.everything: - if isinstance(thing, OverriddenInstructionPlaceHolder): - continue if isinstance(thing, parsing.Macro) and thing.name in self.instrs: continue instr, popped, pushed = self.get_stack_effect_info(thing) @@ -393,8 +390,6 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No for thing in self.everything: format: str | None = None match thing: - case OverriddenInstructionPlaceHolder(): - continue case parsing.InstDef(): format = self.instrs[thing.name].instr_fmt case parsing.Macro(): @@ -492,8 +487,6 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No # Write metadata for each instruction for thing in self.everything: match thing: - case OverriddenInstructionPlaceHolder(): - continue case parsing.InstDef(): self.write_metadata_for_inst(self.instrs[thing.name]) case parsing.Macro(): @@ -774,8 +767,6 @@ def write_instructions( n_macros = 0 for thing in self.everything: match thing: - case OverriddenInstructionPlaceHolder(): - self.write_overridden_instr_place_holder(thing) case parsing.InstDef(): pass case parsing.Macro(): @@ -836,14 +827,6 @@ def write_abstract_interpreter_instructions( file=sys.stderr, ) - def write_overridden_instr_place_holder( - self, place_holder: OverriddenInstructionPlaceHolder - ) -> None: - self.out.emit("") - self.out.emit( - f"{self.out.comment} TARGET({place_holder.name}) overridden by later definition" - ) - def is_super_instruction(mac: MacroInstruction) -> bool: if ( diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index bd7b7dfbaa8f70..c6b551675e3e7e 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -295,11 +295,6 @@ class PseudoInstruction: instr_flags: InstructionFlags -@dataclasses.dataclass -class OverriddenInstructionPlaceHolder: - name: str - - AnyInstruction = Instruction | MacroInstruction | PseudoInstruction From e0c44377935de3491b2cbe1e5f87f8b336fdc922 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Oct 2023 02:37:28 +0200 Subject: [PATCH 60/73] Add support.MS_WINDOWS constant (#110446) --- Lib/test/_test_embed_set_config.py | 2 +- Lib/test/libregrtest/logger.py | 3 ++- Lib/test/libregrtest/main.py | 4 ++-- Lib/test/libregrtest/run_workers.py | 4 ++-- Lib/test/libregrtest/utils.py | 2 -- Lib/test/pythoninfo.py | 1 + Lib/test/support/__init__.py | 4 +++- Lib/test/test_asyncio/test_subprocess.py | 5 ++--- Lib/test/test_cppext/__init__.py | 3 +-- Lib/test/test_cppext/setup.py | 6 ++---- Lib/test/test_embed.py | 7 ++----- Lib/test/test_faulthandler.py | 4 +--- Lib/test/test_gdb/__init__.py | 3 +-- Lib/test/test_regrtest.py | 4 +--- Lib/test/test_utf8_mode.py | 3 +-- 15 files changed, 22 insertions(+), 33 deletions(-) diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py index 0c016b5d75d734..a2ddd133cf47c8 100644 --- a/Lib/test/_test_embed_set_config.py +++ b/Lib/test/_test_embed_set_config.py @@ -9,9 +9,9 @@ import os import sys import unittest +from test.support import MS_WINDOWS -MS_WINDOWS = (os.name == 'nt') MAX_HASH_SEED = 4294967295 class SetConfigTests(unittest.TestCase): diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py index 2f0c4bf1c84b5c..a125706927393c 100644 --- a/Lib/test/libregrtest/logger.py +++ b/Lib/test/libregrtest/logger.py @@ -1,9 +1,10 @@ import os import time +from test.support import MS_WINDOWS from .results import TestResults from .runtests import RunTests -from .utils import print_warning, MS_WINDOWS +from .utils import print_warning if MS_WINDOWS: from .win_utils import WindowsLoadTracker diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 60179ec7708c1c..cb60d5af732b43 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,7 +7,7 @@ import time from test import support -from test.support import os_helper +from test.support import os_helper, MS_WINDOWS from .cmdline import _parse_args, Namespace from .findtests import findtests, split_test_packages, list_cases @@ -24,7 +24,7 @@ printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, is_cross_compiled, get_host_runner, - MS_WINDOWS, EXIT_TIMEOUT) + EXIT_TIMEOUT) class Regrtest: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 106f9730832e54..16f8331abd32f9 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -13,7 +13,7 @@ from typing import Literal, TextIO from test import support -from test.support import os_helper +from test.support import os_helper, MS_WINDOWS from .logger import Logger from .result import TestResult, State @@ -21,7 +21,7 @@ from .runtests import RunTests, JsonFile, JsonFileType from .single import PROGRESS_MIN_TIME from .utils import ( - StrPath, TestName, MS_WINDOWS, + StrPath, TestName, format_duration, print_warning, count, plural, get_signal_name) from .worker import create_worker_process, USE_PROCESS_GROUP diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index ea2086cd71b173..aac8395da87845 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -19,8 +19,6 @@ from test.support import threading_helper -MS_WINDOWS = (sys.platform == 'win32') - # All temporary files and temporary directories created by libregrtest should # use TMP_PREFIX so cleanup_temp_dir() can remove them all. TMP_PREFIX = 'test_python_' diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 58d906ffc62a53..4f3ebb12ed957d 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -730,6 +730,7 @@ def collect_support(info_add): return attributes = ( + 'MS_WINDOWS', 'has_fork_support', 'has_socket_support', 'has_strftime_extensions', diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f3270d66bd0852..982d0df3b86f48 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -46,7 +46,7 @@ "check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer", "requires_limited_api", "requires_specialization", # sys - "is_jython", "is_android", "is_emscripten", "is_wasi", + "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", "check_impl_detail", "unix_shell", "setswitchinterval", # os "get_pagesize", @@ -509,6 +509,8 @@ def has_no_debug_ranges(): def requires_debug_ranges(reason='requires co_positions / debug_ranges'): return unittest.skipIf(has_no_debug_ranges(), reason) +MS_WINDOWS = (sys.platform == 'win32') + # Is not actually used in tests, but is kept for compatibility. is_jython = sys.platform.startswith('java') diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index dc5a48d500e8d5..179c8cb8cc17cf 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -14,8 +14,7 @@ from test.support import os_helper -MS_WINDOWS = (sys.platform == 'win32') -if MS_WINDOWS: +if support.MS_WINDOWS: import msvcrt else: from asyncio import unix_events @@ -283,7 +282,7 @@ def test_stdin_broken_pipe(self): rfd, wfd = os.pipe() self.addCleanup(os.close, rfd) self.addCleanup(os.close, wfd) - if MS_WINDOWS: + if support.MS_WINDOWS: handle = msvcrt.get_osfhandle(rfd) os.set_handle_inheritable(handle, True) code = textwrap.dedent(f''' diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index 25b6fc64a03a51..4d9ee3cb2228ae 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -9,7 +9,6 @@ from test import support -MS_WINDOWS = (sys.platform == 'win32') SOURCE = os.path.join(os.path.dirname(__file__), 'extension.cpp') SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') @@ -30,7 +29,7 @@ def test_build_cpp03(self): # With MSVC, the linker fails with: cannot open file 'python311.lib' # https://github.com/python/cpython/pull/32175#issuecomment-1111175897 - @unittest.skipIf(MS_WINDOWS, 'test fails on Windows') + @unittest.skipIf(support.MS_WINDOWS, 'test fails on Windows') # Building and running an extension in clang sanitizing mode is not # straightforward @unittest.skipIf( diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index 976633bc33889c..c7ba1efb4dd05a 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -4,15 +4,13 @@ import shlex import sys import sysconfig +from test import support from setuptools import setup, Extension -MS_WINDOWS = (sys.platform == 'win32') - - SOURCE = 'extension.cpp' -if not MS_WINDOWS: +if not support.MS_WINDOWS: # C++ compiler flags for GCC and clang CPPFLAGS = [ # gh-91321: The purpose of _testcppext extension is to check that building diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 46c9b03c4178e9..06f2d8b9a3621f 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1,8 +1,6 @@ # Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs) from test import support -from test.support import import_helper -from test.support import os_helper -from test.support import requires_specialization +from test.support import import_helper, os_helper, MS_WINDOWS import unittest from collections import namedtuple @@ -21,7 +19,6 @@ if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") -MS_WINDOWS = (os.name == 'nt') MACOS = (sys.platform == 'darwin') PYMEM_ALLOCATOR_NOT_SET = 0 PYMEM_ALLOCATOR_DEBUG = 2 @@ -348,7 +345,7 @@ def test_simple_initialization_api(self): out, err = self.run_embedded_interpreter("test_repeated_simple_init") self.assertEqual(out, 'Finalized\n' * INIT_LOOPS) - @requires_specialization + @support.requires_specialization def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self): # https://github.com/python/cpython/issues/92031 diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 0b8299a32b03c0..d0473500a17735 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -7,8 +7,7 @@ import subprocess import sys from test import support -from test.support import os_helper -from test.support import script_helper, is_android +from test.support import os_helper, script_helper, is_android, MS_WINDOWS import tempfile import unittest from textwrap import dedent @@ -22,7 +21,6 @@ raise unittest.SkipTest("test module requires subprocess") TIMEOUT = 0.5 -MS_WINDOWS = (os.name == 'nt') def expected_traceback(lineno1, lineno2, header, min_count=1): diff --git a/Lib/test/test_gdb/__init__.py b/Lib/test/test_gdb/__init__.py index d74075e456792d..99557739af6748 100644 --- a/Lib/test/test_gdb/__init__.py +++ b/Lib/test/test_gdb/__init__.py @@ -9,8 +9,7 @@ from test import support -MS_WINDOWS = (os.name == 'nt') -if MS_WINDOWS: +if support.MS_WINDOWS: # On Windows, Python is usually built by MSVC. Passing /p:DebugSymbols=true # option to MSBuild produces PDB debug symbols, but gdb doesn't support PDB # debug symbol files. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index de2c4317e71439..cfe8b579703cb6 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -42,8 +42,6 @@ EXITCODE_RERUN_FAIL = 5 EXITCODE_INTERRUPTED = 130 -MS_WINDOWS = (sys.platform == 'win32') - TEST_INTERRUPTED = textwrap.dedent(""" from signal import SIGINT, raise_signal try: @@ -2072,7 +2070,7 @@ def test_crash(self): self.check_executed_tests(output, testname, failed=[testname], stats=0, parallel=True) - if not MS_WINDOWS: + if not support.MS_WINDOWS: exitcode = -int(signal.SIGSEGV) self.assertIn(f"Exit code {exitcode} (SIGSEGV)", output) self.check_line(output, "just before crash!", full=True, regex=False) diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py index ec29ba6d51b127..f66881044e16df 100644 --- a/Lib/test/test_utf8_mode.py +++ b/Lib/test/test_utf8_mode.py @@ -9,10 +9,9 @@ import unittest from test import support from test.support.script_helper import assert_python_ok, assert_python_failure -from test.support import os_helper +from test.support import os_helper, MS_WINDOWS -MS_WINDOWS = (sys.platform == 'win32') POSIX_LOCALES = ('C', 'POSIX') VXWORKS = (sys.platform == "vxworks") From a155f9f3427578ca5706d27e20bd0576f0395073 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Oct 2023 02:46:52 +0200 Subject: [PATCH 61/73] gh-103053: Fix make check-clean-src: check "python" program (#110449) "make check-clean-src" now also checks if the "python" program is found in the source directory: fail with an error if it does exist. --- Makefile.pre.in | 3 ++- .../next/Build/2023-10-06-02-15-23.gh-issue-103053.--7JUF.rst | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2023-10-06-02-15-23.gh-issue-103053.--7JUF.rst diff --git a/Makefile.pre.in b/Makefile.pre.in index f1f5c8557e7ffc..40951a8af7a6e0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -624,7 +624,8 @@ build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ .PHONY: check-clean-src check-clean-src: @if test -n "$(VPATH)" -a \( \ - -f "$(srcdir)/Programs/python.o" \ + -f "$(srcdir)/$(BUILDPYTHON)" \ + -o -f "$(srcdir)/Programs/python.o" \ -o -f "$(srcdir)\Python/frozen_modules/importlib._bootstrap.h" \ \); then \ echo "Error: The source directory ($(srcdir)) is not clean" ; \ diff --git a/Misc/NEWS.d/next/Build/2023-10-06-02-15-23.gh-issue-103053.--7JUF.rst b/Misc/NEWS.d/next/Build/2023-10-06-02-15-23.gh-issue-103053.--7JUF.rst new file mode 100644 index 00000000000000..81aa21357287c7 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-10-06-02-15-23.gh-issue-103053.--7JUF.rst @@ -0,0 +1,3 @@ +"make check-clean-src" now also checks if the "python" program is found in +the source directory: fail with an error if it does exist. Patch by Victor +Stinner. From a4baa9e8ac62cac3ea6363b15ea585b1998ea1f9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Oct 2023 03:08:34 +0200 Subject: [PATCH 62/73] gh-103053: Fix test_tools.test_freeze on FreeBSD (#110451) Fix test_tools.test_freeze on FreeBSD: run "make distclean" instead of "make clean" in the copied source directory to remove also the "python" program. Other test_freeze changes: * Log executed commands and directories, and the current directory. * No longer uses make -C option to change the directory, instead use subprocess cwd parameter. --- ...-10-06-02-32-18.gh-issue-103053.VfxBLI.rst | 3 ++ Tools/freeze/test/freeze.py | 33 +++++++++++-------- 2 files changed, 22 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-10-06-02-32-18.gh-issue-103053.VfxBLI.rst diff --git a/Misc/NEWS.d/next/Tests/2023-10-06-02-32-18.gh-issue-103053.VfxBLI.rst b/Misc/NEWS.d/next/Tests/2023-10-06-02-32-18.gh-issue-103053.VfxBLI.rst new file mode 100644 index 00000000000000..90a7ca512c97dd --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-10-06-02-32-18.gh-issue-103053.VfxBLI.rst @@ -0,0 +1,3 @@ +Fix test_tools.test_freeze on FreeBSD: run "make distclean" instead of "make +clean" in the copied source directory to remove also the "python" program. +Patch by Victor Stinner. diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index 9030ad4d4e5f93..1e3687dbb807a6 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -27,8 +27,10 @@ class UnsupportedError(Exception): """The operation isn't supported.""" -def _run_quiet(cmd, cwd=None): - #print(f'# {" ".join(shlex.quote(a) for a in cmd)}') +def _run_quiet(cmd, *, cwd=None): + if cwd: + print('+', 'cd', cwd, flush=True) + print('+', shlex.join(cmd), flush=True) try: return subprocess.run( cmd, @@ -48,8 +50,8 @@ def _run_quiet(cmd, cwd=None): raise -def _run_stdout(cmd, cwd=None): - proc = _run_quiet(cmd, cwd) +def _run_stdout(cmd): + proc = _run_quiet(cmd) return proc.stdout.strip() @@ -91,13 +93,18 @@ def copy_source_tree(newroot, oldroot): shutil.copytree(oldroot, newroot, ignore=support.copy_python_src_ignore) if os.path.exists(os.path.join(newroot, 'Makefile')): - _run_quiet([MAKE, 'clean'], newroot) + # Out-of-tree builds require a clean srcdir. "make clean" keeps + # the "python" program, so use "make distclean" instead. + _run_quiet([MAKE, 'distclean'], cwd=newroot) ################################## # freezing def prepare(script=None, outdir=None): + print() + print("cwd:", os.getcwd()) + if not outdir: outdir = OUTDIR os.makedirs(outdir, exist_ok=True) @@ -125,7 +132,7 @@ def prepare(script=None, outdir=None): ensure_opt(cmd, 'cache-file', os.path.join(outdir, 'python-config.cache')) prefix = os.path.join(outdir, 'python-installation') ensure_opt(cmd, 'prefix', prefix) - _run_quiet(cmd, builddir) + _run_quiet(cmd, cwd=builddir) if not MAKE: raise UnsupportedError('make') @@ -135,20 +142,18 @@ def prepare(script=None, outdir=None): # this test is most often run as part of the whole suite with a lot # of other tests running in parallel, from 1-2 vCPU systems up to # people's NNN core beasts. Don't attempt to use it all. - parallel = f'-j{cores*2//3}' + jobs = cores * 2 // 3 + parallel = f'-j{jobs}' else: parallel = '-j2' # Build python. print(f'building python {parallel=} in {builddir}...') - if os.path.exists(os.path.join(srcdir, 'Makefile')): - # Out-of-tree builds require a clean srcdir. - _run_quiet([MAKE, '-C', srcdir, 'clean']) - _run_quiet([MAKE, '-C', builddir, parallel]) + _run_quiet([MAKE, parallel], cwd=builddir) # Install the build. print(f'installing python into {prefix}...') - _run_quiet([MAKE, '-C', builddir, 'install']) + _run_quiet([MAKE, 'install'], cwd=builddir) python = os.path.join(prefix, 'bin', 'python3') return outdir, scriptfile, python @@ -161,8 +166,8 @@ def freeze(python, scriptfile, outdir): print(f'freezing {scriptfile}...') os.makedirs(outdir, exist_ok=True) # Use -E to ignore PYTHONSAFEPATH - _run_quiet([python, '-E', FREEZE, '-o', outdir, scriptfile], outdir) - _run_quiet([MAKE, '-C', os.path.dirname(scriptfile)]) + _run_quiet([python, '-E', FREEZE, '-o', outdir, scriptfile], cwd=outdir) + _run_quiet([MAKE], cwd=os.path.dirname(scriptfile)) name = os.path.basename(scriptfile).rpartition('.')[0] executable = os.path.join(outdir, name) From d023d4166b255023dac448305270350030101481 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Oct 2023 15:44:53 +0200 Subject: [PATCH 63/73] gh-110184: Fix subprocess test_pipesize_default() (#110465) For proc.stdin, get the size of the read end of the test pipe. Use subprocess context manager ("with proc:"). --- Lib/test/test_subprocess.py | 41 +++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index d95ef72b0da47a..a865df1082fba3 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -749,31 +749,36 @@ def test_pipesizes(self): @unittest.skipUnless(fcntl and hasattr(fcntl, 'F_GETPIPE_SZ'), 'fcntl.F_GETPIPE_SZ required for test.') def test_pipesize_default(self): - p = subprocess.Popen( + proc = subprocess.Popen( [sys.executable, "-c", 'import sys; sys.stdin.read(); sys.stdout.write("out"); ' 'sys.stderr.write("error!")'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, pipesize=-1) - try: - fp_r, fp_w = os.pipe() + + with proc: try: - default_pipesize = fcntl.fcntl(fp_w, fcntl.F_GETPIPE_SZ) - for fifo in [p.stdin, p.stdout, p.stderr]: - self.assertEqual( - fcntl.fcntl(fifo.fileno(), fcntl.F_GETPIPE_SZ), - default_pipesize) + fp_r, fp_w = os.pipe() + try: + default_read_pipesize = fcntl.fcntl(fp_r, fcntl.F_GETPIPE_SZ) + default_write_pipesize = fcntl.fcntl(fp_w, fcntl.F_GETPIPE_SZ) + finally: + os.close(fp_r) + os.close(fp_w) + + self.assertEqual( + fcntl.fcntl(proc.stdin.fileno(), fcntl.F_GETPIPE_SZ), + default_read_pipesize) + self.assertEqual( + fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETPIPE_SZ), + default_write_pipesize) + self.assertEqual( + fcntl.fcntl(proc.stderr.fileno(), fcntl.F_GETPIPE_SZ), + default_write_pipesize) + # On other platforms we cannot test the pipe size (yet). But above + # code using pipesize=-1 should not crash. finally: - os.close(fp_r) - os.close(fp_w) - # On other platforms we cannot test the pipe size (yet). But above - # code using pipesize=-1 should not crash. - p.stdin.close() - p.stdout.close() - p.stderr.close() - finally: - p.kill() - p.wait() + proc.kill() def test_env(self): newenv = os.environ.copy() From 02cdaefe9a5fe63bfe6584688a23b9a8229d57ac Mon Sep 17 00:00:00 2001 From: InSync <122007197+InSyncWithFoo@users.noreply.github.com> Date: Fri, 6 Oct 2023 21:25:02 +0700 Subject: [PATCH 64/73] Fix typo in Doc/library/textwrap.rst (#110328) "One problem with this is algorithm is that [...]" -> "One problem with this algorithm is that [...]" --- Doc/library/textwrap.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index e2952ce3cc2ca3..7445410f91808c 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -238,7 +238,7 @@ hyphenated words; only then will long words be broken if necessary, unless However, the sentence detection algorithm is imperfect: it assumes that a sentence ending consists of a lowercase letter followed by one of ``'.'``, ``'!'``, or ``'?'``, possibly followed by one of ``'"'`` or ``"'"``, - followed by a space. One problem with this is algorithm is that it is + followed by a space. One problem with this algorithm is that it is unable to detect the difference between "Dr." in :: [...] Dr. Frankenstein's monster [...] From 201dc11aeb4699de3c5ebaea9798796c30087bcc Mon Sep 17 00:00:00 2001 From: jtranquilli <76231120+jtranquilli@users.noreply.github.com> Date: Fri, 6 Oct 2023 11:12:15 -0400 Subject: [PATCH 65/73] gh-109286: Update Windows installer to use SQLite 3.43.1 (#110403) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Erlend E. Aasland --- .../next/Windows/2023-10-05-15-23-23.gh-issue-109286.N8OzMg.rst | 1 + PCbuild/get_externals.bat | 2 +- PCbuild/python.props | 2 +- PCbuild/readme.txt | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-10-05-15-23-23.gh-issue-109286.N8OzMg.rst diff --git a/Misc/NEWS.d/next/Windows/2023-10-05-15-23-23.gh-issue-109286.N8OzMg.rst b/Misc/NEWS.d/next/Windows/2023-10-05-15-23-23.gh-issue-109286.N8OzMg.rst new file mode 100644 index 00000000000000..14a2aff70803ce --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-10-05-15-23-23.gh-issue-109286.N8OzMg.rst @@ -0,0 +1 @@ +Update Windows installer to use SQLite 3.43.1. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index fc43ce2b1835d0..94437f054d788c 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.11 -set libraries=%libraries% sqlite-3.42.0.0 +set libraries=%libraries% sqlite-3.43.1.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.0 set libraries=%libraries% xz-5.2.5 diff --git a/PCbuild/python.props b/PCbuild/python.props index 35c6e92be45dc9..496bc3dd4cf794 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -68,7 +68,7 @@ - $(ExternalsDir)sqlite-3.42.0.0\ + $(ExternalsDir)sqlite-3.43.1.0\ $(ExternalsDir)bzip2-1.0.8\ $(ExternalsDir)xz-5.2.5\ $(ExternalsDir)libffi-3.4.4\ diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 98b37014907604..175ce918ac20f6 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -189,7 +189,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.42.0, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.43.1, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter From f013b475047b2e9d377feda9f2e16e5cdef824d7 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 6 Oct 2023 15:57:18 -0700 Subject: [PATCH 66/73] gh-110489: Optimise math.ceil for known exact float (#108801) This matches a similar optimisation done for math.floor in https://github.com/python/cpython/pull/21072 --- ...023-10-06-22-30-25.gh-issue-110489.rI2n8A.rst | 1 + Modules/mathmodule.c | 16 +++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-06-22-30-25.gh-issue-110489.rI2n8A.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-06-22-30-25.gh-issue-110489.rI2n8A.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-06-22-30-25.gh-issue-110489.rI2n8A.rst new file mode 100644 index 00000000000000..6252d00cc583d3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-06-22-30-25.gh-issue-110489.rI2n8A.rst @@ -0,0 +1 @@ +Optimise :func:`math.ceil` when the input is exactly a float, resulting in about a 10% improvement. diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 2d896e7fe333a4..a4d94665592351 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -1125,8 +1125,12 @@ static PyObject * math_ceil(PyObject *module, PyObject *number) /*[clinic end generated code: output=6c3b8a78bc201c67 input=2725352806399cab]*/ { + double x; - if (!PyFloat_CheckExact(number)) { + if (PyFloat_CheckExact(number)) { + x = PyFloat_AS_DOUBLE(number); + } + else { math_module_state *state = get_math_module_state(module); PyObject *method = _PyObject_LookupSpecial(number, state->str___ceil__); if (method != NULL) { @@ -1136,11 +1140,10 @@ math_ceil(PyObject *module, PyObject *number) } if (PyErr_Occurred()) return NULL; + x = PyFloat_AsDouble(number); + if (x == -1.0 && PyErr_Occurred()) + return NULL; } - double x = PyFloat_AsDouble(number); - if (x == -1.0 && PyErr_Occurred()) - return NULL; - return PyLong_FromDouble(ceil(x)); } @@ -1196,8 +1199,7 @@ math_floor(PyObject *module, PyObject *number) if (PyFloat_CheckExact(number)) { x = PyFloat_AS_DOUBLE(number); } - else - { + else { math_module_state *state = get_math_module_state(module); PyObject *method = _PyObject_LookupSpecial(number, state->str___floor__); if (method != NULL) { From 5fd8821cf8eb1fe2e8575f8c7cc747cf78855a88 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 6 Oct 2023 16:12:19 -0700 Subject: [PATCH 67/73] GH-110455: Guard `assert(tstate->thread_id > 0)` with `#ifndef HAVE_PTHREAD_STUBS` (GH-110487) --- .../2023-10-06-12-00-43.gh-issue-110455.8BjNGg.rst | 3 +++ Python/pystate.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-06-12-00-43.gh-issue-110455.8BjNGg.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-06-12-00-43.gh-issue-110455.8BjNGg.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-06-12-00-43.gh-issue-110455.8BjNGg.rst new file mode 100644 index 00000000000000..47bf17da757d12 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-06-12-00-43.gh-issue-110455.8BjNGg.rst @@ -0,0 +1,3 @@ +Guard ``assert(tstate->thread_id > 0)`` with ``#ifndef HAVE_PTHREAD_STUBS``. +This allows for for pydebug builds to work under WASI which (currently) +lacks thread support. diff --git a/Python/pystate.c b/Python/pystate.c index a024ae7e3806a6..849a4eec86af83 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -263,10 +263,10 @@ static void unbind_tstate(PyThreadState *tstate) { assert(tstate != NULL); - // XXX assert(tstate_is_alive(tstate)); assert(tstate_is_bound(tstate)); - // XXX assert(!tstate->_status.active); +#ifndef HAVE_PTHREAD_STUBS assert(tstate->thread_id > 0); +#endif #ifdef PY_HAVE_THREAD_NATIVE_ID assert(tstate->native_thread_id > 0); #endif From de1052245f67d5c5a5dbb4f39449f7687f58fd78 Mon Sep 17 00:00:00 2001 From: Charlie Zhao Date: Sat, 7 Oct 2023 07:15:19 +0800 Subject: [PATCH 68/73] gh-106078: Suppress the warning caused by multi-phase initialization of `decimal` (#107524) --- Modules/_decimal/_decimal.c | 8 +++++++- Tools/c-analyzer/cpython/ignored.tsv | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index b49ea3cbb410ef..99ff9aa17b179c 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -5877,6 +5877,7 @@ cfunc_noargs(PyTypeObject *t, const char *name) return NULL; } +static int minalloc_is_set = 0; static int _decimal_exec(PyObject *m) @@ -5899,7 +5900,12 @@ _decimal_exec(PyObject *m) mpd_reallocfunc = PyMem_Realloc; mpd_callocfunc = mpd_callocfunc_em; mpd_free = PyMem_Free; - mpd_setminalloc(_Py_DEC_MINALLOC); + + /* Suppress the warning caused by multi-phase initialization */ + if (!minalloc_is_set) { + mpd_setminalloc(_Py_DEC_MINALLOC); + minalloc_is_set = 1; + } decimal_state *state = get_module_state(m); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index f9911643332b5e..ca656f0760ab4f 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -166,6 +166,9 @@ Python/pylifecycle.c fatal_error reentrant - # explicitly protected, internal-only Modules/_xxinterpchannelsmodule.c - _globals - +# set once during module init +Modules/_decimal/_decimal.c - minalloc_is_set - + ################################## ## not significant From 92ca90b7629c070ebc3b08e6f03db0bb552634e3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 6 Oct 2023 17:52:22 -0600 Subject: [PATCH 69/73] gh-76785: Support Running Some Functions in Subinterpreters (gh-110251) This specifically refers to `test.support.interpreters.Interpreter.run()`. --- Lib/test/support/interpreters.py | 18 +- Lib/test/test__xxsubinterpreters.py | 105 +++++++++ Modules/_xxsubinterpretersmodule.c | 342 ++++++++++++++++++++++++++-- 3 files changed, 439 insertions(+), 26 deletions(-) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index 3b501614bc4b4d..5aba36950f0549 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -91,12 +91,26 @@ def close(self): """ return _interpreters.destroy(self._id) + # XXX Rename "run" to "exec"? def run(self, src_str, /, *, channels=None): """Run the given source code in the interpreter. - This blocks the current Python thread until done. + This is essentially the same as calling the builtin "exec" + with this interpreter, using the __dict__ of its __main__ + module as both globals and locals. + + There is no return value. + + If the code raises an unhandled exception then a RunFailedError + is raised, which summarizes the unhandled exception. The actual + exception is discarded because objects cannot be shared between + interpreters. + + This blocks the current Python thread until done. During + that time, the previous interpreter is allowed to run + in other threads. """ - _interpreters.run_string(self._id, src_str, channels) + _interpreters.exec(self._id, src_str, channels) def create_channel(): diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index ac2280eb7090e9..e3c917aa2eb19d 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -925,5 +925,110 @@ def f(): self.assertEqual(retcode, 0) +class RunFuncTests(TestBase): + + def setUp(self): + super().setUp() + self.id = interpreters.create() + + def test_success(self): + r, w = os.pipe() + def script(): + global w + import contextlib + with open(w, 'w', encoding="utf-8") as spipe: + with contextlib.redirect_stdout(spipe): + print('it worked!', end='') + interpreters.run_func(self.id, script, shared=dict(w=w)) + + with open(r, encoding="utf-8") as outfile: + out = outfile.read() + + self.assertEqual(out, 'it worked!') + + def test_in_thread(self): + r, w = os.pipe() + def script(): + global w + import contextlib + with open(w, 'w', encoding="utf-8") as spipe: + with contextlib.redirect_stdout(spipe): + print('it worked!', end='') + def f(): + interpreters.run_func(self.id, script, shared=dict(w=w)) + t = threading.Thread(target=f) + t.start() + t.join() + + with open(r, encoding="utf-8") as outfile: + out = outfile.read() + + self.assertEqual(out, 'it worked!') + + def test_code_object(self): + r, w = os.pipe() + + def script(): + global w + import contextlib + with open(w, 'w', encoding="utf-8") as spipe: + with contextlib.redirect_stdout(spipe): + print('it worked!', end='') + code = script.__code__ + interpreters.run_func(self.id, code, shared=dict(w=w)) + + with open(r, encoding="utf-8") as outfile: + out = outfile.read() + + self.assertEqual(out, 'it worked!') + + def test_closure(self): + spam = True + def script(): + assert spam + + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + # XXX This hasn't been fixed yet. + @unittest.expectedFailure + def test_return_value(self): + def script(): + return 'spam' + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + def test_args(self): + with self.subTest('args'): + def script(a, b=0): + assert a == b + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + with self.subTest('*args'): + def script(*args): + assert not args + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + with self.subTest('**kwargs'): + def script(**kwargs): + assert not kwargs + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + with self.subTest('kwonly'): + def script(*, spam=True): + assert spam + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + with self.subTest('posonly'): + def script(spam, /): + assert spam + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + if __name__ == '__main__': unittest.main() diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index bca16ac8a62eca..12c98ea61fb7ac 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -10,6 +10,7 @@ #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #include "interpreteridobject.h" +#include "marshal.h" // PyMarshal_ReadObjectFromString() #define MODULE_NAME "_xxsubinterpreters" @@ -366,6 +367,98 @@ _sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass) } +/* Python code **************************************************************/ + +static const char * +check_code_str(PyUnicodeObject *text) +{ + assert(text != NULL); + if (PyUnicode_GET_LENGTH(text) == 0) { + return "too short"; + } + + // XXX Verify that it parses? + + return NULL; +} + +static const char * +check_code_object(PyCodeObject *code) +{ + assert(code != NULL); + if (code->co_argcount > 0 + || code->co_posonlyargcount > 0 + || code->co_kwonlyargcount > 0 + || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) + { + return "arguments not supported"; + } + if (code->co_ncellvars > 0) { + return "closures not supported"; + } + // We trust that no code objects under co_consts have unbound cell vars. + + if (code->co_executors != NULL + || code->_co_instrumentation_version > 0) + { + return "only basic functions are supported"; + } + if (code->_co_monitoring != NULL) { + return "only basic functions are supported"; + } + if (code->co_extra != NULL) { + return "only basic functions are supported"; + } + + return NULL; +} + +#define RUN_TEXT 1 +#define RUN_CODE 2 + +static const char * +get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) +{ + const char *codestr = NULL; + Py_ssize_t len = -1; + PyObject *bytes_obj = NULL; + int flags = 0; + + if (PyUnicode_Check(arg)) { + assert(PyUnicode_CheckExact(arg) + && (check_code_str((PyUnicodeObject *)arg) == NULL)); + codestr = PyUnicode_AsUTF8AndSize(arg, &len); + if (codestr == NULL) { + return NULL; + } + if (strlen(codestr) != (size_t)len) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + return NULL; + } + flags = RUN_TEXT; + } + else { + assert(PyCode_Check(arg) + && (check_code_object((PyCodeObject *)arg) == NULL)); + flags = RUN_CODE; + + // Serialize the code object. + bytes_obj = PyMarshal_WriteObjectToString(arg, Py_MARSHAL_VERSION); + if (bytes_obj == NULL) { + return NULL; + } + codestr = PyBytes_AS_STRING(bytes_obj); + len = PyBytes_GET_SIZE(bytes_obj); + } + + *flags_p = flags; + *bytes_p = bytes_obj; + *len_p = len; + return codestr; +} + + /* interpreter-specific code ************************************************/ static int @@ -393,8 +486,9 @@ exceptions_init(PyObject *mod) } static int -_run_script(PyInterpreterState *interp, const char *codestr, - _sharedns *shared, _sharedexception *sharedexc) +_run_script(PyInterpreterState *interp, + const char *codestr, Py_ssize_t codestrlen, + _sharedns *shared, _sharedexception *sharedexc, int flags) { int errcode = ERR_NOT_SET; @@ -428,8 +522,21 @@ _run_script(PyInterpreterState *interp, const char *codestr, } } - // Run the string (see PyRun_SimpleStringFlags). - PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + // Run the script/code/etc. + PyObject *result = NULL; + if (flags & RUN_TEXT) { + result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + } + else if (flags & RUN_CODE) { + PyObject *code = PyMarshal_ReadObjectFromString(codestr, codestrlen); + if (code != NULL) { + result = PyEval_EvalCode(code, ns, ns); + Py_DECREF(code); + } + } + else { + Py_UNREACHABLE(); + } Py_DECREF(ns); if (result == NULL) { goto error; @@ -465,8 +572,9 @@ _run_script(PyInterpreterState *interp, const char *codestr, } static int -_run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, - const char *codestr, PyObject *shareables) +_run_in_interpreter(PyObject *mod, PyInterpreterState *interp, + const char *codestr, Py_ssize_t codestrlen, + PyObject *shareables, int flags) { module_state *state = get_module_state(mod); assert(state != NULL); @@ -488,7 +596,7 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, // Run the script. _sharedexception exc = (_sharedexception){ .interp = interp }; - int result = _run_script(interp, codestr, shared, &exc); + int result = _run_script(interp, codestr, codestrlen, shared, &exc, flags); // Switch back. if (save_tstate != NULL) { @@ -695,49 +803,231 @@ PyDoc_STRVAR(get_main_doc, Return the ID of main interpreter."); +static PyUnicodeObject * +convert_script_arg(PyObject *arg, const char *fname, const char *displayname, + const char *expected) +{ + PyUnicodeObject *str = NULL; + if (PyUnicode_CheckExact(arg)) { + str = (PyUnicodeObject *)Py_NewRef(arg); + } + else if (PyUnicode_Check(arg)) { + // XXX str = PyUnicode_FromObject(arg); + str = (PyUnicodeObject *)Py_NewRef(arg); + } + else { + _PyArg_BadArgument(fname, displayname, expected, arg); + return NULL; + } + + const char *err = check_code_str(str); + if (err != NULL) { + Py_DECREF(str); + PyErr_Format(PyExc_ValueError, + "%.200s(): bad script text (%s)", fname, err); + return NULL; + } + + return str; +} + +static PyCodeObject * +convert_code_arg(PyObject *arg, const char *fname, const char *displayname, + const char *expected) +{ + const char *kind = NULL; + PyCodeObject *code = NULL; + if (PyFunction_Check(arg)) { + if (PyFunction_GetClosure(arg) != NULL) { + PyErr_Format(PyExc_ValueError, + "%.200s(): closures not supported", fname); + return NULL; + } + code = (PyCodeObject *)PyFunction_GetCode(arg); + if (code == NULL) { + if (PyErr_Occurred()) { + // This chains. + PyErr_Format(PyExc_ValueError, + "%.200s(): bad func", fname); + } + else { + PyErr_Format(PyExc_ValueError, + "%.200s(): func.__code__ missing", fname); + } + return NULL; + } + Py_INCREF(code); + kind = "func"; + } + else if (PyCode_Check(arg)) { + code = (PyCodeObject *)Py_NewRef(arg); + kind = "code object"; + } + else { + _PyArg_BadArgument(fname, displayname, expected, arg); + return NULL; + } + + const char *err = check_code_object(code); + if (err != NULL) { + Py_DECREF(code); + PyErr_Format(PyExc_ValueError, + "%.200s(): bad %s (%s)", fname, kind, err); + return NULL; + } + + return code; +} + +static int +_interp_exec(PyObject *self, + PyObject *id_arg, PyObject *code_arg, PyObject *shared_arg) +{ + // Look up the interpreter. + PyInterpreterState *interp = PyInterpreterID_LookUp(id_arg); + if (interp == NULL) { + return -1; + } + + // Extract code. + Py_ssize_t codestrlen = -1; + PyObject *bytes_obj = NULL; + int flags = 0; + const char *codestr = get_code_str(code_arg, + &codestrlen, &bytes_obj, &flags); + if (codestr == NULL) { + return -1; + } + + // Run the code in the interpreter. + int res = _run_in_interpreter(self, interp, codestr, codestrlen, + shared_arg, flags); + Py_XDECREF(bytes_obj); + if (res != 0) { + return -1; + } + + return 0; +} + static PyObject * -interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) +interp_exec(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "script", "shared", NULL}; + static char *kwlist[] = {"id", "code", "shared", NULL}; PyObject *id, *code; PyObject *shared = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O:run_string", kwlist, + "OO|O:" MODULE_NAME ".exec", kwlist, &id, &code, &shared)) { return NULL; } - // Look up the interpreter. - PyInterpreterState *interp = PyInterpreterID_LookUp(id); - if (interp == NULL) { + const char *expected = "a string, a function, or a code object"; + if (PyUnicode_Check(code)) { + code = (PyObject *)convert_script_arg(code, MODULE_NAME ".exec", + "argument 2", expected); + } + else { + code = (PyObject *)convert_code_arg(code, MODULE_NAME ".exec", + "argument 2", expected); + } + if (code == NULL) { return NULL; } - // Extract code. - Py_ssize_t size; - const char *codestr = PyUnicode_AsUTF8AndSize(code, &size); - if (codestr == NULL) { + int res = _interp_exec(self, id, code, shared); + Py_DECREF(code); + if (res < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(exec_doc, +"exec(id, code, shared=None)\n\ +\n\ +Execute the provided code in the identified interpreter.\n\ +This is equivalent to running the builtin exec() under the target\n\ +interpreter, using the __dict__ of its __main__ module as both\n\ +globals and locals.\n\ +\n\ +\"code\" may be a string containing the text of a Python script.\n\ +\n\ +Functions (and code objects) are also supported, with some restrictions.\n\ +The code/function must not take any arguments or be a closure\n\ +(i.e. have cell vars). Methods and other callables are not supported.\n\ +\n\ +If a function is provided, its code object is used and all its state\n\ +is ignored, including its __globals__ dict."); + +static PyObject * +interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "script", "shared", NULL}; + PyObject *id, *script; + PyObject *shared = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OU|O:" MODULE_NAME ".run_string", kwlist, + &id, &script, &shared)) { return NULL; } - if (strlen(codestr) != (size_t)size) { - PyErr_SetString(PyExc_ValueError, - "source code string cannot contain null bytes"); + + script = (PyObject *)convert_script_arg(script, MODULE_NAME ".exec", + "argument 2", "a string"); + if (script == NULL) { return NULL; } - // Run the code in the interpreter. - if (_run_script_in_interpreter(self, interp, codestr, shared) != 0) { + int res = _interp_exec(self, id, (PyObject *)script, shared); + Py_DECREF(script); + if (res < 0) { return NULL; } Py_RETURN_NONE; } PyDoc_STRVAR(run_string_doc, -"run_string(id, script, shared)\n\ +"run_string(id, script, shared=None)\n\ \n\ Execute the provided string in the identified interpreter.\n\ \n\ -See PyRun_SimpleStrings."); +(See " MODULE_NAME ".exec()."); + +static PyObject * +interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "func", "shared", NULL}; + PyObject *id, *func; + PyObject *shared = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO|O:" MODULE_NAME ".run_func", kwlist, + &id, &func, &shared)) { + return NULL; + } + + PyCodeObject *code = convert_code_arg(func, MODULE_NAME ".exec", + "argument 2", + "a function or a code object"); + if (code == NULL) { + return NULL; + } + + int res = _interp_exec(self, id, (PyObject *)code, shared); + Py_DECREF(code); + if (res < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(run_func_doc, +"run_func(id, func, shared=None)\n\ +\n\ +Execute the body of the provided function in the identified interpreter.\n\ +Code objects are also supported. In both cases, closures and args\n\ +are not supported. Methods and other callables are not supported either.\n\ +\n\ +(See " MODULE_NAME ".exec()."); static PyObject * @@ -804,8 +1094,12 @@ static PyMethodDef module_functions[] = { {"is_running", _PyCFunction_CAST(interp_is_running), METH_VARARGS | METH_KEYWORDS, is_running_doc}, + {"exec", _PyCFunction_CAST(interp_exec), + METH_VARARGS | METH_KEYWORDS, exec_doc}, {"run_string", _PyCFunction_CAST(interp_run_string), METH_VARARGS | METH_KEYWORDS, run_string_doc}, + {"run_func", _PyCFunction_CAST(interp_run_func), + METH_VARARGS | METH_KEYWORDS, run_func_doc}, {"is_shareable", _PyCFunction_CAST(object_is_shareable), METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, From 1aad4fc5dba993899621de86ae5955883448d6f6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 7 Oct 2023 16:00:28 +0300 Subject: [PATCH 70/73] gh-109864: Make test_gettext tests order independent (GH-109866) --- Lib/test/test_gettext.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index 8430fc234d00ee..2650ce501c309d 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -115,6 +115,12 @@ MMOFILE = os.path.join(LOCALEDIR, 'metadata.mo') +def reset_gettext(): + gettext._localedirs.clear() + gettext._current_domain = 'messages' + gettext._translations.clear() + + class GettextBaseTest(unittest.TestCase): def setUp(self): self.addCleanup(os_helper.rmtree, os.path.split(LOCALEDIR)[0]) @@ -132,7 +138,8 @@ def setUp(self): fp.write(base64.decodebytes(MMO_DATA)) self.env = self.enterContext(os_helper.EnvironmentVarGuard()) self.env['LANGUAGE'] = 'xx' - gettext._translations.clear() + reset_gettext() + self.addCleanup(reset_gettext) GNU_MO_DATA_ISSUE_17898 = b'''\ @@ -312,6 +319,10 @@ def test_multiline_strings(self): class PluralFormsTestCase(GettextBaseTest): def setUp(self): GettextBaseTest.setUp(self) + self.localedir = os.curdir + # Set up the bindings + gettext.bindtextdomain('gettext', self.localedir) + gettext.textdomain('gettext') self.mofile = MOFILE def test_plural_forms1(self): @@ -355,7 +366,7 @@ def test_plural_context_forms2(self): x = t.npgettext('With context', 'There is %s file', 'There are %s files', 2) eq(x, 'Hay %s ficheros (context)') - x = gettext.pgettext('With context', 'There is %s file') + x = t.pgettext('With context', 'There is %s file') eq(x, 'Hay %s fichero (context)') # Examples from http://www.gnu.org/software/gettext/manual/gettext.html From b987fdb19b981ef6e7f71b41790b5ed4e2064646 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 7 Oct 2023 16:01:39 +0300 Subject: [PATCH 71/73] gh-109848: Make test_rot13_func in test_codecs independent (GH-109850) --- Lib/test/test_codecs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index c899111e1e4c16..ff511a625a0194 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3637,9 +3637,10 @@ class Rot13UtilTest(unittest.TestCase): $ echo "Hello World" | python -m encodings.rot_13 """ def test_rot13_func(self): + from encodings.rot_13 import rot13 infile = io.StringIO('Gb or, be abg gb or, gung vf gur dhrfgvba') outfile = io.StringIO() - encodings.rot_13.rot13(infile, outfile) + rot13(infile, outfile) outfile.seek(0) plain_text = outfile.read() self.assertEqual( From 64f158e7b09e67d0bf5c8603ff88c86ed4e8f8fd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 7 Oct 2023 17:59:16 +0200 Subject: [PATCH 72/73] gh-110397: Add Py_IsFinalizing() to the stable ABI (#110441) --- Doc/data/stable_abi.dat | 1 + Include/cpython/pylifecycle.h | 2 -- Include/pylifecycle.h | 4 ++++ Lib/test/test_stable_abi_ctypes.py | 1 + Misc/stable_abi.toml | 2 ++ PC/python3dll.c | 1 + 6 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index c189c78238f40f..5bccd5edf586f4 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -840,6 +840,7 @@ function,Py_Initialize,3.2,, function,Py_InitializeEx,3.2,, function,Py_Is,3.10,, function,Py_IsFalse,3.10,, +function,Py_IsFinalizing,3.13,, function,Py_IsInitialized,3.2,, function,Py_IsNone,3.10,, function,Py_IsTrue,3.10,, diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index 11b280afa8435b..d425a233f71000 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -81,5 +81,3 @@ PyAPI_FUNC(PyStatus) Py_NewInterpreterFromConfig( typedef void (*atexit_datacallbackfunc)(void *); PyAPI_FUNC(int) PyUnstable_AtExit( PyInterpreterState *, atexit_datacallbackfunc, void *); - -PyAPI_FUNC(int) Py_IsFinalizing(void); diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 34f32a5000e9d5..c1e2bc5e323358 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -60,6 +60,10 @@ PyAPI_FUNC(PyOS_sighandler_t) PyOS_setsig(int, PyOS_sighandler_t); PyAPI_DATA(const unsigned long) Py_Version; #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 +PyAPI_FUNC(int) Py_IsFinalizing(void); +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYLIFECYCLE_H # include "cpython/pylifecycle.h" diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 94f817f8e1d159..4691687ed9d391 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -851,6 +851,7 @@ def test_windows_feature_macros(self): "Py_InitializeEx", "Py_Is", "Py_IsFalse", + "Py_IsFinalizing", "Py_IsInitialized", "Py_IsNone", "Py_IsTrue", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 8df3f85e61eec6..469fd27b622344 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2460,3 +2460,5 @@ added = '3.13' [function.PyMapping_HasKeyStringWithError] added = '3.13' +[function.Py_IsFinalizing] + added = '3.13' diff --git a/PC/python3dll.c b/PC/python3dll.c index 2c1cc8098ce856..785d6886f39f6d 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -69,6 +69,7 @@ EXPORT_FUNC(Py_Initialize) EXPORT_FUNC(Py_InitializeEx) EXPORT_FUNC(Py_Is) EXPORT_FUNC(Py_IsFalse) +EXPORT_FUNC(Py_IsFinalizing) EXPORT_FUNC(Py_IsInitialized) EXPORT_FUNC(Py_IsNone) EXPORT_FUNC(Py_IsTrue) From de2a4036cbfd5e41a5bdd2b81122b7765729af83 Mon Sep 17 00:00:00 2001 From: Masaru Tsuchiyama Date: Sun, 8 Oct 2023 02:33:22 +0900 Subject: [PATCH 73/73] gh-108277: Add os.timerfd_create() function (#108382) Add wrapper for timerfd_create, timerfd_settime, and timerfd_gettime to os module. Co-authored-by: Serhiy Storchaka Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Erlend E. Aasland Co-authored-by: Victor Stinner --- Doc/howto/index.rst | 1 + Doc/howto/timerfd.rst | 230 ++++++++++ Doc/library/os.rst | 211 ++++++++++ Doc/whatsnew/3.13.rst | 8 + .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + Include/internal/pycore_time.h | 6 +- .../internal/pycore_unicodeobject_generated.h | 3 + Lib/test/test_os.py | 350 ++++++++++++++++ ...-08-23-22-08-32.gh-issue-108277.KLV-6T.rst | 1 + Modules/clinic/posixmodule.c.h | 392 +++++++++++++++++- Modules/posixmodule.c | 248 +++++++++++ Python/pytime.c | 14 +- configure | 50 +++ configure.ac | 9 +- pyconfig.h.in | 6 + 17 files changed, 1527 insertions(+), 5 deletions(-) create mode 100644 Doc/howto/timerfd.rst create mode 100644 Misc/NEWS.d/next/Library/2023-08-23-22-08-32.gh-issue-108277.KLV-6T.rst diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index f521276a5a83c5..9e321be04eb81b 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -33,4 +33,5 @@ Currently, the HOWTOs are: perf_profiling.rst annotations.rst isolating-extensions.rst + timerfd.rst diff --git a/Doc/howto/timerfd.rst b/Doc/howto/timerfd.rst new file mode 100644 index 00000000000000..98f0294f9d082d --- /dev/null +++ b/Doc/howto/timerfd.rst @@ -0,0 +1,230 @@ +.. _timerfd-howto: + +***************************** + timer file descriptor HOWTO +***************************** + +:Release: 1.13 + +This HOWTO discusses Python's support for the linux timer file descriptor. + + +Examples +======== + +The following example shows how to use a timer file descriptor +to execute a function twice a second: + +.. code-block:: python + + # Practical scripts should use really use a non-blocking timer, + # we use a blocking timer here for simplicity. + import os, time + + # Create the timer file descriptor + fd = os.timerfd_create(time.CLOCK_REALTIME) + + # Start the timer in 1 second, with an interval of half a second + os.timerfd_settime(fd, initial=1, interval=0.5) + + try: + # Process timer events four times. + for _ in range(4): + # read() will block until the timer expires + _ = os.read(fd, 8) + print("Timer expired") + finally: + # Remember to close the timer file descriptor! + os.close(fd) + +To avoid the precision loss caused by the :class:`float` type, +timer file descriptors allow specifying initial expiration and interval +in integer nanoseconds with ``_ns`` variants of the functions. + +This example shows how :func:`~select.epoll` can be used with timer file +descriptors to wait until the file descriptor is ready for reading: + +.. code-block:: python + + import os, time, select, socket, sys + + # Create an epoll object + ep = select.epoll() + + # In this example, use loopback address to send "stop" command to the server. + # + # $ telnet 127.0.0.1 1234 + # Trying 127.0.0.1... + # Connected to 127.0.0.1. + # Escape character is '^]'. + # stop + # Connection closed by foreign host. + # + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 1234)) + sock.setblocking(False) + sock.listen(1) + ep.register(sock, select.EPOLLIN) + + # Create timer file descriptors in non-blocking mode. + num = 3 + fds = [] + for _ in range(num): + fd = os.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + fds.append(fd) + # Register the timer file descriptor for read events + ep.register(fd, select.EPOLLIN) + + # Start the timer with os.timerfd_settime_ns() in nanoseconds. + # Timer 1 fires every 0.25 seconds; timer 2 every 0.5 seconds; etc + for i, fd in enumerate(fds, start=1): + one_sec_in_nsec = 10**9 + i = i * one_sec_in_nsec + os.timerfd_settime_ns(fd, initial=i//4, interval=i//4) + + timeout = 3 + try: + conn = None + is_active = True + while is_active: + # Wait for the timer to expire for 3 seconds. + # epoll.poll() returns a list of (fd, event) pairs. + # fd is a file descriptor. + # sock and conn[=returned value of socket.accept()] are socket objects, not file descriptors. + # So use sock.fileno() and conn.fileno() to get the file descriptors. + events = ep.poll(timeout) + + # If more than one timer file descriptors are ready for reading at once, + # epoll.poll() returns a list of (fd, event) pairs. + # + # In this example settings, + # 1st timer fires every 0.25 seconds in 0.25 seconds. (0.25, 0.5, 0.75, 1.0, ...) + # 2nd timer every 0.5 seconds in 0.5 seconds. (0.5, 1.0, 1.5, 2.0, ...) + # 3rd timer every 0.75 seconds in 0.75 seconds. (0.75, 1.5, 2.25, 3.0, ...) + # + # In 0.25 seconds, only 1st timer fires. + # In 0.5 seconds, 1st timer and 2nd timer fires at once. + # In 0.75 seconds, 1st timer and 3rd timer fires at once. + # In 1.5 seconds, 1st timer, 2nd timer and 3rd timer fires at once. + # + # If a timer file descriptor is signaled more than once since + # the last os.read() call, os.read() returns the nubmer of signaled + # as host order of class bytes. + print(f"Signaled events={events}") + for fd, event in events: + if event & select.EPOLLIN: + if fd == sock.fileno(): + # Check if there is a connection request. + print(f"Accepting connection {fd}") + conn, addr = sock.accept() + conn.setblocking(False) + print(f"Accepted connection {conn} from {addr}") + ep.register(conn, select.EPOLLIN) + elif conn and fd == conn.fileno(): + # Check if there is data to read. + print(f"Reading data {fd}") + data = conn.recv(1024) + if data: + # You should catch UnicodeDecodeError exception for safety. + cmd = data.decode() + if cmd.startswith("stop"): + print(f"Stopping server") + is_active = False + else: + print(f"Unknown command: {cmd}") + else: + # No more data, close connection + print(f"Closing connection {fd}") + ep.unregister(conn) + conn.close() + conn = None + elif fd in fds: + print(f"Reading timer {fd}") + count = int.from_bytes(os.read(fd, 8), byteorder=sys.byteorder) + print(f"Timer {fds.index(fd) + 1} expired {count} times") + else: + print(f"Unknown file descriptor {fd}") + finally: + for fd in fds: + ep.unregister(fd) + os.close(fd) + ep.close() + +This example shows how :func:`~select.select` can be used with timer file +descriptors to wait until the file descriptor is ready for reading: + +.. code-block:: python + + import os, time, select, socket, sys + + # In this example, use loopback address to send "stop" command to the server. + # + # $ telnet 127.0.0.1 1234 + # Trying 127.0.0.1... + # Connected to 127.0.0.1. + # Escape character is '^]'. + # stop + # Connection closed by foreign host. + # + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 1234)) + sock.setblocking(False) + sock.listen(1) + + # Create timer file descriptors in non-blocking mode. + num = 3 + fds = [os.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + for _ in range(num)] + select_fds = fds + [sock] + + # Start the timers with os.timerfd_settime() in seconds. + # Timer 1 fires every 0.25 seconds; timer 2 every 0.5 seconds; etc + for i, fd in enumerate(fds, start=1): + os.timerfd_settime(fd, initial=i/4, interval=i/4) + + timeout = 3 + try: + conn = None + is_active = True + while is_active: + # Wait for the timer to expire for 3 seconds. + # select.select() returns a list of file descriptors or objects. + rfd, wfd, xfd = select.select(select_fds, select_fds, select_fds, timeout) + for fd in rfd: + if fd == sock: + # Check if there is a connection request. + print(f"Accepting connection {fd}") + conn, addr = sock.accept() + conn.setblocking(False) + print(f"Accepted connection {conn} from {addr}") + select_fds.append(conn) + elif conn and fd == conn: + # Check if there is data to read. + print(f"Reading data {fd}") + data = conn.recv(1024) + if data: + # You should catch UnicodeDecodeError exception for safety. + cmd = data.decode() + if cmd.startswith("stop"): + print(f"Stopping server") + is_active = False + else: + print(f"Unknown command: {cmd}") + else: + # No more data, close connection + print(f"Closing connection {fd}") + select_fds.remove(conn) + conn.close() + conn = None + elif fd in fds: + print(f"Reading timer {fd}") + count = int.from_bytes(os.read(fd, 8), byteorder=sys.byteorder) + print(f"Timer {fds.index(fd) + 1} expired {count} times") + else: + print(f"Unknown file descriptor {fd}") + finally: + for fd in fds: + os.close(fd) + sock.close() + sock = None + diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 141ab0bff5b4bf..a1595dfbc060f3 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3781,6 +3781,217 @@ features: .. versionadded:: 3.10 +Timer File Descriptors +~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 3.13 + +These functions provide support for Linux's *timer file descriptor* API. +Naturally, they are all only available on Linux. + +.. function:: timerfd_create(clockid, /, *, flags=0) + + Create and return a timer file descriptor (*timerfd*). + + The file descriptor returned by :func:`timerfd_create` supports: + + - :func:`read` + - :func:`~select.select` + - :func:`~select.poll`. + + The file descriptor's :func:`read` method can be called with a buffer size + of 8. If the timer has already expired one or more times, :func:`read` + returns the number of expirations with the host's endianness, which may be + converted to an :class:`int` by ``int.from_bytes(x, byteorder=sys.byteorder)``. + + :func:`~select.select` and :func:`~select.poll` can be used to wait until + timer expires and the file descriptor is readable. + + *clockid* must be a valid :ref:`clock ID `, + as defined in the :py:mod:`time` module: + + - :const:`time.CLOCK_REALTIME` + - :const:`time.CLOCK_MONOTONIC` + - :const:`time.CLOCK_BOOTTIME` (Since Linux 3.15 for timerfd_create) + + If *clockid* is :const:`time.CLOCK_REALTIME`, a settable system-wide + real-time clock is used. If system clock is changed, timer setting need + to be updated. To cancel timer when system clock is changed, see + :const:`TFD_TIMER_CANCEL_ON_SET`. + + If *clockid* is :const:`time.CLOCK_MONOTONIC`, a non-settable monotonically + increasing clock is used. Even if the system clock is changed, the timer + setting will not be affected. + + If *clockid* is :const:`time.CLOCK_BOOTTIME`, same as :const:`time.CLOCK_MONOTONIC` + except it includes any time that the system is suspended. + + The file descriptor's behaviour can be modified by specifying a *flags* value. + Any of the following variables may used, combined using bitwise OR + (the ``|`` operator): + + - :const:`TFD_NONBLOCK` + - :const:`TFD_CLOEXEC` + + If :const:`TFD_NONBLOCK` is not set as a flag, :func:`read` blocks until + the timer expires. If it is set as a flag, :func:`read` doesn't block, but + If there hasn't been an expiration since the last call to read, + :func:`read` raises :class:`OSError` with ``errno`` is set to + :const:`errno.EAGAIN`. + + :const:`TFD_CLOEXEC` is always set by Python automatically. + + The file descriptor must be closed with :func:`os.close` when it is no + longer needed, or else the file descriptor will be leaked. + + .. seealso:: The :manpage:`timerfd_create(2)` man page. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + + +.. function:: timerfd_settime(fd, /, *, flags=flags, initial=0.0, interval=0.0) + + Alter a timer file descriptor's internal timer. + This function operates the same interval timer as :func:`timerfd_settime_ns`. + + *fd* must be a valid timer file descriptor. + + The timer's behaviour can be modified by specifying a *flags* value. + Any of the following variables may used, combined using bitwise OR + (the ``|`` operator): + + - :const:`TFD_TIMER_ABSTIME` + - :const:`TFD_TIMER_CANCEL_ON_SET` + + The timer is disabled by setting *initial* to zero (``0``). + If *initial* is equal to or greater than zero, the timer is enabled. + If *initial* is less than zero, it raises an :class:`OSError` exception + with ``errno`` set to :const:`errno.EINVAL` + + By default the timer will fire when *initial* seconds have elapsed. + (If *initial* is zero, timer will fire immediately.) + + However, if the :const:`TFD_TIMER_ABSTIME` flag is set, + the timer will fire when the timer's clock + (set by *clockid* in :func:`timerfd_create`) reaches *initial* seconds. + + The timer's interval is set by the *interval* :py:class:`float`. + If *interval* is zero, the timer only fires once, on the initial expiration. + If *interval* is greater than zero, the timer fires every time *interval* + seconds have elapsed since the previous expiration. + If *interval* is less than zero, it raises :class:`OSError` with ``errno`` + set to :const:`errno.EINVAL` + + If the :const:`TFD_TIMER_CANCEL_ON_SET` flag is set along with + :const:`TFD_TIMER_ABSTIME` and the clock for this timer is + :const:`time.CLOCK_REALTIME`, the timer is marked as cancelable if the + real-time clock is changed discontinuously. Reading the descriptor is + aborted with the error ECANCELED. + + Linux manages system clock as UTC. A daylight-savings time transition is + done by changing time offset only and doesn't cause discontinuous system + clock change. + + Discontinuous system clock change will be caused by the following events: + + - ``settimeofday`` + - ``clock_settime`` + - set the system date and time by ``date`` command + + Return a two-item tuple of (``next_expiration``, ``interval``) from + the previous timer state, before this function executed. + + .. seealso:: + + :manpage:`timerfd_create(2)`, :manpage:`timerfd_settime(2)`, + :manpage:`settimeofday(2)`, :manpage:`clock_settime(2)`, + and :manpage:`date(1)`. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + + +.. function:: timerfd_settime_ns(fd, /, *, flags=0, initial=0, interval=0) + + Similar to :func:`timerfd_settime`, but use time as nanoseconds. + This function operates the same interval timer as :func:`timerfd_settime`. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + + +.. function:: timerfd_gettime(fd, /) + + Return a two-item tuple of floats (``next_expiration``, ``interval``). + + ``next_expiration`` denotes the relative time until next the timer next fires, + regardless of if the :const:`TFD_TIMER_ABSTIME` flag is set. + + ``interval`` denotes the timer's interval. + If zero, the timer will only fire once, after ``next_expiration`` seconds + have elapsed. + + .. seealso:: :manpage:`timerfd_gettime(2)` + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + + +.. function:: timerfd_gettime_ns(fd, /) + + Similar to :func:`timerfd_gettime`, but return time as nanoseconds. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + +.. data:: TFD_NONBLOCK + + A flag for the :func:`timerfd_create` function, + which sets the :const:`O_NONBLOCK` status flag for the new timer file + descriptor. If :const:`TFD_NONBLOCK` is not set as a flag, :func:`read` blocks. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + +.. data:: TFD_CLOEXEC + + A flag for the :func:`timerfd_create` function, + If :const:`TFD_CLOEXEC` is set as a flag, set close-on-exec flag for new file + descriptor. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + +.. data:: TFD_TIMER_ABSTIME + + A flag for the :func:`timerfd_settime` and :func:`timerfd_settime_ns` functions. + If this flag is set, *initial* is interpreted as an absolute value on the + timer's clock (in UTC seconds or nanoseconds since the Unix Epoch). + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + +.. data:: TFD_TIMER_CANCEL_ON_SET + + A flag for the :func:`timerfd_settime` and :func:`timerfd_settime_ns` + functions along with :const:`TFD_TIMER_ABSTIME`. + The timer is cancelled when the time of the underlying clock changes + discontinuously. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + + Linux extended attributes ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d5987ae31ce68d..73975b055c240b 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -180,6 +180,14 @@ os usable by the calling thread of the current process. (Contributed by Victor Stinner in :gh:`109649`.) +* Add a low level interface for Linux's timer notification file descriptors + via :func:`os.timerfd_create`, + :func:`os.timerfd_settime`, :func:`os.timerfd_settime_ns`, + :func:`os.timerfd_gettime`, and :func:`os.timerfd_gettime_ns`, + :const:`os.TFD_NONBLOCK`, :const:`os.TFD_CLOEXEC`, + :const:`os.TFD_TIMER_ABSTIME`, and :const:`os.TFD_TIMER_CANCEL_ON_SET` + (Contributed by Masaru Tsuchiyama in :gh:`108277`.) + pathlib ------- diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 6361f5d1100231..8fb22f70505808 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -993,6 +993,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(instructions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(intern)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(intersection)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(interval)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(is_running)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isatty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isinstance)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 504008d67fe9cd..39ffda8419a77d 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -482,6 +482,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(instructions) STRUCT_FOR_ID(intern) STRUCT_FOR_ID(intersection) + STRUCT_FOR_ID(interval) STRUCT_FOR_ID(is_running) STRUCT_FOR_ID(isatty) STRUCT_FOR_ID(isinstance) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 8cc3287ce35e5b..43a8243bf41b7e 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -991,6 +991,7 @@ extern "C" { INIT_ID(instructions), \ INIT_ID(intern), \ INIT_ID(intersection), \ + INIT_ID(interval), \ INIT_ID(is_running), \ INIT_ID(isatty), \ INIT_ID(isinstance), \ diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index d8ea63a7242ccc..46713f91d190ff 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -143,6 +143,10 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimespec( // Export for '_socket' shared extension. PyAPI_FUNC(_PyTime_t) _PyTime_FromSeconds(int seconds); +// Create a timestamp from a number of seconds in double. +// Export for '_socket' shared extension. +PyAPI_FUNC(_PyTime_t) _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round); + // Macro to create a timestamp from a number of seconds, no integer overflow. // Only use the macro for small values, prefer _PyTime_FromSeconds(). #define _PYTIME_FROMSECONDS(seconds) \ @@ -241,7 +245,7 @@ PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) // Create a timestamp from a timespec structure. // Raise an exception and return -1 on overflow, return 0 on success. -extern int _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts); +extern int _PyTime_FromTimespec(_PyTime_t *tp, const struct timespec *ts); // Convert a timestamp to a timespec structure (nanosecond resolution). // tv_nsec is always positive. diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 50400db2919a73..729d54bbb951e7 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1287,6 +1287,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(intersection); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(interval); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(is_running); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 669e27c0473af0..5149b0d3884417 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3926,6 +3926,356 @@ def test_eventfd_select(self): self.assertEqual((rfd, wfd, xfd), ([fd], [], [])) os.eventfd_read(fd) +@unittest.skipUnless(hasattr(os, 'timerfd_create'), 'requires os.timerfd_create') +@support.requires_linux_version(2, 6, 30) +class TimerfdTests(unittest.TestCase): + def timerfd_create(self, *args, **kwargs): + fd = os.timerfd_create(*args, **kwargs) + self.assertGreaterEqual(fd, 0) + self.assertFalse(os.get_inheritable(fd)) + self.addCleanup(os.close, fd) + return fd + + def test_timerfd_initval(self): + fd = self.timerfd_create(time.CLOCK_REALTIME) + + initial_expiration = 0.25 + interval = 0.125 + + # 1st call + next_expiration, interval2 = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) + self.assertAlmostEqual(interval2, 0.0, places=3) + self.assertAlmostEqual(next_expiration, 0.0, places=3) + + # 2nd call + next_expiration, interval2 = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) + self.assertAlmostEqual(interval2, interval, places=3) + self.assertAlmostEqual(next_expiration, initial_expiration, places=3) + + # timerfd_gettime + next_expiration, interval2 = os.timerfd_gettime(fd) + self.assertAlmostEqual(interval2, interval, places=3) + self.assertAlmostEqual(next_expiration, initial_expiration, places=3) + + def test_timerfd_non_blocking(self): + size = 8 # read 8 bytes + fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + + # 0.1 second later + initial_expiration = 0.1 + _, _ = os.timerfd_settime(fd, initial=initial_expiration, interval=0) + + # read() raises OSError with errno is EAGAIN for non-blocking timer. + with self.assertRaises(OSError) as ctx: + _ = os.read(fd, size) + self.assertEqual(ctx.exception.errno, errno.EAGAIN) + + # Wait more than 0.1 seconds + time.sleep(initial_expiration + 0.1) + + # confirm if timerfd is readable and read() returns 1 as bytes. + n = os.read(fd, size) + count_signaled = int.from_bytes(n, byteorder=sys.byteorder) + self.assertEqual(count_signaled, 1) + + def test_timerfd_negative(self): + one_sec_in_nsec = 10**9 + fd = self.timerfd_create(time.CLOCK_REALTIME) + + # Any of 'initial' and 'interval' is negative value. + for initial, interval in ( (-1, 0), (1, -1), (-1, -1), (-0.1, 0), (1, -0.1), (-0.1, -0.1)): + for flags in (0, os.TFD_TIMER_ABSTIME, os.TFD_TIMER_ABSTIME|os.TFD_TIMER_CANCEL_ON_SET): + with self.subTest(flags=flags, initial=initial, interval=interval): + with self.assertRaises(OSError) as context: + _, _ = os.timerfd_settime(fd, flags=flags, initial=initial, interval=interval) + self.assertEqual(context.exception.errno, errno.EINVAL) + + with self.assertRaises(OSError) as context: + initial_ns = int( one_sec_in_nsec * initial ) + interval_ns = int( one_sec_in_nsec * interval ) + _, _ = os.timerfd_settime_ns(fd, flags=flags, initial=initial_ns, interval=interval_ns) + self.assertEqual(context.exception.errno, errno.EINVAL) + + def test_timerfd_interval(self): + size = 8 # read 8 bytes + fd = self.timerfd_create(time.CLOCK_REALTIME) + + # 1 second + initial_expiration = 1 + # 0.5 second + interval = 0.5 + + _, _ = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) + + # timerfd_gettime + next_expiration, interval2 = os.timerfd_gettime(fd) + self.assertAlmostEqual(interval2, interval, places=3) + self.assertAlmostEqual(next_expiration, initial_expiration, places=3) + + count = 3 + t = time.perf_counter() + for _ in range(count): + n = os.read(fd, size) + count_signaled = int.from_bytes(n, byteorder=sys.byteorder) + self.assertEqual(count_signaled, 1) + t = time.perf_counter() - t + + total_time = initial_expiration + interval * (count - 1) + self.assertGreater(t, total_time) + + # wait 3.5 time of interval + time.sleep( (count+0.5) * interval) + n = os.read(fd, size) + count_signaled = int.from_bytes(n, byteorder=sys.byteorder) + self.assertEqual(count_signaled, count) + + def test_timerfd_TFD_TIMER_ABSTIME(self): + size = 8 # read 8 bytes + fd = self.timerfd_create(time.CLOCK_REALTIME) + + now = time.clock_gettime(time.CLOCK_REALTIME) + + # 1 second later from now. + offset = 1 + initial_expiration = now + offset + # not interval timer + interval = 0 + + _, _ = os.timerfd_settime(fd, flags=os.TFD_TIMER_ABSTIME, initial=initial_expiration, interval=interval) + + # timerfd_gettime + # Note: timerfd_gettime returns relative values even if TFD_TIMER_ABSTIME is specified. + next_expiration, interval2 = os.timerfd_gettime(fd) + self.assertAlmostEqual(interval2, interval, places=3) + self.assertAlmostEqual(next_expiration, offset, places=3) + + t = time.perf_counter() + n = os.read(fd, size) + count_signaled = int.from_bytes(n, byteorder=sys.byteorder) + t = time.perf_counter() - t + self.assertEqual(count_signaled, 1) + + self.assertGreater(t, offset) + + def test_timerfd_select(self): + size = 8 # read 8 bytes + fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + + rfd, wfd, xfd = select.select([fd], [fd], [fd], 0) + self.assertEqual((rfd, wfd, xfd), ([], [], [])) + + # 0.25 second + initial_expiration = 0.25 + # every 0.125 second + interval = 0.125 + + _, _ = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) + + count = 3 + t = time.perf_counter() + for _ in range(count): + rfd, wfd, xfd = select.select([fd], [fd], [fd], initial_expiration + interval) + self.assertEqual((rfd, wfd, xfd), ([fd], [], [])) + n = os.read(fd, size) + count_signaled = int.from_bytes(n, byteorder=sys.byteorder) + self.assertEqual(count_signaled, 1) + t = time.perf_counter() - t + + total_time = initial_expiration + interval * (count - 1) + self.assertGreater(t, total_time) + + def test_timerfd_epoll(self): + size = 8 # read 8 bytes + fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + + ep = select.epoll() + ep.register(fd, select.EPOLLIN) + self.addCleanup(ep.close) + + # 0.25 second + initial_expiration = 0.25 + # every 0.125 second + interval = 0.125 + + _, _ = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) + + count = 3 + t = time.perf_counter() + for i in range(count): + timeout_margin = interval + if i == 0: + timeout = initial_expiration + interval + timeout_margin + else: + timeout = interval + timeout_margin + # epoll timeout is in seconds. + events = ep.poll(timeout) + self.assertEqual(events, [(fd, select.EPOLLIN)]) + n = os.read(fd, size) + count_signaled = int.from_bytes(n, byteorder=sys.byteorder) + self.assertEqual(count_signaled, 1) + + t = time.perf_counter() - t + + total_time = initial_expiration + interval * (count - 1) + self.assertGreater(t, total_time) + ep.unregister(fd) + + def test_timerfd_ns_initval(self): + one_sec_in_nsec = 10**9 + limit_error = one_sec_in_nsec // 10**3 + fd = self.timerfd_create(time.CLOCK_REALTIME) + + # 1st call + initial_expiration_ns = 0 + interval_ns = one_sec_in_nsec // 1000 + next_expiration_ns, interval_ns2 = os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) + self.assertEqual(interval_ns2, 0) + self.assertEqual(next_expiration_ns, 0) + + # 2nd call + next_expiration_ns, interval_ns2 = os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) + self.assertEqual(interval_ns2, interval_ns) + self.assertEqual(next_expiration_ns, initial_expiration_ns) + + # timerfd_gettime + next_expiration_ns, interval_ns2 = os.timerfd_gettime_ns(fd) + self.assertEqual(interval_ns2, interval_ns) + self.assertLessEqual(next_expiration_ns, initial_expiration_ns) + + self.assertAlmostEqual(next_expiration_ns, initial_expiration_ns, delta=limit_error) + + def test_timerfd_ns_interval(self): + size = 8 # read 8 bytes + one_sec_in_nsec = 10**9 + limit_error = one_sec_in_nsec // 10**3 + fd = self.timerfd_create(time.CLOCK_REALTIME) + + # 1 second + initial_expiration_ns = one_sec_in_nsec + # every 0.5 second + interval_ns = one_sec_in_nsec // 2 + + _, _ = os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) + + # timerfd_gettime + next_expiration_ns, interval_ns2 = os.timerfd_gettime_ns(fd) + self.assertEqual(interval_ns2, interval_ns) + self.assertLessEqual(next_expiration_ns, initial_expiration_ns) + + count = 3 + t = time.perf_counter_ns() + for _ in range(count): + n = os.read(fd, size) + count_signaled = int.from_bytes(n, byteorder=sys.byteorder) + self.assertEqual(count_signaled, 1) + t = time.perf_counter_ns() - t + + total_time_ns = initial_expiration_ns + interval_ns * (count - 1) + self.assertGreater(t, total_time_ns) + + # wait 3.5 time of interval + time.sleep( (count+0.5) * interval_ns / one_sec_in_nsec) + n = os.read(fd, size) + count_signaled = int.from_bytes(n, byteorder=sys.byteorder) + self.assertEqual(count_signaled, count) + + + def test_timerfd_ns_TFD_TIMER_ABSTIME(self): + size = 8 # read 8 bytes + one_sec_in_nsec = 10**9 + limit_error = one_sec_in_nsec // 10**3 + fd = self.timerfd_create(time.CLOCK_REALTIME) + + now_ns = time.clock_gettime_ns(time.CLOCK_REALTIME) + + # 1 second later from now. + offset_ns = one_sec_in_nsec + initial_expiration_ns = now_ns + offset_ns + # not interval timer + interval_ns = 0 + + _, _ = os.timerfd_settime_ns(fd, flags=os.TFD_TIMER_ABSTIME, initial=initial_expiration_ns, interval=interval_ns) + + # timerfd_gettime + # Note: timerfd_gettime returns relative values even if TFD_TIMER_ABSTIME is specified. + next_expiration_ns, interval_ns2 = os.timerfd_gettime_ns(fd) + self.assertLess(abs(interval_ns2 - interval_ns), limit_error) + self.assertLess(abs(next_expiration_ns - offset_ns), limit_error) + + t = time.perf_counter_ns() + n = os.read(fd, size) + count_signaled = int.from_bytes(n, byteorder=sys.byteorder) + t = time.perf_counter_ns() - t + self.assertEqual(count_signaled, 1) + + self.assertGreater(t, offset_ns) + + def test_timerfd_ns_select(self): + size = 8 # read 8 bytes + one_sec_in_nsec = 10**9 + + fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + + rfd, wfd, xfd = select.select([fd], [fd], [fd], 0) + self.assertEqual((rfd, wfd, xfd), ([], [], [])) + + # 0.25 second + initial_expiration_ns = one_sec_in_nsec // 4 + # every 0.125 second + interval_ns = one_sec_in_nsec // 8 + + _, _ = os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) + + count = 3 + t = time.perf_counter_ns() + for _ in range(count): + rfd, wfd, xfd = select.select([fd], [fd], [fd], (initial_expiration_ns + interval_ns) / 1e9 ) + self.assertEqual((rfd, wfd, xfd), ([fd], [], [])) + n = os.read(fd, size) + count_signaled = int.from_bytes(n, byteorder=sys.byteorder) + self.assertEqual(count_signaled, 1) + t = time.perf_counter_ns() - t + + total_time_ns = initial_expiration_ns + interval_ns * (count - 1) + self.assertGreater(t, total_time_ns) + + def test_timerfd_ns_epoll(self): + size = 8 # read 8 bytes + one_sec_in_nsec = 10**9 + fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + + ep = select.epoll() + ep.register(fd, select.EPOLLIN) + self.addCleanup(ep.close) + + # 0.25 second + initial_expiration_ns = one_sec_in_nsec // 4 + # every 0.125 second + interval_ns = one_sec_in_nsec // 8 + + _, _ = os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) + + count = 3 + t = time.perf_counter_ns() + for i in range(count): + timeout_margin_ns = interval_ns + if i == 0: + timeout_ns = initial_expiration_ns + interval_ns + timeout_margin_ns + else: + timeout_ns = interval_ns + timeout_margin_ns + + # epoll timeout is in seconds. + events = ep.poll(timeout_ns / one_sec_in_nsec) + self.assertEqual(events, [(fd, select.EPOLLIN)]) + n = os.read(fd, size) + count_signaled = int.from_bytes(n, byteorder=sys.byteorder) + self.assertEqual(count_signaled, 1) + + t = time.perf_counter_ns() - t + + total_time = initial_expiration_ns + interval_ns * (count - 1) + self.assertGreater(t, total_time) + ep.unregister(fd) class OSErrorTests(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Library/2023-08-23-22-08-32.gh-issue-108277.KLV-6T.rst b/Misc/NEWS.d/next/Library/2023-08-23-22-08-32.gh-issue-108277.KLV-6T.rst new file mode 100644 index 00000000000000..6f99e0b33237d7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-23-22-08-32.gh-issue-108277.KLV-6T.rst @@ -0,0 +1 @@ +Add :func:`os.timerfd_create`, :func:`os.timerfd_settime`, :func:`os.timerfd_gettime`, :func:`os.timerfd_settime_ns`, and :func:`os.timerfd_gettime_ns` to provide a low level interface for Linux's timer notification file descriptor. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 0238d3a2f23149..179132754f9aeb 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -6022,6 +6022,376 @@ os_times(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(HAVE_TIMES) */ +#if defined(HAVE_TIMERFD_CREATE) + +PyDoc_STRVAR(os_timerfd_create__doc__, +"timerfd_create($module, clockid, /, *, flags=0)\n" +"--\n" +"\n" +"Create and return a timer file descriptor.\n" +"\n" +" clockid\n" +" A valid clock ID constant as timer file descriptor.\n" +"\n" +" time.CLOCK_REALTIME\n" +" time.CLOCK_MONOTONIC\n" +" time.CLOCK_BOOTTIME\n" +" flags\n" +" 0 or a bit mask of os.TFD_NONBLOCK or os.TFD_CLOEXEC.\n" +"\n" +" os.TFD_NONBLOCK\n" +" If *TFD_NONBLOCK* is set as a flag, read doesn\'t blocks.\n" +" If *TFD_NONBLOCK* is not set as a flag, read block until the timer fires.\n" +"\n" +" os.TFD_CLOEXEC\n" +" If *TFD_CLOEXEC* is set as a flag, enable the close-on-exec flag"); + +#define OS_TIMERFD_CREATE_METHODDEF \ + {"timerfd_create", _PyCFunction_CAST(os_timerfd_create), METH_FASTCALL|METH_KEYWORDS, os_timerfd_create__doc__}, + +static PyObject * +os_timerfd_create_impl(PyObject *module, int clockid, int flags); + +static PyObject * +os_timerfd_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(flags), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "flags", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "timerfd_create", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + int clockid; + int flags = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + clockid = PyLong_AsInt(args[0]); + if (clockid == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + flags = PyLong_AsInt(args[1]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = os_timerfd_create_impl(module, clockid, flags); + +exit: + return return_value; +} + +#endif /* defined(HAVE_TIMERFD_CREATE) */ + +#if defined(HAVE_TIMERFD_CREATE) + +PyDoc_STRVAR(os_timerfd_settime__doc__, +"timerfd_settime($module, fd, /, *, flags=0, initial=0.0, interval=0.0)\n" +"--\n" +"\n" +"Alter a timer file descriptor\'s internal timer in seconds.\n" +"\n" +" fd\n" +" A timer file descriptor.\n" +" flags\n" +" 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET.\n" +" initial\n" +" The initial expiration time, in seconds.\n" +" interval\n" +" The timer\'s interval, in seconds."); + +#define OS_TIMERFD_SETTIME_METHODDEF \ + {"timerfd_settime", _PyCFunction_CAST(os_timerfd_settime), METH_FASTCALL|METH_KEYWORDS, os_timerfd_settime__doc__}, + +static PyObject * +os_timerfd_settime_impl(PyObject *module, int fd, int flags, double initial, + double interval); + +static PyObject * +os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(flags), &_Py_ID(initial), &_Py_ID(interval), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "flags", "initial", "interval", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "timerfd_settime", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + int fd; + int flags = 0; + double initial = 0.0; + double interval = 0.0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[1]) { + flags = PyLong_AsInt(args[1]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[2]) { + if (PyFloat_CheckExact(args[2])) { + initial = PyFloat_AS_DOUBLE(args[2]); + } + else + { + initial = PyFloat_AsDouble(args[2]); + if (initial == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (PyFloat_CheckExact(args[3])) { + interval = PyFloat_AS_DOUBLE(args[3]); + } + else + { + interval = PyFloat_AsDouble(args[3]); + if (interval == -1.0 && PyErr_Occurred()) { + goto exit; + } + } +skip_optional_kwonly: + return_value = os_timerfd_settime_impl(module, fd, flags, initial, interval); + +exit: + return return_value; +} + +#endif /* defined(HAVE_TIMERFD_CREATE) */ + +#if defined(HAVE_TIMERFD_CREATE) + +PyDoc_STRVAR(os_timerfd_settime_ns__doc__, +"timerfd_settime_ns($module, fd, /, *, flags=0, initial=0, interval=0)\n" +"--\n" +"\n" +"Alter a timer file descriptor\'s internal timer in nanoseconds.\n" +"\n" +" fd\n" +" A timer file descriptor.\n" +" flags\n" +" 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET.\n" +" initial\n" +" initial expiration timing in seconds.\n" +" interval\n" +" interval for the timer in seconds."); + +#define OS_TIMERFD_SETTIME_NS_METHODDEF \ + {"timerfd_settime_ns", _PyCFunction_CAST(os_timerfd_settime_ns), METH_FASTCALL|METH_KEYWORDS, os_timerfd_settime_ns__doc__}, + +static PyObject * +os_timerfd_settime_ns_impl(PyObject *module, int fd, int flags, + long long initial, long long interval); + +static PyObject * +os_timerfd_settime_ns(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(flags), &_Py_ID(initial), &_Py_ID(interval), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "flags", "initial", "interval", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "timerfd_settime_ns", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + int fd; + int flags = 0; + long long initial = 0; + long long interval = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[1]) { + flags = PyLong_AsInt(args[1]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[2]) { + initial = PyLong_AsLongLong(args[2]); + if (initial == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + interval = PyLong_AsLongLong(args[3]); + if (interval == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = os_timerfd_settime_ns_impl(module, fd, flags, initial, interval); + +exit: + return return_value; +} + +#endif /* defined(HAVE_TIMERFD_CREATE) */ + +#if defined(HAVE_TIMERFD_CREATE) + +PyDoc_STRVAR(os_timerfd_gettime__doc__, +"timerfd_gettime($module, fd, /)\n" +"--\n" +"\n" +"Return a tuple of a timer file descriptor\'s (interval, next expiration) in float seconds.\n" +"\n" +" fd\n" +" A timer file descriptor."); + +#define OS_TIMERFD_GETTIME_METHODDEF \ + {"timerfd_gettime", (PyCFunction)os_timerfd_gettime, METH_O, os_timerfd_gettime__doc__}, + +static PyObject * +os_timerfd_gettime_impl(PyObject *module, int fd); + +static PyObject * +os_timerfd_gettime(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + goto exit; + } + return_value = os_timerfd_gettime_impl(module, fd); + +exit: + return return_value; +} + +#endif /* defined(HAVE_TIMERFD_CREATE) */ + +#if defined(HAVE_TIMERFD_CREATE) + +PyDoc_STRVAR(os_timerfd_gettime_ns__doc__, +"timerfd_gettime_ns($module, fd, /)\n" +"--\n" +"\n" +"Return a tuple of a timer file descriptor\'s (interval, next expiration) in nanoseconds.\n" +"\n" +" fd\n" +" A timer file descriptor."); + +#define OS_TIMERFD_GETTIME_NS_METHODDEF \ + {"timerfd_gettime_ns", (PyCFunction)os_timerfd_gettime_ns, METH_O, os_timerfd_gettime_ns__doc__}, + +static PyObject * +os_timerfd_gettime_ns_impl(PyObject *module, int fd); + +static PyObject * +os_timerfd_gettime_ns(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + goto exit; + } + return_value = os_timerfd_gettime_ns_impl(module, fd); + +exit: + return return_value; +} + +#endif /* defined(HAVE_TIMERFD_CREATE) */ + #if defined(HAVE_GETSID) PyDoc_STRVAR(os_getsid__doc__, @@ -11761,6 +12131,26 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS_TIMES_METHODDEF #endif /* !defined(OS_TIMES_METHODDEF) */ +#ifndef OS_TIMERFD_CREATE_METHODDEF + #define OS_TIMERFD_CREATE_METHODDEF +#endif /* !defined(OS_TIMERFD_CREATE_METHODDEF) */ + +#ifndef OS_TIMERFD_SETTIME_METHODDEF + #define OS_TIMERFD_SETTIME_METHODDEF +#endif /* !defined(OS_TIMERFD_SETTIME_METHODDEF) */ + +#ifndef OS_TIMERFD_SETTIME_NS_METHODDEF + #define OS_TIMERFD_SETTIME_NS_METHODDEF +#endif /* !defined(OS_TIMERFD_SETTIME_NS_METHODDEF) */ + +#ifndef OS_TIMERFD_GETTIME_METHODDEF + #define OS_TIMERFD_GETTIME_METHODDEF +#endif /* !defined(OS_TIMERFD_GETTIME_METHODDEF) */ + +#ifndef OS_TIMERFD_GETTIME_NS_METHODDEF + #define OS_TIMERFD_GETTIME_NS_METHODDEF +#endif /* !defined(OS_TIMERFD_GETTIME_NS_METHODDEF) */ + #ifndef OS_GETSID_METHODDEF #define OS_GETSID_METHODDEF #endif /* !defined(OS_GETSID_METHODDEF) */ @@ -12024,4 +12414,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=a36904281a8a7507 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7c3058135ed49d20 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2c32a45a53277f..0975ef71d44be5 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -550,6 +550,11 @@ extern char *ctermid_r(char *); # include #endif +/* timerfd_create() */ +#ifdef HAVE_SYS_TIMERFD_H +# include +#endif + #ifdef _Py_MEMORY_SANITIZER # include #endif @@ -10096,6 +10101,227 @@ os_times_impl(PyObject *module) #endif /* HAVE_TIMES */ +#if defined(HAVE_TIMERFD_CREATE) +#define ONE_SECOND_IN_NS (1000 * 1000 * 1000) +#define EXTRACT_NSEC(value) (long)( ( (double)(value) - (time_t)(value) ) * 1e9) +#define CONVERT_SEC_AND_NSEC_TO_DOUBLE(sec, nsec) ( (double)(sec) + (double)(nsec) * 1e-9 ) + +static PyObject * +build_itimerspec(const struct itimerspec* curr_value) +{ + double _value = CONVERT_SEC_AND_NSEC_TO_DOUBLE(curr_value->it_value.tv_sec, + curr_value->it_value.tv_nsec); + PyObject *value = PyFloat_FromDouble(_value); + if (value == NULL) { + return NULL; + } + double _interval = CONVERT_SEC_AND_NSEC_TO_DOUBLE(curr_value->it_interval.tv_sec, + curr_value->it_interval.tv_nsec); + PyObject *interval = PyFloat_FromDouble(_interval); + if (interval == NULL) { + Py_DECREF(value); + return NULL; + } + PyObject *tuple = PyTuple_Pack(2, value, interval); + Py_DECREF(interval); + Py_DECREF(value); + return tuple; +} + +static PyObject * +build_itimerspec_ns(const struct itimerspec* curr_value) +{ + _PyTime_t value, interval; + if (_PyTime_FromTimespec(&value, &curr_value->it_value) < 0) { + return NULL; + } + if (_PyTime_FromTimespec(&interval, &curr_value->it_interval) < 0) { + return NULL; + } + return Py_BuildValue("LL", value, interval); +} + +/*[clinic input] +os.timerfd_create + + clockid: int + A valid clock ID constant as timer file descriptor. + + time.CLOCK_REALTIME + time.CLOCK_MONOTONIC + time.CLOCK_BOOTTIME + / + * + flags: int = 0 + 0 or a bit mask of os.TFD_NONBLOCK or os.TFD_CLOEXEC. + + os.TFD_NONBLOCK + If *TFD_NONBLOCK* is set as a flag, read doesn't blocks. + If *TFD_NONBLOCK* is not set as a flag, read block until the timer fires. + + os.TFD_CLOEXEC + If *TFD_CLOEXEC* is set as a flag, enable the close-on-exec flag + +Create and return a timer file descriptor. +[clinic start generated code]*/ + +static PyObject * +os_timerfd_create_impl(PyObject *module, int clockid, int flags) +/*[clinic end generated code: output=1caae80fb168004a input=64b7020c5ac0b8f4]*/ + +{ + int fd; + Py_BEGIN_ALLOW_THREADS + flags |= TFD_CLOEXEC; // PEP 446: always create non-inheritable FD + fd = timerfd_create(clockid, flags); + Py_END_ALLOW_THREADS + if (fd == -1) { + return PyErr_SetFromErrno(PyExc_OSError); + } + return PyLong_FromLong(fd); +} + +/*[clinic input] +os.timerfd_settime + + fd: fildes + A timer file descriptor. + / + * + flags: int = 0 + 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET. + initial: double = 0.0 + The initial expiration time, in seconds. + interval: double = 0.0 + The timer's interval, in seconds. + +Alter a timer file descriptor's internal timer in seconds. +[clinic start generated code]*/ + +static PyObject * +os_timerfd_settime_impl(PyObject *module, int fd, int flags, double initial, + double interval) +/*[clinic end generated code: output=0dda31115317adb9 input=6c24e47e7a4d799e]*/ +{ + struct itimerspec new_value; + struct itimerspec old_value; + int result; + if (_PyTime_AsTimespec(_PyTime_FromSecondsDouble(initial, _PyTime_ROUND_FLOOR), &new_value.it_value) < 0) { + PyErr_SetString(PyExc_ValueError, "invalid initial value"); + return NULL; + } + if (_PyTime_AsTimespec(_PyTime_FromSecondsDouble(interval, _PyTime_ROUND_FLOOR), &new_value.it_interval) < 0) { + PyErr_SetString(PyExc_ValueError, "invalid interval value"); + return NULL; + } + Py_BEGIN_ALLOW_THREADS + result = timerfd_settime(fd, flags, &new_value, &old_value); + Py_END_ALLOW_THREADS + if (result == -1) { + return PyErr_SetFromErrno(PyExc_OSError); + } + return build_itimerspec(&old_value); +} + + +/*[clinic input] +os.timerfd_settime_ns + + fd: fildes + A timer file descriptor. + / + * + flags: int = 0 + 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET. + initial: long_long = 0 + initial expiration timing in seconds. + interval: long_long = 0 + interval for the timer in seconds. + +Alter a timer file descriptor's internal timer in nanoseconds. +[clinic start generated code]*/ + +static PyObject * +os_timerfd_settime_ns_impl(PyObject *module, int fd, int flags, + long long initial, long long interval) +/*[clinic end generated code: output=6273ec7d7b4cc0b3 input=261e105d6e42f5bc]*/ +{ + struct itimerspec new_value; + struct itimerspec old_value; + int result; + if (_PyTime_AsTimespec(initial, &new_value.it_value) < 0) { + PyErr_SetString(PyExc_ValueError, "invalid initial value"); + return NULL; + } + if (_PyTime_AsTimespec(interval, &new_value.it_interval) < 0) { + PyErr_SetString(PyExc_ValueError, "invalid interval value"); + return NULL; + } + Py_BEGIN_ALLOW_THREADS + result = timerfd_settime(fd, flags, &new_value, &old_value); + Py_END_ALLOW_THREADS + if (result == -1) { + return PyErr_SetFromErrno(PyExc_OSError); + } + return build_itimerspec_ns(&old_value); +} + +/*[clinic input] +os.timerfd_gettime + + fd: fildes + A timer file descriptor. + / + +Return a tuple of a timer file descriptor's (interval, next expiration) in float seconds. +[clinic start generated code]*/ + +static PyObject * +os_timerfd_gettime_impl(PyObject *module, int fd) +/*[clinic end generated code: output=ec5a94a66cfe6ab4 input=8148e3430870da1c]*/ +{ + struct itimerspec curr_value; + int result; + Py_BEGIN_ALLOW_THREADS + result = timerfd_gettime(fd, &curr_value); + Py_END_ALLOW_THREADS + if (result == -1) { + return PyErr_SetFromErrno(PyExc_OSError); + } + return build_itimerspec(&curr_value); +} + + +/*[clinic input] +os.timerfd_gettime_ns + + fd: fildes + A timer file descriptor. + / + +Return a tuple of a timer file descriptor's (interval, next expiration) in nanoseconds. +[clinic start generated code]*/ + +static PyObject * +os_timerfd_gettime_ns_impl(PyObject *module, int fd) +/*[clinic end generated code: output=580633a4465f39fe input=a825443e4c6b40ac]*/ +{ + struct itimerspec curr_value; + int result; + Py_BEGIN_ALLOW_THREADS + result = timerfd_gettime(fd, &curr_value); + Py_END_ALLOW_THREADS + if (result == -1) { + return PyErr_SetFromErrno(PyExc_OSError); + } + return build_itimerspec_ns(&curr_value); +} + +#undef ONE_SECOND_IN_NS +#undef EXTRACT_NSEC + +#endif /* HAVE_TIMERFD_CREATE */ + #ifdef HAVE_GETSID /*[clinic input] os.getsid @@ -16028,6 +16254,11 @@ static PyMethodDef posix_methods[] = { OS_WAITSTATUS_TO_EXITCODE_METHODDEF OS_SETNS_METHODDEF OS_UNSHARE_METHODDEF + OS_TIMERFD_CREATE_METHODDEF + OS_TIMERFD_SETTIME_METHODDEF + OS_TIMERFD_SETTIME_NS_METHODDEF + OS_TIMERFD_GETTIME_METHODDEF + OS_TIMERFD_GETTIME_NS_METHODDEF OS__PATH_ISDEVDRIVE_METHODDEF OS__PATH_ISDIR_METHODDEF @@ -16343,6 +16574,19 @@ all_ins(PyObject *m) if (PyModule_AddIntMacro(m, SF_NOCACHE)) return -1; #endif +#ifdef TFD_NONBLOCK + if (PyModule_AddIntMacro(m, TFD_NONBLOCK)) return -1; +#endif +#ifdef TFD_CLOEXEC + if (PyModule_AddIntMacro(m, TFD_CLOEXEC)) return -1; +#endif +#ifdef TFD_TIMER_ABSTIME + if (PyModule_AddIntMacro(m, TFD_TIMER_ABSTIME)) return -1; +#endif +#ifdef TFD_TIMER_CANCEL_ON_SET + if (PyModule_AddIntMacro(m, TFD_TIMER_CANCEL_ON_SET)) return -1; +#endif + /* constants for posix_fadvise */ #ifdef POSIX_FADV_NORMAL if (PyModule_AddIntMacro(m, POSIX_FADV_NORMAL)) return -1; @@ -16741,6 +16985,10 @@ static const struct have_function { {"HAVE_EVENTFD", NULL}, #endif +#ifdef HAVE_TIMERFD_CREATE + {"HAVE_TIMERFD_CREATE", NULL}, +#endif + #ifdef HAVE_FACCESSAT { "HAVE_FACCESSAT", probe_faccessat }, #endif diff --git a/Python/pytime.c b/Python/pytime.c index d1e29e57d362f6..e4813d4a9c2a2a 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -470,7 +470,7 @@ _PyTime_FromNanosecondsObject(_PyTime_t *tp, PyObject *obj) #ifdef HAVE_CLOCK_GETTIME static int -pytime_fromtimespec(_PyTime_t *tp, struct timespec *ts, int raise_exc) +pytime_fromtimespec(_PyTime_t *tp, const struct timespec *ts, int raise_exc) { _PyTime_t t, tv_nsec; @@ -493,7 +493,7 @@ pytime_fromtimespec(_PyTime_t *tp, struct timespec *ts, int raise_exc) } int -_PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts) +_PyTime_FromTimespec(_PyTime_t *tp, const struct timespec *ts) { return pytime_fromtimespec(tp, ts, 1); } @@ -635,6 +635,16 @@ _PyTime_AsNanosecondsObject(_PyTime_t t) return PyLong_FromLongLong((long long)ns); } +_PyTime_t +_PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round) +{ + _PyTime_t tp; + if(pytime_from_double(&tp, seconds, round, SEC_TO_NS) < 0) { + return -1; + } + return tp; +} + static _PyTime_t pytime_divide_round_up(const _PyTime_t t, const _PyTime_t k) diff --git a/configure b/configure index 0e5f3f64c680b2..7c5fdec4c93aa9 100755 --- a/configure +++ b/configure @@ -10709,6 +10709,12 @@ if test "x$ac_cv_header_sys_times_h" = xyes then : printf "%s\n" "#define HAVE_SYS_TIMES_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "sys/timerfd.h" "ac_cv_header_sys_timerfd_h" "$ac_includes_default" +if test "x$ac_cv_header_sys_timerfd_h" = xyes +then : + printf "%s\n" "#define HAVE_SYS_TIMERFD_H 1" >>confdefs.h + fi ac_fn_c_check_header_compile "$LINENO" "sys/types.h" "ac_cv_header_sys_types_h" "$ac_includes_default" if test "x$ac_cv_header_sys_types_h" = xyes @@ -18781,6 +18787,50 @@ fi + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for timerfd_create" >&5 +printf %s "checking for timerfd_create... " >&6; } +if test ${ac_cv_func_timerfd_create+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#ifdef HAVE_SYS_TIMERFD_H +#include +#endif + +int +main (void) +{ +void *x=timerfd_create + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_func_timerfd_create=yes +else $as_nop + ac_cv_func_timerfd_create=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_timerfd_create" >&5 +printf "%s\n" "$ac_cv_func_timerfd_create" >&6; } + if test "x$ac_cv_func_timerfd_create" = xyes +then : + +printf "%s\n" "#define HAVE_TIMERFD_CREATE 1" >>confdefs.h + +fi + + + + # On some systems (eg. FreeBSD 5), we would find a definition of the # functions ctermid_r, setgroups in the library, but no prototype # (e.g. because we use _XOPEN_SOURCE). See whether we can take their diff --git a/configure.ac b/configure.ac index 493868130414ee..6093afa0926053 100644 --- a/configure.ac +++ b/configure.ac @@ -2691,7 +2691,7 @@ AC_CHECK_HEADERS([ \ sys/endian.h sys/epoll.h sys/event.h sys/eventfd.h sys/file.h sys/ioctl.h sys/kern_control.h \ sys/loadavg.h sys/lock.h sys/memfd.h sys/mkdev.h sys/mman.h sys/modem.h sys/param.h sys/poll.h \ sys/random.h sys/resource.h sys/select.h sys/sendfile.h sys/socket.h sys/soundcard.h sys/stat.h \ - sys/statvfs.h sys/sys_domain.h sys/syscall.h sys/sysmacros.h sys/termio.h sys/time.h sys/times.h \ + sys/statvfs.h sys/sys_domain.h sys/syscall.h sys/sysmacros.h sys/termio.h sys/time.h sys/times.h sys/timerfd.h \ sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h sys/xattr.h sysexits.h syslog.h \ termios.h util.h utime.h utmp.h \ ]) @@ -4744,6 +4744,13 @@ PY_CHECK_FUNC([eventfd], [ #endif ]) +PY_CHECK_FUNC([timerfd_create], [ +#ifdef HAVE_SYS_TIMERFD_H +#include +#endif +], +[HAVE_TIMERFD_CREATE]) + # On some systems (eg. FreeBSD 5), we would find a definition of the # functions ctermid_r, setgroups in the library, but no prototype # (e.g. because we use _XOPEN_SOURCE). See whether we can take their diff --git a/pyconfig.h.in b/pyconfig.h.in index c2c75c96dcaad1..9924a9011ed4ed 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1363,6 +1363,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_TERMIO_H +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_TIMERFD_H + /* Define to 1 if you have the header file. */ #undef HAVE_SYS_TIMES_H @@ -1405,6 +1408,9 @@ /* Define to 1 if you have the `timegm' function. */ #undef HAVE_TIMEGM +/* Define if you have the 'timerfd_create' function. */ +#undef HAVE_TIMERFD_CREATE + /* Define to 1 if you have the `times' function. */ #undef HAVE_TIMES