From 710c01be94ca7b2c96407f41f0e37e24701008b4 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 19 Apr 2024 06:40:28 +0900 Subject: [PATCH 01/37] gh-112069: Make PySet_GET_SIZE to be atomic safe. (gh-118053) gh-112069: Make PySet_GET_SIZE to be atomic operation --- Include/cpython/setobject.h | 4 ++++ Objects/setobject.c | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Include/cpython/setobject.h b/Include/cpython/setobject.h index 1778c778a05324..89565cb29212fc 100644 --- a/Include/cpython/setobject.h +++ b/Include/cpython/setobject.h @@ -62,6 +62,10 @@ typedef struct { (assert(PyAnySet_Check(so)), _Py_CAST(PySetObject*, so)) static inline Py_ssize_t PySet_GET_SIZE(PyObject *so) { +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&(_PySet_CAST(so)->used)); +#else return _PySet_CAST(so)->used; +#endif } #define PySet_GET_SIZE(so) PySet_GET_SIZE(_PyObject_CAST(so)) diff --git a/Objects/setobject.c b/Objects/setobject.c index 7af0ae166f9da3..d5030cec2d6206 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2080,7 +2080,6 @@ set_issuperset_impl(PySetObject *so, PyObject *other) Py_RETURN_TRUE; } -// TODO: Make thread-safe in free-threaded builds static PyObject * set_richcompare(PySetObject *v, PyObject *w, int op) { From 398abdd6fa5b6b15c0570c75321cd7df9573a5b7 Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Fri, 19 Apr 2024 00:28:12 -0300 Subject: [PATCH 02/37] Use "Contributed by" in a couple of occurrences of 3.12 whatsnew (#118070) --- Doc/whatsnew/3.12.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index f2ef4efcb378bc..ce3d9ec6a29de8 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -726,7 +726,7 @@ inspect * Add :func:`inspect.markcoroutinefunction` to mark sync functions that return a :term:`coroutine` for use with :func:`inspect.iscoroutinefunction`. - (Contributed Carlton Gibson in :gh:`99247`.) + (Contributed by Carlton Gibson in :gh:`99247`.) * Add :func:`inspect.getasyncgenstate` and :func:`inspect.getasyncgenlocals` for determining the current state of asynchronous generators. @@ -751,8 +751,8 @@ math (Contributed by Raymond Hettinger in :gh:`100485`.) * Extend :func:`math.nextafter` to include a *steps* argument - for moving up or down multiple steps at a time. - (By Matthias Goergens, Mark Dickinson, and Raymond Hettinger in :gh:`94906`.) + for moving up or down multiple steps at a time. (Contributed by + Matthias Goergens, Mark Dickinson, and Raymond Hettinger in :gh:`94906`.) os -- From a09e47299217fe98615aec8318e13a744307d52a Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 18 Apr 2024 20:50:09 -0700 Subject: [PATCH 03/37] gh-117535: Change unknown filename of warnings from `sys` to `` (#118018) --- Lib/test/test_warnings/__init__.py | 4 ++-- Lib/warnings.py | 4 ++-- .../Library/2024-04-18-00-35-11.gh-issue-117535.0m6SIM.rst | 1 + Python/_warnings.c | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-18-00-35-11.gh-issue-117535.0m6SIM.rst diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 50b0f3fff04c57..b768631846e240 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -489,7 +489,7 @@ def test_stacklevel(self): warning_tests.inner("spam7", stacklevel=9999) self.assertEqual(os.path.basename(w[-1].filename), - "sys") + "") def test_stacklevel_import(self): # Issue #24305: With stacklevel=2, module-level warnings should work. @@ -1388,7 +1388,7 @@ def test_late_resource_warning(self): # Issue #21925: Emitting a ResourceWarning late during the Python # shutdown must be logged. - expected = b"sys:1: ResourceWarning: unclosed file " + expected = b":0: ResourceWarning: unclosed file " # don't import the warnings module # (_warnings will try to import it) diff --git a/Lib/warnings.py b/Lib/warnings.py index 4ad6ad027192e8..20a39d54bf7e6a 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -332,8 +332,8 @@ def warn(message, category=None, stacklevel=1, source=None, raise ValueError except ValueError: globals = sys.__dict__ - filename = "sys" - lineno = 1 + filename = "" + lineno = 0 else: globals = frame.f_globals filename = frame.f_code.co_filename diff --git a/Misc/NEWS.d/next/Library/2024-04-18-00-35-11.gh-issue-117535.0m6SIM.rst b/Misc/NEWS.d/next/Library/2024-04-18-00-35-11.gh-issue-117535.0m6SIM.rst new file mode 100644 index 00000000000000..72423506ccc07b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-18-00-35-11.gh-issue-117535.0m6SIM.rst @@ -0,0 +1 @@ +Change the unknown filename of :mod:`warnings` from ``sys`` to ```` to clarify that it's not a real filename. diff --git a/Python/_warnings.c b/Python/_warnings.c index 4c520252aa12a8..2ba704dcaa79b2 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -894,8 +894,8 @@ setup_context(Py_ssize_t stack_level, if (f == NULL) { globals = interp->sysdict; - *filename = PyUnicode_FromString("sys"); - *lineno = 1; + *filename = PyUnicode_FromString(""); + *lineno = 0; } else { globals = f->f_frame->f_globals; From 60787b8a4e80b9a3e67de781e6803364410ab285 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 18 Apr 2024 21:31:53 -0700 Subject: [PATCH 04/37] Docs: Fix CVE link (#118077) --- Doc/c-api/init.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index cbc03bfd18ea3c..8725ce085145aa 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -699,7 +699,7 @@ Process-wide parameters It is recommended that applications embedding the Python interpreter for purposes other than executing a single script pass ``0`` as *updatepath*, and update :data:`sys.path` themselves if desired. - See `CVE-2008-5983 `_. + See :cve:`2008-5983`. On versions before 3.1.3, you can achieve the same effect by manually popping the first :data:`sys.path` element after having called From 6099fdf733c2d67c05f0f64c9980fb621741baed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Apr 2024 22:23:49 -0700 Subject: [PATCH 05/37] build(deps): bump hypothesis from 6.98.15 to 6.100.0 in /Tools (#117416) Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 6.98.15 to 6.100.0. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-6.98.15...hypothesis-python-6.100.0) --- updated-dependencies: - dependency-name: hypothesis dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Tools/requirements-hypothesis.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 1a45d1c431dd11..8cfa0df523dd16 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.98.15 +hypothesis==6.100.0 From fefd5d97111364afa027ae580c3244f427dda59d Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Fri, 19 Apr 2024 07:36:24 +0200 Subject: [PATCH 06/37] gh-64588: Clarify the difference between mu and xbar in statistics docs (#117333) Thanks Davin Potts for the clarification idea. --- Doc/library/statistics.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 873ccd650f45cd..cc72396964342e 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -501,9 +501,9 @@ However, for reading convenience, most of the examples show sorted sequences. variance indicates that the data is spread out; a small variance indicates it is clustered closely around the mean. - If the optional second argument *mu* is given, it is typically the mean of - the *data*. It can also be used to compute the second moment around a - point that is not the mean. If it is missing or ``None`` (the default), + If the optional second argument *mu* is given, it should be the *population* + mean of the *data*. It can also be used to compute the second moment around + a point that is not the mean. If it is missing or ``None`` (the default), the arithmetic mean is automatically calculated. Use this function to calculate the variance from the entire population. To @@ -573,8 +573,8 @@ However, for reading convenience, most of the examples show sorted sequences. the data is spread out; a small variance indicates it is clustered closely around the mean. - If the optional second argument *xbar* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is + If the optional second argument *xbar* is given, it should be the *sample* + mean of *data*. If it is missing or ``None`` (the default), the mean is automatically calculated. Use this function when your data is a sample from a population. To calculate @@ -590,8 +590,8 @@ However, for reading convenience, most of the examples show sorted sequences. >>> variance(data) 1.3720238095238095 - If you have already calculated the mean of your data, you can pass it as the - optional second argument *xbar* to avoid recalculation: + If you have already calculated the sample mean of your data, you can pass it + as the optional second argument *xbar* to avoid recalculation: .. doctest:: From d3bd6b5f3f48731715e21fe132b8e65a4e5f6ce8 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 19 Apr 2024 09:25:07 +0100 Subject: [PATCH 07/37] GH-115419: Improve list of escaping functions (GH-118054) --- Include/internal/pycore_opcode_metadata.h | 24 +++++++++++------------ Include/internal/pycore_uop_metadata.h | 24 +++++++++++------------ Python/optimizer_analysis.c | 4 +++- Tools/cases_generator/analyzer.py | 10 ++++++++++ 4 files changed, 37 insertions(+), 25 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index aa87dc413876f0..e75de9d12e0c55 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -984,8 +984,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1000,8 +1000,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, @@ -1009,7 +1009,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1035,7 +1035,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, @@ -1103,7 +1103,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1124,18 +1124,18 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ESCAPES_FLAG }, + [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, 0 }, [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1146,7 +1146,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index e5a99421c241e0..481d7415d19ad1 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -78,12 +78,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG, [_STORE_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_POP_FRAME] = HAS_ESCAPES_FLAG, + [_POP_FRAME] = 0, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -109,7 +109,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -147,9 +147,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, [_GUARD_DORV_NO_DICT] = HAS_DEOPT_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = 0, - [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG, + [_STORE_ATTR_SLOT] = 0, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_COMPARE_OP_STR] = HAS_ARG_FLAG, [_IS_OP] = HAS_ARG_FLAG, @@ -192,18 +192,18 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_PEP_523] = HAS_DEOPT_FLAG, [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_0] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_1] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_0] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_1] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = 0, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, + [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 90444e33568b9d..efc5b3c39b6e76 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -543,7 +543,9 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) return pc + 1; default: { - bool needs_ip = false; + /* _PUSH_FRAME doesn't escape or error, but it + * does need the IP for the return address */ + bool needs_ip = opcode == _PUSH_FRAME; if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { needs_ip = true; may_have_escaped = true; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index e38ab3c9047039..d17b2b9b024b99 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -358,6 +358,7 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "_PyObject_InlineValues", "_PyDictValues_AddToInsertionOrder", "Py_DECREF", + "Py_XDECREF", "_Py_DECREF_SPECIALIZED", "DECREF_INPUTS_AND_REUSE_FLOAT", "PyUnicode_Append", @@ -365,6 +366,7 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "Py_SIZE", "Py_TYPE", "PyList_GET_ITEM", + "PyList_SET_ITEM", "PyTuple_GET_ITEM", "PyList_GET_SIZE", "PyTuple_GET_SIZE", @@ -400,8 +402,14 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "PySlice_New", "_Py_LeaveRecursiveCallPy", "CALL_STAT_INC", + "STAT_INC", "maybe_lltrace_resume_frame", "_PyUnicode_JoinArray", + "_PyEval_FrameClearAndPop", + "_PyFrame_StackPush", + "PyCell_New", + "PyFloat_AS_DOUBLE", + "_PyFrame_PushUnchecked", ) ESCAPING_FUNCTIONS = ( @@ -427,6 +435,8 @@ def makes_escaping_api_call(instr: parser.InstDef) -> bool: continue if tkn.text in ESCAPING_FUNCTIONS: return True + if tkn.text == "tp_vectorcall": + return True if not tkn.text.startswith("Py") and not tkn.text.startswith("_Py"): continue if tkn.text.endswith("Check"): From 7e6fa5fceddc3f4721c036116c7a11533c8b23b3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 19 Apr 2024 09:26:42 +0100 Subject: [PATCH 08/37] GH-116202: Incorporate invalidation check into _START_EXECUTOR. (GH-118044) --- Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 1 + Python/executor_cases.c.h | 4 ++++ Python/optimizer.c | 4 +--- Python/optimizer_analysis.c | 3 +++ 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 481d7415d19ad1..44ede3e77c68e1 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -240,7 +240,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_START_EXECUTOR] = 0, + [_START_EXECUTOR] = HAS_DEOPT_FLAG, [_FATAL_ERROR] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, [_DEOPT] = 0, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d6fb66a7be34ac..c34d702f06418e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4181,6 +4181,7 @@ dummy_func( #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif + DEOPT_IF(!((_PyExecutorObject *)executor)->vm_data.valid); } tier2 op(_FATAL_ERROR, (--)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a3447da00477ca..fccff24a418586 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4137,6 +4137,10 @@ #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif + if (!((_PyExecutorObject *)executor)->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } diff --git a/Python/optimizer.c b/Python/optimizer.c index 5c69d9d5de92eb..bb537c9111a51f 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1109,8 +1109,6 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil assert(next_exit == -1); assert(dest == executor->trace); assert(dest->opcode == _START_EXECUTOR); - dest->oparg = 0; - dest->target = 0; _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -1314,7 +1312,7 @@ counter_optimize( } _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; _PyUOpInstruction buffer[5] = { - { .opcode = _START_EXECUTOR }, + { .opcode = _START_EXECUTOR, .jump_target = 4, .format=UOP_FORMAT_JUMP }, { .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self }, { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, { .opcode = _EXIT_TRACE, .jump_target = 4, .format=UOP_FORMAT_JUMP }, diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index efc5b3c39b6e76..155f7026b041b0 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -497,6 +497,9 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) for (int pc = 0; pc < buffer_size; pc++) { int opcode = buffer[pc].opcode; switch (opcode) { + case _START_EXECUTOR: + may_have_escaped = false; + break; case _SET_IP: buffer[pc].opcode = _NOP; last_set_ip = pc; From 4605a197bd84da1a232bd835d8e8e654f2fef220 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 19 Apr 2024 10:41:37 +0200 Subject: [PATCH 09/37] gh-117518: Clarify PyTuple_GetItem() borrowed reference in the doc (GH-117920) --- Doc/c-api/tuple.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index b3710560ebe7ac..0d68a360f347f8 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -59,6 +59,12 @@ Tuple Objects Return the object at position *pos* in the tuple pointed to by *p*. If *pos* is negative or out of bounds, return ``NULL`` and set an :exc:`IndexError` exception. + The returned reference is borrowed from the tuple *p* + (that is: it is only valid as long as you hold a reference to *p*). + To get a :term:`strong reference`, use + :c:func:`Py_NewRef(PyTuple_GetItem(...)) ` + or :c:func:`PySequence_GetItem`. + .. c:function:: PyObject* PyTuple_GET_ITEM(PyObject *p, Py_ssize_t pos) From 5d544365742a117027747306e2d4473f3b73d921 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 19 Apr 2024 02:29:23 -0700 Subject: [PATCH 10/37] gh-116935: Document that heap types need to support garbage collection (GH-118021) --- Doc/c-api/typeobj.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index e66ab01878cac0..1105b943325077 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1034,7 +1034,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) the type, and the type object is INCREF'ed when a new instance is created, and DECREF'ed when an instance is destroyed (this does not apply to instances of subtypes; only the type referenced by the instance's ob_type gets INCREF'ed or - DECREF'ed). + DECREF'ed). Heap types should also :ref:`support garbage collection ` + as they can form a reference cycle with their own module object. **Inheritance:** From 8a01fd7b9bb27c7d284e2f0152713a8619fd34a3 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 19 Apr 2024 11:57:31 +0100 Subject: [PATCH 11/37] gh-115775: Add whatsnew entry about __static_attributes__ (GH-117909) Co-authored-by: Kirill Podoprigora Co-authored-by: Petr Viktorin Co-authored-by: Jelle Zijlstra --- Doc/library/stdtypes.rst | 7 +++++++ Doc/reference/datamodel.rst | 5 +++++ Doc/whatsnew/3.13.rst | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index fc613d4dbe1b5c..6c13bd015d5691 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5542,6 +5542,13 @@ types, where they are relevant. Some of these are not reported by the [, , , ] +.. attribute:: class.__static_attributes__ + + A tuple containing names of attributes of this class which are accessed + through ``self.X`` from any function in its body. + + .. versionadded:: 3.13 + .. _int_max_str_digits: Integer string conversion length limitation diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 6d6395a21f65d2..5e1558362ffaa0 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -970,6 +970,7 @@ A class object can be called (see above) to yield a class instance (see below). single: __doc__ (class attribute) single: __annotations__ (class attribute) single: __type_params__ (class attribute) + single: __static_attributes__ (class attribute) Special attributes: @@ -1000,6 +1001,10 @@ Special attributes: A tuple containing the :ref:`type parameters ` of a :ref:`generic class `. + :attr:`~class.__static_attributes__` + A tuple containing names of attributes of this class which are accessed + through ``self.X`` from any function in its body. + Class instances --------------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 0ea27a081b2d2d..cf9853f94872c3 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -115,6 +115,11 @@ Improved Error Messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? +* Classes have a new :attr:`~class.__static_attributes__` attribute, populated by the compiler, + with a tuple of names of attributes of this class which are accessed + through ``self.X`` from any function in its body. (Contributed by Irit Katriel + in :gh:`115775`.) + Incremental Garbage Collection ------------------------------ From 7c6cc00211772cc2afe0bc5e996b6d28f925d133 Mon Sep 17 00:00:00 2001 From: lit Date: Fri, 19 Apr 2024 19:28:46 +0800 Subject: [PATCH 12/37] gh-88035: update doc-string of `epoch` in timemodule.c (GH-118076) Follow #88035, update doc-string of epoch in timemodule.c The epoch is `January 1st, 1970 on all platforms`, according to current documentation. --- Modules/timemodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 2ec5aff235c293..3211c7530da0b2 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1896,8 +1896,8 @@ PyDoc_STRVAR(module_doc, There are two standard representations of time. One is the number\n\ of seconds since the Epoch, in UTC (a.k.a. GMT). It may be an integer\n\ or a floating point number (to represent fractions of seconds).\n\ -The Epoch is system-defined; on Unix, it is generally January 1st, 1970.\n\ -The actual value can be retrieved by calling gmtime(0).\n\ +The epoch is the point where the time starts, the return value of time.gmtime(0).\n\ +It is January 1, 1970, 00:00:00 (UTC) on all platforms.\n\ \n\ The other representation is a tuple of 9 integers giving local time.\n\ The tuple items are:\n\ From 3e7d990a09f0928050b2b0c85f724c2bce13fcbb Mon Sep 17 00:00:00 2001 From: Rostyslav Lobov <90602764+Motorenger@users.noreply.github.com> Date: Fri, 19 Apr 2024 07:33:13 -0400 Subject: [PATCH 13/37] setobject: remove out of date docstring info (GH-118048) --- Objects/setobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index d5030cec2d6206..0d88f4ff922d24 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2,7 +2,7 @@ /* set object implementation Written and maintained by Raymond D. Hettinger - Derived from Lib/sets.py and Objects/dictobject.c. + Derived from Objects/dictobject.c. The basic lookup function used by all operations. This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. From 15b3555e4a47ec925c965778a415dc11f0f981fd Mon Sep 17 00:00:00 2001 From: lyc8503 Date: Fri, 19 Apr 2024 19:41:51 +0800 Subject: [PATCH 14/37] gh-116931: Add fileobj parameter check for Tarfile.addfile (GH-117988) Tarfile.addfile now throws an ValueError when the user passes in a non-zero size tarinfo but does not provide a fileobj, instead of writing an incomplete entry. --- Doc/library/tarfile.rst | 10 +++++++--- Lib/tarfile.py | 11 +++++++---- Lib/test/test_tarfile.py | 12 ++++++++++-- .../2024-04-17-21-28-24.gh-issue-116931._AS09h.rst | 1 + 4 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-17-21-28-24.gh-issue-116931._AS09h.rst diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 2134293a0bb0de..afcad9bf722934 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -637,11 +637,15 @@ be finalized; only the internally used file object will be closed. See the .. method:: TarFile.addfile(tarinfo, fileobj=None) - Add the :class:`TarInfo` object *tarinfo* to the archive. If *fileobj* is given, - it should be a :term:`binary file`, and - ``tarinfo.size`` bytes are read from it and added to the archive. You can + Add the :class:`TarInfo` object *tarinfo* to the archive. If *tarinfo* represents + a non zero-size regular file, the *fileobj* argument should be a :term:`binary file`, + and ``tarinfo.size`` bytes are read from it and added to the archive. You can create :class:`TarInfo` objects directly, or by using :meth:`gettarinfo`. + .. versionchanged:: 3.13 + + *fileobj* must be given for non-zero-sized regular files. + .. method:: TarFile.gettarinfo(name=None, arcname=None, fileobj=None) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 149b1c3b1bda28..78bb10c80820d7 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2214,13 +2214,16 @@ def add(self, name, arcname=None, recursive=True, *, filter=None): self.addfile(tarinfo) def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is - given, it should be a binary file, and tarinfo.size bytes are read - from it and added to the archive. You can create TarInfo objects - directly, or by using gettarinfo(). + """Add the TarInfo object `tarinfo' to the archive. If `tarinfo' represents + a non zero-size regular file, the `fileobj' argument should be a binary file, + and tarinfo.size bytes are read from it and added to the archive. + You can create TarInfo objects directly, or by using gettarinfo(). """ self._check("awx") + if fileobj is None and tarinfo.isreg() and tarinfo.size != 0: + raise ValueError("fileobj not provided for non zero-size regular file") + tarinfo = copy.copy(tarinfo) buf = tarinfo.tobuf(self.format, self.encoding, self.errors) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index f18dcc02b23856..c2abc7ef9698fb 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1612,6 +1612,12 @@ def write(self, data): pax_headers={'non': 'empty'}) self.assertFalse(f.closed) + def test_missing_fileobj(self): + with tarfile.open(tmpname, self.mode) as tar: + tarinfo = tar.gettarinfo(tarname) + with self.assertRaises(ValueError): + tar.addfile(tarinfo) + class GzipWriteTest(GzipTest, WriteTest): pass @@ -3283,7 +3289,8 @@ def test_add(self): tar = tarfile.open(fileobj=bio, mode='w', format=tarformat) tarinfo = tar.gettarinfo(tarname) try: - tar.addfile(tarinfo) + with open(tarname, 'rb') as f: + tar.addfile(tarinfo, f) except Exception: if tarformat == tarfile.USTAR_FORMAT: # In the old, limited format, adding might fail for @@ -3298,7 +3305,8 @@ def test_add(self): replaced = tarinfo.replace(**{attr_name: None}) with self.assertRaisesRegex(ValueError, f"{attr_name}"): - tar.addfile(replaced) + with open(tarname, 'rb') as f: + tar.addfile(replaced, f) def test_list(self): # Change some metadata to None, then compare list() output diff --git a/Misc/NEWS.d/next/Library/2024-04-17-21-28-24.gh-issue-116931._AS09h.rst b/Misc/NEWS.d/next/Library/2024-04-17-21-28-24.gh-issue-116931._AS09h.rst new file mode 100644 index 00000000000000..98df8f59f71b76 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-17-21-28-24.gh-issue-116931._AS09h.rst @@ -0,0 +1 @@ +Add parameter *fileobj* check for :func:`tarfile.addfile()` From 1e3e7ce11e3b0fc76e981db85d27019d6d210bbc Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 19 Apr 2024 14:03:44 +0100 Subject: [PATCH 15/37] gh-114053: Fix bad interaction of PEP-695, PEP-563 and ``get_type_hints`` (#118009) Co-authored-by: Jelle Zijlstra --- Doc/library/typing.rst | 4 ++- Lib/test/test_typing.py | 26 +++++++++++++- Lib/test/typinganndata/ann_module695.py | 22 ++++++++++++ Lib/typing.py | 35 ++++++++++++++----- ...-04-17-22-00-15.gh-issue-114053._JBV4D.rst | 4 +++ 5 files changed, 81 insertions(+), 10 deletions(-) create mode 100644 Lib/test/typinganndata/ann_module695.py create mode 100644 Misc/NEWS.d/next/Library/2024-04-17-22-00-15.gh-issue-114053._JBV4D.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 31cf225ebf8fab..2e71be7eaf8d03 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -3024,7 +3024,9 @@ Introspection helpers This is often the same as ``obj.__annotations__``. In addition, forward references encoded as string literals are handled by evaluating - them in ``globals`` and ``locals`` namespaces. For a class ``C``, return + them in ``globals``, ``locals`` and (where applicable) + :ref:`type parameter ` namespaces. + For a class ``C``, return a dictionary constructed by merging all the ``__annotations__`` along ``C.__mro__`` in reverse order. diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index bae0a8480b994f..58781e52aca6d8 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -46,7 +46,7 @@ import types from test.support import captured_stderr, cpython_only, infinite_recursion -from test.typinganndata import mod_generics_cache, _typed_dict_helper +from test.typinganndata import ann_module695, mod_generics_cache, _typed_dict_helper CANNOT_SUBCLASS_TYPE = 'Cannot subclass special typing classes' @@ -4641,6 +4641,30 @@ def f(x: X): ... {'x': list[list[ForwardRef('X')]]} ) + def test_pep695_generic_with_future_annotations(self): + hints_for_A = get_type_hints(ann_module695.A) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(hints_for_A["x"], A_type_params[0]) + self.assertEqual(hints_for_A["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(hints_for_A["z"].__args__[0], A_type_params[2]) + + hints_for_B = get_type_hints(ann_module695.B) + self.assertEqual(hints_for_B.keys(), {"x", "y", "z"}) + self.assertEqual( + set(hints_for_B.values()) ^ set(ann_module695.B.__type_params__), + set() + ) + + hints_for_generic_function = get_type_hints(ann_module695.generic_function) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + hints_for_generic_function.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(hints_for_generic_function["x"], func_t_params[0]) + self.assertEqual(hints_for_generic_function["y"], Unpack[func_t_params[1]]) + self.assertIs(hints_for_generic_function["z"].__origin__, func_t_params[2]) + self.assertIs(hints_for_generic_function["zz"].__origin__, func_t_params[2]) + def test_extended_generic_rules_subclassing(self): class T1(Tuple[T, KT]): ... class T2(Tuple[T, ...]): ... diff --git a/Lib/test/typinganndata/ann_module695.py b/Lib/test/typinganndata/ann_module695.py new file mode 100644 index 00000000000000..2ede9fe382564f --- /dev/null +++ b/Lib/test/typinganndata/ann_module695.py @@ -0,0 +1,22 @@ +from __future__ import annotations +from typing import Callable + + +class A[T, *Ts, **P]: + x: T + y: tuple[*Ts] + z: Callable[P, str] + + +class B[T, *Ts, **P]: + T = int + Ts = str + P = bytes + x: T + y: Ts + z: P + + +def generic_function[T, *Ts, **P]( + x: T, *y: *Ts, z: P.args, zz: P.kwargs +) -> None: ... diff --git a/Lib/typing.py b/Lib/typing.py index 231492cdcc01cf..a0b68f593ca0d9 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -399,7 +399,8 @@ def inner(*args, **kwds): return decorator -def _eval_type(t, globalns, localns, recursive_guard=frozenset()): + +def _eval_type(t, globalns, localns, type_params, *, recursive_guard=frozenset()): """Evaluate all forward references in the given type t. For use of globalns and localns see the docstring for get_type_hints(). @@ -407,7 +408,7 @@ def _eval_type(t, globalns, localns, recursive_guard=frozenset()): ForwardRef. """ if isinstance(t, ForwardRef): - return t._evaluate(globalns, localns, recursive_guard) + return t._evaluate(globalns, localns, type_params, recursive_guard=recursive_guard) if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): if isinstance(t, GenericAlias): args = tuple( @@ -421,7 +422,13 @@ def _eval_type(t, globalns, localns, recursive_guard=frozenset()): t = t.__origin__[args] if is_unpacked: t = Unpack[t] - ev_args = tuple(_eval_type(a, globalns, localns, recursive_guard) for a in t.__args__) + + ev_args = tuple( + _eval_type( + a, globalns, localns, type_params, recursive_guard=recursive_guard + ) + for a in t.__args__ + ) if ev_args == t.__args__: return t if isinstance(t, GenericAlias): @@ -974,7 +981,7 @@ def __init__(self, arg, is_argument=True, module=None, *, is_class=False): self.__forward_is_class__ = is_class self.__forward_module__ = module - def _evaluate(self, globalns, localns, recursive_guard): + def _evaluate(self, globalns, localns, type_params, *, recursive_guard): if self.__forward_arg__ in recursive_guard: return self if not self.__forward_evaluated__ or localns is not globalns: @@ -988,14 +995,25 @@ def _evaluate(self, globalns, localns, recursive_guard): globalns = getattr( sys.modules.get(self.__forward_module__, None), '__dict__', globalns ) + if type_params: + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + locals_to_pass = {param.__name__: param for param in type_params} | localns + else: + locals_to_pass = localns type_ = _type_check( - eval(self.__forward_code__, globalns, localns), + eval(self.__forward_code__, globalns, locals_to_pass), "Forward references must evaluate to types.", is_argument=self.__forward_is_argument__, allow_special_forms=self.__forward_is_class__, ) self.__forward_value__ = _eval_type( - type_, globalns, localns, recursive_guard | {self.__forward_arg__} + type_, + globalns, + localns, + type_params, + recursive_guard=(recursive_guard | {self.__forward_arg__}), ) self.__forward_evaluated__ = True return self.__forward_value__ @@ -2334,7 +2352,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): value = type(None) if isinstance(value, str): value = ForwardRef(value, is_argument=False, is_class=True) - value = _eval_type(value, base_globals, base_locals) + value = _eval_type(value, base_globals, base_locals, base.__type_params__) hints[name] = value return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} @@ -2360,6 +2378,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): raise TypeError('{!r} is not a module, class, method, ' 'or function.'.format(obj)) hints = dict(hints) + type_params = getattr(obj, "__type_params__", ()) for name, value in hints.items(): if value is None: value = type(None) @@ -2371,7 +2390,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - hints[name] = _eval_type(value, globalns, localns) + hints[name] = _eval_type(value, globalns, localns, type_params) return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} diff --git a/Misc/NEWS.d/next/Library/2024-04-17-22-00-15.gh-issue-114053._JBV4D.rst b/Misc/NEWS.d/next/Library/2024-04-17-22-00-15.gh-issue-114053._JBV4D.rst new file mode 100644 index 00000000000000..827b2d656b4d38 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-17-22-00-15.gh-issue-114053._JBV4D.rst @@ -0,0 +1,4 @@ +Fix erroneous :exc:`NameError` when calling :func:`typing.get_type_hints` on +a class that made use of :pep:`695` type parameters in a module that had +``from __future__ import annotations`` at the top of the file. Patch by Alex +Waygood. From 8d4a244f1516dcde23becc2a273d30c202237598 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Fri, 19 Apr 2024 18:38:13 +0300 Subject: [PATCH 16/37] gh-118079: Fix ``requires_singlephase_init`` helper (#118081) Before this PR tests decorated with a `requires_singlephase_init` helper did not run because of an incorrect call to the `requires_gil_enabled` helper. --- Lib/test/test_import/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 4726619b08edc4..469d1fbe59aaa2 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -159,9 +159,8 @@ def meth(self, _meth=meth): finally: restore__testsinglephase() meth = cpython_only(meth) - # gh-117649: free-threaded build does not currently support single-phase - # init modules in subinterpreters. - meth = requires_gil_enabled(meth) + msg = "gh-117694: free-threaded build does not currently support single-phase init modules in sub-interpreters" + meth = requires_gil_enabled(msg)(meth) return unittest.skipIf(_testsinglephase is None, 'test requires _testsinglephase module')(meth) From 1e4a4c4897d0f45b1f594bc429284c82efe49188 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Fri, 19 Apr 2024 09:25:08 -0700 Subject: [PATCH 17/37] gh-117657: use relaxed loads for checking dict keys immortality (#118067) Use relaxed load to check if dictkeys are immortal --- Objects/dictobject.c | 4 ++-- Tools/tsan/suppressions_free_threading.txt | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 58f34c32a87ea9..c3516dff973745 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -441,7 +441,7 @@ static void free_keys_object(PyDictKeysObject *keys, bool use_qsbr); static inline void dictkeys_incref(PyDictKeysObject *dk) { - if (dk->dk_refcnt == _Py_IMMORTAL_REFCNT) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_IMMORTAL_REFCNT) { return; } #ifdef Py_REF_DEBUG @@ -453,7 +453,7 @@ dictkeys_incref(PyDictKeysObject *dk) static inline void dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) { - if (dk->dk_refcnt == _Py_IMMORTAL_REFCNT) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_IMMORTAL_REFCNT) { return; } assert(dk->dk_refcnt > 0); diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 80191d6c2484e6..1408103ba80f96 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -29,8 +29,6 @@ race:_PyType_HasFeature race:assign_version_tag race:compare_unicode_unicode race:delitem_common -race:dictkeys_decref -race:dictkeys_incref race:dictresize race:gc_collect_main race:gc_restore_tid From 2aa11cca115add03f39cb6cd7299135ecf4d4d82 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 19 Apr 2024 21:25:54 +0300 Subject: [PATCH 18/37] gh-118100: Improve links in `ast.rst` (#118101) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/ast.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index d10f3f275c0eaf..8d407321092eef 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2536,7 +2536,8 @@ to stdout. Otherwise, the content is read from stdin. code that generated them. This is helpful for tools that make source code transformations. - `leoAst.py `_ unifies the + `leoAst.py `_ + unifies the token-based and parse-tree-based views of python programs by inserting two-way links between tokens and ast nodes. @@ -2548,4 +2549,4 @@ to stdout. Otherwise, the content is read from stdin. `Parso `_ is a Python parser that supports error recovery and round-trip parsing for different Python versions (in multiple Python versions). Parso is also able to list multiple syntax errors - in your python file. + in your Python file. From ab99438900c26bdbbdfb411be43d1493f3b00ab7 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Sat, 20 Apr 2024 02:56:33 +0800 Subject: [PATCH 19/37] gh-114099: Modify preprocessor symbol usage to support older macOS SDKs (GH-118073) Co-authored-by: Joshua Root jmr@macports.org --- .../macOS/2024-04-19-08-40-00.gh-issue-114099._iDfrQ.rst | 1 + Misc/platform_triplet.c | 8 +++++--- Modules/_testexternalinspection.c | 4 ++++ Python/marshal.c | 3 ++- 4 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2024-04-19-08-40-00.gh-issue-114099._iDfrQ.rst diff --git a/Misc/NEWS.d/next/macOS/2024-04-19-08-40-00.gh-issue-114099._iDfrQ.rst b/Misc/NEWS.d/next/macOS/2024-04-19-08-40-00.gh-issue-114099._iDfrQ.rst new file mode 100644 index 00000000000000..f9af0629ed9434 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-04-19-08-40-00.gh-issue-114099._iDfrQ.rst @@ -0,0 +1 @@ +iOS preprocessor symbol usage was made compatible with older macOS SDKs. diff --git a/Misc/platform_triplet.c b/Misc/platform_triplet.c index 06b03bfa9a266a..ec0857a4a998c0 100644 --- a/Misc/platform_triplet.c +++ b/Misc/platform_triplet.c @@ -246,8 +246,9 @@ PLATFORM_TRIPLET=i386-gnu # endif #elif defined(__APPLE__) # include "TargetConditionals.h" -# if TARGET_OS_IOS -# if TARGET_OS_SIMULATOR +// Older macOS SDKs do not define TARGET_OS_* +# if defined(TARGET_OS_IOS) && TARGET_OS_IOS +# if defined(TARGET_OS_SIMULATOR) && TARGET_OS_SIMULATOR # if __x86_64__ PLATFORM_TRIPLET=x86_64-iphonesimulator # else @@ -256,7 +257,8 @@ PLATFORM_TRIPLET=arm64-iphonesimulator # else PLATFORM_TRIPLET=arm64-iphoneos # endif -# elif TARGET_OS_OSX +// Older macOS SDKs do not define TARGET_OS_OSX +# elif !defined(TARGET_OS_OSX) || TARGET_OS_OSX PLATFORM_TRIPLET=darwin # else # error unknown Apple platform diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index bd77f0cd0f1fc7..e2f96cdad5c58e 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -17,6 +17,10 @@ #if defined(__APPLE__) # include +// Older macOS SDKs do not define TARGET_OS_OSX +# if !defined(TARGET_OS_OSX) +# define TARGET_OS_OSX 1 +# endif # if TARGET_OS_OSX # include # include diff --git a/Python/marshal.c b/Python/marshal.c index 4274f90206b240..4bd8bb1d3a9308 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -42,7 +42,8 @@ module marshal #elif defined(__wasi__) # define MAX_MARSHAL_STACK_DEPTH 1500 // TARGET_OS_IPHONE covers any non-macOS Apple platform. -#elif defined(__APPLE__) && TARGET_OS_IPHONE +// It won't be defined on older macOS SDKs +#elif defined(__APPLE__) && defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE # define MAX_MARSHAL_STACK_DEPTH 1500 #else # define MAX_MARSHAL_STACK_DEPTH 2000 From b624490deedc37d02e5f29d19ff5e222df3a64dd Mon Sep 17 00:00:00 2001 From: Noah Kim Date: Fri, 19 Apr 2024 15:19:12 -0400 Subject: [PATCH 20/37] Fix a typo in dictobject.c documentation (#117515) --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index c3516dff973745..4e696419eb5eb0 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1876,7 +1876,7 @@ actually be smaller than the old one. If a table is split (its keys and hashes are shared, its values are not), then the values are temporarily copied into the table, it is resized as a combined table, then the me_value slots in the old table are NULLed out. -After resizing a table is always combined. +After resizing, a table is always combined. This function supports: - Unicode split -> Unicode combined or Generic From b45af00bad3449b85c8f54ba7a9474ca1f52de69 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Fri, 19 Apr 2024 14:21:01 -0700 Subject: [PATCH 21/37] [gh-117657] _Py_MergeZeroLocalRefcount isn't loading ob_ref_shared with strong enough semantics (#118111) Use acquire for load of ob_ref_shared --- Objects/object.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/object.c b/Objects/object.c index 214e7c5b567928..73a1927263cdcb 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -374,7 +374,7 @@ _Py_MergeZeroLocalRefcount(PyObject *op) assert(op->ob_ref_local == 0); _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); - Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + Py_ssize_t shared = _Py_atomic_load_ssize_acquire(&op->ob_ref_shared); if (shared == 0) { // Fast-path: shared refcount is zero (including flags) _Py_Dealloc(op); From 07525c9a85e7fa04db565c6e6e6605ff6099dcb7 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Fri, 19 Apr 2024 14:47:42 -0700 Subject: [PATCH 22/37] gh-116818: Make `sys.settrace`, `sys.setprofile`, and monitoring thread-safe (#116775) Makes sys.settrace, sys.setprofile, and monitoring generally thread-safe. Mostly uses a stop-the-world approach and synchronization around the code object's _co_instrumentation_version. There may be a little bit of extra synchronization around the monitoring data that's required to be TSAN clean. --- Include/cpython/pyatomic.h | 8 + Include/cpython/pyatomic_gcc.h | 8 + Include/cpython/pyatomic_msc.h | 25 ++ Include/cpython/pyatomic_std.h | 16 ++ Include/internal/pycore_ceval_state.h | 1 + Include/internal/pycore_gc.h | 2 +- .../internal/pycore_pyatomic_ft_wrappers.h | 14 ++ Lib/test/test_free_threading/__init__.py | 7 + .../test_free_threading/test_monitoring.py | 232 ++++++++++++++++++ Makefile.pre.in | 1 + Python/bytecodes.c | 17 +- Python/ceval.c | 1 + Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 18 +- Python/instrumentation.c | 141 +++++++++-- Python/legacy_tracing.c | 94 ++++--- Python/pystate.c | 1 + Tools/jit/template.c | 1 + 18 files changed, 528 insertions(+), 61 deletions(-) create mode 100644 Lib/test/test_free_threading/__init__.py create mode 100644 Lib/test/test_free_threading/test_monitoring.py diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index c3e132d3877ca5..69083f1d9dd0c2 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -465,10 +465,16 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, static inline void * _Py_atomic_load_ptr_acquire(const void *obj); +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj); + // Stores `*obj = value` (release operation) static inline void _Py_atomic_store_ptr_release(void *obj, void *value); +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); @@ -491,6 +497,8 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); + + // --- _Py_atomic_fence ------------------------------------------------------ // Sequential consistency fence. C11 fences have complex semantics. When diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 0b40f81bd8736d..af78a94c736545 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -492,10 +492,18 @@ static inline void * _Py_atomic_load_ptr_acquire(const void *obj) { return (void *)__atomic_load_n((void **)obj, __ATOMIC_ACQUIRE); } +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ return (uintptr_t)__atomic_load_n((uintptr_t *)obj, __ATOMIC_ACQUIRE); } + static inline void _Py_atomic_store_ptr_release(void *obj, void *value) { __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 3205e253b28546..212cd7817d01c5 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -914,6 +914,18 @@ _Py_atomic_load_ptr_acquire(const void *obj) #endif } +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uintptr_t volatile *)obj; +#elif defined(_M_ARM64) + return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uintptr_acquire" +#endif +} + static inline void _Py_atomic_store_ptr_release(void *obj, void *value) { @@ -926,6 +938,19 @@ _Py_atomic_store_ptr_release(void *obj, void *value) #endif } +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uintptr_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_uintptr_release" +#endif +} + static inline void _Py_atomic_store_int_release(int *obj, int value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index ef34bb0b77dfe5..6a77eae536d8dd 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -863,6 +863,14 @@ _Py_atomic_load_ptr_acquire(const void *obj) memory_order_acquire); } +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uintptr_t)*)obj, + memory_order_acquire); +} + static inline void _Py_atomic_store_ptr_release(void *obj, void *value) { @@ -871,6 +879,14 @@ _Py_atomic_store_ptr_release(void *obj, void *value) memory_order_release); } +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uintptr_t)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_int_release(int *obj, int value) { diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index b453328f15649e..168295534e036c 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -63,6 +63,7 @@ struct _ceval_runtime_state { } perf; /* Pending calls to be made only on the main thread. */ struct _pending_calls pending_mainthread; + PyMutex sys_trace_profile_mutex; }; #ifdef PY_HAVE_PERF_TRAMPOLINE diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 60020b5c01f8a6..9e465fdd86279f 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -93,7 +93,7 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) { * threads and needs special purpose when freeing due to * the possibility of in-flight lock-free reads occurring. * Objects with this bit that are GC objects will automatically - * delay-freed by PyObject_GC_Del. */ + * delay-freed by PyObject_GC_Del. */ static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) { return (op->ob_gc_bits & _PyGC_BITS_SHARED_INLINE) != 0; } diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 2514f51f1b0086..fed5d6e0ec2c54 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -26,20 +26,34 @@ extern "C" { _Py_atomic_load_ssize_relaxed(&value) #define FT_ATOMIC_STORE_PTR(value, new_value) \ _Py_atomic_store_ptr(&value, new_value) +#define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) \ + _Py_atomic_load_ptr_acquire(&value) +#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) \ + _Py_atomic_load_uintptr_acquire(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ _Py_atomic_store_ptr_relaxed(&value, new_value) #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ _Py_atomic_store_ptr_release(&value, new_value) +#define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) \ + _Py_atomic_store_uintptr_release(&value, new_value) #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) \ + _Py_atomic_store_uint8_relaxed(&value, new_value) + #else #define FT_ATOMIC_LOAD_PTR(value) value #define FT_ATOMIC_LOAD_SSIZE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value #define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value + #endif #ifdef __cplusplus diff --git a/Lib/test/test_free_threading/__init__.py b/Lib/test/test_free_threading/__init__.py new file mode 100644 index 00000000000000..9a89d27ba9f979 --- /dev/null +++ b/Lib/test/test_free_threading/__init__.py @@ -0,0 +1,7 @@ +import os + +from test import support + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py new file mode 100644 index 00000000000000..e170840ca3cb27 --- /dev/null +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -0,0 +1,232 @@ +"""Tests monitoring, sys.settrace, and sys.setprofile in a multi-threaded +environmenet to verify things are thread-safe in a free-threaded build""" + +import sys +import time +import unittest +import weakref + +from sys import monitoring +from test.support import is_wasi +from threading import Thread +from unittest import TestCase + + +class InstrumentationMultiThreadedMixin: + if not hasattr(sys, "gettotalrefcount"): + thread_count = 50 + func_count = 1000 + fib = 15 + else: + # Run a little faster in debug builds... + thread_count = 25 + func_count = 500 + fib = 15 + + def after_threads(self): + """Runs once after all the threads have started""" + pass + + def during_threads(self): + """Runs repeatedly while the threads are still running""" + pass + + def work(self, n, funcs): + """Fibonacci function which also calls a bunch of random functions""" + for func in funcs: + func() + if n < 2: + return n + return self.work(n - 1, funcs) + self.work(n - 2, funcs) + + def start_work(self, n, funcs): + # With the GIL builds we need to make sure that the hooks have + # a chance to run as it's possible to run w/o releasing the GIL. + time.sleep(1) + self.work(n, funcs) + + def after_test(self): + """Runs once after the test is done""" + pass + + def test_instrumentation(self): + # Setup a bunch of functions which will need instrumentation... + funcs = [] + for i in range(self.func_count): + x = {} + exec("def f(): pass", x) + funcs.append(x["f"]) + + threads = [] + for i in range(self.thread_count): + # Each thread gets a copy of the func list to avoid contention + t = Thread(target=self.start_work, args=(self.fib, list(funcs))) + t.start() + threads.append(t) + + self.after_threads() + + while True: + any_alive = False + for t in threads: + if t.is_alive(): + any_alive = True + break + + if not any_alive: + break + + self.during_threads() + + self.after_test() + + +class MonitoringTestMixin: + def setUp(self): + for i in range(6): + if monitoring.get_tool(i) is None: + self.tool_id = i + monitoring.use_tool_id(i, self.__class__.__name__) + break + + def tearDown(self): + monitoring.free_tool_id(self.tool_id) + + +@unittest.skipIf(is_wasi, "WASI has no threads.") +class SetPreTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Sets tracing one time after the threads have started""" + + def setUp(self): + super().setUp() + self.called = False + + def after_test(self): + self.assertTrue(self.called) + + def trace_func(self, frame, event, arg): + self.called = True + return self.trace_func + + def after_threads(self): + sys.settrace(self.trace_func) + + +@unittest.skipIf(is_wasi, "WASI has no threads.") +class MonitoringMultiThreaded( + MonitoringTestMixin, InstrumentationMultiThreadedMixin, TestCase +): + """Uses sys.monitoring and repeatedly toggles instrumentation on and off""" + + def setUp(self): + super().setUp() + self.set = False + self.called = False + monitoring.register_callback( + self.tool_id, monitoring.events.LINE, self.callback + ) + + def tearDown(self): + monitoring.set_events(self.tool_id, 0) + super().tearDown() + + def callback(self, *args): + self.called = True + + def after_test(self): + self.assertTrue(self.called) + + def during_threads(self): + if self.set: + monitoring.set_events( + self.tool_id, monitoring.events.CALL | monitoring.events.LINE + ) + else: + monitoring.set_events(self.tool_id, 0) + self.set = not self.set + + +@unittest.skipIf(is_wasi, "WASI has no threads.") +class SetTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Uses sys.settrace and repeatedly toggles instrumentation on and off""" + + def setUp(self): + self.set = False + self.called = False + + def after_test(self): + self.assertTrue(self.called) + + def tearDown(self): + sys.settrace(None) + + def trace_func(self, frame, event, arg): + self.called = True + return self.trace_func + + def during_threads(self): + if self.set: + sys.settrace(self.trace_func) + else: + sys.settrace(None) + self.set = not self.set + + +@unittest.skipIf(is_wasi, "WASI has no threads.") +class SetProfileMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Uses sys.setprofile and repeatedly toggles instrumentation on and off""" + thread_count = 25 + func_count = 200 + fib = 15 + + def setUp(self): + self.set = False + self.called = False + + def after_test(self): + self.assertTrue(self.called) + + def tearDown(self): + sys.setprofile(None) + + def trace_func(self, frame, event, arg): + self.called = True + return self.trace_func + + def during_threads(self): + if self.set: + sys.setprofile(self.trace_func) + else: + sys.setprofile(None) + self.set = not self.set + + +@unittest.skipIf(is_wasi, "WASI has no threads.") +class MonitoringMisc(MonitoringTestMixin, TestCase): + def register_callback(self): + def callback(*args): + pass + + for i in range(200): + monitoring.register_callback(self.tool_id, monitoring.events.LINE, callback) + + self.refs.append(weakref.ref(callback)) + + def test_register_callback(self): + self.refs = [] + threads = [] + for i in range(50): + t = Thread(target=self.register_callback) + t.start() + threads.append(t) + + for thread in threads: + thread.join() + + monitoring.register_callback(self.tool_id, monitoring.events.LINE, None) + for ref in self.refs: + self.assertEqual(ref(), None) + + +if __name__ == "__main__": + unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index fd8678cdaf8207..f7c21a380caa99 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2366,6 +2366,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_doctest \ test/test_email \ test/test_email/data \ + test/test_free_threading \ test/test_future_stmt \ test/test_gdb \ test/test_import \ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c34d702f06418e..c1fbd3c7d26e01 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -20,6 +20,7 @@ #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_opcode_metadata.h" // uop names #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject @@ -150,10 +151,11 @@ dummy_func( uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + PyCodeObject *code = _PyFrame_GetCode(frame); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); assert((code_version & 255) == 0); if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + int err = _Py_Instrument(code, tstate->interp); ERROR_IF(err, error); next_instr = this_instr; } @@ -171,14 +173,14 @@ dummy_func( _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); DEOPT_IF(eval_breaker != version); } inst(INSTRUMENTED_RESUME, (--)) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { ERROR_NO_POP(); @@ -2377,7 +2379,14 @@ dummy_func( }; tier1 inst(ENTER_EXECUTOR, (--)) { + int prevoparg = oparg; CHECK_EVAL_BREAKER(); + if (this_instr->op.code != ENTER_EXECUTOR || + this_instr->op.arg != prevoparg) { + next_instr = this_instr; + DISPATCH(); + } + PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); diff --git a/Python/ceval.c b/Python/ceval.c index b88e555ded5c2e..2f217c5f33c6ce 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -20,6 +20,7 @@ #include "pycore_opcode_metadata.h" // EXTRA_CASES #include "pycore_optimizer.h" // _PyUOpExecutor_Type #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index fccff24a418586..df87f9178f17cf 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -21,7 +21,7 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UOP_STAT_INC(uopcode, miss); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a7764b0ec12e10..a426d9e208492e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2498,10 +2498,17 @@ } TARGET(ENTER_EXECUTOR) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); + int prevoparg = oparg; CHECK_EVAL_BREAKER(); + if (this_instr->op.code != ENTER_EXECUTOR || + this_instr->op.arg != prevoparg) { + next_instr = this_instr; + DISPATCH(); + } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); @@ -3267,7 +3274,7 @@ next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { goto error; @@ -4927,10 +4934,11 @@ uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + PyCodeObject *code = _PyFrame_GetCode(frame); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); assert((code_version & 255) == 0); if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + int err = _Py_Instrument(code, tstate->interp); if (err) goto error; next_instr = this_instr; } @@ -4953,7 +4961,7 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); DEOPT_IF(eval_breaker != version, RESUME); DISPATCH(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 3866144a19bf74..71efeff077633d 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -6,6 +6,7 @@ #include "pycore_call.h" #include "pycore_ceval.h" // _PY_EVAL_EVENTS_BITS #include "pycore_code.h" // _PyCode_Clear_Executors() +#include "pycore_critical_section.h" #include "pycore_frame.h" #include "pycore_interp.h" #include "pycore_long.h" @@ -13,12 +14,43 @@ #include "pycore_namespace.h" #include "pycore_object.h" #include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, _PyOpcode_Caches +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_UINTPTR_RELEASE #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() /* Uncomment this to dump debugging output when assertions fail */ // #define INSTRUMENT_DEBUG 1 +#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) + +#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj) \ + if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); \ + } +#define ASSERT_WORLD_STOPPED() assert(_PyInterpreterState_GET()->stoptheworld.world_stopped); + +#else + +#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj) +#define ASSERT_WORLD_STOPPED() + +#endif + +#ifdef Py_GIL_DISABLED + +#define LOCK_CODE(code) \ + assert(!_PyInterpreterState_GET()->stoptheworld.world_stopped); \ + Py_BEGIN_CRITICAL_SECTION(code) + +#define UNLOCK_CODE() Py_END_CRITICAL_SECTION() + +#else + +#define LOCK_CODE(code) +#define UNLOCK_CODE() + +#endif + PyObject _PyInstrumentation_DISABLE = _PyObject_HEAD_INIT(&PyBaseObject_Type); PyObject _PyInstrumentation_MISSING = _PyObject_HEAD_INIT(&PyBaseObject_Type); @@ -278,6 +310,8 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) int _PyInstruction_GetLength(PyCodeObject *code, int offset) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + int opcode = _PyCode_CODE(code)[offset].op.code; assert(opcode != 0); assert(opcode != RESERVED); @@ -424,6 +458,7 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) } #define CHECK(test) do { \ + ASSERT_WORLD_STOPPED_OR_LOCKED(code); \ if (!(test)) { \ dump_instrumentation_data(code, i, stderr); \ } \ @@ -449,6 +484,8 @@ valid_opcode(int opcode) static void sanity_check_instrumentation(PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + _PyCoMonitoringData *data = code->_co_monitoring; if (data == NULL) { return; @@ -718,6 +755,7 @@ instrument_per_instruction(PyCodeObject *code, int i) static void remove_tools(PyCodeObject * code, int offset, int event, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event)); @@ -752,6 +790,8 @@ tools_is_subset_for_event(PyCodeObject * code, int event, int tools) static void remove_line_tools(PyCodeObject * code, int offset, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + assert(code->_co_monitoring); if (code->_co_monitoring->line_tools) { @@ -774,6 +814,7 @@ remove_line_tools(PyCodeObject * code, int offset, int tools) static void add_tools(PyCodeObject * code, int offset, int event, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event)); @@ -794,6 +835,8 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) static void add_line_tools(PyCodeObject * code, int offset, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools)); assert(code->_co_monitoring); if (code->_co_monitoring->line_tools) { @@ -810,6 +853,8 @@ add_line_tools(PyCodeObject * code, int offset, int tools) static void add_per_instruction_tools(PyCodeObject * code, int offset, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools)); assert(code->_co_monitoring); if (code->_co_monitoring->per_instruction_tools) { @@ -826,6 +871,8 @@ add_per_instruction_tools(PyCodeObject * code, int offset, int tools) static void remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + assert(code->_co_monitoring); if (code->_co_monitoring->per_instruction_tools) { uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset]; @@ -1056,7 +1103,9 @@ call_instrumentation_vector( break; } else { + LOCK_CODE(code); remove_tools(code, offset, event, 1 << tool); + UNLOCK_CODE(); } } } @@ -1189,6 +1238,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, goto done; } } + uint8_t tools = code->_co_monitoring->line_tools != NULL ? code->_co_monitoring->line_tools[i] : (interp->monitors.tools[PY_MONITORING_EVENT_LINE] | @@ -1249,7 +1299,9 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, } else { /* DISABLE */ + LOCK_CODE(code); remove_line_tools(code, i, 1 << tool); + UNLOCK_CODE(); } } while (tools); Py_DECREF(line_obj); @@ -1305,7 +1357,9 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* } else { /* DISABLE */ + LOCK_CODE(code); remove_per_instruction_tools(code, offset, 1 << tool); + UNLOCK_CODE(); } } Py_DECREF(offset_obj); @@ -1320,15 +1374,18 @@ _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) PyInterpreterState *is = _PyInterpreterState_GET(); assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); assert(0 <= event_id && event_id < _PY_MONITORING_EVENTS); - PyObject *callback = is->monitoring_callables[tool_id][event_id]; - is->monitoring_callables[tool_id][event_id] = Py_XNewRef(obj); + PyObject *callback = _Py_atomic_exchange_ptr(&is->monitoring_callables[tool_id][event_id], + Py_XNewRef(obj)); + return callback; } static void initialize_tools(PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); uint8_t* tools = code->_co_monitoring->tools; + assert(tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { @@ -1384,7 +1441,9 @@ initialize_tools(PyCodeObject *code) static void initialize_lines(PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; + assert(line_data != NULL); int code_len = (int)Py_SIZE(code); PyCodeAddressRange range; @@ -1501,7 +1560,9 @@ initialize_lines(PyCodeObject *code) static void initialize_line_tools(PyCodeObject *code, _Py_LocalMonitors *all_events) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); uint8_t *line_tools = code->_co_monitoring->line_tools; + assert(line_tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { @@ -1512,6 +1573,7 @@ initialize_line_tools(PyCodeObject *code, _Py_LocalMonitors *all_events) static int allocate_instrumentation_data(PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); if (code->_co_monitoring == NULL) { code->_co_monitoring = PyMem_Malloc(sizeof(_PyCoMonitoringData)); @@ -1533,6 +1595,8 @@ allocate_instrumentation_data(PyCodeObject *code) static int update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + int code_len = (int)Py_SIZE(code); if (allocate_instrumentation_data(code)) { return -1; @@ -1594,9 +1658,11 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) return 0; } -int -_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) +static int +instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + if (is_version_up_to_date(code, interp)) { assert( interp->ceval.instrumentation_version == 0 || @@ -1636,12 +1702,8 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) assert(monitors_are_empty(monitors_and(new_events, removed_events))); } code->_co_monitoring->active_monitors = active_events; - code->_co_instrumentation_version = global_version(interp); if (monitors_are_empty(new_events) && monitors_are_empty(removed_events)) { -#ifdef INSTRUMENT_DEBUG - sanity_check_instrumentation(code); -#endif - return 0; + goto done; } /* Insert instrumentation */ for (int i = code->_co_firsttraceable; i < code_len; i+= _PyInstruction_GetLength(code, i)) { @@ -1730,12 +1792,26 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) i += _PyInstruction_GetLength(code, i); } } +done: + FT_ATOMIC_STORE_UINTPTR_RELEASE(code->_co_instrumentation_version, + global_version(interp)); + #ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); #endif return 0; } +int +_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) +{ + int res; + LOCK_CODE(code); + res = instrument_lock_held(code, interp); + UNLOCK_CODE(); + return res; +} + #define C_RETURN_EVENTS \ ((1 << PY_MONITORING_EVENT_C_RETURN) | \ (1 << PY_MONITORING_EVENT_C_RAISE)) @@ -1746,6 +1822,8 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) static int instrument_all_executing_code_objects(PyInterpreterState *interp) { + ASSERT_WORLD_STOPPED(); + _PyRuntimeState *runtime = &_PyRuntime; HEAD_LOCK(runtime); PyThreadState* ts = PyInterpreterState_ThreadHead(interp); @@ -1754,7 +1832,7 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { _PyInterpreterFrame *frame = ts->current_frame; while (frame) { if (frame->owner != FRAME_OWNED_BY_CSTACK) { - if (_Py_Instrument(_PyFrame_GetCode(frame), interp)) { + if (instrument_lock_held(_PyFrame_GetCode(frame), interp)) { return -1; } } @@ -1817,19 +1895,27 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) if (check_tool(interp, tool_id)) { return -1; } + + int res; + _PyEval_StopTheWorld(interp); uint32_t existing_events = get_events(&interp->monitors, tool_id); if (existing_events == events) { - return 0; + res = 0; + goto done; } set_events(&interp->monitors, tool_id, events); uint32_t new_version = global_version(interp) + MONITORING_VERSION_INCREMENT; if (new_version == 0) { PyErr_Format(PyExc_OverflowError, "events set too many times"); - return -1; + res = -1; + goto done; } set_global_version(tstate, new_version); _Py_Executors_InvalidateAll(interp, 1); - return instrument_all_executing_code_objects(interp); + res = instrument_all_executing_code_objects(interp); +done: + _PyEval_StartTheWorld(interp); + return res; } int @@ -1845,24 +1931,33 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent if (check_tool(interp, tool_id)) { return -1; } + + int res; + LOCK_CODE(code); if (allocate_instrumentation_data(code)) { - return -1; + res = -1; + goto done; } + _Py_LocalMonitors *local = &code->_co_monitoring->local_monitors; uint32_t existing_events = get_local_events(local, tool_id); if (existing_events == events) { - return 0; + res = 0; + goto done; } set_local_events(local, tool_id, events); if (is_version_up_to_date(code, interp)) { /* Force instrumentation update */ code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT; } + _Py_Executors_InvalidateDependency(interp, code, 1); - if (_Py_Instrument(code, interp)) { - return -1; - } - return 0; + + res = instrument_lock_held(code, interp); + +done: + UNLOCK_CODE(); + return res; } int @@ -2158,15 +2253,21 @@ monitoring_restart_events_impl(PyObject *module) */ PyThreadState *tstate = _PyThreadState_GET(); PyInterpreterState *interp = tstate->interp; + + _PyEval_StopTheWorld(interp); uint32_t restart_version = global_version(interp) + MONITORING_VERSION_INCREMENT; uint32_t new_version = restart_version + MONITORING_VERSION_INCREMENT; if (new_version <= MONITORING_VERSION_INCREMENT) { + _PyEval_StartTheWorld(interp); PyErr_Format(PyExc_OverflowError, "events set too many times"); return NULL; } interp->last_restart_version = restart_version; set_global_version(tstate, new_version); - if (instrument_all_executing_code_objects(interp)) { + int res = instrument_all_executing_code_objects(interp); + _PyEval_StartTheWorld(interp); + + if (res) { return NULL; } Py_RETURN_NONE; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index ccbb3eb3f7c82a..d7aae7d2343ac2 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -16,6 +16,13 @@ typedef struct _PyLegacyEventHandler { int event; } _PyLegacyEventHandler; +#ifdef Py_GIL_DISABLED +#define LOCK_SETUP() PyMutex_Lock(&_PyRuntime.ceval.sys_trace_profile_mutex); +#define UNLOCK_SETUP() PyMutex_Unlock(&_PyRuntime.ceval.sys_trace_profile_mutex); +#else +#define LOCK_SETUP() +#define UNLOCK_SETUP() +#endif /* The Py_tracefunc function expects the following arguments: * obj: the trace object (PyObject *) * frame: the current frame (PyFrameObject *) @@ -414,19 +421,10 @@ is_tstate_valid(PyThreadState *tstate) } #endif -int -_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +static Py_ssize_t +setup_profile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg, PyObject **old_profileobj) { - assert(is_tstate_valid(tstate)); - /* The caller must hold the GIL */ - assert(PyGILState_Check()); - - /* Call _PySys_Audit() in the context of the current thread state, - even if tstate is not the current thread state. */ - PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { - return -1; - } + *old_profileobj = NULL; /* Setup PEP 669 monitoring callbacks and events. */ if (!tstate->interp->sys_profile_initialized) { tstate->interp->sys_profile_initialized = true; @@ -469,25 +467,15 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) int delta = (func != NULL) - (tstate->c_profilefunc != NULL); tstate->c_profilefunc = func; - PyObject *old_profileobj = tstate->c_profileobj; + *old_profileobj = tstate->c_profileobj; tstate->c_profileobj = Py_XNewRef(arg); - Py_XDECREF(old_profileobj); tstate->interp->sys_profiling_threads += delta; assert(tstate->interp->sys_profiling_threads >= 0); - - uint32_t events = 0; - if (tstate->interp->sys_profiling_threads) { - events = - (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | - (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | - (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | - (1 << PY_MONITORING_EVENT_PY_THROW); - } - return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); + return tstate->interp->sys_profiling_threads; } int -_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) { assert(is_tstate_valid(tstate)); /* The caller must hold the GIL */ @@ -496,11 +484,32 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) /* Call _PySys_Audit() in the context of the current thread state, even if tstate is not the current thread state. */ PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { + if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { return -1; } - assert(tstate->interp->sys_tracing_threads >= 0); + // needs to be decref'd outside of the lock + PyObject *old_profileobj; + LOCK_SETUP(); + Py_ssize_t profiling_threads = setup_profile(tstate, func, arg, &old_profileobj); + UNLOCK_SETUP(); + Py_XDECREF(old_profileobj); + + uint32_t events = 0; + if (profiling_threads) { + events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | + (1 << PY_MONITORING_EVENT_PY_THROW); + } + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); +} + +static Py_ssize_t +setup_tracing(PyThreadState *tstate, Py_tracefunc func, PyObject *arg, PyObject **old_traceobj) +{ + *old_traceobj = NULL; /* Setup PEP 669 monitoring callbacks and events. */ if (!tstate->interp->sys_trace_initialized) { tstate->interp->sys_trace_initialized = true; @@ -553,14 +562,39 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) int delta = (func != NULL) - (tstate->c_tracefunc != NULL); tstate->c_tracefunc = func; - PyObject *old_traceobj = tstate->c_traceobj; + *old_traceobj = tstate->c_traceobj; tstate->c_traceobj = Py_XNewRef(arg); - Py_XDECREF(old_traceobj); tstate->interp->sys_tracing_threads += delta; assert(tstate->interp->sys_tracing_threads >= 0); + return tstate->interp->sys_tracing_threads; +} + +int +_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ + assert(is_tstate_valid(tstate)); + /* The caller must hold the GIL */ + assert(PyGILState_Check()); + + /* Call _PySys_Audit() in the context of the current thread state, + even if tstate is not the current thread state. */ + PyThreadState *current_tstate = _PyThreadState_GET(); + if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { + return -1; + } + assert(tstate->interp->sys_tracing_threads >= 0); + // needs to be decref'd outside of the lock + PyObject *old_traceobj; + LOCK_SETUP(); + Py_ssize_t tracing_threads = setup_tracing(tstate, func, arg, &old_traceobj); + UNLOCK_SETUP(); + Py_XDECREF(old_traceobj); + if (tracing_threads < 0) { + return -1; + } uint32_t events = 0; - if (tstate->interp->sys_tracing_threads) { + if (tracing_threads) { events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | diff --git a/Python/pystate.c b/Python/pystate.c index 37480df88aeb72..06806bd75fbcb2 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -399,6 +399,7 @@ _Py_COMP_DIAG_POP &(runtime)->unicode_state.ids.mutex, \ &(runtime)->imports.extensions.mutex, \ &(runtime)->ceval.pending_mainthread.mutex, \ + &(runtime)->ceval.sys_trace_profile_mutex, \ &(runtime)->atexit.mutex, \ &(runtime)->audit_hooks.mutex, \ &(runtime)->allocators.mutex, \ diff --git a/Tools/jit/template.c b/Tools/jit/template.c index b195aff377b3b5..228dc83254d678 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -12,6 +12,7 @@ #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_range.h" #include "pycore_setobject.h" #include "pycore_sliceobject.h" From d8f350309ded3130c43f0d2809dcb8ec13112320 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 19 Apr 2024 15:30:52 -0700 Subject: [PATCH 23/37] GH-115874: Fix segfault in FutureIter_dealloc (GH-117741) --- ...4-04-13-18-59-25.gh-issue-115874.c3xG-E.rst | 1 + Modules/_asynciomodule.c | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-59-25.gh-issue-115874.c3xG-E.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-59-25.gh-issue-115874.c3xG-E.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-59-25.gh-issue-115874.c3xG-E.rst new file mode 100644 index 00000000000000..5a6fa4cedd9fe4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-59-25.gh-issue-115874.c3xG-E.rst @@ -0,0 +1 @@ +Fixed a possible segfault during garbage collection of ``_asyncio.FutureIter`` objects diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index b886051186de9c..0873d32a9ec1a5 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1601,11 +1601,25 @@ static void FutureIter_dealloc(futureiterobject *it) { PyTypeObject *tp = Py_TYPE(it); - asyncio_state *state = get_asyncio_state_by_def((PyObject *)it); + + // FutureIter is a heap type so any subclass must also be a heap type. + assert(_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); + + PyObject *module = ((PyHeapTypeObject*)tp)->ht_module; + asyncio_state *state = NULL; + PyObject_GC_UnTrack(it); tp->tp_clear((PyObject *)it); - if (state->fi_freelist_len < FI_FREELIST_MAXLEN) { + // GH-115874: We can't use PyType_GetModuleByDef here as the type might have + // already been cleared, which is also why we must check if ht_module != NULL. + // Due to this restriction, subclasses that belong to a different module + // will not be able to use the free list. + if (module && _PyModule_GetDef(module) == &_asynciomodule) { + state = get_asyncio_state(module); + } + + if (state && state->fi_freelist_len < FI_FREELIST_MAXLEN) { state->fi_freelist_len++; it->future = (FutureObj*) state->fi_freelist; state->fi_freelist = it; From 15fbd53ba96be4b6a5abd94ceada684493c36bdd Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 20 Apr 2024 17:46:52 +0100 Subject: [PATCH 24/37] GH-112855: Speed up `pathlib.PurePath` pickling (#112856) The second item in the tuple returned from `__reduce__()` is a tuple of arguments to supply to path constructor. Previously we returned the `parts` tuple here, which entailed joining, parsing and normalising the path object, and produced a compact pickle representation. With this patch, we instead return a tuple of paths that were originally given to the path constructor. This makes pickling much faster (at the expense of compactness). It's worth noting that, in the olden times, pathlib performed this parsing/normalization up-front in every case, and so using `parts` for pickling was almost free. Nowadays pathlib only parses/normalises paths when it's necessary or advantageous to do so (e.g. computing a path parent, or iterating over a directory, respectively). --- Lib/pathlib/__init__.py | 4 +--- .../Library/2023-12-07-20-05-54.gh-issue-112855.ph4ehh.rst | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-12-07-20-05-54.gh-issue-112855.ph4ehh.rst diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index a4721fbe813962..f03f317ef6c16a 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -169,9 +169,7 @@ def __rtruediv__(self, key): return NotImplemented def __reduce__(self): - # Using the parts tuple helps share interned path parts - # when pickling related paths. - return (self.__class__, self.parts) + return self.__class__, tuple(self._raw_paths) def __repr__(self): return "{}({!r})".format(self.__class__.__name__, self.as_posix()) diff --git a/Misc/NEWS.d/next/Library/2023-12-07-20-05-54.gh-issue-112855.ph4ehh.rst b/Misc/NEWS.d/next/Library/2023-12-07-20-05-54.gh-issue-112855.ph4ehh.rst new file mode 100644 index 00000000000000..6badc7a9dc1c94 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-07-20-05-54.gh-issue-112855.ph4ehh.rst @@ -0,0 +1,2 @@ +Speed up pickling of :class:`pathlib.PurePath` objects. Patch by Barney +Gale. From 1558d993166636f371c1003107ec979db6744f21 Mon Sep 17 00:00:00 2001 From: Quazi Irfan Date: Sat, 20 Apr 2024 21:42:51 -0400 Subject: [PATCH 25/37] Clarifying nonlocal doc: SyntaxError is raised if nearest enclosing scope is global (#114009) Co-authored-by: Jelle Zijlstra --- Doc/reference/executionmodel.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index cea3a56ba51644..ed50faed6c940d 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -139,8 +139,9 @@ namespace. Names are resolved in the top-level namespace by searching the global namespace, i.e. the namespace of the module containing the code block, and the builtins namespace, the namespace of the module :mod:`builtins`. The global namespace is searched first. If the names are not found there, the -builtins namespace is searched. The :keyword:`!global` statement must precede -all uses of the listed names. +builtins namespace is searched next. If the names are also not found in the +builtins namespace, new variables are created in the global namespace. +The global statement must precede all uses of the listed names. The :keyword:`global` statement has the same scope as a name binding operation in the same block. If the nearest enclosing scope for a free variable contains From 92c84ef831c8276d591130f1c2c51289e2ba203a Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sun, 21 Apr 2024 05:25:39 +0300 Subject: [PATCH 26/37] ``Objects/typeobject.c``: Fix typo (#118126) --- Objects/typeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2f356388785665..970c82d2a17ada 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5671,7 +5671,7 @@ type_clear(PyObject *self) the dict, so that other objects caught in a reference cycle don't start calling destroyed methods. - Otherwise, the we need to clear tp_mro, which is + Otherwise, we need to clear tp_mro, which is part of a hard cycle (its first element is the class itself) that won't be broken otherwise (it's a tuple and tuples don't have a tp_clear handler). From df987331d896031799f33cb0f3a7b5c5712cb4e7 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Sun, 21 Apr 2024 11:04:08 +0800 Subject: [PATCH 27/37] gh-114099: Formalize Tier 3 status of iOS (GH-118020) --- Doc/whatsnew/3.13.rst | 16 ++++++++++++++++ Misc/ACKS | 1 + 2 files changed, 17 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index cf9853f94872c3..0f0fca5d23597b 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -127,6 +127,22 @@ Incremental Garbage Collection This means that maximum pause times are reduced by an order of magnitude or more for larger heaps. +Support For Mobile Platforms +---------------------------- + +* iOS is now a :pep:`11` supported platform. ``arm64-apple-ios`` + (iPhone and iPad devices released after 2013) and + ``arm64-apple-ios-simulator`` (Xcode iOS simulator running on Apple Silicon + hardware) are now tier 3 platforms. + + ``x86_64-apple-ios-simulator`` (Xcode iOS simulator running on older x86_64 + hardware) is not a tier 3 supported platform, but will be supported on a + best-effort basis. + + See :pep:`730`: for more details. + + (PEP written and implementation contributed by Russell Keith-Magee in + :gh:`114099`.) Other Language Changes ====================== diff --git a/Misc/ACKS b/Misc/ACKS index 76d30b257b4693..eaa7453aaade3e 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -930,6 +930,7 @@ Hiroaki Kawai Dmitry Kazakov Brian Kearns Sebastien Keim +Russell Keith-Magee Ryan Kelly Hugo van Kemenade Dan Kenigsberg From 5fa5b7facbcd1f725e51daf31c321e02b7db3f02 Mon Sep 17 00:00:00 2001 From: Andrew Cassidy Date: Sat, 20 Apr 2024 23:52:58 -0700 Subject: [PATCH 28/37] gh-91629 Use conf.d configs and fish_add_path to set the PATH when installing for the Fish shell. (GH-91630) Co-authored-by: Erlend E. Aasland --- Mac/BuildScript/scripts/postflight.patch-profile | 13 +++++++------ .../2022-04-17-01-07-42.gh-issue-91629.YBGAAt.rst | 1 + 2 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2022-04-17-01-07-42.gh-issue-91629.YBGAAt.rst diff --git a/Mac/BuildScript/scripts/postflight.patch-profile b/Mac/BuildScript/scripts/postflight.patch-profile index 68b8e4bb044e10..9caf62211ddd16 100755 --- a/Mac/BuildScript/scripts/postflight.patch-profile +++ b/Mac/BuildScript/scripts/postflight.patch-profile @@ -77,16 +77,17 @@ bash) fi ;; fish) - CONFIG_DIR="${HOME}/.config/fish" - RC="${CONFIG_DIR}/config.fish" + CONFIG_DIR="${HOME}/.config/fish/conf.d/" + RC="${CONFIG_DIR}/python-${PYVER}.fish" mkdir -p "$CONFIG_DIR" if [ -f "${RC}" ]; then cp -fp "${RC}" "${RC}.pysave" fi - echo "" >> "${RC}" - echo "# Setting PATH for Python ${PYVER}" >> "${RC}" - echo "# The original version is saved in ${RC}.pysave" >> "${RC}" - echo "set -x PATH \"${PYTHON_ROOT}/bin\" \"\$PATH\"" >> "${RC}" + echo "# Setting PATH for Python ${PYVER}" > "${RC}" + if [ -f "${RC}.pysave" ]; then + echo "# The original version is saved in ${RC}.pysave" >> "${RC}" + fi + echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" if [ `id -ur` = 0 ]; then chown "${USER}" "${RC}" fi diff --git a/Misc/NEWS.d/next/macOS/2022-04-17-01-07-42.gh-issue-91629.YBGAAt.rst b/Misc/NEWS.d/next/macOS/2022-04-17-01-07-42.gh-issue-91629.YBGAAt.rst new file mode 100644 index 00000000000000..13f3336c4a6ed8 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2022-04-17-01-07-42.gh-issue-91629.YBGAAt.rst @@ -0,0 +1 @@ +Use :file:`~/.config/fish/conf.d` configs and :program:`fish_add_path` to set :envvar:`PATH` when installing for the Fish shell. From ccda73828473576c57d1bb31774f56542d6e8964 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sun, 21 Apr 2024 10:08:32 +0300 Subject: [PATCH 29/37] gh-118121: Fix `test_doctest.test_look_in_unwrapped` (#118122) --- Lib/test/test_doctest/test_doctest.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index f71d62cc174d6b..cba4b16d544a20 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -2545,7 +2545,7 @@ def __call__(self, *args, **kwargs): self.func(*args, **kwargs) @Wrapper -def test_look_in_unwrapped(): +def wrapped(): """ Docstrings in wrapped functions must be detected as well. @@ -2553,6 +2553,21 @@ def test_look_in_unwrapped(): 'one other test' """ +def test_look_in_unwrapped(): + """ + Ensure that wrapped doctests work correctly. + + >>> import doctest + >>> doctest.run_docstring_examples( + ... wrapped, {}, name=wrapped.__name__, verbose=True) + Finding tests in wrapped + Trying: + 'one other test' + Expecting: + 'one other test' + ok + """ + @doctest_skip_if(support.check_impl_detail(cpython=False)) def test_wrapped_c_func(): """ From 51ef89cd9acbfbfa7fd8d82f0c6baa388bb650c9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 21 Apr 2024 11:46:39 +0300 Subject: [PATCH 30/37] gh-115961: Add name and mode attributes for compressed file-like objects (GH-116036) * Add name and mode attributes for compressed and archived file-like objects in modules bz2, lzma, tarfile and zipfile. * Change the value of the mode attribute of GzipFile from integer (1 or 2) to string ('rb' or 'wb'). * Change the value of the mode attribute of ZipExtFile from 'r' to 'rb'. --- Doc/library/bz2.rst | 15 +++- Doc/library/gzip.rst | 15 ++-- Doc/library/lzma.rst | 16 ++++- Doc/library/tarfile.rst | 4 ++ Doc/library/zipfile.rst | 10 +++ Doc/whatsnew/3.13.rst | 11 +++ Lib/bz2.py | 18 +++-- Lib/gzip.py | 3 +- Lib/lzma.py | 18 +++-- Lib/tarfile.py | 4 ++ Lib/test/test_bz2.py | 72 +++++++++++++++++-- Lib/test/test_gzip.py | 2 + Lib/test/test_lzma.py | 55 +++++++++++++- Lib/test/test_tarfile.py | 12 ++-- Lib/test/test_zipfile/test_core.py | 9 ++- Lib/zipfile/__init__.py | 12 +++- ...-02-28-10-41-24.gh-issue-115961.P-_DU0.rst | 7 ++ 17 files changed, 246 insertions(+), 37 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-28-10-41-24.gh-issue-115961.P-_DU0.rst diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index 5e0aa3e493f224..eaf0a096455fad 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -91,7 +91,7 @@ The :mod:`bz2` module contains: and :meth:`~io.IOBase.truncate`. Iteration and the :keyword:`with` statement are supported. - :class:`BZ2File` also provides the following methods: + :class:`BZ2File` also provides the following methods and attributes: .. method:: peek([n]) @@ -148,6 +148,19 @@ The :mod:`bz2` module contains: .. versionadded:: 3.3 + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. versionadded:: 3.13 + + .. attribute:: name + + The bzip2 file name. Equivalent to the :attr:`~io.FileIO.name` + attribute of the underlying :term:`file object`. + + .. versionadded:: 3.13 + .. versionchanged:: 3.1 Support for the :keyword:`with` statement was added. diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 044be8c1c1bf41..e6106e72a83d25 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -133,6 +133,13 @@ The module defines the following items: .. versionadded:: 3.2 + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. versionchanged:: 3.13 + In previous versions it was an integer ``1`` or ``2``. + .. attribute:: mtime When decompressing, this attribute is set to the last timestamp in the most @@ -168,14 +175,14 @@ The module defines the following items: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. versionchanged:: 3.12 - Remove the ``filename`` attribute, use the :attr:`~GzipFile.name` - attribute instead. - .. deprecated:: 3.9 Opening :class:`GzipFile` for writing without specifying the *mode* argument is deprecated. + .. versionchanged:: 3.12 + Remove the ``filename`` attribute, use the :attr:`~GzipFile.name` + attribute instead. + .. function:: compress(data, compresslevel=9, *, mtime=None) diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 0d69c3bc01d1e2..74bf2670f9def6 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -104,7 +104,7 @@ Reading and writing compressed files and :meth:`~io.IOBase.truncate`. Iteration and the :keyword:`with` statement are supported. - The following method is also provided: + The following method and attributes are also provided: .. method:: peek(size=-1) @@ -117,6 +117,20 @@ Reading and writing compressed files file object (e.g. if the :class:`LZMAFile` was constructed by passing a file object for *filename*). + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. versionadded:: 3.13 + + .. attribute:: name + + The lzma file name. Equivalent to the :attr:`~io.FileIO.name` + attribute of the underlying :term:`file object`. + + .. versionadded:: 3.13 + + .. versionchanged:: 3.4 Added support for the ``"x"`` and ``"xb"`` modes. diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index afcad9bf722934..cf21dee83e6e7a 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -565,6 +565,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.3 Return an :class:`io.BufferedReader` object. + .. versionchanged:: 3.13 + The returned :class:`io.BufferedReader` object has the :attr:`!mode` + attribute which is always equal to ``'rb'``. + .. attribute:: TarFile.errorlevel :type: int diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index ee53f162ac9080..6192689c3a1194 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -301,6 +301,10 @@ ZipFile Objects attempting to read or write other files in the ZIP file will raise a :exc:`ValueError`. + In both cases the file-like object has also attributes :attr:`!name`, + which is equivalent to the name of a file within the archive, and + :attr:`!mode`, which is ``'rb'`` or ``'wb'`` depending on the input mode. + When writing a file, if the file size is not known in advance but may exceed 2 GiB, pass ``force_zip64=True`` to ensure that the header format is capable of supporting large files. If the file size is known in advance, @@ -325,6 +329,12 @@ ZipFile Objects Calling :meth:`.open` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.13 + Added attributes :attr:`!name` and :attr:`!mode` for the writeable + file-like object. + The value of the :attr:`!mode` attribute for the readable file-like + object was changed from ``'r'`` to ``'rb'``. + .. method:: ZipFile.extract(member, path=None, pwd=None) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 0f0fca5d23597b..5be562030b507b 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -206,6 +206,11 @@ Other Language Changes (Contributed by Victor Stinner in :gh:`114570`.) +* Added :attr:`!name` and :attr:`!mode` attributes for compressed + and archived file-like objects in modules :mod:`bz2`, :mod:`lzma`, + :mod:`tarfile` and :mod:`zipfile`. + (Contributed by Serhiy Storchaka in :gh:`115961`.) + * Allow controlling Expat >=2.6.0 reparse deferral (:cve:`2023-52425`) by adding five new methods: @@ -1605,6 +1610,12 @@ Changes in the Python API than directories only. Users may add a trailing slash to match only directories. +* The value of the :attr:`!mode` attribute of :class:`gzip.GzipFile` was + changed from integer (``1`` or ``2``) to string (``'rb'`` or ``'wb'``). + The value of the :attr:`!mode` attribute of the readable file-like object + returned by :meth:`zipfile.ZipFile.open` was changed from ``'r'`` to ``'rb'``. + (Contributed by Serhiy Storchaka in :gh:`115961`.) + * :c:func:`!PyCode_GetFirstFree` is an unstable API now and has been renamed to :c:func:`PyUnstable_Code_GetFirstFree`. (Contributed by Bogdan Romanyuk in :gh:`115781`.) diff --git a/Lib/bz2.py b/Lib/bz2.py index fabe4f73c8d808..2420cd019069b4 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -17,7 +17,7 @@ from _bz2 import BZ2Compressor, BZ2Decompressor -_MODE_CLOSED = 0 +# Value 0 no longer used _MODE_READ = 1 # Value 2 no longer used _MODE_WRITE = 3 @@ -54,7 +54,7 @@ def __init__(self, filename, mode="r", *, compresslevel=9): """ self._fp = None self._closefp = False - self._mode = _MODE_CLOSED + self._mode = None if not (1 <= compresslevel <= 9): raise ValueError("compresslevel must be between 1 and 9") @@ -100,7 +100,7 @@ def close(self): May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ - if self._mode == _MODE_CLOSED: + if self.closed: return try: if self._mode == _MODE_READ: @@ -115,13 +115,21 @@ def close(self): finally: self._fp = None self._closefp = False - self._mode = _MODE_CLOSED self._buffer = None @property def closed(self): """True if this file is closed.""" - return self._mode == _MODE_CLOSED + return self._fp is None + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' def fileno(self): """Return the file descriptor for the underlying file.""" diff --git a/Lib/gzip.py b/Lib/gzip.py index 1d6faaa82c6a68..0d19c84c59cfa7 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -15,7 +15,8 @@ FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 -READ, WRITE = 1, 2 +READ = 'rb' +WRITE = 'wb' _COMPRESS_LEVEL_FAST = 1 _COMPRESS_LEVEL_TRADEOFF = 6 diff --git a/Lib/lzma.py b/Lib/lzma.py index 800f52198fbb79..c1e3d33deb69a1 100644 --- a/Lib/lzma.py +++ b/Lib/lzma.py @@ -29,7 +29,7 @@ import _compression -_MODE_CLOSED = 0 +# Value 0 no longer used _MODE_READ = 1 # Value 2 no longer used _MODE_WRITE = 3 @@ -92,7 +92,7 @@ def __init__(self, filename=None, mode="r", *, """ self._fp = None self._closefp = False - self._mode = _MODE_CLOSED + self._mode = None if mode in ("r", "rb"): if check != -1: @@ -137,7 +137,7 @@ def close(self): May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ - if self._mode == _MODE_CLOSED: + if self.closed: return try: if self._mode == _MODE_READ: @@ -153,12 +153,20 @@ def close(self): finally: self._fp = None self._closefp = False - self._mode = _MODE_CLOSED @property def closed(self): """True if this file is closed.""" - return self._mode == _MODE_CLOSED + return self._fp is None + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' def fileno(self): """Return the file descriptor for the underlying file.""" diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 78bb10c80820d7..5fc6183ffcf93c 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -636,6 +636,10 @@ def __init__(self, fileobj, offset, size, name, blockinfo=None): def flush(self): pass + @property + def mode(self): + return 'rb' + def readable(self): return True diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 772f0eacce28f5..e4d1381be5f340 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -540,40 +540,54 @@ def testMultiStreamOrdering(self): def testOpenFilename(self): with BZ2File(self.filename, "wb") as f: f.write(b'content') + self.assertEqual(f.name, self.filename) self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) with BZ2File(self.filename, "ab") as f: f.write(b'appendix') + self.assertEqual(f.name, self.filename) self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) with BZ2File(self.filename, 'rb') as f: self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, self.filename) self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'rb') self.assertIs(f.readable(), True) self.assertIs(f.writable(), False) self.assertIs(f.seekable(), True) self.assertIs(f.closed, False) self.assertIs(f.closed, True) with self.assertRaises(ValueError): - f.fileno() + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -582,13 +596,18 @@ def testOpenFileWithName(self): with open(self.filename, 'wb') as raw: with BZ2File(raw, 'wb') as f: f.write(b'content') + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -596,13 +615,18 @@ def testOpenFileWithName(self): with open(self.filename, 'ab') as raw: with BZ2File(raw, 'ab') as f: f.write(b'appendix') + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -610,14 +634,18 @@ def testOpenFileWithName(self): with open(self.filename, 'rb') as raw: with BZ2File(raw, 'rb') as f: self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') self.assertIs(f.readable(), True) self.assertIs(f.writable(), False) self.assertIs(f.seekable(), True) self.assertIs(f.closed, False) self.assertIs(f.closed, True) with self.assertRaises(ValueError): - f.fileno() + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -626,61 +654,91 @@ def testOpenFileWithoutName(self): bio = BytesIO() with BZ2File(bio, 'wb') as f: f.write(b'content') + with self.assertRaises(AttributeError): + f.name self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) with BZ2File(bio, 'ab') as f: f.write(b'appendix') + with self.assertRaises(AttributeError): + f.name self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) bio.seek(0) with BZ2File(bio, 'rb') as f: self.assertEqual(f.read(), b'contentappendix') + with self.assertRaises(AttributeError): + f.name self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'rb') with self.assertRaises(ValueError): - f.fileno() + f.name + self.assertRaises(ValueError, f.fileno) def testOpenFileWithIntName(self): fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) with open(fd, 'wb') as raw: with BZ2File(raw, 'wb') as f: f.write(b'content') + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) with open(fd, 'ab') as raw: with BZ2File(raw, 'ab') as f: f.write(b'appendix') + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) fd = os.open(self.filename, os.O_RDONLY) with open(fd, 'rb') as raw: with BZ2File(raw, 'rb') as f: self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') with self.assertRaises(ValueError): - f.fileno() + f.name + self.assertRaises(ValueError, f.fileno) def testOpenBytesFilename(self): str_filename = self.filename bytes_filename = os.fsencode(str_filename) with BZ2File(bytes_filename, "wb") as f: f.write(self.DATA) + self.assertEqual(f.name, bytes_filename) with BZ2File(bytes_filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, bytes_filename) # Sanity check that we are actually operating on the right file. with BZ2File(str_filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, str_filename) def testOpenPathLikeFilename(self): filename = FakePath(self.filename) with BZ2File(filename, "wb") as f: f.write(self.DATA) + self.assertEqual(f.name, self.filename) with BZ2File(filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, self.filename) def testDecompressLimited(self): """Decompressed data buffering should be limited""" @@ -701,6 +759,9 @@ def testReadBytesIO(self): with BZ2File(bio) as bz2f: self.assertRaises(TypeError, bz2f.read, float()) self.assertEqual(bz2f.read(), self.TEXT) + with self.assertRaises(AttributeError): + bz2.name + self.assertEqual(bz2f.mode, 'rb') self.assertFalse(bio.closed) def testPeekBytesIO(self): @@ -716,6 +777,9 @@ def testWriteBytesIO(self): with BZ2File(bio, "w") as bz2f: self.assertRaises(TypeError, bz2f.write) bz2f.write(self.TEXT) + with self.assertRaises(AttributeError): + bz2.name + self.assertEqual(bz2f.mode, 'wb') self.assertEqual(ext_decompress(bio.getvalue()), self.TEXT) self.assertFalse(bio.closed) diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index d220c7d06e50c9..cf801278da9e9b 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -587,6 +587,8 @@ def test_fileobj_from_fdopen(self): self.assertRaises(AttributeError, f.fileno) def test_fileobj_mode(self): + self.assertEqual(gzip.READ, 'rb') + self.assertEqual(gzip.WRITE, 'wb') gzip.GzipFile(self.filename, "wb").close() with open(self.filename, "r+b") as f: with gzip.GzipFile(fileobj=f, mode='r') as g: diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index db290e139327e0..22478c14fb4a65 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -551,19 +551,25 @@ def test_init_with_PathLike_filename(self): with TempFile(filename, COMPRESSED_XZ): with LZMAFile(filename) as f: self.assertEqual(f.read(), INPUT) + self.assertEqual(f.name, TESTFN) with LZMAFile(filename, "a") as f: f.write(INPUT) + self.assertEqual(f.name, TESTFN) with LZMAFile(filename) as f: self.assertEqual(f.read(), INPUT * 2) + self.assertEqual(f.name, TESTFN) def test_init_with_filename(self): with TempFile(TESTFN, COMPRESSED_XZ): with LZMAFile(TESTFN) as f: - pass + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'rb') with LZMAFile(TESTFN, "w") as f: - pass + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'wb') with LZMAFile(TESTFN, "a") as f: - pass + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'wb') def test_init_mode(self): with TempFile(TESTFN): @@ -586,6 +592,7 @@ def test_init_with_x_mode(self): unlink(TESTFN) with LZMAFile(TESTFN, mode) as f: pass + self.assertEqual(f.mode, 'wb') with self.assertRaises(FileExistsError): LZMAFile(TESTFN, mode) @@ -865,13 +872,18 @@ def test_read_from_file(self): with LZMAFile(TESTFN) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertEqual(f.name, TESTFN) self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'rb') self.assertIs(f.readable(), True) self.assertIs(f.writable(), False) self.assertIs(f.seekable(), True) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -882,6 +894,7 @@ def test_read_from_file_with_bytes_filename(self): with LZMAFile(bytes_filename) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertEqual(f.name, bytes_filename) def test_read_from_fileobj(self): with TempFile(TESTFN, COMPRESSED_XZ): @@ -889,13 +902,18 @@ def test_read_from_fileobj(self): with LZMAFile(raw) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') self.assertIs(f.readable(), True) self.assertIs(f.writable(), False) self.assertIs(f.seekable(), True) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -907,13 +925,18 @@ def test_read_from_fileobj_with_int_name(self): with LZMAFile(raw) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') self.assertIs(f.readable(), True) self.assertIs(f.writable(), False) self.assertIs(f.seekable(), True) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -1045,6 +1068,8 @@ def test_write(self): with BytesIO() as dst: with LZMAFile(dst, "w") as f: f.write(INPUT) + with self.assertRaises(AttributeError): + f.name expected = lzma.compress(INPUT) self.assertEqual(dst.getvalue(), expected) with BytesIO() as dst: @@ -1081,23 +1106,31 @@ def test_write_append(self): with BytesIO() as dst: with LZMAFile(dst, "w") as f: f.write(part1) + self.assertEqual(f.mode, 'wb') with LZMAFile(dst, "a") as f: f.write(part2) + self.assertEqual(f.mode, 'wb') with LZMAFile(dst, "a") as f: f.write(part3) + self.assertEqual(f.mode, 'wb') self.assertEqual(dst.getvalue(), expected) def test_write_to_file(self): try: with LZMAFile(TESTFN, "w") as f: f.write(INPUT) + self.assertEqual(f.name, TESTFN) self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -1113,6 +1146,7 @@ def test_write_to_file_with_bytes_filename(self): try: with LZMAFile(bytes_filename, "w") as f: f.write(INPUT) + self.assertEqual(f.name, bytes_filename) expected = lzma.compress(INPUT) with open(TESTFN, "rb") as f: self.assertEqual(f.read(), expected) @@ -1124,13 +1158,18 @@ def test_write_to_fileobj(self): with open(TESTFN, "wb") as raw: with LZMAFile(raw, "w") as f: f.write(INPUT) + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -1147,13 +1186,18 @@ def test_write_to_fileobj_with_int_name(self): with open(fd, 'wb') as raw: with LZMAFile(raw, "w") as f: f.write(INPUT) + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -1172,10 +1216,13 @@ def test_write_append_to_file(self): try: with LZMAFile(TESTFN, "w") as f: f.write(part1) + self.assertEqual(f.mode, 'wb') with LZMAFile(TESTFN, "a") as f: f.write(part2) + self.assertEqual(f.mode, 'wb') with LZMAFile(TESTFN, "a") as f: f.write(part3) + self.assertEqual(f.mode, 'wb') with open(TESTFN, "rb") as f: self.assertEqual(f.read(), expected) finally: @@ -1373,11 +1420,13 @@ def test_with_pathlike_filename(self): with TempFile(filename): with lzma.open(filename, "wb") as f: f.write(INPUT) + self.assertEqual(f.name, TESTFN) with open(filename, "rb") as f: file_data = lzma.decompress(f.read()) self.assertEqual(file_data, INPUT) with lzma.open(filename, "rb") as f: self.assertEqual(f.read(), INPUT) + self.assertEqual(f.name, TESTFN) def test_bad_params(self): # Test invalid parameter combinations. diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index c2abc7ef9698fb..c9c1097963a885 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -513,6 +513,7 @@ def test_extractfile_attrs(self): with self.tar.extractfile(file) as fobj: self.assertEqual(fobj.name, 'ustar/regtype') self.assertRaises(AttributeError, fobj.fileno) + self.assertEqual(fobj.mode, 'rb') self.assertIs(fobj.readable(), True) self.assertIs(fobj.writable(), False) if self.is_stream: @@ -523,6 +524,7 @@ def test_extractfile_attrs(self): self.assertIs(fobj.closed, True) self.assertEqual(fobj.name, 'ustar/regtype') self.assertRaises(AttributeError, fobj.fileno) + self.assertEqual(fobj.mode, 'rb') self.assertIs(fobj.readable(), True) self.assertIs(fobj.writable(), False) if self.is_stream: @@ -533,11 +535,8 @@ def test_extractfile_attrs(self): class MiscReadTestBase(CommonReadTest): is_stream = False - def requires_name_attribute(self): - pass def test_no_name_argument(self): - self.requires_name_attribute() with open(self.tarname, "rb") as fobj: self.assertIsInstance(fobj.name, str) with tarfile.open(fileobj=fobj, mode=self.mode) as tar: @@ -570,7 +569,6 @@ def test_int_name_attribute(self): self.assertIsNone(tar.name) def test_bytes_name_attribute(self): - self.requires_name_attribute() tarname = os.fsencode(self.tarname) with open(tarname, 'rb') as fobj: self.assertIsInstance(fobj.name, bytes) @@ -839,12 +837,10 @@ class GzipMiscReadTest(GzipTest, MiscReadTestBase, unittest.TestCase): pass class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase): - def requires_name_attribute(self): - self.skipTest("BZ2File have no name attribute") + pass class LzmaMiscReadTest(LzmaTest, MiscReadTestBase, unittest.TestCase): - def requires_name_attribute(self): - self.skipTest("LZMAFile have no name attribute") + pass class StreamReadTest(CommonReadTest, unittest.TestCase): diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index a605aa1f14fe3f..423974aada4ac1 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -389,7 +389,6 @@ def test_repr(self): with zipfp.open(fname) as zipopen: r = repr(zipopen) self.assertIn('name=%r' % fname, r) - self.assertIn("mode='r'", r) if self.compression != zipfile.ZIP_STORED: self.assertIn('compress_type=', r) self.assertIn('[closed]', repr(zipopen)) @@ -455,14 +454,14 @@ def test_zipextfile_attrs(self): with zipfp.open(fname) as fid: self.assertEqual(fid.name, fname) self.assertRaises(io.UnsupportedOperation, fid.fileno) - self.assertEqual(fid.mode, 'r') + self.assertEqual(fid.mode, 'rb') self.assertIs(fid.readable(), True) self.assertIs(fid.writable(), False) self.assertIs(fid.seekable(), True) self.assertIs(fid.closed, False) self.assertIs(fid.closed, True) self.assertEqual(fid.name, fname) - self.assertEqual(fid.mode, 'r') + self.assertEqual(fid.mode, 'rb') self.assertRaises(io.UnsupportedOperation, fid.fileno) self.assertRaises(ValueError, fid.readable) self.assertIs(fid.writable(), False) @@ -1308,12 +1307,16 @@ def test_zipwritefile_attrs(self): fname = "somefile.txt" with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: with zipfp.open(fname, 'w') as fid: + self.assertEqual(fid.name, fname) self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'wb') self.assertIs(fid.readable(), False) self.assertIs(fid.writable(), True) self.assertIs(fid.seekable(), False) self.assertIs(fid.closed, False) self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'wb') self.assertRaises(io.UnsupportedOperation, fid.fileno) self.assertIs(fid.readable(), False) self.assertIs(fid.writable(), True) diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index e4603b559f5962..31ef9bb1ad925e 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -940,7 +940,7 @@ def __repr__(self): result = ['<%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)] if not self.closed: - result.append(' name=%r mode=%r' % (self.name, self.mode)) + result.append(' name=%r' % (self.name,)) if self._compress_type != ZIP_STORED: result.append(' compress_type=%s' % compressor_names.get(self._compress_type, @@ -1217,6 +1217,14 @@ def __init__(self, zf, zinfo, zip64): def _fileobj(self): return self._zipfile.fp + @property + def name(self): + return self._zinfo.filename + + @property + def mode(self): + return 'wb' + def writable(self): return True @@ -1687,7 +1695,7 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): else: pwd = None - return ZipExtFile(zef_file, mode, zinfo, pwd, True) + return ZipExtFile(zef_file, mode + 'b', zinfo, pwd, True) except: zef_file.close() raise diff --git a/Misc/NEWS.d/next/Library/2024-02-28-10-41-24.gh-issue-115961.P-_DU0.rst b/Misc/NEWS.d/next/Library/2024-02-28-10-41-24.gh-issue-115961.P-_DU0.rst new file mode 100644 index 00000000000000..eef7eb8687b44f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-28-10-41-24.gh-issue-115961.P-_DU0.rst @@ -0,0 +1,7 @@ +Added :attr:`!name` and :attr:`!mode` attributes for compressed and archived +file-like objects in modules :mod:`bz2`, :mod:`lzma`, :mod:`tarfile` and +:mod:`zipfile`. The value of the :attr:`!mode` attribute of +:class:`gzip.GzipFile` was changed from integer (``1`` or ``2``) to string +(``'rb'`` or ``'wb'``). The value of the :attr:`!mode` attribute of the +readable file-like object returned by :meth:`zipfile.ZipFile.open` was +changed from ``'r'`` to ``'rb'``. From 1446024124fb98c3051199760380685f8a2fd127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 21 Apr 2024 20:03:46 +0200 Subject: [PATCH 31/37] Docs: replace Harry Potter reference with Monty Python (#118130) --- Doc/library/doctest.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index a643a0e7e313bf..5f7d10a6dce037 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -800,18 +800,18 @@ guarantee about output. For example, when printing a set, Python doesn't guarantee that the element is printed in any particular order, so a test like :: >>> foo() - {"Hermione", "Harry"} + {"spam", "eggs"} is vulnerable! One workaround is to do :: - >>> foo() == {"Hermione", "Harry"} + >>> foo() == {"spam", "eggs"} True instead. Another is to do :: >>> d = sorted(foo()) >>> d - ['Harry', 'Hermione'] + ['eggs', 'spam'] There are others, but you get the idea. From 8b541c017ea92040add608b3e0ef8dc85e9e6060 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Sun, 21 Apr 2024 22:57:05 -0700 Subject: [PATCH 32/37] gh-112075: Make instance attributes stored in inline "dict" thread safe (#114742) Make instance attributes stored in inline "dict" thread safe on free-threaded builds --- Include/cpython/object.h | 1 + Include/internal/pycore_dict.h | 19 +- Include/internal/pycore_object.h | 16 +- .../internal/pycore_pyatomic_ft_wrappers.h | 14 + Lib/test/test_class.py | 9 + Objects/dictobject.c | 385 ++++++++++++++---- Objects/object.c | 45 +- Objects/typeobject.c | 30 +- Python/bytecodes.c | 15 +- Python/executor_cases.c.h | 10 +- Python/generated_cases.c.h | 13 +- Python/specialize.c | 3 +- Tools/cases_generator/analyzer.py | 1 + 13 files changed, 419 insertions(+), 142 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 2797051281f3b4..a8f57827a964cd 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -493,6 +493,7 @@ do { \ PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); +PyAPI_FUNC(void) _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict); PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj); #define TYPE_MAX_WATCHERS 8 diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index fba0dfc40714ec..f33026dbd6be58 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -1,4 +1,3 @@ - #ifndef Py_INTERNAL_DICT_H #define Py_INTERNAL_DICT_H #ifdef __cplusplus @@ -9,9 +8,10 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState -#include "pycore_identifier.h" // _Py_Identifier -#include "pycore_object.h" // PyManagedDictPointer +#include "pycore_freelist.h" // _PyFreeListState +#include "pycore_identifier.h" // _Py_Identifier +#include "pycore_object.h" // PyManagedDictPointer +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); @@ -249,7 +249,7 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK); } -extern PyDictObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj); +extern PyDictObject *_PyObject_MaterializeManagedDict(PyObject *obj); PyAPI_FUNC(PyObject *)_PyDict_FromItems( PyObject *const *keys, Py_ssize_t keys_offset, @@ -277,7 +277,6 @@ _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) static inline size_t shared_keys_usable_size(PyDictKeysObject *keys) { -#ifdef Py_GIL_DISABLED // dk_usable will decrease for each instance that is created and each // value that is added. dk_nentries will increase for each value that // is added. We want to always return the right value or larger. @@ -285,11 +284,9 @@ shared_keys_usable_size(PyDictKeysObject *keys) // second, and conversely here we read dk_usable first and dk_entries // second (to avoid the case where we read entries before the increment // and read usable after the decrement) - return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) + - _Py_atomic_load_ssize_acquire(&keys->dk_nentries)); -#else - return (size_t)keys->dk_nentries + (size_t)keys->dk_usable; -#endif + Py_ssize_t dk_usable = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_usable); + Py_ssize_t dk_nentries = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_nentries); + return dk_nentries + dk_usable; } static inline size_t diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 512f7a35f50e38..88b052f4544b15 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -12,6 +12,7 @@ extern "C" { #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_emscripten_trampoline.h" // _PyCFunction_TrampolineCall() #include "pycore_interp.h" // PyInterpreterState.gc +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED #include "pycore_pystate.h" // _PyInterpreterState_GET() /* Check if an object is consistent. For example, ensure that the reference @@ -659,10 +660,10 @@ extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); -extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, - PyObject *name, PyObject *value); -PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, - PyObject *name); +extern int _PyObject_StoreInstanceAttribute(PyObject *obj, + PyObject *name, PyObject *value); +extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, + PyObject **attr); #ifdef Py_GIL_DISABLED # define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-1) @@ -683,6 +684,13 @@ _PyObject_ManagedDictPointer(PyObject *obj) return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET); } +static inline PyDictObject * +_PyObject_GetManagedDict(PyObject *obj) +{ + PyManagedDictPointer *dorv = _PyObject_ManagedDictPointer(obj); + return (PyDictObject *)FT_ATOMIC_LOAD_PTR_RELAXED(dorv->dict); +} + static inline PyDictValues * _PyObject_InlineValues(PyObject *obj) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index fed5d6e0ec2c54..bbfc462a733d0e 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -21,7 +21,10 @@ extern "C" { #ifdef Py_GIL_DISABLED #define FT_ATOMIC_LOAD_PTR(value) _Py_atomic_load_ptr(&value) +#define FT_ATOMIC_STORE_PTR(value, new_value) _Py_atomic_store_ptr(&value, new_value) #define FT_ATOMIC_LOAD_SSIZE(value) _Py_atomic_load_ssize(&value) +#define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) \ + _Py_atomic_load_ssize_acquire(&value) #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) \ _Py_atomic_load_ssize_relaxed(&value) #define FT_ATOMIC_STORE_PTR(value, new_value) \ @@ -30,6 +33,12 @@ extern "C" { _Py_atomic_load_ptr_acquire(&value) #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) \ _Py_atomic_load_uintptr_acquire(&value) +#define FT_ATOMIC_LOAD_PTR_RELAXED(value) \ + _Py_atomic_load_ptr_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT8(value) \ + _Py_atomic_load_uint8(&value) +#define FT_ATOMIC_STORE_UINT8(value, new_value) \ + _Py_atomic_store_uint8(&value, new_value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ _Py_atomic_store_ptr_relaxed(&value, new_value) #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ @@ -43,11 +52,16 @@ extern "C" { #else #define FT_ATOMIC_LOAD_PTR(value) value +#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value #define FT_ATOMIC_LOAD_SSIZE(value) value +#define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value #define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_PTR_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT8(value) value +#define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index a9cfd8df691845..5885db84b66b01 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -873,6 +873,15 @@ def __init__(self): obj.foo = None # Aborted here self.assertEqual(obj.__dict__, {"foo":None}) + def test_store_attr_deleted_dict(self): + class Foo: + pass + + f = Foo() + del f.__dict__ + f.a = 3 + self.assertEqual(f.a, 3) + if __name__ == '__main__': unittest.main() diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 4e696419eb5eb0..2644516bc30770 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1752,7 +1752,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_MODIFIED, mp, key, value); if (_PyDict_HasSplitTable(mp)) { - mp->ma_values->values[ix] = value; + STORE_SPLIT_VALUE(mp, ix, value); if (old_value == NULL) { _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); mp->ma_used++; @@ -2514,7 +2514,7 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, mp->ma_version_tag = new_version; if (_PyDict_HasSplitTable(mp)) { assert(old_value == mp->ma_values->values[ix]); - mp->ma_values->values[ix] = NULL; + STORE_SPLIT_VALUE(mp, ix, NULL); assert(ix < SHARED_KEYS_MAX_SIZE); /* Update order */ delete_index_from_values(mp->ma_values, ix); @@ -4226,7 +4226,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu assert(_PyDict_HasSplitTable(mp)); assert(mp->ma_values->values[ix] == NULL); MAINTAIN_TRACKING(mp, key, value); - mp->ma_values->values[ix] = Py_NewRef(value); + STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); mp->ma_used++; mp->ma_version_tag = new_version; @@ -6616,28 +6616,79 @@ make_dict_from_instance_attributes(PyInterpreterState *interp, return res; } -PyDictObject * -_PyObject_MakeDictFromInstanceAttributes(PyObject *obj) +static PyDictObject * +materialize_managed_dict_lock_held(PyObject *obj) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); + + OBJECT_STAT_INC(dict_materialized_on_request); + PyDictValues *values = _PyObject_InlineValues(obj); PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); OBJECT_STAT_INC(dict_materialized_on_request); - return make_dict_from_instance_attributes(interp, keys, values); + PyDictObject *dict = make_dict_from_instance_attributes(interp, keys, values); + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)dict); + return dict; } +PyDictObject * +_PyObject_MaterializeManagedDict(PyObject *obj) +{ + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict != NULL) { + return dict; + } + Py_BEGIN_CRITICAL_SECTION(obj); -int -_PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, +#ifdef Py_GIL_DISABLED + dict = _PyObject_GetManagedDict(obj); + if (dict != NULL) { + // We raced with another thread creating the dict + goto exit; + } +#endif + dict = materialize_managed_dict_lock_held(obj); + +#ifdef Py_GIL_DISABLED +exit: +#endif + Py_END_CRITICAL_SECTION(); + return dict; +} + +static int +set_or_del_lock_held(PyDictObject *dict, PyObject *name, PyObject *value) +{ + if (value == NULL) { + Py_hash_t hash; + if (!PyUnicode_CheckExact(name) || (hash = unicode_get_hash(name)) == -1) { + hash = PyObject_Hash(name); + if (hash == -1) + return -1; + } + return delitem_knownhash_lock_held((PyObject *)dict, name, hash); + } else { + return setitem_lock_held(dict, name, value); + } +} + +// Called with either the object's lock or the dict's lock held +// depending on whether or not a dict has been materialized for +// the object. +static int +store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value) { - PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); assert(values != NULL); assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); Py_ssize_t ix = DKIX_EMPTY; + PyDictObject *dict = _PyObject_GetManagedDict(obj); + assert(dict == NULL || ((PyDictObject *)dict)->ma_values == values); if (PyUnicode_CheckExact(name)) { Py_hash_t hash = unicode_get_hash(name); if (hash == -1) { @@ -6674,25 +6725,33 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, } #endif } - PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict; + if (ix == DKIX_EMPTY) { + int res; if (dict == NULL) { - dict = make_dict_from_instance_attributes( - interp, keys, values); - if (dict == NULL) { + // Make the dict but don't publish it in the object + // so that no one else will see it. + dict = make_dict_from_instance_attributes(PyInterpreterState_Get(), keys, values); + if (dict == NULL || + set_or_del_lock_held(dict, name, value) < 0) { + Py_XDECREF(dict); return -1; } - _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict; - } - if (value == NULL) { - return PyDict_DelItem((PyObject *)dict, name); - } - else { - return PyDict_SetItem((PyObject *)dict, name, value); + + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)dict); + return 0; } + + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict); + + res = set_or_del_lock_held (dict, name, value); + return res; } + PyObject *old_value = values->values[ix]; - values->values[ix] = Py_XNewRef(value); + FT_ATOMIC_STORE_PTR_RELEASE(values->values[ix], Py_XNewRef(value)); + if (old_value == NULL) { if (value == NULL) { PyErr_Format(PyExc_AttributeError, @@ -6719,6 +6778,72 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, return 0; } +static inline int +store_instance_attr_dict(PyObject *obj, PyDictObject *dict, PyObject *name, PyObject *value) +{ + PyDictValues *values = _PyObject_InlineValues(obj); + int res; + Py_BEGIN_CRITICAL_SECTION(dict); + if (dict->ma_values == values) { + res = store_instance_attr_lock_held(obj, values, name, value); + } + else { + res = set_or_del_lock_held(dict, name, value); + } + Py_END_CRITICAL_SECTION(); + return res; +} + +int +_PyObject_StoreInstanceAttribute(PyObject *obj, PyObject *name, PyObject *value) +{ + PyDictValues *values = _PyObject_InlineValues(obj); + if (!FT_ATOMIC_LOAD_UINT8(values->valid)) { + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + dict = (PyDictObject *)PyObject_GenericGetDict(obj, NULL); + if (dict == NULL) { + return -1; + } + int res = store_instance_attr_dict(obj, dict, name, value); + Py_DECREF(dict); + return res; + } + return store_instance_attr_dict(obj, dict, name, value); + } + +#ifdef Py_GIL_DISABLED + // We have a valid inline values, at least for now... There are two potential + // races with having the values become invalid. One is the dictionary + // being detached from the object. The other is if someone is inserting + // into the dictionary directly and therefore causing it to resize. + // + // If we haven't materialized the dictionary yet we lock on the object, which + // will also be used to prevent the dictionary from being materialized while + // we're doing the insertion. If we race and the dictionary gets created + // then we'll need to release the object lock and lock the dictionary to + // prevent resizing. + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + int res; + Py_BEGIN_CRITICAL_SECTION(obj); + dict = _PyObject_GetManagedDict(obj); + + if (dict == NULL) { + res = store_instance_attr_lock_held(obj, values, name, value); + } + Py_END_CRITICAL_SECTION(); + + if (dict == NULL) { + return res; + } + } + return store_instance_attr_dict(obj, dict, name, value); +#else + return store_instance_attr_lock_held(obj, values, name, value); +#endif +} + /* Sanity check for managed dicts */ #if 0 #define CHECK(val) assert(val); if (!(val)) { return 0; } @@ -6750,19 +6875,79 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj) } #endif -PyObject * -_PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, - PyObject *name) +// Attempts to get an instance attribute from the inline values. Returns true +// if successful, or false if the caller needs to lookup in the dictionary. +bool +_PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr) { assert(PyUnicode_CheckExact(name)); + PyDictValues *values = _PyObject_InlineValues(obj); + if (!FT_ATOMIC_LOAD_UINT8(values->valid)) { + return false; + } + PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); Py_ssize_t ix = _PyDictKeys_StringLookup(keys, name); if (ix == DKIX_EMPTY) { - return NULL; + *attr = NULL; + return true; + } + +#ifdef Py_GIL_DISABLED + PyObject *value = _Py_atomic_load_ptr_relaxed(&values->values[ix]); + if (value == NULL || _Py_TryIncrefCompare(&values->values[ix], value)) { + *attr = value; + return true; + } + + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + // No dict, lock the object to prevent one from being + // materialized... + bool success = false; + Py_BEGIN_CRITICAL_SECTION(obj); + + dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + // Still no dict, we can read from the values + assert(values->valid); + value = values->values[ix]; + *attr = Py_XNewRef(value); + success = true; + } + + Py_END_CRITICAL_SECTION(); + + if (success) { + return true; + } } + + // We have a dictionary, we'll need to lock it to prevent + // the values from being resized. + assert(dict != NULL); + + bool success; + Py_BEGIN_CRITICAL_SECTION(dict); + + if (dict->ma_values == values && FT_ATOMIC_LOAD_UINT8(values->valid)) { + value = _Py_atomic_load_ptr_relaxed(&values->values[ix]); + *attr = Py_XNewRef(value); + success = true; + } else { + // Caller needs to lookup from the dictionary + success = false; + } + + Py_END_CRITICAL_SECTION(); + + return success; +#else PyObject *value = values->values[ix]; - return Py_XNewRef(value); + *attr = Py_XNewRef(value); + return true; +#endif } int @@ -6775,20 +6960,19 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) PyDictObject *dict; if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { PyDictValues *values = _PyObject_InlineValues(obj); - if (values->valid) { + if (FT_ATOMIC_LOAD_UINT8(values->valid)) { PyDictKeysObject *keys = CACHED_KEYS(tp); for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - if (values->values[i] != NULL) { + if (FT_ATOMIC_LOAD_PTR_RELAXED(values->values[i]) != NULL) { return 0; } } return 1; } - dict = _PyObject_ManagedDictPointer(obj)->dict; + dict = _PyObject_GetManagedDict(obj); } else if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj); - dict = managed_dict->dict; + dict = _PyObject_GetManagedDict(obj); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -6820,53 +7004,115 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) return 0; } +static void +set_dict_inline_values(PyObject *obj, PyDictObject *new_dict) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); + + PyDictValues *values = _PyObject_InlineValues(obj); + + Py_XINCREF(new_dict); + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, new_dict); + + if (values->valid) { + FT_ATOMIC_STORE_UINT8(values->valid, 0); + for (Py_ssize_t i = 0; i < values->capacity; i++) { + Py_CLEAR(values->values[i]); + } + } +} + void -PyObject_ClearManagedDict(PyObject *obj) +_PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict) { assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); assert(_PyObject_InlineValuesConsistencyCheck(obj)); PyTypeObject *tp = Py_TYPE(obj); if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { - PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict; - if (dict) { - _PyDict_DetachFromObject(dict, obj); - _PyObject_ManagedDictPointer(obj)->dict = NULL; - Py_DECREF(dict); - } - else { - PyDictValues *values = _PyObject_InlineValues(obj); - if (values->valid) { - for (Py_ssize_t i = 0; i < values->capacity; i++) { - Py_CLEAR(values->values[i]); - } - values->valid = 0; + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { +#ifdef Py_GIL_DISABLED + Py_BEGIN_CRITICAL_SECTION(obj); + + dict = _PyObject_ManagedDictPointer(obj)->dict; + if (dict == NULL) { + set_dict_inline_values(obj, (PyDictObject *)new_dict); + } + + Py_END_CRITICAL_SECTION(); + + if (dict == NULL) { + return; } +#else + set_dict_inline_values(obj, (PyDictObject *)new_dict); + return; +#endif } + + Py_BEGIN_CRITICAL_SECTION2(dict, obj); + + // We've locked dict, but the actual dict could have changed + // since we locked it. + dict = _PyObject_ManagedDictPointer(obj)->dict; + + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)Py_XNewRef(new_dict)); + + _PyDict_DetachFromObject(dict, obj); + + Py_END_CRITICAL_SECTION2(); + + Py_XDECREF(dict); } else { - Py_CLEAR(_PyObject_ManagedDictPointer(obj)->dict); + PyDictObject *dict; + + Py_BEGIN_CRITICAL_SECTION(obj); + + dict = _PyObject_ManagedDictPointer(obj)->dict; + + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)Py_XNewRef(new_dict)); + + Py_END_CRITICAL_SECTION(); + + Py_XDECREF(dict); } assert(_PyObject_InlineValuesConsistencyCheck(obj)); } +void +PyObject_ClearManagedDict(PyObject *obj) +{ + _PyObject_SetManagedDict(obj, NULL); +} + int _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) { - assert(_PyObject_ManagedDictPointer(obj)->dict == mp); - assert(_PyObject_InlineValuesConsistencyCheck(obj)); - if (mp->ma_values == NULL || mp->ma_values != _PyObject_InlineValues(obj)) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); + + if (FT_ATOMIC_LOAD_PTR_RELAXED(mp->ma_values) != _PyObject_InlineValues(obj)) { return 0; } + + // We could be called with an unlocked dict when the caller knows the + // values are already detached, so we assert after inline values check. + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); assert(mp->ma_values->embedded == 1); assert(mp->ma_values->valid == 1); assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - Py_BEGIN_CRITICAL_SECTION(mp); - mp->ma_values = copy_values(mp->ma_values); - _PyObject_InlineValues(obj)->valid = 0; - Py_END_CRITICAL_SECTION(); - if (mp->ma_values == NULL) { + + PyDictValues *values = copy_values(mp->ma_values); + + if (values == NULL) { return -1; } + mp->ma_values = values; + + FT_ATOMIC_STORE_UINT8(_PyObject_InlineValues(obj)->valid, 0); + assert(_PyObject_InlineValuesConsistencyCheck(obj)); ASSERT_CONSISTENT(mp); return 0; @@ -6877,29 +7123,28 @@ PyObject_GenericGetDict(PyObject *obj, void *context) { PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); + PyDictObject *dict; if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { - PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); - PyDictObject *dict = managed_dict->dict; + dict = _PyObject_GetManagedDict(obj); if (dict == NULL && (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && - _PyObject_InlineValues(obj)->valid - ) { - PyDictValues *values = _PyObject_InlineValues(obj); - OBJECT_STAT_INC(dict_materialized_on_request); - dict = make_dict_from_instance_attributes( - interp, CACHED_KEYS(tp), values); - if (dict != NULL) { - managed_dict->dict = (PyDictObject *)dict; - } + FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(obj)->valid)) { + dict = _PyObject_MaterializeManagedDict(obj); } - else { - dict = managed_dict->dict; + else if (dict == NULL) { + Py_BEGIN_CRITICAL_SECTION(obj); + + // Check again that we're not racing with someone else creating the dict + dict = _PyObject_GetManagedDict(obj); if (dict == NULL) { - dictkeys_incref(CACHED_KEYS(tp)); OBJECT_STAT_INC(dict_materialized_on_request); + dictkeys_incref(CACHED_KEYS(tp)); dict = (PyDictObject *)new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - managed_dict->dict = (PyDictObject *)dict; + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)dict); } + + Py_END_CRITICAL_SECTION(); } return Py_XNewRef((PyObject *)dict); } @@ -7109,7 +7354,7 @@ _PyObject_InlineValuesConsistencyCheck(PyObject *obj) return 1; } assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = (PyDictObject *)_PyObject_ManagedDictPointer(obj)->dict; + PyDictObject *dict = _PyObject_GetManagedDict(obj); if (dict == NULL) { return 1; } diff --git a/Objects/object.c b/Objects/object.c index 73a1927263cdcb..91bb0114cbfc32 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -6,6 +6,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() #include "pycore_context.h" // _PyContextTokenMissing_Type +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() @@ -25,6 +26,7 @@ #include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_initialize_generic #include "pycore_unionobject.h" // _PyUnion_Type + #ifdef Py_LIMITED_API // Prevent recursive call _Py_IncRef() <=> Py_INCREF() # error "Py_LIMITED_API macro must not be defined" @@ -1403,16 +1405,15 @@ _PyObject_GetDictPtr(PyObject *obj) if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return _PyObject_ComputedDictPointer(obj); } - PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); - if (managed_dict->dict == NULL && Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) { - PyDictObject *dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(obj); + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL && Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + dict = _PyObject_MaterializeManagedDict(obj); if (dict == NULL) { PyErr_Clear(); return NULL; } - managed_dict->dict = dict; } - return (PyObject **)&managed_dict->dict; + return (PyObject **)&_PyObject_ManagedDictPointer(obj)->dict; } PyObject * @@ -1480,10 +1481,9 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } } } - PyObject *dict; - if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) { - PyDictValues *values = _PyObject_InlineValues(obj); - PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name); + PyObject *dict, *attr; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_TryGetInstanceAttribute(obj, name, &attr)) { if (attr != NULL) { *method = attr; Py_XDECREF(descr); @@ -1492,8 +1492,7 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) dict = NULL; } else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj); - dict = (PyObject *)managed_dict->dict; + dict = (PyObject *)_PyObject_GetManagedDict(obj); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -1586,26 +1585,23 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, } } if (dict == NULL) { - if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) { - PyDictValues *values = _PyObject_InlineValues(obj); - if (PyUnicode_CheckExact(name)) { - res = _PyObject_GetInstanceAttribute(obj, values, name); + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES)) { + if (PyUnicode_CheckExact(name) && + _PyObject_TryGetInstanceAttribute(obj, name, &res)) { if (res != NULL) { goto done; } } else { - dict = (PyObject *)_PyObject_MakeDictFromInstanceAttributes(obj); + dict = (PyObject *)_PyObject_MaterializeManagedDict(obj); if (dict == NULL) { res = NULL; goto done; } - _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict; } } else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj); - dict = (PyObject *)managed_dict->dict; + dict = (PyObject *)_PyObject_GetManagedDict(obj); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -1700,12 +1696,13 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, if (dict == NULL) { PyObject **dictptr; - if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) { - res = _PyObject_StoreInstanceAttribute( - obj, _PyObject_InlineValues(obj), name, value); + + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES)) { + res = _PyObject_StoreInstanceAttribute(obj, name, value); goto error_check; } - else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + + if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); dictptr = (PyObject **)&managed_dict->dict; } @@ -1779,7 +1776,7 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) PyObject **dictptr = _PyObject_GetDictPtr(obj); if (dictptr == NULL) { if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) && - _PyObject_ManagedDictPointer(obj)->dict == NULL + _PyObject_GetManagedDict(obj) == NULL ) { /* Was unable to convert to dict */ PyErr_NoMemory(); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 970c82d2a17ada..808e11fcbaf1ff 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3165,9 +3165,9 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context) "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } + if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyObject_ClearManagedDict(obj); - _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)Py_XNewRef(value); + _PyObject_SetManagedDict(obj, value); } else { dictptr = _PyObject_ComputedDictPointer(obj); @@ -6194,15 +6194,27 @@ object_set_class(PyObject *self, PyObject *value, void *closure) /* Changing the class will change the implicit dict keys, * so we must materialize the dictionary first. */ if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) { - PyDictObject *dict = _PyObject_ManagedDictPointer(self)->dict; + PyDictObject *dict = _PyObject_MaterializeManagedDict(self); if (dict == NULL) { - dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(self); - if (dict == NULL) { - return -1; - } - _PyObject_ManagedDictPointer(self)->dict = dict; + return -1; + } + + bool error = false; + + Py_BEGIN_CRITICAL_SECTION2(self, dict); + + // If we raced after materialization and replaced the dict + // then the materialized dict should no longer have the + // inline values in which case detach is a nop. + assert(_PyObject_GetManagedDict(self) == dict || + dict->ma_values != _PyObject_InlineValues(self)); + + if (_PyDict_DetachFromObject(dict, self) < 0) { + error = true; } - if (_PyDict_DetachFromObject(dict, self)) { + + Py_END_CRITICAL_SECTION2(); + if (error) { return -1; } } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c1fbd3c7d26e01..b7511b9107fdf6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1947,15 +1947,13 @@ dummy_func( op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); - PyDictObject *dict = managed_dict->dict; + PyDictObject *dict = _PyObject_GetManagedDict(owner); DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); } op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { - PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); - PyDictObject *dict = managed_dict->dict; + PyDictObject *dict = _PyObject_GetManagedDict(owner); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { @@ -2072,14 +2070,15 @@ dummy_func( op(_GUARD_DORV_NO_DICT, (owner -- owner)) { assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict); + DEOPT_IF(_PyObject_GetManagedDict(owner)); DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0); } op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) { STAT_INC(STORE_ATTR, hit); - assert(_PyObject_ManagedDictPointer(owner)->dict == NULL); + assert(_PyObject_GetManagedDict(owner) == NULL); PyDictValues *values = _PyObject_InlineValues(owner); + PyObject *old_value = values->values[index]; values->values[index] = value; if (old_value == NULL) { @@ -2088,6 +2087,7 @@ dummy_func( else { Py_DECREF(old_value); } + Py_DECREF(owner); } @@ -2102,8 +2102,7 @@ dummy_func( assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); - PyDictObject *dict = managed_dict->dict; + PyDictObject *dict = _PyObject_GetManagedDict(owner); DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index df87f9178f17cf..841ce8cbedb3fb 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1998,8 +1998,7 @@ PyObject *owner; owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); - PyDictObject *dict = managed_dict->dict; + PyDictObject *dict = _PyObject_GetManagedDict(owner); if (dict == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2015,8 +2014,7 @@ oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t hint = (uint16_t)CURRENT_OPERAND(); - PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); - PyDictObject *dict = managed_dict->dict; + PyDictObject *dict = _PyObject_GetManagedDict(owner); if (hint >= (size_t)dict->ma_keys->dk_nentries) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2159,7 +2157,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_ManagedDictPointer(owner)->dict) { + if (_PyObject_GetManagedDict(owner)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -2177,7 +2175,7 @@ value = stack_pointer[-2]; uint16_t index = (uint16_t)CURRENT_OPERAND(); STAT_INC(STORE_ATTR, hit); - assert(_PyObject_ManagedDictPointer(owner)->dict == NULL); + assert(_PyObject_GetManagedDict(owner) == NULL); PyDictValues *values = _PyObject_InlineValues(owner); PyObject *old_value = values->values[index]; values->values[index] = value; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a426d9e208492e..058cac8bedd917 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4017,16 +4017,14 @@ // _CHECK_ATTR_WITH_HINT { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); - PyDictObject *dict = managed_dict->dict; + PyDictObject *dict = _PyObject_GetManagedDict(owner); DEOPT_IF(dict == NULL, LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); } // _LOAD_ATTR_WITH_HINT { uint16_t hint = read_u16(&this_instr[4].cache); - PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); - PyDictObject *dict = managed_dict->dict; + PyDictObject *dict = _PyObject_GetManagedDict(owner); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { @@ -5309,7 +5307,7 @@ { assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict, STORE_ATTR); + DEOPT_IF(_PyObject_GetManagedDict(owner), STORE_ATTR); DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0, STORE_ATTR); } // _STORE_ATTR_INSTANCE_VALUE @@ -5317,7 +5315,7 @@ { uint16_t index = read_u16(&this_instr[4].cache); STAT_INC(STORE_ATTR, hit); - assert(_PyObject_ManagedDictPointer(owner)->dict == NULL); + assert(_PyObject_GetManagedDict(owner) == NULL); PyDictValues *values = _PyObject_InlineValues(owner); PyObject *old_value = values->values[index]; values->values[index] = value; @@ -5380,8 +5378,7 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); - PyDictObject *dict = managed_dict->dict; + PyDictObject *dict = _PyObject_GetManagedDict(owner); DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); diff --git a/Python/specialize.c b/Python/specialize.c index 5e14bb56b30036..ee51781372166a 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -852,8 +852,7 @@ specialize_dict_access( instr->op.code = values_op; } else { - PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); - PyDictObject *dict = managed_dict->dict; + PyDictObject *dict = _PyObject_GetManagedDict(owner); if (dict == NULL || !PyDict_CheckExact(dict)) { SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); return 0; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index d17b2b9b024b99..18cefa08328804 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -354,6 +354,7 @@ def has_error_without_pop(op: parser.InstDef) -> bool: NON_ESCAPING_FUNCTIONS = ( "Py_INCREF", "_PyManagedDictPointer_IsValues", + "_PyObject_GetManagedDict", "_PyObject_ManagedDictPointer", "_PyObject_InlineValues", "_PyDictValues_AddToInsertionOrder", From 550483b7e6c54b2a25d4db0c4ca41bd9c1132f93 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 22 Apr 2024 08:43:20 +0200 Subject: [PATCH 33/37] gh-117995: Don't raise DeprecationWarnings for indexed nameless params (#118001) Filter out '?NNN' placeholders when looking for named params. Co-authored-by: AN Long --- Lib/test/test_sqlite3/test_dbapi.py | 14 ++++++++++++++ .../2024-04-17-19-41-59.gh-issue-117995.Vt76Rv.rst | 2 ++ Modules/_sqlite/cursor.c | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-17-19-41-59.gh-issue-117995.Vt76Rv.rst diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 4182de246a071b..6d8744ca5f7969 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -28,6 +28,7 @@ import threading import unittest import urllib.parse +import warnings from test.support import ( SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess, @@ -887,6 +888,19 @@ def test_execute_named_param_and_sequence(self): self.cu.execute(query, params) self.assertEqual(cm.filename, __file__) + def test_execute_indexed_nameless_params(self): + # See gh-117995: "'?1' is considered a named placeholder" + for query, params, expected in ( + ("select ?1, ?2", (1, 2), (1, 2)), + ("select ?2, ?1", (1, 2), (2, 1)), + ): + with self.subTest(query=query, params=params): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + cu = self.cu.execute(query, params) + actual, = cu.fetchall() + self.assertEqual(actual, expected) + def test_execute_too_many_params(self): category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER msg = "too many SQL variables" diff --git a/Misc/NEWS.d/next/Library/2024-04-17-19-41-59.gh-issue-117995.Vt76Rv.rst b/Misc/NEWS.d/next/Library/2024-04-17-19-41-59.gh-issue-117995.Vt76Rv.rst new file mode 100644 index 00000000000000..a289939d33e830 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-17-19-41-59.gh-issue-117995.Vt76Rv.rst @@ -0,0 +1,2 @@ +Don't raise :exc:`DeprecationWarning` when a :term:`sequence` of parameters +is used to bind indexed, nameless placeholders. See also :gh:`100668`. diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index f95df612328e57..950596ea82b568 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -669,7 +669,7 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, } for (i = 0; i < num_params; i++) { const char *name = sqlite3_bind_parameter_name(self->st, i+1); - if (name != NULL) { + if (name != NULL && name[0] != '?') { int ret = PyErr_WarnFormat(PyExc_DeprecationWarning, 1, "Binding %d ('%s') is a named parameter, but you " "supplied a sequence which requires nameless (qmark) " From ceb6038b053c403bed3ca3a8bd17b7e3fc9aab7d Mon Sep 17 00:00:00 2001 From: Kerim Kabirov Date: Mon, 22 Apr 2024 12:28:21 +0200 Subject: [PATCH 34/37] gh-115986 Improve pprint documentation accuracy (#117403) Co-authored-by: Jelle Zijlstra --- Doc/library/pprint.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index eebd270a096ba5..6dfea25d755f75 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -19,9 +19,8 @@ such as files, sockets or classes are included, as well as many other objects which are not representable as Python literals. The formatted representation keeps objects on a single line if it can, and -breaks them onto multiple lines if they don't fit within the allowed width. -Construct :class:`PrettyPrinter` objects explicitly if you need to adjust the -width constraint. +breaks them onto multiple lines if they don't fit within the allowed width, +adjustable by the *width* parameter defaulting to 80 characters. Dictionaries are sorted by key before the display is computed. From a6647d16abf4dd65997865e857371673238e60bf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 22 Apr 2024 13:34:06 +0100 Subject: [PATCH 35/37] GH-115480: Reduce guard strength for binary ops when type of one operand is known already (GH-118050) --- Include/internal/pycore_optimizer.h | 1 + Include/internal/pycore_uop_ids.h | 168 +++++++++++++------------ Include/internal/pycore_uop_metadata.h | 16 +++ Lib/test/test_capi/test_opt.py | 44 ++++++- Python/bytecodes.c | 16 +++ Python/executor_cases.c.h | 40 ++++++ Python/optimizer_analysis.c | 1 + Python/optimizer_bytecodes.c | 51 ++++++-- Python/optimizer_cases.c.h | 68 ++++++++-- Python/optimizer_symbols.c | 14 ++- 10 files changed, 316 insertions(+), 103 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 44cafe61b75596..c0a76e85350541 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -98,6 +98,7 @@ extern bool _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ); extern bool _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val); extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); extern int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym); +extern PyTypeObject *_Py_uop_sym_get_type(_Py_UopsSymbol *sym); extern int _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 3e4dd8b4009cd4..f0558743b32f5e 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -118,17 +118,21 @@ extern "C" { #define _GUARD_IS_NOT_NONE_POP 356 #define _GUARD_IS_TRUE_POP 357 #define _GUARD_KEYS_VERSION 358 -#define _GUARD_NOT_EXHAUSTED_LIST 359 -#define _GUARD_NOT_EXHAUSTED_RANGE 360 -#define _GUARD_NOT_EXHAUSTED_TUPLE 361 -#define _GUARD_TYPE_VERSION 362 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 363 -#define _INIT_CALL_PY_EXACT_ARGS 364 -#define _INIT_CALL_PY_EXACT_ARGS_0 365 -#define _INIT_CALL_PY_EXACT_ARGS_1 366 -#define _INIT_CALL_PY_EXACT_ARGS_2 367 -#define _INIT_CALL_PY_EXACT_ARGS_3 368 -#define _INIT_CALL_PY_EXACT_ARGS_4 369 +#define _GUARD_NOS_FLOAT 359 +#define _GUARD_NOS_INT 360 +#define _GUARD_NOT_EXHAUSTED_LIST 361 +#define _GUARD_NOT_EXHAUSTED_RANGE 362 +#define _GUARD_NOT_EXHAUSTED_TUPLE 363 +#define _GUARD_TOS_FLOAT 364 +#define _GUARD_TOS_INT 365 +#define _GUARD_TYPE_VERSION 366 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 367 +#define _INIT_CALL_PY_EXACT_ARGS 368 +#define _INIT_CALL_PY_EXACT_ARGS_0 369 +#define _INIT_CALL_PY_EXACT_ARGS_1 370 +#define _INIT_CALL_PY_EXACT_ARGS_2 371 +#define _INIT_CALL_PY_EXACT_ARGS_3 372 +#define _INIT_CALL_PY_EXACT_ARGS_4 373 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -145,65 +149,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 370 -#define _IS_NONE 371 +#define _INTERNAL_INCREMENT_OPT_COUNTER 374 +#define _IS_NONE 375 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 372 -#define _ITER_CHECK_RANGE 373 -#define _ITER_CHECK_TUPLE 374 -#define _ITER_JUMP_LIST 375 -#define _ITER_JUMP_RANGE 376 -#define _ITER_JUMP_TUPLE 377 -#define _ITER_NEXT_LIST 378 -#define _ITER_NEXT_RANGE 379 -#define _ITER_NEXT_TUPLE 380 -#define _JUMP_TO_TOP 381 +#define _ITER_CHECK_LIST 376 +#define _ITER_CHECK_RANGE 377 +#define _ITER_CHECK_TUPLE 378 +#define _ITER_JUMP_LIST 379 +#define _ITER_JUMP_RANGE 380 +#define _ITER_JUMP_TUPLE 381 +#define _ITER_NEXT_LIST 382 +#define _ITER_NEXT_RANGE 383 +#define _ITER_NEXT_TUPLE 384 +#define _JUMP_TO_TOP 385 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 382 -#define _LOAD_ATTR_CLASS 383 -#define _LOAD_ATTR_CLASS_0 384 -#define _LOAD_ATTR_CLASS_1 385 +#define _LOAD_ATTR 386 +#define _LOAD_ATTR_CLASS 387 +#define _LOAD_ATTR_CLASS_0 388 +#define _LOAD_ATTR_CLASS_1 389 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 386 -#define _LOAD_ATTR_INSTANCE_VALUE_0 387 -#define _LOAD_ATTR_INSTANCE_VALUE_1 388 -#define _LOAD_ATTR_METHOD_LAZY_DICT 389 -#define _LOAD_ATTR_METHOD_NO_DICT 390 -#define _LOAD_ATTR_METHOD_WITH_VALUES 391 -#define _LOAD_ATTR_MODULE 392 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 393 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 394 +#define _LOAD_ATTR_INSTANCE_VALUE 390 +#define _LOAD_ATTR_INSTANCE_VALUE_0 391 +#define _LOAD_ATTR_INSTANCE_VALUE_1 392 +#define _LOAD_ATTR_METHOD_LAZY_DICT 393 +#define _LOAD_ATTR_METHOD_NO_DICT 394 +#define _LOAD_ATTR_METHOD_WITH_VALUES 395 +#define _LOAD_ATTR_MODULE 396 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 397 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 398 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 395 -#define _LOAD_ATTR_SLOT_0 396 -#define _LOAD_ATTR_SLOT_1 397 -#define _LOAD_ATTR_WITH_HINT 398 +#define _LOAD_ATTR_SLOT 399 +#define _LOAD_ATTR_SLOT_0 400 +#define _LOAD_ATTR_SLOT_1 401 +#define _LOAD_ATTR_WITH_HINT 402 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 399 -#define _LOAD_CONST_INLINE_BORROW 400 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 401 -#define _LOAD_CONST_INLINE_WITH_NULL 402 +#define _LOAD_CONST_INLINE 403 +#define _LOAD_CONST_INLINE_BORROW 404 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 405 +#define _LOAD_CONST_INLINE_WITH_NULL 406 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 403 -#define _LOAD_FAST_0 404 -#define _LOAD_FAST_1 405 -#define _LOAD_FAST_2 406 -#define _LOAD_FAST_3 407 -#define _LOAD_FAST_4 408 -#define _LOAD_FAST_5 409 -#define _LOAD_FAST_6 410 -#define _LOAD_FAST_7 411 +#define _LOAD_FAST 407 +#define _LOAD_FAST_0 408 +#define _LOAD_FAST_1 409 +#define _LOAD_FAST_2 410 +#define _LOAD_FAST_3 411 +#define _LOAD_FAST_4 412 +#define _LOAD_FAST_5 413 +#define _LOAD_FAST_6 414 +#define _LOAD_FAST_7 415 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 412 -#define _LOAD_GLOBAL_BUILTINS 413 -#define _LOAD_GLOBAL_MODULE 414 +#define _LOAD_GLOBAL 416 +#define _LOAD_GLOBAL_BUILTINS 417 +#define _LOAD_GLOBAL_MODULE 418 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -217,49 +221,49 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 415 -#define _POP_JUMP_IF_FALSE 416 -#define _POP_JUMP_IF_TRUE 417 +#define _POP_FRAME 419 +#define _POP_JUMP_IF_FALSE 420 +#define _POP_JUMP_IF_TRUE 421 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 418 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 422 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 419 +#define _PUSH_FRAME 423 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 420 +#define _REPLACE_WITH_TRUE 424 #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 421 -#define _SEND 422 +#define _SAVE_RETURN_OFFSET 425 +#define _SEND 426 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _SIDE_EXIT 423 -#define _START_EXECUTOR 424 -#define _STORE_ATTR 425 -#define _STORE_ATTR_INSTANCE_VALUE 426 -#define _STORE_ATTR_SLOT 427 +#define _SIDE_EXIT 427 +#define _START_EXECUTOR 428 +#define _STORE_ATTR 429 +#define _STORE_ATTR_INSTANCE_VALUE 430 +#define _STORE_ATTR_SLOT 431 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 428 -#define _STORE_FAST_0 429 -#define _STORE_FAST_1 430 -#define _STORE_FAST_2 431 -#define _STORE_FAST_3 432 -#define _STORE_FAST_4 433 -#define _STORE_FAST_5 434 -#define _STORE_FAST_6 435 -#define _STORE_FAST_7 436 +#define _STORE_FAST 432 +#define _STORE_FAST_0 433 +#define _STORE_FAST_1 434 +#define _STORE_FAST_2 435 +#define _STORE_FAST_3 436 +#define _STORE_FAST_4 437 +#define _STORE_FAST_5 438 +#define _STORE_FAST_6 439 +#define _STORE_FAST_7 440 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 437 +#define _STORE_SUBSCR 441 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 438 +#define _TO_BOOL 442 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -269,12 +273,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 439 +#define _UNPACK_SEQUENCE 443 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 439 +#define MAX_UOP_ID 443 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 44ede3e77c68e1..4d15be6317d615 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -59,10 +59,14 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_REPLACE_WITH_TRUE] = 0, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BOTH_INT] = HAS_EXIT_FLAG, + [_GUARD_NOS_INT] = HAS_EXIT_FLAG, + [_GUARD_TOS_INT] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG, + [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, + [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, @@ -352,9 +356,13 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", + [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", + [_GUARD_NOS_INT] = "_GUARD_NOS_INT", [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", + [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", + [_GUARD_TOS_INT] = "_GUARD_TOS_INT", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", @@ -566,6 +574,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GUARD_BOTH_INT: return 2; + case _GUARD_NOS_INT: + return 2; + case _GUARD_TOS_INT: + return 1; case _BINARY_OP_MULTIPLY_INT: return 2; case _BINARY_OP_ADD_INT: @@ -574,6 +586,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _GUARD_BOTH_FLOAT: return 2; + case _GUARD_NOS_FLOAT: + return 2; + case _GUARD_TOS_FLOAT: + return 1; case _BINARY_OP_MULTIPLY_FLOAT: return 2; case _BINARY_OP_ADD_FLOAT: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 28d18739b6d4a5..ae23eadb8aafa0 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -903,10 +903,50 @@ def testfunc(n): self.assertTrue(res) self.assertIsNotNone(ex) uops = get_opnames(ex) - guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] - self.assertLessEqual(len(guard_both_float_count), 1) + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertLessEqual(len(guard_both_int_count), 1) self.assertIn("_COMPARE_OP_INT", uops) + def test_compare_op_type_propagation_int_partial(self): + def testfunc(n): + a = 1 + for _ in range(n): + if a > 2: + x = 0 + if a < 2: + x = 1 + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_left_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertLessEqual(len(guard_left_int_count), 1) + self.assertEqual(len(guard_both_int_count), 0) + self.assertIn("_COMPARE_OP_INT", uops) + + def test_compare_op_type_propagation_float_partial(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + if a > 2.0: + x = 0 + if a < 2.0: + x = 1 + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_left_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_left_float_count), 1) + self.assertEqual(len(guard_both_float_count), 0) + self.assertIn("_COMPARE_OP_FLOAT", uops) + def test_compare_op_type_propagation_unicode(self): def testfunc(n): a = "" diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b7511b9107fdf6..4541eb635da015 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -426,6 +426,14 @@ dummy_func( EXIT_IF(!PyLong_CheckExact(right)); } + op(_GUARD_NOS_INT, (left, unused -- left, unused)) { + EXIT_IF(!PyLong_CheckExact(left)); + } + + op(_GUARD_TOS_INT, (value -- value)) { + EXIT_IF(!PyLong_CheckExact(value)); + } + pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); @@ -462,6 +470,14 @@ dummy_func( EXIT_IF(!PyFloat_CheckExact(right)); } + op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { + EXIT_IF(!PyFloat_CheckExact(left)); + } + + op(_GUARD_TOS_FLOAT, (value -- value)) { + EXIT_IF(!PyFloat_CheckExact(value)); + } + pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); double dres = diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 841ce8cbedb3fb..43b022107a9ae6 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -447,6 +447,26 @@ break; } + case _GUARD_NOS_INT: { + PyObject *left; + left = stack_pointer[-2]; + if (!PyLong_CheckExact(left)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_TOS_INT: { + PyObject *value; + value = stack_pointer[-1]; + if (!PyLong_CheckExact(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + case _BINARY_OP_MULTIPLY_INT: { PyObject *right; PyObject *left; @@ -511,6 +531,26 @@ break; } + case _GUARD_NOS_FLOAT: { + PyObject *left; + left = stack_pointer[-2]; + if (!PyFloat_CheckExact(left)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_TOS_FLOAT: { + PyObject *value; + value = stack_pointer[-1]; + if (!PyFloat_CheckExact(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + case _BINARY_OP_MULTIPLY_FLOAT: { PyObject *right; PyObject *left; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 155f7026b041b0..76de6e50f1f786 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -320,6 +320,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_new_const _Py_uop_sym_new_const #define sym_new_null _Py_uop_sym_new_null #define sym_has_type _Py_uop_sym_has_type +#define sym_get_type _Py_uop_sym_get_type #define sym_matches_type _Py_uop_sym_matches_type #define sym_set_null _Py_uop_sym_set_null #define sym_set_non_null _Py_uop_sym_set_non_null diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index f119b8e20719fa..481fb8387af416 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -21,6 +21,7 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define sym_new_const _Py_uop_sym_new_const #define sym_new_null _Py_uop_sym_new_null #define sym_matches_type _Py_uop_sym_matches_type +#define sym_get_type _Py_uop_sym_get_type #define sym_has_type _Py_uop_sym_has_type #define sym_set_null _Py_uop_sym_set_null #define sym_set_non_null _Py_uop_sym_set_non_null @@ -99,9 +100,18 @@ dummy_func(void) { } op(_GUARD_BOTH_INT, (left, right -- left, right)) { - if (sym_matches_type(left, &PyLong_Type) && - sym_matches_type(right, &PyLong_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + if (sym_matches_type(left, &PyLong_Type)) { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_INT, 0, 0); + } + } + else { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0); + } } if (!sym_set_type(left, &PyLong_Type)) { goto hit_bottom; @@ -112,9 +122,18 @@ dummy_func(void) { } op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - if (sym_matches_type(left, &PyFloat_Type) && - sym_matches_type(right, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + if (sym_matches_type(left, &PyFloat_Type)) { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_FLOAT, 0, 0); + } + } + else { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0); + } } if (!sym_set_type(left, &PyFloat_Type)) { goto hit_bottom; @@ -137,6 +156,25 @@ dummy_func(void) { } } + op(_BINARY_OP, (left, right -- res)) { + PyTypeObject *ltype = sym_get_type(left); + PyTypeObject *rtype = sym_get_type(right); + if (ltype != NULL && (ltype == &PyLong_Type || ltype == &PyFloat_Type) && + rtype != NULL && (rtype == &PyLong_Type || rtype == &PyFloat_Type)) + { + if (oparg != NB_TRUE_DIVIDE && oparg != NB_INPLACE_TRUE_DIVIDE && + ltype == &PyLong_Type && rtype == &PyLong_Type) { + /* If both inputs are ints and the op is not division the result is an int */ + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + } + else { + /* For any other op combining ints/floats the result is a float */ + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } + } + OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + } + op(_BINARY_OP_ADD_INT, (left, right -- res)) { if (sym_is_const(left) && sym_is_const(right) && sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) @@ -424,7 +462,6 @@ dummy_func(void) { OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); } - op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { assert(oparg > 0); top = bottom; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 50f335e0c8a0a2..0a7d96d30ad3e8 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -225,9 +225,18 @@ _Py_UopsSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_matches_type(left, &PyLong_Type) && - sym_matches_type(right, &PyLong_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + if (sym_matches_type(left, &PyLong_Type)) { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_INT, 0, 0); + } + } + else { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0); + } } if (!sym_set_type(left, &PyLong_Type)) { goto hit_bottom; @@ -238,6 +247,14 @@ break; } + case _GUARD_NOS_INT: { + break; + } + + case _GUARD_TOS_INT: { + break; + } + case _BINARY_OP_MULTIPLY_INT: { _Py_UopsSymbol *right; _Py_UopsSymbol *left; @@ -333,9 +350,18 @@ _Py_UopsSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_matches_type(left, &PyFloat_Type) && - sym_matches_type(right, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + if (sym_matches_type(left, &PyFloat_Type)) { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_FLOAT, 0, 0); + } + } + else { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0); + } } if (!sym_set_type(left, &PyFloat_Type)) { goto hit_bottom; @@ -346,6 +372,14 @@ break; } + case _GUARD_NOS_FLOAT: { + break; + } + + case _GUARD_TOS_FLOAT: { + break; + } + case _BINARY_OP_MULTIPLY_FLOAT: { _Py_UopsSymbol *right; _Py_UopsSymbol *left; @@ -1852,9 +1886,27 @@ } case _BINARY_OP: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyTypeObject *ltype = sym_get_type(left); + PyTypeObject *rtype = sym_get_type(right); + if (ltype != NULL && (ltype == &PyLong_Type || ltype == &PyFloat_Type) && + rtype != NULL && (rtype == &PyLong_Type || rtype == &PyFloat_Type)) + { + if (oparg != NB_TRUE_DIVIDE && oparg != NB_INPLACE_TRUE_DIVIDE && + ltype == &PyLong_Type && rtype == &PyLong_Type) { + /* If both inputs are ints and the op is not division the result is an int */ + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + } + else { + /* For any other op combining ints/floats the result is a float */ + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } + } + OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); stack_pointer[-2] = res; stack_pointer += -1; break; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 86b0d4d395afa2..204599b08766c3 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -231,6 +231,15 @@ _Py_uop_sym_new_null(_Py_UOpsContext *ctx) return null_sym; } +PyTypeObject * +_Py_uop_sym_get_type(_Py_UopsSymbol *sym) +{ + if (_Py_uop_sym_is_bottom(sym)) { + return NULL; + } + return sym->typ; +} + bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym) { @@ -244,10 +253,7 @@ bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); - if (_Py_uop_sym_is_bottom(sym)) { - return false; - } - return sym->typ == typ; + return _Py_uop_sym_get_type(sym) == typ; } int From 287d939ed4445089e8312ab44110cbb6b6306a5c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 22 Apr 2024 16:27:47 +0300 Subject: [PATCH 36/37] gh-118148: Improve tests for shutil.make_archive() (GH-118149) --- Lib/test/test_shutil.py | 247 ++++++++++++++++++++++++++++------------ 1 file changed, 176 insertions(+), 71 deletions(-) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 60e88d57b2b23d..3aa0ec6952339b 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1615,42 +1615,6 @@ class TestArchives(BaseTest, unittest.TestCase): ### shutil.make_archive - @support.requires_zlib() - def test_make_tarball(self): - # creating something to tar - root_dir, base_dir = self._create_files('') - - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - tarball = make_archive(rel_base_name, 'gztar', root_dir, '.') - - # check if the compressed tarball was created - self.assertEqual(tarball, base_name + '.tar.gz') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r:gz') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) - - # trying an uncompressed one - with os_helper.change_cwd(work_dir), no_chdir: - tarball = make_archive(rel_base_name, 'tar', root_dir, '.') - self.assertEqual(tarball, base_name + '.tar') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) - def _tarinfo(self, path): with tarfile.open(path) as tar: names = tar.getnames() @@ -1671,6 +1635,92 @@ def _create_files(self, base_dir='dist'): write_file((root_dir, 'outer'), 'xxx') return root_dir, base_dir + @support.requires_zlib() + def test_make_tarfile(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir) + # check if the compressed tarball was created + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'tar', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + # check if the uncompressed tarball was created + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'tar', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist/sub', 'dist/sub/file3']) + + @support.requires_zlib() + def test_make_tarfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'gztar') + self.assertEqual(archive, base_name + '.tar.gz') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r:gz') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', base_dir=base_dir) + self.assertEqual(archive, base_name + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + def test_make_tarfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + @support.requires_zlib() @unittest.skipUnless(shutil.which('tar'), 'Need the tar command to run') @@ -1720,40 +1770,89 @@ def test_tarfile_vs_tar(self): @support.requires_zlib() def test_make_zipfile(self): - # creating something to zip root_dir, base_dir = self._create_files() + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'zip', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'zip', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/sub/', 'dist/sub/file3']) - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir) + @support.requires_zlib() + def test_make_zipfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'zip') + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + root_dir, base_dir = self._create_files() + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', base_dir=base_dir) + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3', - 'outer']) - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir, base_dir) - - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3']) + @support.requires_zlib() + def test_make_zipfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) @support.requires_zlib() @unittest.skipUnless(shutil.which('zip'), @@ -1923,17 +2022,19 @@ def archiver(base_name, base_dir, **kw): unregister_archive_format('xxx') def test_make_tarfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'tar'), 'test.tar') self.assertTrue(os.path.isfile('test.tar')) @support.requires_zlib() def test_make_zipfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'zip'), 'test.zip') self.assertTrue(os.path.isfile('test.zip')) @@ -1954,10 +2055,11 @@ def test_register_archive_format(self): self.assertNotIn('xxx', formats) def test_make_tarfile_rootdir_nodir(self): - # GH-99203 + # GH-99203: Test with root_dir is not a real directory. self.addCleanup(os_helper.unlink, f'{TESTFN}.tar') for dry_run in (False, True): with self.subTest(dry_run=dry_run): + # root_dir does not exist. tmp_dir = self.mkdtemp() nonexisting_file = os.path.join(tmp_dir, 'nonexisting') with self.assertRaises(FileNotFoundError) as cm: @@ -1966,6 +2068,7 @@ def test_make_tarfile_rootdir_nodir(self): self.assertEqual(cm.exception.filename, nonexisting_file) self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + # root_dir is a file. tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) os.close(tmp_fd) with self.assertRaises(NotADirectoryError) as cm: @@ -1976,10 +2079,11 @@ def test_make_tarfile_rootdir_nodir(self): @support.requires_zlib() def test_make_zipfile_rootdir_nodir(self): - # GH-99203 + # GH-99203: Test with root_dir is not a real directory. self.addCleanup(os_helper.unlink, f'{TESTFN}.zip') for dry_run in (False, True): with self.subTest(dry_run=dry_run): + # root_dir does not exist. tmp_dir = self.mkdtemp() nonexisting_file = os.path.join(tmp_dir, 'nonexisting') with self.assertRaises(FileNotFoundError) as cm: @@ -1988,6 +2092,7 @@ def test_make_zipfile_rootdir_nodir(self): self.assertEqual(cm.exception.filename, nonexisting_file) self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + # root_dir is a file. tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) os.close(tmp_fd) with self.assertRaises(NotADirectoryError) as cm: From 78ba4cb758ba1e40d27af6bc2fa15ed3e33a29d2 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Mon, 22 Apr 2024 16:57:46 +0200 Subject: [PATCH 37/37] gh-118030: Group definitions for `ParamSpecArgs` and `ParamSpecKwargs` in `typing.rst` (#118154) --- Doc/library/typing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 2e71be7eaf8d03..d816e6368f40d2 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1972,7 +1972,7 @@ without the dedicated syntax, as documented below. * :ref:`annotating-callables` .. data:: ParamSpecArgs -.. data:: ParamSpecKwargs + ParamSpecKwargs Arguments and keyword arguments attributes of a :class:`ParamSpec`. The ``P.args`` attribute of a ``ParamSpec`` is an instance of ``ParamSpecArgs``,