From a1dbf2ea69acc6ccee6292709af1dadd55c068be Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 11 Sep 2024 12:05:46 +0300 Subject: [PATCH 01/50] gh-77894: Fix a crash when the GC breaks a loop containing a memoryview (GH-123898) Now a memoryview object can only be cleared if there are no buffers that refer it. --- Lib/test/pickletester.py | 4 +- Lib/test/test_memoryview.py | 27 ++++++++- ...4-09-10-13-27-16.gh-issue-77894.ZC-Olu.rst | 4 ++ Objects/memoryobject.c | 55 +++++++++---------- 4 files changed, 57 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-10-13-27-16.gh-issue-77894.ZC-Olu.rst diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index c7dbd9978941de..334d4dfebdf893 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2323,12 +2323,10 @@ def test_picklebuffer_error(self): 'PickleBuffer can only be pickled with protocol >= 5') def test_non_continuous_buffer(self): - if self.pickler is pickle._Pickler: - self.skipTest('CRASHES (see gh-122306)') for proto in protocols[5:]: with self.subTest(proto=proto): pb = pickle.PickleBuffer(memoryview(b"foobar")[::2]) - with self.assertRaises(pickle.PicklingError): + with self.assertRaises((pickle.PicklingError, BufferError)): self.dumps(pb, proto) def test_buffer_callback_error(self): diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 0eb2a367603cfc..2d4bf5f1408df8 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -18,6 +18,10 @@ from test.support import import_helper +class MyObject: + pass + + class AbstractMemoryTests: source_bytes = b"abcdef" @@ -228,8 +232,6 @@ def __init__(self, base): self.m = memoryview(base) class MySource(tp): pass - class MyObject: - pass # Create a reference cycle through a memoryview object. # This exercises mbuf_clear(). @@ -656,5 +658,26 @@ def __bool__(self): m[0] = MyBool() self.assertEqual(ba[:8], b'\0'*8) + def test_buffer_reference_loop(self): + m = memoryview(b'abc').__buffer__(0) + o = MyObject() + o.m = m + o.o = o + wr = weakref.ref(o) + del m, o + gc.collect() + self.assertIsNone(wr()) + + def test_picklebuffer_reference_loop(self): + pb = pickle.PickleBuffer(memoryview(b'abc')) + o = MyObject() + o.pb = pb + o.o = o + wr = weakref.ref(o) + del pb, o + gc.collect() + self.assertIsNone(wr()) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-10-13-27-16.gh-issue-77894.ZC-Olu.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-10-13-27-16.gh-issue-77894.ZC-Olu.rst new file mode 100644 index 00000000000000..a714033dd296b9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-10-13-27-16.gh-issue-77894.ZC-Olu.rst @@ -0,0 +1,4 @@ +Fix possible crash in the garbage collector when it tries to break a +reference loop containing a :class:`memoryview` object. Now a +:class:`!memoryview` object can only be cleared if there are no buffers that +refer it. diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 498a37c1a3d869..a2472d4873641d 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -109,8 +109,6 @@ mbuf_release(_PyManagedBufferObject *self) if (self->flags&_Py_MANAGED_BUFFER_RELEASED) return; - /* NOTE: at this point self->exports can still be > 0 if this function - is called from mbuf_clear() to break up a reference cycle. */ self->flags |= _Py_MANAGED_BUFFER_RELEASED; /* PyBuffer_Release() decrements master->obj and sets it to NULL. */ @@ -1096,32 +1094,19 @@ PyBuffer_ToContiguous(void *buf, const Py_buffer *src, Py_ssize_t len, char orde /* Inform the managed buffer that this particular memoryview will not access the underlying buffer again. If no other memoryviews are registered with the managed buffer, the underlying buffer is released instantly and - marked as inaccessible for both the memoryview and the managed buffer. - - This function fails if the memoryview itself has exported buffers. */ -static int + marked as inaccessible for both the memoryview and the managed buffer. */ +static void _memory_release(PyMemoryViewObject *self) { + assert(self->exports == 0); if (self->flags & _Py_MEMORYVIEW_RELEASED) - return 0; + return; - if (self->exports == 0) { - self->flags |= _Py_MEMORYVIEW_RELEASED; - assert(self->mbuf->exports > 0); - if (--self->mbuf->exports == 0) - mbuf_release(self->mbuf); - return 0; + self->flags |= _Py_MEMORYVIEW_RELEASED; + assert(self->mbuf->exports > 0); + if (--self->mbuf->exports == 0) { + mbuf_release(self->mbuf); } - if (self->exports > 0) { - PyErr_Format(PyExc_BufferError, - "memoryview has %zd exported buffer%s", self->exports, - self->exports==1 ? "" : "s"); - return -1; - } - - PyErr_SetString(PyExc_SystemError, - "_memory_release(): negative export count"); - return -1; } /*[clinic input] @@ -1134,9 +1119,21 @@ static PyObject * memoryview_release_impl(PyMemoryViewObject *self) /*[clinic end generated code: output=d0b7e3ba95b7fcb9 input=bc71d1d51f4a52f0]*/ { - if (_memory_release(self) < 0) + if (self->exports == 0) { + _memory_release(self); + Py_RETURN_NONE; + } + + if (self->exports > 0) { + PyErr_Format(PyExc_BufferError, + "memoryview has %zd exported buffer%s", self->exports, + self->exports==1 ? "" : "s"); return NULL; - Py_RETURN_NONE; + } + + PyErr_SetString(PyExc_SystemError, + "memoryview: negative export count"); + return NULL; } static void @@ -1145,7 +1142,7 @@ memory_dealloc(PyObject *_self) PyMemoryViewObject *self = (PyMemoryViewObject *)_self; assert(self->exports == 0); _PyObject_GC_UNTRACK(self); - (void)_memory_release(self); + _memory_release(self); Py_CLEAR(self->mbuf); if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); @@ -1164,8 +1161,10 @@ static int memory_clear(PyObject *_self) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; - (void)_memory_release(self); - Py_CLEAR(self->mbuf); + if (self->exports == 0) { + _memory_release(self); + Py_CLEAR(self->mbuf); + } return 0; } From d2b9b6f919e92184420c8e13d078e83447ce7917 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 11 Sep 2024 12:21:05 +0300 Subject: [PATCH 02/50] gh-123811: test that round() can return signed zero (#123829) --- Lib/test/test_float.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 42fc99131472c5..048bb14509064b 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -863,7 +863,7 @@ def test_short_repr(self): self.assertEqual(repr(float(negs)), str(float(negs))) @support.requires_IEEE_754 -class RoundTestCase(unittest.TestCase): +class RoundTestCase(unittest.TestCase, FloatsAreIdenticalMixin): def test_inf_nan(self): self.assertRaises(OverflowError, round, INF) @@ -893,10 +893,10 @@ def test_large_n(self): def test_small_n(self): for n in [-308, -309, -400, 1-2**31, -2**31, -2**31-1, -2**100]: - self.assertEqual(round(123.456, n), 0.0) - self.assertEqual(round(-123.456, n), -0.0) - self.assertEqual(round(1e300, n), 0.0) - self.assertEqual(round(1e-320, n), 0.0) + self.assertFloatsAreIdentical(round(123.456, n), 0.0) + self.assertFloatsAreIdentical(round(-123.456, n), -0.0) + self.assertFloatsAreIdentical(round(1e300, n), 0.0) + self.assertFloatsAreIdentical(round(1e-320, n), 0.0) def test_overflow(self): self.assertRaises(OverflowError, round, 1.6e308, -308) From e9eedf19c99475b1940bbbbdc8816b51da3968e7 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Wed, 11 Sep 2024 19:44:46 +0800 Subject: [PATCH 03/50] Fix invisible character typo (#123933) Remove accidental addition of zero-width character (U+FEFF) reported by @jaraco: - https://github.com/python/cpython/commit/c3f4a6b52418d9b9f091f864cb6340d0d5fc6966#commitcomment-146456562 --- Lib/zipfile/_path/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py index 5079db90648006..c0e53e273cfaac 100644 --- a/Lib/zipfile/_path/__init__.py +++ b/Lib/zipfile/_path/__init__.py @@ -280,7 +280,7 @@ class Path: >>> str(path.parent) 'mem' - If the zipfile has no filename, such attributes are not + If the zipfile has no filename, such attributes are not valid and accessing them will raise an Exception. >>> zf.filename = None From c8d1dbef5b770b647aa7ff45fd5b269bc7629d0b Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 11 Sep 2024 16:37:35 +0300 Subject: [PATCH 04/50] gh-123919: Fix null handling in `_freeze_module.c` (#123920) --- Programs/_freeze_module.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 2a462a42cdad7c..891e4256e897ab 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -110,6 +110,9 @@ static PyObject * compile_and_marshal(const char *name, const char *text) { char *filename = (char *) malloc(strlen(name) + 10); + if (filename == NULL) { + return PyErr_NoMemory(); + } sprintf(filename, "", name); PyObject *code = Py_CompileStringExFlags(text, filename, Py_file_input, NULL, 0); @@ -133,6 +136,9 @@ get_varname(const char *name, const char *prefix) { size_t n = strlen(prefix); char *varname = (char *) malloc(strlen(name) + n + 1); + if (varname == NULL) { + return NULL; + } (void)strcpy(varname, prefix); for (size_t i = 0; name[i] != '\0'; i++) { if (name[i] == '.') { @@ -178,6 +184,11 @@ write_frozen(const char *outpath, const char *inpath, const char *name, fprintf(outfile, "%s\n", header); char *arrayname = get_varname(name, "_Py_M__"); + if (arrayname == NULL) { + fprintf(stderr, "memory error: could not allocate varname\n"); + fclose(outfile); + return -1; + } write_code(outfile, marshalled, arrayname); free(arrayname); From 6e23c89fcdd02b08fa6e9fa70d6e90763ddfc327 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:39:53 +0100 Subject: [PATCH 05/50] gh-123942: add missing test for docstring-handling code in ast_opt.c (#123943) --- Lib/test/test_compile.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 658ef11f7bf44b..7d6ddba1adc87e 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -870,6 +870,32 @@ def unused_code_at_end(): 'RETURN_CONST', list(dis.get_instructions(unused_code_at_end))[-1].opname) + @support.cpython_only + def test_docstring(self): + src = textwrap.dedent(""" + def with_docstring(): + "docstring" + + def with_fstring(): + f"not docstring" + + def with_const_expression(): + "also" + " not docstring" + """) + + for opt in [0, 1, 2]: + with self.subTest(opt=opt): + code = compile(src, "", "exec", optimize=opt) + ns = {} + exec(code, ns) + + if opt < 2: + self.assertEqual(ns['with_docstring'].__doc__, "docstring") + else: + self.assertIsNone(ns['with_docstring'].__doc__) + self.assertIsNone(ns['with_fstring'].__doc__) + self.assertIsNone(ns['with_const_expression'].__doc__) + @support.cpython_only def test_docstring_omitted(self): # See gh-115347 From 5436d8b9c397c48d9b0d5f9d4ad5e1d5a5d500f6 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 11 Sep 2024 07:49:59 -0700 Subject: [PATCH 06/50] gh-119180: Documentation for PEP 649 and 749 (#122235) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Carol Willing --- Doc/glossary.rst | 20 +- Doc/howto/annotations.rst | 21 +- Doc/library/__future__.rst | 14 +- Doc/library/annotationlib.rst | 349 +++++++++++++++++++++++++++++++ Doc/library/inspect.rst | 69 ++---- Doc/library/python.rst | 1 + Doc/library/typing.rst | 111 +++++++++- Doc/reference/compound_stmts.rst | 46 +++- Doc/reference/datamodel.rst | 104 ++++++++- Doc/reference/executionmodel.rst | 23 +- Doc/reference/simple_stmts.rst | 26 +-- 11 files changed, 680 insertions(+), 104 deletions(-) create mode 100644 Doc/library/annotationlib.rst diff --git a/Doc/glossary.rst b/Doc/glossary.rst index d9f9392c327f5c..ea2f4da093caa0 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -36,6 +36,12 @@ Glossary and loaders (in the :mod:`importlib.abc` module). You can create your own ABCs with the :mod:`abc` module. + annotate function + A function that can be called to retrieve the :term:`annotations ` + of an object. This function is accessible as the :attr:`~object.__annotate__` + attribute of functions, classes, and modules. Annotate functions are a + subset of :term:`evaluate functions `. + annotation A label associated with a variable, a class attribute or a function parameter or return value, @@ -43,12 +49,11 @@ Glossary Annotations of local variables cannot be accessed at runtime, but annotations of global variables, class attributes, and functions - are stored in the :attr:`__annotations__` - special attribute of modules, classes, and functions, - respectively. + can be retrieved by calling :func:`annotationlib.get_annotations` + on modules, classes, and functions, respectively. - See :term:`variable annotation`, :term:`function annotation`, :pep:`484` - and :pep:`526`, which describe this functionality. + See :term:`variable annotation`, :term:`function annotation`, :pep:`484`, + :pep:`526`, and :pep:`649`, which describe this functionality. Also see :ref:`annotations-howto` for best practices on working with annotations. @@ -366,6 +371,11 @@ Glossary statements. The technique contrasts with the :term:`LBYL` style common to many other languages such as C. + evaluate function + A function that can be called to evaluate a lazily evaluated attribute + of an object, such as the value of type aliases created with the :keyword:`type` + statement. + expression A piece of syntax which can be evaluated to some value. In other words, an expression is an accumulation of expression elements like literals, diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index be8c7e6c827f57..e9fc563f1b5880 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -34,11 +34,16 @@ Accessing The Annotations Dict Of An Object In Python 3.10 And Newer Python 3.10 adds a new function to the standard library: :func:`inspect.get_annotations`. In Python versions 3.10 -and newer, calling this function is the best practice for +through 3.13, calling this function is the best practice for accessing the annotations dict of any object that supports annotations. This function can also "un-stringize" stringized annotations for you. +In Python 3.14, there is a new :mod:`annotationlib` module +with functionality for working with annotations. This +includes a :func:`annotationlib.get_annotations` function, +which supersedes :func:`inspect.get_annotations`. + If for some reason :func:`inspect.get_annotations` isn't viable for your use case, you may access the ``__annotations__`` data member manually. Best practice @@ -184,7 +189,11 @@ Best Practices For ``__annotations__`` In Any Python Version * If you do assign directly to the ``__annotations__`` member of an object, you should always set it to a ``dict`` object. -* If you directly access the ``__annotations__`` member +* You should avoid accessing ``__annotations__`` directly on any object. + Instead, use :func:`annotationlib.get_annotations` (Python 3.14+) + or :func:`inspect.get_annotations` (Python 3.10+). + +* If you do directly access the ``__annotations__`` member of an object, you should ensure that it's a dictionary before attempting to examine its contents. @@ -231,3 +240,11 @@ itself be quoted. In effect the annotation is quoted This prints ``{'a': "'str'"}``. This shouldn't really be considered a "quirk"; it's mentioned here simply because it might be surprising. + +If you use a class with a custom metaclass and access ``__annotations__`` +on the class, you may observe unexpected behavior; see +:pep:`749 <749#pep749-metaclasses>` for some examples. You can avoid these +quirks by using :func:`annotationlib.get_annotations` on Python 3.14+ or +:func:`inspect.get_annotations` on Python 3.10+. On earlier versions of +Python, you can avoid these bugs by accessing the annotations from the +class's ``__dict__`` (e.g., ``cls.__dict__.get('__annotations__', None)``). diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 1ebff4409b1e95..6a1179434acd5a 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -64,8 +64,10 @@ language using this mechanism: | generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | | | | | *StopIteration handling inside generators* | +------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: | -| | | | *Postponed evaluation of annotations* | +| annotations | 3.7.0b1 | Never [1]_ | :pep:`563`: | +| | | | *Postponed evaluation of annotations*, | +| | | | :pep:`649`: *Deferred evalutation of | +| | | | annotations using descriptors* | +------------------+-------------+--------------+---------------------------------------------+ .. XXX Adding a new entry? Remember to update simple_stmts.rst, too. @@ -115,11 +117,9 @@ language using this mechanism: .. [1] ``from __future__ import annotations`` was previously scheduled to - become mandatory in Python 3.10, but the Python Steering Council - twice decided to delay the change - (`announcement for Python 3.10 `__; - `announcement for Python 3.11 `__). - No final decision has been made yet. See also :pep:`563` and :pep:`649`. + become mandatory in Python 3.10, but the change was delayed and ultimately + canceled. This feature will eventually be deprecated and removed. See + :pep:`649` and :pep:`749`. .. seealso:: diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst new file mode 100644 index 00000000000000..ecf56ed50b6a2a --- /dev/null +++ b/Doc/library/annotationlib.rst @@ -0,0 +1,349 @@ +:mod:`!annotationlib` --- Functionality for introspecting annotations +===================================================================== + +.. module:: annotationlib + :synopsis: Functionality for introspecting annotations + + +**Source code:** :source:`Lib/annotationlib.py` + +.. testsetup:: default + + import annotationlib + from annotationlib import * + +-------------- + +The :mod:`!annotationlib` module provides tools for introspecting +:term:`annotations ` on modules, classes, and functions. + +Annotations are :ref:`lazily evaluated ` and often contain +forward references to objects that are not yet defined when the annotation +is created. This module provides a set of low-level tools that can be used to retrieve annotations in a reliable way, even +in the presence of forward references and other edge cases. + +This module supports retrieving annotations in three main formats +(see :class:`Format`), each of which works best for different use cases: + +* :attr:`~Format.VALUE` evaluates the annotations and returns their value. + This is most straightforward to work with, but it may raise errors, + for example if the annotations contain references to undefined names. +* :attr:`~Format.FORWARDREF` returns :class:`ForwardRef` objects + for annotations that cannot be resolved, allowing you to inspect the + annotations without evaluating them. This is useful when you need to + work with annotations that may contain unresolved forward references. +* :attr:`~Format.SOURCE` returns the annotations as a string, similar + to how it would appear in the source file. This is useful for documentation + generators that want to display annotations in a readable way. + +The :func:`get_annotations` function is the main entry point for +retrieving annotations. Given a function, class, or module, it returns +an annotations dictionary in the requested format. This module also provides +functionality for working directly with the :term:`annotate function` +that is used to evaluate annotations, such as :func:`get_annotate_function` +and :func:`call_annotate_function`, as well as the +:func:`call_evaluate_function` function for working with +:term:`evaluate functions `. + + +.. seealso:: + + :pep:`649` proposed the current model for how annotations work in Python. + + :pep:`749` expanded on various aspects of :pep:`649` and introduced the + :mod:`!annotationlib` module. + + :ref:`annotations-howto` provides best practices for working with + annotations. + + :pypi:`typing-extensions` provides a backport of :func:`get_annotations` + that works on earlier versions of Python. + +Annotation semantics +-------------------- + +The way annotations are evaluated has changed over the history of Python 3, +and currently still depends on a :ref:`future import `. +There have been execution models for annotations: + +* *Stock semantics* (default in Python 3.0 through 3.13; see :pep:`3107` + and :pep:`526`): Annotations are evaluated eagerly, as they are + encountered in the source code. +* *Stringified annotations* (used with ``from __future__ import annotations`` + in Python 3.7 and newer; see :pep:`563`): Annotations are stored as + strings only. +* *Deferred evaluation* (default in Python 3.14 and newer; see :pep:`649` and + :pep:`749`): Annotations are evaluated lazily, only when they are accessed. + +As an example, consider the following program:: + + def func(a: Cls) -> None: + print(a) + + class Cls: pass + + print(func.__annotations__) + +This will behave as follows: + +* Under stock semantics (Python 3.13 and earlier), it will throw a + :exc:`NameError` at the line where ``func`` is defined, + because ``Cls`` is an undefined name at that point. +* Under stringified annotations (if ``from __future__ import annotations`` + is used), it will print ``{'a': 'Cls', 'return': 'None'}``. +* Under deferred evaluation (Python 3.14 and later), it will print + ``{'a': , 'return': None}``. + +Stock semantics were used when function annotations were first introduced +in Python 3.0 (by :pep:`3107`) because this was the simplest, most obvious +way to implement annotations. The same execution model was used when variable +annotations were introduced in Python 3.6 (by :pep:`526`). However, +stock semantics caused problems when using annotations as type hints, +such as a need to refer to names that are not yet defined when the +annotation is encountered. In addition, there were performance problems +with executing annotations at module import time. Therefore, in Python 3.7, +:pep:`563` introduced the ability to store annotations as strings using the +``from __future__ import annotations`` syntax. The plan at the time was to +eventually make this behavior the default, but a problem appeared: +stringified annotations are more difficult to process for those who +introspect annotations at runtime. An alternative proposal, :pep:`649`, +introduced the third execution model, deferred evaluation, and was implemented +in Python 3.14. Stringified annotations are still used if +``from __future__ import annotations`` is present, but this behavior will +eventually be removed. + +Classes +------- + +.. class:: Format + + An :class:`~enum.IntEnum` describing the formats in which annotations + can be returned. Members of the enum, or their equivalent integer values, + can be passed to :func:`get_annotations` and other functions in this + module, as well as to :attr:`~object.__annotate__` functions. + + .. attribute:: VALUE + :value: 1 + + Values are the result of evaluating the annotation expressions. + + .. attribute:: FORWARDREF + :value: 2 + + Values are real annotation values (as per :attr:`Format.VALUE` format) + for defined values, and :class:`ForwardRef` proxies for undefined + values. Real objects may contain references to, :class:`ForwardRef` + proxy objects. + + .. attribute:: SOURCE + :value: 3 + + Values are the text string of the annotation as it appears in the + source code, up to modifications including, but not restricted to, + whitespace normalizations and constant values optimizations. + + The exact values of these strings may change in future versions of Python. + + .. versionadded:: 3.14 + +.. class:: ForwardRef + + A proxy object for forward references in annotations. + + Instances of this class are returned when the :attr:`~Format.FORWARDREF` + format is used and annotations contain a name that cannot be resolved. + This can happen when a forward reference is used in an annotation, such as + when a class is referenced before it is defined. + + .. attribute:: __forward_arg__ + + A string containing the code that was evaluated to produce the + :class:`~ForwardRef`. The string may not be exactly equivalent + to the original source. + + .. method:: evaluate(*, globals=None, locals=None, type_params=None, owner=None) + + Evaluate the forward reference, returning its value. + + This may throw an exception, such as :exc:`NameError`, if the forward + reference refers to names that do not exist. The arguments to this + method can be used to provide bindings for names that would otherwise + be undefined. + + :class:`~ForwardRef` instances returned by :func:`get_annotations` + retain references to information about the scope they originated from, + so calling this method with no further arguments may be sufficient to + evaluate such objects. :class:`~ForwardRef` instances created by other + means may not have any information about their scope, so passing + arguments to this method may be necessary to evaluate them successfully. + + *globals* and *locals* are passed to :func:`eval`, representing + the global and local namespaces in which the name is evaluated. + *type_params*, if given, must be a tuple of + :ref:`type parameters ` that are in scope while the forward + reference is being evaluated. *owner* is the object that owns the + annotation from which the forward reference derives, usually a function, + class, or module. + + .. important:: + + Once a :class:`~ForwardRef` instance has been evaluated, it caches + the evaluated value, and future calls to :meth:`evaluate` will return + the cached value, regardless of the parameters passed in. + + .. versionadded:: 3.14 + + +Functions +--------- + +.. function:: call_annotate_function(annotate, format, *, owner=None) + + Call the :term:`annotate function` *annotate* with the given *format*, + a member of the :class:`Format` enum, and return the annotations + dictionary produced by the function. + + This helper function is required because annotate functions generated by + the compiler for functions, classes, and modules only support + the :attr:`~Format.VALUE` format when called directly. + To support other formats, this function calls the annotate function + in a special environment that allows it to produce annotations in the + other formats. This is a useful building block when implementing + functionality that needs to partially evaluate annotations while a class + is being constructed. + + *owner* is the object that owns the annotation function, usually + a function, class, or module. If provided, it is used in the + :attr:`~Format.FORWARDREF` format to produce a :class:`ForwardRef` + object that carries more information. + + .. seealso:: + + :PEP:`PEP 649 <649#the-stringizer-and-the-fake-globals-environment>` + contains an explanation of the implementation technique used by this + function. + + .. versionadded:: 3.14 + +.. function:: call_evaluate_function(evaluate, format, *, owner=None) + + Call the :term:`evaluate function` *evaluate* with the given *format*, + a member of the :class:`Format` enum, and return the value produced by + the function. This is similar to :func:`call_annotate_function`, + but the latter always returns a dictionary mapping strings to annotations, + while this function returns a single value. + + This is intended for use with the evaluate functions generated for lazily + evaluated elements related to type aliases and type parameters: + + * :meth:`typing.TypeAliasType.evaluate_value`, the value of type aliases + * :meth:`typing.TypeVar.evaluate_bound`, the bound of type variables + * :meth:`typing.TypeVar.evaluate_constraints`, the constraints of + type variables + * :meth:`typing.TypeVar.evaluate_default`, the default value of + type variables + * :meth:`typing.ParamSpec.evaluate_default`, the default value of + parameter specifications + * :meth:`typing.TypeVarTuple.evaluate_default`, the default value of + type variable tuples + + *owner* is the object that owns the evaluate function, such as the type + alias or type variable object. + + *format* can be used to control the format in which the value is returned: + + .. doctest:: + + >>> type Alias = undefined + >>> call_evaluate_function(Alias.evaluate_value, Format.VALUE) + Traceback (most recent call last): + ... + NameError: name 'undefined' is not defined + >>> call_evaluate_function(Alias.evaluate_value, Format.FORWARDREF) + ForwardRef('undefined') + >>> call_evaluate_function(Alias.evaluate_value, Format.SOURCE) + 'undefined' + + .. versionadded:: 3.14 + +.. function:: get_annotate_function(obj) + + Retrieve the :term:`annotate function` for *obj*. Return :const:`!None` + if *obj* does not have an annotate function. + + This is usually equivalent to accessing the :attr:`~object.__annotate__` + attribute of *obj*, but direct access to the attribute may return the wrong + object in certain situations involving metaclasses. This function should be + used instead of accessing the attribute directly. + + .. versionadded:: 3.14 + +.. function:: get_annotations(obj, *, globals=None, locals=None, eval_str=False, format=Format.VALUE) + + Compute the annotations dict for an object. + + *obj* may be a callable, class, module, or other object with + :attr:`~object.__annotate__` and :attr:`~object.__annotations__` attributes. + Passing in an object of any other type raises :exc:`TypeError`. + + The *format* parameter controls the format in which annotations are returned, + and must be a member of the :class:`Format` enum or its integer equivalent. + + Returns a dict. :func:`!get_annotations` returns a new dict every time + it's called; calling it twice on the same object will return two + different but equivalent dicts. + + This function handles several details for you: + + * If *eval_str* is true, values of type :class:`!str` will + be un-stringized using :func:`eval`. This is intended + for use with stringized annotations + (``from __future__ import annotations``). It is an error + to set *eval_str* to true with formats other than :attr:`Format.VALUE`. + * If *obj* doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes, as well as annotations + on metaclasses. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using ``getattr()`` and ``dict.get()`` for safety. + + *eval_str* controls whether or not values of type :class:`!str` are + replaced with the result of calling :func:`eval` on those values: + + * If eval_str is true, :func:`eval` is called on values of type + :class:`!str`. (Note that :func:`!get_annotations` doesn't catch + exceptions; if :func:`eval()` raises an exception, it will unwind + the stack past the :func:`!get_annotations` call.) + * If *eval_str* is false (the default), values of type :class:`!str` are + unchanged. + + *globals* and *locals* are passed in to :func:`eval`; see the documentation + for :func:`eval` for more information. If *globals* or *locals* + is :const:`!None`, this function may replace that value with a + context-specific default, contingent on ``type(obj)``: + + * If *obj* is a module, *globals* defaults to ``obj.__dict__``. + * If *obj* is a class, *globals* defaults to + ``sys.modules[obj.__module__].__dict__`` and *locals* defaults + to the *obj* class namespace. + * If *obj* is a callable, *globals* defaults to + :attr:`obj.__globals__ `, + although if *obj* is a wrapped function (using + :func:`functools.update_wrapper`) or a :class:`functools.partial` object, + it is unwrapped until a non-wrapped function is found. + + Calling :func:`!get_annotations` is best practice for accessing the + annotations dict of any object. See :ref:`annotations-howto` for + more information on annotations best practices. + + .. doctest:: + + >>> def f(a: int, b: str) -> float: + ... pass + >>> get_annotations(f) + {'a': , 'b': , 'return': } + + .. versionadded:: 3.14 diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index d19e779a52a6df..f55824ab5f6cd9 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -718,19 +718,19 @@ function. Accepts a wide range of Python callables, from plain functions and classes to :func:`functools.partial` objects. - For objects defined in modules using stringized annotations - (``from __future__ import annotations``), :func:`signature` will + If some of the annotations are strings (e.g., because + ``from __future__ import annotations`` was used), :func:`signature` will attempt to automatically un-stringize the annotations using - :func:`get_annotations`. The + :func:`annotationlib.get_annotations`. The *globals*, *locals*, and *eval_str* parameters are passed - into :func:`get_annotations` when resolving the - annotations; see the documentation for :func:`get_annotations` + into :func:`!annotationlib.get_annotations` when resolving the + annotations; see the documentation for :func:`!annotationlib.get_annotations` for instructions on how to use these parameters. Raises :exc:`ValueError` if no signature can be provided, and :exc:`TypeError` if that type of object is not supported. Also, if the annotations are stringized, and *eval_str* is not false, - the ``eval()`` call(s) to un-stringize the annotations in :func:`get_annotations` + the ``eval()`` call(s) to un-stringize the annotations in :func:`annotationlib.get_annotations` could potentially raise any kind of exception. A slash(/) in the signature of a function denotes that the parameters prior @@ -1247,62 +1247,19 @@ Classes and functions .. versionadded:: 3.4 -.. function:: get_annotations(obj, *, globals=None, locals=None, eval_str=False) +.. function:: get_annotations(obj, *, globals=None, locals=None, eval_str=False, format=annotationlib.Format.VALUE) Compute the annotations dict for an object. - ``obj`` may be a callable, class, or module. - Passing in an object of any other type raises :exc:`TypeError`. - - Returns a dict. ``get_annotations()`` returns a new dict every time - it's called; calling it twice on the same object will return two - different but equivalent dicts. - - This function handles several details for you: - - * If ``eval_str`` is true, values of type ``str`` will - be un-stringized using :func:`eval`. This is intended - for use with stringized annotations - (``from __future__ import annotations``). - * If ``obj`` doesn't have an annotations dict, returns an - empty dict. (Functions and methods always have an - annotations dict; classes, modules, and other types of - callables may not.) - * Ignores inherited annotations on classes. If a class - doesn't have its own annotations dict, returns an empty dict. - * All accesses to object members and dict values are done - using ``getattr()`` and ``dict.get()`` for safety. - * Always, always, always returns a freshly created dict. - - ``eval_str`` controls whether or not values of type ``str`` are replaced - with the result of calling :func:`eval` on those values: - - * If eval_str is true, :func:`eval` is called on values of type ``str``. - (Note that ``get_annotations`` doesn't catch exceptions; if :func:`eval` - raises an exception, it will unwind the stack past the ``get_annotations`` - call.) - * If eval_str is false (the default), values of type ``str`` are unchanged. - - ``globals`` and ``locals`` are passed in to :func:`eval`; see the documentation - for :func:`eval` for more information. If ``globals`` or ``locals`` - is ``None``, this function may replace that value with a context-specific - default, contingent on ``type(obj)``: - - * If ``obj`` is a module, ``globals`` defaults to ``obj.__dict__``. - * If ``obj`` is a class, ``globals`` defaults to - ``sys.modules[obj.__module__].__dict__`` and ``locals`` defaults - to the ``obj`` class namespace. - * If ``obj`` is a callable, ``globals`` defaults to - :attr:`obj.__globals__ `, - although if ``obj`` is a wrapped function (using - :func:`functools.update_wrapper`) it is first unwrapped. - - Calling ``get_annotations`` is best practice for accessing the - annotations dict of any object. See :ref:`annotations-howto` for - more information on annotations best practices. + This is an alias for :func:`annotationlib.get_annotations`; see the documentation + of that function for more information. .. versionadded:: 3.10 + .. versionchanged:: 3.14 + This function is now an alias for :func:`annotationlib.get_annotations`. + Calling it as ``inspect.get_annotations`` will continue to work. + .. _inspect-stack: diff --git a/Doc/library/python.rst b/Doc/library/python.rst index 610435999d9f48..c2c231af7c3033 100644 --- a/Doc/library/python.rst +++ b/Doc/library/python.rst @@ -25,4 +25,5 @@ overview: __future__.rst gc.rst inspect.rst + annotationlib.rst site.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index f8b533a5fc87aa..cb5b46f7167376 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1825,6 +1825,16 @@ without the dedicated syntax, as documented below. the bound is evaluated only when the attribute is accessed, not when the type variable is created (see :ref:`lazy-evaluation`). + .. method:: evaluate_bound + + An :term:`evaluate function` corresponding to the :attr:`~TypeVar.__bound__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~TypeVar.__bound__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + .. attribute:: __constraints__ A tuple containing the constraints of the type variable, if any. @@ -1835,6 +1845,16 @@ without the dedicated syntax, as documented below. the constraints are evaluated only when the attribute is accessed, not when the type variable is created (see :ref:`lazy-evaluation`). + .. method:: evaluate_constraints + + An :term:`evaluate function` corresponding to the :attr:`~TypeVar.__constraints__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~TypeVar.__constraints__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + .. attribute:: __default__ The default value of the type variable, or :data:`typing.NoDefault` if it @@ -1842,6 +1862,16 @@ without the dedicated syntax, as documented below. .. versionadded:: 3.13 + .. method:: evaluate_default + + An :term:`evaluate function` corresponding to the :attr:`~TypeVar.__default__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~TypeVar.__default__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + .. method:: has_default() Return whether or not the type variable has a default value. This is equivalent @@ -1980,6 +2010,16 @@ without the dedicated syntax, as documented below. .. versionadded:: 3.13 + .. method:: evaluate_default + + An :term:`evaluate function` corresponding to the :attr:`~TypeVarTuple.__default__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~TypeVarTuple.__default__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + .. method:: has_default() Return whether or not the type variable tuple has a default value. This is equivalent @@ -2076,6 +2116,16 @@ without the dedicated syntax, as documented below. .. versionadded:: 3.13 + .. method:: evaluate_default + + An :term:`evaluate function` corresponding to the :attr:`~ParamSpec.__default__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~ParamSpec.__default__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + .. method:: has_default() Return whether or not the parameter specification has a default value. This is equivalent @@ -2200,6 +2250,32 @@ without the dedicated syntax, as documented below. >>> Recursive.__value__ Mutually + .. method:: evaluate_value + + An :term:`evaluate function` corresponding to the :attr:`__value__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`__value__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format: + + .. doctest:: + + >>> type Alias = undefined + >>> Alias.__value__ + Traceback (most recent call last): + ... + NameError: name 'undefined' is not defined + >>> from annotationlib import Format, call_evaluate_function + >>> Alias.evaluate_value(Format.VALUE) + Traceback (most recent call last): + ... + NameError: name 'undefined' is not defined + >>> call_evaluate_function(Alias.evaluate_value, Format.FORWARDREF) + ForwardRef('undefined') + + .. versionadded:: 3.14 + + Other special directives """""""""""""""""""""""" @@ -3306,7 +3382,7 @@ Introspection helpers Class used for internal typing representation of string forward references. For example, ``List["SomeClass"]`` is implicitly transformed into - ``List[ForwardRef("SomeClass")]``. ``ForwardRef`` should not be instantiated by + ``List[ForwardRef("SomeClass")]``. :class:`!ForwardRef` should not be instantiated by a user, but may be used by introspection tools. .. note:: @@ -3316,6 +3392,39 @@ Introspection helpers .. versionadded:: 3.7.4 + .. versionchanged:: 3.14 + This is now an alias for :class:`annotationlib.ForwardRef`. + +.. function:: evaluate_forward_ref(forward_ref, *, owner=None, globals=None, locals=None, type_params=None, format=annotationlib.Format.VALUE) + + Evaluate an :class:`annotationlib.ForwardRef` as a :term:`type hint`. + + This is similar to calling :meth:`annotationlib.ForwardRef.evaluate`, + but unlike that method, :func:`!evaluate_forward_ref` also: + + * Recursively evaluates forward references nested within the type hint. + * Raises :exc:`TypeError` when it encounters certain objects that are + not valid type hints. + * Replaces type hints that evaluate to :const:`!None` with + :class:`types.NoneType`. + * Supports the :attr:`~annotationlib.Format.FORWARDREF` and + :attr:`~annotationlib.Format.SOURCE` formats. + + *forward_ref* must be an instance of :class:`~annotationlib.ForwardRef`. + *owner*, if given, should be the object that holds the annotations that + the forward reference derived from, such as a module, class object, or function. + It is used to infer the namespaces to use for looking up names. + *globals* and *locals* can also be explicitly given to provide + the global and local namespaces. + *type_params* is a tuple of :ref:`type parameters ` that + are in scope when evaluating the forward reference. + This parameter must be provided (though it may be an empty tuple) if *owner* + is not given and the forward reference does not already have an owner set. + *format* specifies the format of the annotation and is a member of + the :class:`annotationlib.Format` enum. + + .. versionadded:: 3.14 + .. data:: NoDefault A sentinel object used to indicate that a type parameter has no default diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 46ee3a174f3d0f..d31fbf87b739dc 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -1329,13 +1329,7 @@ following the parameter name. Any parameter may have an annotation, even those ``*identifier`` or ``**identifier``. Functions may have "return" annotation of the form "``-> expression``" after the parameter list. These annotations can be any valid Python expression. The presence of annotations does not change the -semantics of a function. The annotation values are available as values of -a dictionary keyed by the parameters' names in the :attr:`__annotations__` -attribute of the function object. If the ``annotations`` import from -:mod:`__future__` is used, annotations are preserved as strings at runtime which -enables postponed evaluation. Otherwise, they are evaluated when the function -definition is executed. In this case annotations may be evaluated in -a different order than they appear in the source code. +semantics of a function. See :ref:`annotations` for more information on annotations. .. index:: pair: lambda; expression @@ -1852,6 +1846,44 @@ Here, ``annotation-def`` (not a real keyword) indicates an :ref:`annotation scope `. The capitalized names like ``TYPE_PARAMS_OF_ListOrSet`` are not actually bound at runtime. +.. _annotations: + +Annotations +=========== + +.. versionchanged:: 3.14 + Annotations are now lazily evaluated by default. + +Variables and function parameters may carry :term:`annotations `, +created by adding a colon after the name, followed by an expression:: + + x: annotation = 1 + def f(param: annotation): ... + +Functions may also carry a return annotation following an arrow:: + + def f() -> annotation: ... + +Annotations are conventionally used for :term:`type hints `, but this +is not enforced by the language, and in general annotations may contain arbitrary +expressions. The presence of annotations does not change the runtime semantics of +the code, except if some mechanism is used that introspects and uses the annotations +(such as :mod:`dataclasses` or :func:`functools.singledispatch`). + +By default, annotations are lazily evaluated in a :ref:`annotation scope `. +This means that they are not evaluated when the code containing the annotation is evaluated. +Instead, the interpreter saves information that can be used to evaluate the annotation later +if requested. The :mod:`annotationlib` module provides tools for evaluating annotations. + +If the :ref:`future statement ` ``from __future__ import annotations`` is present, +all annotations are instead stored as strings:: + + >>> from __future__ import annotations + >>> def f(param: annotation): ... + >>> f.__annotations__ + {'param': 'annotation'} + + .. rubric:: Footnotes .. [#] The exception is propagated to the invocation stack unless diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index ee9703819e2674..21aee0b6d0e3c5 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -581,6 +581,7 @@ Special writable attributes single: __defaults__ (function attribute) single: __code__ (function attribute) single: __annotations__ (function attribute) + single: __annotate__ (function attribute) single: __kwdefaults__ (function attribute) single: __type_params__ (function attribute) @@ -628,7 +629,17 @@ Most of these attributes check the type of the assigned value: :term:`parameters `. The keys of the dictionary are the parameter names, and ``'return'`` for the return annotation, if provided. - See also: :ref:`annotations-howto`. + See also: :attr:`object.__annotations__`. + + .. versionchanged:: 3.14 + Annotations are now :ref:`lazily evaluated `. + See :pep:`649`. + + * - .. attribute:: function.__annotate__ + - The :term:`annotate function` for this function, or ``None`` + if the function has no annotations. See :attr:`object.__annotate__`. + + .. versionadded:: 3.14 * - .. attribute:: function.__kwdefaults__ - A :class:`dictionary ` containing defaults for keyword-only @@ -881,6 +892,7 @@ Attribute assignment updates the module's namespace dictionary, e.g., single: __doc__ (module attribute) single: __file__ (module attribute) single: __annotations__ (module attribute) + single: __annotate__ (module attribute) pair: module; namespace Predefined (writable) attributes: @@ -901,11 +913,21 @@ Predefined (writable) attributes: loaded dynamically from a shared library, it's the pathname of the shared library file. - :attr:`__annotations__` + :attr:`~object.__annotations__` A dictionary containing :term:`variable annotations ` collected during module body execution. For best practices on working - with :attr:`__annotations__`, please see :ref:`annotations-howto`. + with :attr:`!__annotations__`, see :mod:`annotationlib`. + + .. versionchanged:: 3.14 + Annotations are now :ref:`lazily evaluated `. + See :pep:`649`. + + :attr:`~object.__annotate__` + The :term:`annotate function` for this module, or ``None`` + if the module has no annotations. See :attr:`object.__annotate__`. + + .. versionadded:: 3.14 .. index:: single: __dict__ (module attribute) @@ -969,6 +991,7 @@ A class object can be called (see above) to yield a class instance (see below). single: __bases__ (class attribute) single: __doc__ (class attribute) single: __annotations__ (class attribute) + single: __annotate__ (class attribute) single: __type_params__ (class attribute) single: __static_attributes__ (class attribute) single: __firstlineno__ (class attribute) @@ -991,12 +1014,36 @@ Special attributes: :attr:`__doc__` The class's documentation string, or ``None`` if undefined. - :attr:`__annotations__` + :attr:`~object.__annotations__` A dictionary containing :term:`variable annotations ` collected during class body execution. For best practices on - working with :attr:`__annotations__`, please see - :ref:`annotations-howto`. + working with :attr:`~object.__annotations__`, please see + :mod:`annotationlib`. + + .. warning:: + + Accessing the :attr:`~object.__annotations__` attribute of a class + object directly may yield incorrect results in the presence of + metaclasses. Use :func:`annotationlib.get_annotations` to + retrieve class annotations safely. + + .. versionchanged:: 3.14 + Annotations are now :ref:`lazily evaluated `. + See :pep:`649`. + + :attr:`~object.__annotate__` + The :term:`annotate function` for this class, or ``None`` + if the class has no annotations. See :attr:`object.__annotate__`. + + .. warning:: + + Accessing the :attr:`~object.__annotate__` attribute of a class + object directly may yield incorrect results in the presence of + metaclasses. Use :func:`annotationlib.get_annotate_function` to + retrieve the annotate function safely. + + .. versionadded:: 3.14 :attr:`__type_params__` A tuple containing the :ref:`type parameters ` of @@ -3253,6 +3300,51 @@ implement the protocol in Python. :class:`collections.abc.Buffer` ABC for buffer types. +Annotations +----------- + +Functions, classes, and modules may contain :term:`annotations `, +which are a way to associate information (usually :term:`type hints `) +with a symbol. + +.. attribute:: object.__annotations__ + + This attribute contains the annotations for an object. It is + :ref:`lazily evaluated `, so accessing the attribute may + execute arbitrary code and raise exceptions. If evaluation is successful, the + attribute is set to a dictionary mapping from variable names to annotations. + + .. versionchanged:: 3.14 + Annotations are now lazily evaluated. + +.. method:: object.__annotate__(format) + + An :term:`annotate function`. + Returns a new dictionary object mapping attribute/parameter names to their annotation values. + + Takes a format parameter specifying the format in which annotations values should be provided. + It must be a member of the :class:`annotationlib.Format` enum, or an integer with + a value corresponding to a member of the enum. + + If an annotate function doesn't support the requested format, it must raise + :exc:`NotImplementedError`. Annotate functions must always support + :attr:`~annotationlib.Format.VALUE` format; they must not raise + :exc:`NotImplementedError()` when called with this format. + + When called with :attr:`~annotationlib.Format.VALUE` format, an annotate function may raise + :exc:`NameError`; it must not raise :exc:`!NameError` when called requesting any other format. + + If an object does not have any annotations, :attr:`~object.__annotate__` should preferably be set + to ``None`` (it can’t be deleted), rather than set to a function that returns an empty dict. + + .. versionadded:: 3.14 + +.. seealso:: + + :pep:`649` --- Deferred evaluation of annotation using descriptors + Introduces lazy evaluation of annotations and the ``__annotate__`` function. + + .. _special-lookup: Special method lookup diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index f24e1537af39ed..a02b5153ef0620 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -190,14 +190,15 @@ However, the following will succeed:: Annotation scopes ----------------- -:ref:`Type parameter lists ` and :keyword:`type` statements +:term:`Annotations `, :ref:`type parameter lists ` +and :keyword:`type` statements introduce *annotation scopes*, which behave mostly like function scopes, -but with some exceptions discussed below. :term:`Annotations ` -currently do not use annotation scopes, but they are expected to use -annotation scopes in Python 3.13 when :pep:`649` is implemented. +but with some exceptions discussed below. Annotation scopes are used in the following contexts: +* :term:`Function annotations `. +* :term:`Variable annotations `. * Type parameter lists for :ref:`generic type aliases `. * Type parameter lists for :ref:`generic functions `. A generic function's annotations are @@ -236,17 +237,23 @@ Annotation scopes differ from function scopes in the following ways: Annotation scopes are also used for type parameter defaults, as introduced by :pep:`696`. +.. versionchanged:: 3.14 + Annotation scopes are now also used for annotations, as specified in + :pep:`649` and :pep:`749`. + .. _lazy-evaluation: Lazy evaluation --------------- -The values of type aliases created through the :keyword:`type` statement are -*lazily evaluated*. The same applies to the bounds, constraints, and default values of type +Most annotation scopes are *lazily evaluated*. This includes annotations, +the values of type aliases created through the :keyword:`type` statement, and +the bounds, constraints, and default values of type variables created through the :ref:`type parameter syntax `. This means that they are not evaluated when the type alias or type variable is -created. Instead, they are only evaluated when doing so is necessary to resolve -an attribute access. +created, or when the object carrying annotations is created. Instead, they +are only evaluated when necessary, for example when the ``__value__`` +attribute on a type alias is accessed. Example: diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 618664b23f0680..24df4a6ba7b678 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -336,23 +336,21 @@ The difference from normal :ref:`assignment` is that only a single target is all The assignment target is considered "simple" if it consists of a single name that is not enclosed in parentheses. For simple assignment targets, if in class or module scope, -the annotations are evaluated and stored in a special class or module -attribute :attr:`__annotations__` -that is a dictionary mapping from variable names (mangled if private) to -evaluated annotations. This attribute is writable and is automatically -created at the start of class or module body execution, if annotations -are found statically. +the annotations are gathered in a lazily evaluated +:ref:`annotation scope `. The annotations can be +evaluated using the :attr:`~object.__annotations__` attribute of a +class or module, or using the facilities in the :mod:`annotationlib` +module. If the assignment target is not simple (an attribute, subscript node, or -parenthesized name), the annotation is evaluated if -in class or module scope, but not stored. +parenthesized name), the annotation is never evaluated. If a name is annotated in a function scope, then this name is local for that scope. Annotations are never evaluated and stored in function scopes. If the right hand side is present, an annotated -assignment performs the actual assignment before evaluating annotations -(where applicable). If the right hand side is not present for an expression +assignment performs the actual assignment as if there was no annotation +present. If the right hand side is not present for an expression target, then the interpreter evaluates the target except for the last :meth:`~object.__setitem__` or :meth:`~object.__setattr__` call. @@ -373,6 +371,10 @@ target, then the interpreter evaluates the target except for the last regular assignments. Previously, some expressions (like un-parenthesized tuple expressions) caused a syntax error. +.. versionchanged:: 3.14 + Annotations are now lazily evaluated in a separate :ref:`annotation scope `. + If the assignment target is not simple, annotations are never evaluated. + .. _assert: @@ -975,8 +977,8 @@ block textually preceding that :keyword:`!global` statement. Names listed in a :keyword:`global` statement must not be defined as formal parameters, or as targets in :keyword:`with` statements or :keyword:`except` clauses, or in a :keyword:`for` target list, :keyword:`class` -definition, function definition, :keyword:`import` statement, or variable -annotation. +definition, function definition, :keyword:`import` statement, or +:term:`variable annotations `. .. impl-detail:: From ef05801ba0cbf090034df17e2a0420fb42c2d538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 11 Sep 2024 17:43:24 +0200 Subject: [PATCH 07/50] gh-108951: Document how to terminate an asyncio.TaskGroup (#123837) We don't want to add another API, since the recipe is straightforward and rarely needed. The advantage is that we could backport this to the earliest Python version that has taskgroups (3.11, alas in security mode already, so we'll just do 3.12 and 3.13). --- Doc/library/asyncio-task.rst | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index abf1726b34f539..4716a3f9c8ac79 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -414,6 +414,53 @@ reported by :meth:`asyncio.Task.cancelling`. Improved handling of simultaneous internal and external cancellations and correct preservation of cancellation counts. +Terminating a Task Group +------------------------ + +While terminating a task group is not natively supported by the standard +library, termination can be achieved by adding an exception-raising task +to the task group and ignoring the raised exception: + +.. code-block:: python + + import asyncio + from asyncio import TaskGroup + + class TerminateTaskGroup(Exception): + """Exception raised to terminate a task group.""" + + async def force_terminate_task_group(): + """Used to force termination of a task group.""" + raise TerminateTaskGroup() + + async def job(task_id, sleep_time): + print(f'Task {task_id}: start') + await asyncio.sleep(sleep_time) + print(f'Task {task_id}: done') + + async def main(): + try: + async with TaskGroup() as group: + # spawn some tasks + group.create_task(job(1, 0.5)) + group.create_task(job(2, 1.5)) + # sleep for 1 second + await asyncio.sleep(1) + # add an exception-raising task to force the group to terminate + group.create_task(force_terminate_task_group()) + except* TerminateTaskGroup: + pass + + asyncio.run(main()) + +Expected output: + +.. code-block:: text + + Task 1: start + Task 2: start + Task 1: done + Sleeping ======== From 2938c3dec99390087490124c2ef50e1592671e72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 11 Sep 2024 17:49:33 +0200 Subject: [PATCH 08/50] gh-123290: fix reference leaks in the error-branches of `_cursesmodule.c` (#123953) --- Modules/_cursesmodule.c | 358 +++++++++++++++++++++++----------------- 1 file changed, 211 insertions(+), 147 deletions(-) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index b5854e8c33f28a..1d36b40d5236b8 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -716,9 +716,13 @@ PyCursesWindow_New(WINDOW *win, const char *encoding) static void PyCursesWindow_Dealloc(PyCursesWindowObject *wo) { - if (wo->win != stdscr) delwin(wo->win); - if (wo->encoding != NULL) + if (wo->win != stdscr && wo->win != NULL) { + // silently ignore errors in delwin(3) + (void)delwin(wo->win); + } + if (wo->encoding != NULL) { PyMem_Free(wo->encoding); + } PyObject_Free(wo); } @@ -3266,7 +3270,6 @@ _curses_initscr_impl(PyObject *module) /*[clinic end generated code: output=619fb68443810b7b input=514f4bce1821f6b5]*/ { WINDOW *win; - PyCursesWindowObject *winobj; if (initialised) { wrefresh(stdscr); @@ -3284,12 +3287,17 @@ _curses_initscr_impl(PyObject *module) /* This was moved from initcurses() because it core dumped on SGI, where they're not defined until you've called initscr() */ -#define SetDictInt(string,ch) \ - do { \ - PyObject *o = PyLong_FromLong((long) (ch)); \ - if (o && PyDict_SetItemString(ModDict, string, o) == 0) { \ - Py_DECREF(o); \ - } \ +#define SetDictInt(NAME, VALUE) \ + do { \ + PyObject *value = PyLong_FromLong((long)(VALUE)); \ + if (value == NULL) { \ + return NULL; \ + } \ + int rc = PyDict_SetItemString(ModDict, (NAME), value); \ + Py_DECREF(value); \ + if (rc < 0) { \ + return NULL; \ + } \ } while (0) /* Here are some graphic symbols you can use */ @@ -3361,8 +3369,12 @@ _curses_initscr_impl(PyObject *module) SetDictInt("LINES", LINES); SetDictInt("COLS", COLS); +#undef SetDictInt - winobj = (PyCursesWindowObject *)PyCursesWindow_New(win, NULL); + PyCursesWindowObject *winobj = (PyCursesWindowObject *)PyCursesWindow_New(win, NULL); + if (winobj == NULL) { + return NULL; + } screen_encoding = winobj->encoding; return (PyObject *)winobj; } @@ -3970,46 +3982,47 @@ _curses_qiflush_impl(PyObject *module, int flag) static int update_lines_cols(void) { - PyObject *o; - PyObject *m = PyImport_ImportModule("curses"); + PyObject *exposed_module = NULL, *o = NULL; - if (!m) - return 0; + exposed_module = PyImport_ImportModule("curses"); + if (exposed_module == NULL) { + goto error; + } + PyObject *exposed_module_dict = PyModule_GetDict(exposed_module); // borrowed + if (exposed_module_dict == NULL) { + goto error; + } o = PyLong_FromLong(LINES); - if (!o) { - Py_DECREF(m); - return 0; + if (o == NULL) { + goto error; } - if (PyObject_SetAttrString(m, "LINES", o)) { - Py_DECREF(m); - Py_DECREF(o); - return 0; + if (PyDict_SetItemString(exposed_module_dict, "LINES", o) < 0) { + goto error; } - if (PyDict_SetItemString(ModDict, "LINES", o)) { - Py_DECREF(m); - Py_DECREF(o); - return 0; + if (PyDict_SetItemString(ModDict, "LINES", o) < 0) { + goto error; } Py_DECREF(o); + o = PyLong_FromLong(COLS); - if (!o) { - Py_DECREF(m); - return 0; + if (o == NULL) { + goto error; } - if (PyObject_SetAttrString(m, "COLS", o)) { - Py_DECREF(m); - Py_DECREF(o); - return 0; + if (PyDict_SetItemString(exposed_module_dict, "COLS", o) < 0) { + goto error; } - if (PyDict_SetItemString(ModDict, "COLS", o)) { - Py_DECREF(m); - Py_DECREF(o); - return 0; + if (PyDict_SetItemString(ModDict, "COLS", o) < 0) { + goto error; } Py_DECREF(o); - Py_DECREF(m); + Py_DECREF(exposed_module); return 1; + +error: + Py_XDECREF(o); + Py_DECREF(exposed_module); + return 0; } /*[clinic input] @@ -4210,35 +4223,33 @@ static PyObject * _curses_start_color_impl(PyObject *module) /*[clinic end generated code: output=8b772b41d8090ede input=0ca0ecb2b77e1a12]*/ { - int code; - PyObject *c, *cp; - PyCursesInitialised; - code = start_color(); - if (code != ERR) { - initialisedcolors = TRUE; - c = PyLong_FromLong((long) COLORS); - if (c == NULL) - return NULL; - if (PyDict_SetItemString(ModDict, "COLORS", c) < 0) { - Py_DECREF(c); - return NULL; - } - Py_DECREF(c); - cp = PyLong_FromLong((long) COLOR_PAIRS); - if (cp == NULL) - return NULL; - if (PyDict_SetItemString(ModDict, "COLOR_PAIRS", cp) < 0) { - Py_DECREF(cp); - return NULL; - } - Py_DECREF(cp); - Py_RETURN_NONE; - } else { + if (start_color() == ERR) { PyErr_SetString(PyCursesError, "start_color() returned ERR"); return NULL; } + + initialisedcolors = TRUE; + +#define DICT_ADD_INT_VALUE(NAME, VALUE) \ + do { \ + PyObject *value = PyLong_FromLong((long)(VALUE)); \ + if (value == NULL) { \ + return NULL; \ + } \ + int rc = PyDict_SetItemString(ModDict, (NAME), value); \ + Py_DECREF(value); \ + if (rc < 0) { \ + return NULL; \ + } \ + } while (0) + + DICT_ADD_INT_VALUE("COLORS", COLORS); + DICT_ADD_INT_VALUE("COLOR_PAIRS", COLOR_PAIRS); +#undef DICT_ADD_INT_VALUE + + Py_RETURN_NONE; } /*[clinic input] @@ -4595,10 +4606,7 @@ static PyStructSequence_Desc ncurses_version_desc = { static PyObject * make_ncurses_version(PyTypeObject *type) { - PyObject *ncurses_version; - int pos = 0; - - ncurses_version = PyStructSequence_New(type); + PyObject *ncurses_version = PyStructSequence_New(type); if (ncurses_version == NULL) { return NULL; } @@ -4610,18 +4618,20 @@ make_ncurses_version(PyTypeObject *type) minor = NCURSES_VERSION_MINOR; patch = NCURSES_VERSION_PATCH; } -#define SetIntItem(flag) \ - PyStructSequence_SET_ITEM(ncurses_version, pos++, PyLong_FromLong(flag)); \ - if (PyErr_Occurred()) { \ - Py_CLEAR(ncurses_version); \ - return NULL; \ - } - - SetIntItem(major) - SetIntItem(minor) - SetIntItem(patch) -#undef SetIntItem +#define SET_VERSION_COMPONENT(INDEX, VALUE) \ + do { \ + PyObject *o = PyLong_FromLong(VALUE); \ + if (o == NULL) { \ + Py_DECREF(ncurses_version); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(ncurses_version, INDEX, o); \ + } while (0) + SET_VERSION_COMPONENT(0, major); + SET_VERSION_COMPONENT(1, minor); + SET_VERSION_COMPONENT(2, patch); +#undef SET_VERSION_COMPONENT return ncurses_version; } @@ -4753,33 +4763,28 @@ curses_destructor(PyObject *op) PyMem_Free(ptr); } -PyMODINIT_FUNC -PyInit__curses(void) +static int +cursesmodule_exec(PyObject *module) { - PyObject *m, *d, *v, *c_api_object; - /* Initialize object type */ - if (PyType_Ready(&PyCursesWindow_Type) < 0) - return NULL; - - /* Create the module and add the functions */ - m = PyModule_Create(&_cursesmodule); - if (m == NULL) - return NULL; -#ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); -#endif + if (PyType_Ready(&PyCursesWindow_Type) < 0) { + return -1; + } + if (PyModule_AddType(module, &PyCursesWindow_Type) < 0) { + return -1; + } /* Add some symbolic constants to the module */ - d = PyModule_GetDict(m); - if (d == NULL) - return NULL; - ModDict = d; /* For PyCurses_InitScr to use later */ + PyObject *module_dict = PyModule_GetDict(module); + if (module_dict == NULL) { + return -1; + } + ModDict = module_dict; /* For PyCurses_InitScr to use later */ void **PyCurses_API = PyMem_Calloc(PyCurses_API_pointers, sizeof(void *)); if (PyCurses_API == NULL) { PyErr_NoMemory(); - return NULL; + return -1; } /* Initialize the C API pointer array */ PyCurses_API[0] = (void *)Py_NewRef(&PyCursesWindow_Type); @@ -4788,28 +4793,45 @@ PyInit__curses(void) PyCurses_API[3] = (void *)func_PyCursesInitialisedColor; /* Add a capsule for the C API */ - c_api_object = PyCapsule_New(PyCurses_API, PyCurses_CAPSULE_NAME, - curses_destructor); + PyObject *c_api_object = PyCapsule_New(PyCurses_API, PyCurses_CAPSULE_NAME, + curses_destructor); if (c_api_object == NULL) { Py_DECREF(PyCurses_API[0]); PyMem_Free(PyCurses_API); - return NULL; - } - if (PyDict_SetItemString(d, "_C_API", c_api_object) < 0) { - Py_DECREF(c_api_object); - return NULL; + return -1; } + int rc = PyDict_SetItemString(module_dict, "_C_API", c_api_object); Py_DECREF(c_api_object); + if (rc < 0) { + return -1; + } /* For exception curses.error */ PyCursesError = PyErr_NewException("_curses.error", NULL, NULL); - PyDict_SetItemString(d, "error", PyCursesError); + if (PyCursesError == NULL) { + return -1; + } + rc = PyDict_SetItemString(module_dict, "error", PyCursesError); + Py_DECREF(PyCursesError); + if (rc < 0) { + return -1; + } /* Make the version available */ - v = PyBytes_FromString(PyCursesVersion); - PyDict_SetItemString(d, "version", v); - PyDict_SetItemString(d, "__version__", v); - Py_DECREF(v); + PyObject *curses_version = PyBytes_FromString(PyCursesVersion); + if (curses_version == NULL) { + return -1; + } + rc = PyDict_SetItemString(module_dict, "version", curses_version); + if (rc < 0) { + Py_DECREF(curses_version); + return -1; + } + rc = PyDict_SetItemString(module_dict, "__version__", curses_version); + Py_CLEAR(curses_version); + if (rc < 0) { + return -1; + } #ifdef NCURSES_VERSION /* ncurses_version */ @@ -4817,17 +4839,33 @@ PyInit__curses(void) version_type = _PyStructSequence_NewType(&ncurses_version_desc, Py_TPFLAGS_DISALLOW_INSTANTIATION); if (version_type == NULL) { - return NULL; + return -1; } - v = make_ncurses_version(version_type); + PyObject *ncurses_version = make_ncurses_version(version_type); Py_DECREF(version_type); - if (v == NULL) { - return NULL; + if (ncurses_version == NULL) { + return -1; + } + rc = PyDict_SetItemString(module_dict, "ncurses_version", ncurses_version); + Py_CLEAR(ncurses_version); + if (rc < 0) { + return -1; } - PyDict_SetItemString(d, "ncurses_version", v); - Py_DECREF(v); #endif /* NCURSES_VERSION */ +#define SetDictInt(NAME, VALUE) \ + do { \ + PyObject *value = PyLong_FromLong((long)(VALUE)); \ + if (value == NULL) { \ + return -1; \ + } \ + int rc = PyDict_SetItemString(module_dict, (NAME), value); \ + Py_DECREF(value); \ + if (rc < 0) { \ + return -1; \ + } \ + } while (0) + SetDictInt("ERR", ERR); SetDictInt("OK", OK); @@ -4923,43 +4961,69 @@ PyInit__curses(void) SetDictInt("REPORT_MOUSE_POSITION", REPORT_MOUSE_POSITION); #endif /* Now set everything up for KEY_ variables */ - { - int key; - char *key_n; - char *key_n2; - for (key=KEY_MIN;key < KEY_MAX; key++) { - key_n = (char *)keyname(key); - if (key_n == NULL || strcmp(key_n,"UNKNOWN KEY")==0) - continue; - if (strncmp(key_n,"KEY_F(",6)==0) { - char *p1, *p2; - key_n2 = PyMem_Malloc(strlen(key_n)+1); - if (!key_n2) { - PyErr_NoMemory(); - break; - } - p1 = key_n; - p2 = key_n2; - while (*p1) { - if (*p1 != '(' && *p1 != ')') { - *p2 = *p1; - p2++; - } - p1++; + for (int keycode = KEY_MIN; keycode < KEY_MAX; keycode++) { + const char *key_name = keyname(keycode); + if (key_name == NULL || strcmp(key_name, "UNKNOWN KEY") == 0) { + continue; + } + if (strncmp(key_name, "KEY_F(", 6) == 0) { + char *fn_key_name = PyMem_Malloc(strlen(key_name) + 1); + if (!fn_key_name) { + PyErr_NoMemory(); + return -1; + } + const char *p1 = key_name; + char *p2 = fn_key_name; + while (*p1) { + if (*p1 != '(' && *p1 != ')') { + *p2 = *p1; + p2++; } - *p2 = (char)0; - } else - key_n2 = key_n; - SetDictInt(key_n2,key); - if (key_n2 != key_n) - PyMem_Free(key_n2); + p1++; + } + *p2 = (char)0; + PyObject *p_keycode = PyLong_FromLong((long)keycode); + if (p_keycode == NULL) { + PyMem_Free(fn_key_name); + return -1; + } + int rc = PyDict_SetItemString(module_dict, fn_key_name, p_keycode); + Py_DECREF(p_keycode); + PyMem_Free(fn_key_name); + if (rc < 0) { + return -1; + } + } + else { + SetDictInt(key_name, keycode); } - SetDictInt("KEY_MIN", KEY_MIN); - SetDictInt("KEY_MAX", KEY_MAX); } + SetDictInt("KEY_MIN", KEY_MIN); + SetDictInt("KEY_MAX", KEY_MAX); +#undef SetDictInt + return 0; +} - if (PyModule_AddType(m, &PyCursesWindow_Type) < 0) { - return NULL; +PyMODINIT_FUNC +PyInit__curses(void) +{ + // create the module + PyObject *mod = PyModule_Create(&_cursesmodule); + if (mod == NULL) { + goto error; } - return m; +#ifdef Py_GIL_DISABLED + if (PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED) < 0) { + goto error; + } +#endif + // populate the module + if (cursesmodule_exec(mod) < 0) { + goto error; + } + return mod; + +error: + Py_XDECREF(mod); + return NULL; } From e07154fd1e3152a758cf9b476257a4ffdc48dfc6 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 11 Sep 2024 18:02:28 +0100 Subject: [PATCH 09/50] gh-123958: apply docstring removal optimization in ast_opt instead of codegen (#123959) --- Doc/whatsnew/3.14.rst | 4 +- Lib/test/test_compile.py | 6 +++ ...-09-11-15-48-36.gh-issue-123958.5VW2r0.rst | 1 + Python/ast_opt.c | 21 ++++++++++ Python/codegen.c | 38 ++++++++----------- 5 files changed, 47 insertions(+), 23 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-11-15-48-36.gh-issue-123958.5VW2r0.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 7aca6bd2117173..dc2083b241d391 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -123,9 +123,11 @@ ast (Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`.) * Add support for :func:`copy.replace` for AST nodes. - (Contributed by Bénédikt Tran in :gh:`121141`.) +* Docstrings are now removed from an optimized AST in optimization level 2. + (Contributed by Irit Katriel in :gh:`123958`.) + ctypes ------ diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 7d6ddba1adc87e..f22761f0a3af9f 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -876,6 +876,10 @@ def test_docstring(self): def with_docstring(): "docstring" + def two_strings(): + "docstring" + "not docstring" + def with_fstring(): f"not docstring" @@ -891,8 +895,10 @@ def with_const_expression(): if opt < 2: self.assertEqual(ns['with_docstring'].__doc__, "docstring") + self.assertEqual(ns['two_strings'].__doc__, "docstring") else: self.assertIsNone(ns['with_docstring'].__doc__) + self.assertIsNone(ns['two_strings'].__doc__) self.assertIsNone(ns['with_fstring'].__doc__) self.assertIsNone(ns['with_const_expression'].__doc__) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-11-15-48-36.gh-issue-123958.5VW2r0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-11-15-48-36.gh-issue-123958.5VW2r0.rst new file mode 100644 index 00000000000000..fc2623a7cbf789 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-11-15-48-36.gh-issue-123958.5VW2r0.rst @@ -0,0 +1 @@ +docstrings are now removed from the optimized AST in optimization level 2. diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 5a51305d2a7ade..f5b04757e08bf3 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -673,10 +673,31 @@ static int astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimize } +static int +stmt_seq_remove_item(asdl_stmt_seq *stmts, Py_ssize_t idx) +{ + if (idx >= asdl_seq_LEN(stmts)) { + return 0; + } + for (Py_ssize_t i = idx; i < asdl_seq_LEN(stmts) - 1; i++) { + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, i+1); + asdl_seq_SET(stmts, i, st); + } + stmts->size--; + return 1; +} + static int astfold_body(asdl_stmt_seq *stmts, PyArena *ctx_, _PyASTOptimizeState *state) { int docstring = _PyAST_GetDocString(stmts) != NULL; + if (docstring && (state->optimize >= 2)) { + /* remove the docstring */ + if (!stmt_seq_remove_item(stmts, 0)) { + return 0; + } + docstring = 0; + } CALL_SEQ(astfold_stmt, stmt, stmts); if (!docstring && _PyAST_GetDocString(stmts) != NULL) { stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); diff --git a/Python/codegen.c b/Python/codegen.c index ed06724e95d05b..fd2260a538f76d 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -763,19 +763,18 @@ _PyCodegen_Body(compiler *c, location loc, asdl_stmt_seq *stmts) PyObject *docstring = _PyAST_GetDocString(stmts); if (docstring) { first_instr = 1; - /* if not -OO mode, set docstring */ - if (OPTIMIZATION_LEVEL(c) < 2) { - PyObject *cleandoc = _PyCompile_CleanDoc(docstring); - if (cleandoc == NULL) { - return ERROR; - } - stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); - assert(st->kind == Expr_kind); - location loc = LOC(st->v.Expr.value); - ADDOP_LOAD_CONST(c, loc, cleandoc); - Py_DECREF(cleandoc); - RETURN_IF_ERROR(codegen_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); + /* set docstring */ + assert(OPTIMIZATION_LEVEL(c) < 2); + PyObject *cleandoc = _PyCompile_CleanDoc(docstring); + if (cleandoc == NULL) { + return ERROR; } + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); + assert(st->kind == Expr_kind); + location loc = LOC(st->v.Expr.value); + ADDOP_LOAD_CONST(c, loc, cleandoc); + Py_DECREF(cleandoc); + RETURN_IF_ERROR(codegen_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } } for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) { @@ -1230,18 +1229,13 @@ codegen_function_body(compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags Py_ssize_t first_instr = 0; PyObject *docstring = _PyAST_GetDocString(body); + assert(OPTIMIZATION_LEVEL(c) < 2 || docstring == NULL); if (docstring) { first_instr = 1; - /* if not -OO mode, add docstring */ - if (OPTIMIZATION_LEVEL(c) < 2) { - docstring = _PyCompile_CleanDoc(docstring); - if (docstring == NULL) { - _PyCompile_ExitScope(c); - return ERROR; - } - } - else { - docstring = NULL; + docstring = _PyCompile_CleanDoc(docstring); + if (docstring == NULL) { + _PyCompile_ExitScope(c); + return ERROR; } } Py_ssize_t idx = _PyCompile_AddConst(c, docstring ? docstring : Py_None); From eb169f40276a8611827770f02c4b82827c98e00f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 11 Sep 2024 21:05:15 +0300 Subject: [PATCH 10/50] Update Sphinx Lint and fix unnecessary parentheses in `:func:`s (#123960) --- .pre-commit-config.yaml | 8 ++++---- Doc/library/annotationlib.rst | 2 +- .../Library/2024-08-23-22-01-30.gh-issue-76960.vsANPu.rst | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b10be5b6bd9904..1ce4cb81511d8e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.4 + rev: v0.4.10 hooks: - id: ruff name: Run Ruff (lint) on Doc/ @@ -20,7 +20,7 @@ repos: files: ^Doc/ - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black name: Run Black on Tools/jit/ @@ -28,7 +28,7 @@ repos: language_version: python3.12 - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-case-conflict - id: check-merge-conflict @@ -42,7 +42,7 @@ repos: types_or: [c, inc, python, rst] - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v0.9.1 + rev: v1.0.0 hooks: - id: sphinx-lint args: [--enable=default-role] diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index ecf56ed50b6a2a..1e72c5421674bc 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -315,7 +315,7 @@ Functions * If eval_str is true, :func:`eval` is called on values of type :class:`!str`. (Note that :func:`!get_annotations` doesn't catch - exceptions; if :func:`eval()` raises an exception, it will unwind + exceptions; if :func:`eval` raises an exception, it will unwind the stack past the :func:`!get_annotations` call.) * If *eval_str* is false (the default), values of type :class:`!str` are unchanged. diff --git a/Misc/NEWS.d/next/Library/2024-08-23-22-01-30.gh-issue-76960.vsANPu.rst b/Misc/NEWS.d/next/Library/2024-08-23-22-01-30.gh-issue-76960.vsANPu.rst index 7ce2baee3f9ab6..acb0a991e4c93d 100644 --- a/Misc/NEWS.d/next/Library/2024-08-23-22-01-30.gh-issue-76960.vsANPu.rst +++ b/Misc/NEWS.d/next/Library/2024-08-23-22-01-30.gh-issue-76960.vsANPu.rst @@ -1,5 +1,5 @@ Fix :func:`urllib.parse.urljoin` and :func:`urllib.parse.urldefrag` for URIs -containing empty components. For example, :func:`!urljoin()` with relative +containing empty components. For example, :func:`!urljoin` with relative reference "?" now sets empty query and removes fragment. Preserve empty components (authority, params, query, fragment) in :func:`!urljoin`. Preserve empty components (authority, params, query) in :func:`!urldefrag`. From 3bd942f106aa36c261a2d90104c027026b2a8fb6 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 11 Sep 2024 21:25:23 +0300 Subject: [PATCH 11/50] Fix "The Matrix" movie release year in `typing.rst` (#123965) --- Doc/library/typing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index cb5b46f7167376..fa7932061645a4 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1316,7 +1316,7 @@ These can be used as types in annotations. They all support subscription using year: int def mutate_movie(m: Movie) -> None: - m["year"] = 1992 # allowed + m["year"] = 1999 # allowed m["title"] = "The Matrix" # typechecker error There is no runtime checking for this property. From ba687d9481c04fd160795ff8d8568f5c9f877128 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 11 Sep 2024 22:33:07 -0400 Subject: [PATCH 12/50] gh-121735: Fix module-adjacent references in zip files (#123037) * gh-116608: Apply style and compatibility changes from importlib_metadata. * gh-121735: Ensure module-adjacent resources are loadable from a zipfile. * gh-121735: Allow all modules to be processed by the ZipReader. * Add blurb * Remove update-zips script, unneeded. * Remove unnecessary references to removed static fixtures. * Remove zipdata fixtures, unused. --- .gitattributes | 2 - Lib/importlib/resources/readers.py | 6 +- .../resources/data01/__init__.py | 0 .../resources/data01/binary.file | Bin 4 -> 0 bytes .../resources/data01/subdirectory/__init__.py | 0 .../resources/data01/subdirectory/binary.file | 1 - .../resources/data01/utf-16.file | Bin 44 -> 0 bytes .../resources/data01/utf-8.file | 1 - .../resources/data02/__init__.py | 0 .../resources/data02/one/__init__.py | 0 .../resources/data02/one/resource1.txt | 1 - .../subdirectory/subsubdir/resource.txt | 1 - .../resources/data02/two/__init__.py | 0 .../resources/data02/two/resource2.txt | 1 - .../resources/data03/__init__.py | 0 .../data03/namespace/portion1/__init__.py | 0 .../data03/namespace/portion2/__init__.py | 0 .../resources/data03/namespace/resource1.txt | 0 .../resources/namespacedata01/binary.file | Bin 4 -> 0 bytes .../namespacedata01/subdirectory/binary.file | 1 - .../resources/namespacedata01/utf-16.file | Bin 44 -> 0 bytes .../resources/namespacedata01/utf-8.file | 1 - .../test_importlib/resources/test_contents.py | 15 +-- .../test_importlib/resources/test_files.py | 102 +++++++++++------- .../resources/test_functional.py | 30 ++++-- .../test_importlib/resources/test_open.py | 15 +-- .../test_importlib/resources/test_path.py | 5 +- .../test_importlib/resources/test_read.py | 15 ++- .../test_importlib/resources/test_reader.py | 48 ++++----- .../test_importlib/resources/test_resource.py | 58 +++++----- .../test_importlib/resources/update-zips.py | 53 --------- Lib/test/test_importlib/resources/util.py | 72 ++++++++++--- Lib/test/test_importlib/resources/zip.py | 26 ++--- .../resources/zipdata01/__init__.py | 0 .../resources/zipdata01/ziptestdata.zip | Bin 876 -> 0 bytes .../resources/zipdata02/__init__.py | 0 .../resources/zipdata02/ziptestdata.zip | Bin 698 -> 0 bytes Lib/zipimport.py | 12 +-- Makefile.pre.in | 15 --- ...-08-15-09-45-34.gh-issue-121735._1q0qf.rst | 3 + 40 files changed, 223 insertions(+), 261 deletions(-) delete mode 100644 Lib/test/test_importlib/resources/data01/__init__.py delete mode 100644 Lib/test/test_importlib/resources/data01/binary.file delete mode 100644 Lib/test/test_importlib/resources/data01/subdirectory/__init__.py delete mode 100644 Lib/test/test_importlib/resources/data01/subdirectory/binary.file delete mode 100644 Lib/test/test_importlib/resources/data01/utf-16.file delete mode 100644 Lib/test/test_importlib/resources/data01/utf-8.file delete mode 100644 Lib/test/test_importlib/resources/data02/__init__.py delete mode 100644 Lib/test/test_importlib/resources/data02/one/__init__.py delete mode 100644 Lib/test/test_importlib/resources/data02/one/resource1.txt delete mode 100644 Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt delete mode 100644 Lib/test/test_importlib/resources/data02/two/__init__.py delete mode 100644 Lib/test/test_importlib/resources/data02/two/resource2.txt delete mode 100644 Lib/test/test_importlib/resources/data03/__init__.py delete mode 100644 Lib/test/test_importlib/resources/data03/namespace/portion1/__init__.py delete mode 100644 Lib/test/test_importlib/resources/data03/namespace/portion2/__init__.py delete mode 100644 Lib/test/test_importlib/resources/data03/namespace/resource1.txt delete mode 100644 Lib/test/test_importlib/resources/namespacedata01/binary.file delete mode 100644 Lib/test/test_importlib/resources/namespacedata01/subdirectory/binary.file delete mode 100644 Lib/test/test_importlib/resources/namespacedata01/utf-16.file delete mode 100644 Lib/test/test_importlib/resources/namespacedata01/utf-8.file delete mode 100755 Lib/test/test_importlib/resources/update-zips.py delete mode 100644 Lib/test/test_importlib/resources/zipdata01/__init__.py delete mode 100644 Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip delete mode 100644 Lib/test/test_importlib/resources/zipdata02/__init__.py delete mode 100644 Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip create mode 100644 Misc/NEWS.d/next/Library/2024-08-15-09-45-34.gh-issue-121735._1q0qf.rst diff --git a/.gitattributes b/.gitattributes index 5b81d2cb3c90e9..2f5a030981fb94 100644 --- a/.gitattributes +++ b/.gitattributes @@ -27,8 +27,6 @@ Lib/test/cjkencodings/* noeol Lib/test/tokenizedata/coding20731.py noeol Lib/test/decimaltestdata/*.decTest noeol Lib/test/test_email/data/*.txt noeol -Lib/test/test_importlib/resources/data01/* noeol -Lib/test/test_importlib/resources/namespacedata01/* noeol Lib/test/xmltestdata/* noeol # Shell scripts should have LF even on Windows because of Cygwin diff --git a/Lib/importlib/resources/readers.py b/Lib/importlib/resources/readers.py index b86cdeff57c4c2..ccc5abbeb4e56e 100644 --- a/Lib/importlib/resources/readers.py +++ b/Lib/importlib/resources/readers.py @@ -34,8 +34,10 @@ def files(self): class ZipReader(abc.TraversableResources): def __init__(self, loader, module): - _, _, name = module.rpartition('.') - self.prefix = loader.prefix.replace('\\', '/') + name + '/' + self.prefix = loader.prefix.replace('\\', '/') + if loader.is_package(module): + _, _, name = module.rpartition('.') + self.prefix += name + '/' self.archive = loader.archive def open_resource(self, resource): diff --git a/Lib/test/test_importlib/resources/data01/__init__.py b/Lib/test/test_importlib/resources/data01/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data01/binary.file b/Lib/test/test_importlib/resources/data01/binary.file deleted file mode 100644 index eaf36c1daccfdf325514461cd1a2ffbc139b5464..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmZQzWMT#Y01f~L diff --git a/Lib/test/test_importlib/resources/data01/subdirectory/__init__.py b/Lib/test/test_importlib/resources/data01/subdirectory/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data01/subdirectory/binary.file b/Lib/test/test_importlib/resources/data01/subdirectory/binary.file deleted file mode 100644 index 5bd8bb897b1322..00000000000000 --- a/Lib/test/test_importlib/resources/data01/subdirectory/binary.file +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/Lib/test/test_importlib/resources/data01/utf-16.file b/Lib/test/test_importlib/resources/data01/utf-16.file deleted file mode 100644 index 2cb772295ef4b480a8d83725bd5006a0236d8f68..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 44 ucmezW&x0YAAqNQa8FUyF7(y9B7~B|i84MZBfV^^`Xc15@g+Y;liva-T)Ce>H diff --git a/Lib/test/test_importlib/resources/data01/utf-8.file b/Lib/test/test_importlib/resources/data01/utf-8.file deleted file mode 100644 index 1c0132ad90a192..00000000000000 --- a/Lib/test/test_importlib/resources/data01/utf-8.file +++ /dev/null @@ -1 +0,0 @@ -Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/resources/data02/__init__.py b/Lib/test/test_importlib/resources/data02/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data02/one/__init__.py b/Lib/test/test_importlib/resources/data02/one/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data02/one/resource1.txt b/Lib/test/test_importlib/resources/data02/one/resource1.txt deleted file mode 100644 index 61a813e40174a6..00000000000000 --- a/Lib/test/test_importlib/resources/data02/one/resource1.txt +++ /dev/null @@ -1 +0,0 @@ -one resource diff --git a/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt b/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt deleted file mode 100644 index 48f587a2d0ac53..00000000000000 --- a/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt +++ /dev/null @@ -1 +0,0 @@ -a resource \ No newline at end of file diff --git a/Lib/test/test_importlib/resources/data02/two/__init__.py b/Lib/test/test_importlib/resources/data02/two/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data02/two/resource2.txt b/Lib/test/test_importlib/resources/data02/two/resource2.txt deleted file mode 100644 index a80ce46ea362e2..00000000000000 --- a/Lib/test/test_importlib/resources/data02/two/resource2.txt +++ /dev/null @@ -1 +0,0 @@ -two resource diff --git a/Lib/test/test_importlib/resources/data03/__init__.py b/Lib/test/test_importlib/resources/data03/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/portion1/__init__.py b/Lib/test/test_importlib/resources/data03/namespace/portion1/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/portion2/__init__.py b/Lib/test/test_importlib/resources/data03/namespace/portion2/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/resource1.txt b/Lib/test/test_importlib/resources/data03/namespace/resource1.txt deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/namespacedata01/binary.file b/Lib/test/test_importlib/resources/namespacedata01/binary.file deleted file mode 100644 index eaf36c1daccfdf325514461cd1a2ffbc139b5464..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmZQzWMT#Y01f~L diff --git a/Lib/test/test_importlib/resources/namespacedata01/subdirectory/binary.file b/Lib/test/test_importlib/resources/namespacedata01/subdirectory/binary.file deleted file mode 100644 index 100f50643d8d21..00000000000000 --- a/Lib/test/test_importlib/resources/namespacedata01/subdirectory/binary.file +++ /dev/null @@ -1 +0,0 @@ -  \ No newline at end of file diff --git a/Lib/test/test_importlib/resources/namespacedata01/utf-16.file b/Lib/test/test_importlib/resources/namespacedata01/utf-16.file deleted file mode 100644 index 2cb772295ef4b480a8d83725bd5006a0236d8f68..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 44 ucmezW&x0YAAqNQa8FUyF7(y9B7~B|i84MZBfV^^`Xc15@g+Y;liva-T)Ce>H diff --git a/Lib/test/test_importlib/resources/namespacedata01/utf-8.file b/Lib/test/test_importlib/resources/namespacedata01/utf-8.file deleted file mode 100644 index 1c0132ad90a192..00000000000000 --- a/Lib/test/test_importlib/resources/namespacedata01/utf-8.file +++ /dev/null @@ -1 +0,0 @@ -Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/resources/test_contents.py b/Lib/test/test_importlib/resources/test_contents.py index beab67ccc21680..4e4e0e9c337f23 100644 --- a/Lib/test/test_importlib/resources/test_contents.py +++ b/Lib/test/test_importlib/resources/test_contents.py @@ -1,7 +1,6 @@ import unittest from importlib import resources -from . import data01 from . import util @@ -19,16 +18,17 @@ def test_contents(self): assert self.expected <= contents -class ContentsDiskTests(ContentsTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class ContentsDiskTests(ContentsTests, util.DiskSetup, unittest.TestCase): + pass class ContentsZipTests(ContentsTests, util.ZipSetup, unittest.TestCase): pass -class ContentsNamespaceTests(ContentsTests, unittest.TestCase): +class ContentsNamespaceTests(ContentsTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + expected = { # no __init__ because of namespace design 'binary.file', @@ -36,8 +36,3 @@ class ContentsNamespaceTests(ContentsTests, unittest.TestCase): 'utf-16.file', 'utf-8.file', } - - def setUp(self): - from . import namespacedata01 - - self.data = namespacedata01 diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 7df6d03ead7480..08b840834dfd4b 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -6,11 +6,7 @@ from importlib import resources from importlib.resources.abc import Traversable -from . import data01 from . import util -from . import _path -from test.support import os_helper -from test.support import import_helper @contextlib.contextmanager @@ -48,70 +44,96 @@ def test_old_parameter(self): resources.files(package=self.data) -class OpenDiskTests(FilesTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class OpenDiskTests(FilesTests, util.DiskSetup, unittest.TestCase): + pass class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase): pass -class OpenNamespaceTests(FilesTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - - self.data = namespacedata01 +class OpenNamespaceTests(FilesTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase): ZIP_MODULE = 'namespacedata01' -class SiteDir: - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - self.site_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(import_helper.DirsOnSysPath(self.site_dir)) - self.fixtures.enter_context(import_helper.isolated_modules()) +class DirectSpec: + """ + Override behavior of ModuleSetup to write a full spec directly. + """ + + MODULE = 'unused' + + def load_fixture(self, name): + self.tree_on_path(self.spec) -class ModulesFilesTests(SiteDir, unittest.TestCase): +class ModulesFiles: + spec = { + 'mod.py': '', + 'res.txt': 'resources are the best', + } + def test_module_resources(self): """ A module can have resources found adjacent to the module. """ - spec = { - 'mod.py': '', - 'res.txt': 'resources are the best', - } - _path.build(spec, self.site_dir) import mod actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8') - assert actual == spec['res.txt'] + assert actual == self.spec['res.txt'] + + +class ModuleFilesDiskTests(DirectSpec, util.DiskSetup, ModulesFiles, unittest.TestCase): + pass + + +class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.TestCase): + pass + +class ImplicitContextFiles: + set_val = textwrap.dedent( + """ + import importlib.resources as res + val = res.files().joinpath('res.txt').read_text(encoding='utf-8') + """ + ) + spec = { + 'somepkg': { + '__init__.py': set_val, + 'submod.py': set_val, + 'res.txt': 'resources are the best', + }, + } -class ImplicitContextFilesTests(SiteDir, unittest.TestCase): - def test_implicit_files(self): + def test_implicit_files_package(self): """ Without any parameter, files() will infer the location as the caller. """ - spec = { - 'somepkg': { - '__init__.py': textwrap.dedent( - """ - import importlib.resources as res - val = res.files().joinpath('res.txt').read_text(encoding='utf-8') - """ - ), - 'res.txt': 'resources are the best', - }, - } - _path.build(spec, self.site_dir) assert importlib.import_module('somepkg').val == 'resources are the best' + def test_implicit_files_submodule(self): + """ + Without any parameter, files() will infer the location as the caller. + """ + assert importlib.import_module('somepkg.submod').val == 'resources are the best' + + +class ImplicitContextFilesDiskTests( + DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase +): + pass + + +class ImplicitContextFilesZipTests( + DirectSpec, util.ZipSetup, ImplicitContextFiles, unittest.TestCase +): + pass + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_functional.py b/Lib/test/test_importlib/resources/test_functional.py index f65ffe646cf0d4..4317abf3162c52 100644 --- a/Lib/test/test_importlib/resources/test_functional.py +++ b/Lib/test/test_importlib/resources/test_functional.py @@ -1,26 +1,38 @@ import unittest import os +import importlib from test.support import warnings_helper from importlib import resources +from . import util + # Since the functional API forwards to Traversable, we only test # filesystem resources here -- not zip files, namespace packages etc. # We do test for two kinds of Anchor, though. class StringAnchorMixin: - anchor01 = 'test.test_importlib.resources.data01' - anchor02 = 'test.test_importlib.resources.data02' + anchor01 = 'data01' + anchor02 = 'data02' class ModuleAnchorMixin: - from . import data01 as anchor01 - from . import data02 as anchor02 + @property + def anchor01(self): + return importlib.import_module('data01') + + @property + def anchor02(self): + return importlib.import_module('data02') + +class FunctionalAPIBase(util.DiskSetup): + def setUp(self): + super().setUp() + self.load_fixture('data02') -class FunctionalAPIBase: def _gen_resourcetxt_path_parts(self): """Yield various names of a text file in anchor02, each in a subTest""" for path_parts in ( @@ -228,16 +240,16 @@ def test_text_errors(self): class FunctionalAPITest_StringAnchor( - unittest.TestCase, - FunctionalAPIBase, StringAnchorMixin, + FunctionalAPIBase, + unittest.TestCase, ): pass class FunctionalAPITest_ModuleAnchor( - unittest.TestCase, - FunctionalAPIBase, ModuleAnchorMixin, + FunctionalAPIBase, + unittest.TestCase, ): pass diff --git a/Lib/test/test_importlib/resources/test_open.py b/Lib/test/test_importlib/resources/test_open.py index 3b6b2142ef47b1..8c00378ad3cc9c 100644 --- a/Lib/test/test_importlib/resources/test_open.py +++ b/Lib/test/test_importlib/resources/test_open.py @@ -1,7 +1,6 @@ import unittest from importlib import resources -from . import data01 from . import util @@ -65,16 +64,12 @@ def test_open_text_FileNotFoundError(self): target.open(encoding='utf-8') -class OpenDiskTests(OpenTests, unittest.TestCase): - def setUp(self): - self.data = data01 - +class OpenDiskTests(OpenTests, util.DiskSetup, unittest.TestCase): + pass -class OpenDiskNamespaceTests(OpenTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - self.data = namespacedata01 +class OpenDiskNamespaceTests(OpenTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): @@ -82,7 +77,7 @@ class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): class OpenNamespaceZipTests(OpenTests, util.ZipSetup, unittest.TestCase): - ZIP_MODULE = 'namespacedata01' + MODULE = 'namespacedata01' if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/test_path.py b/Lib/test/test_importlib/resources/test_path.py index 90b22905ab8692..378dc7a2baeb23 100644 --- a/Lib/test/test_importlib/resources/test_path.py +++ b/Lib/test/test_importlib/resources/test_path.py @@ -3,7 +3,6 @@ import unittest from importlib import resources -from . import data01 from . import util @@ -25,9 +24,7 @@ def test_reading(self): self.assertEqual('Hello, UTF-8 world!\n', path.read_text(encoding='utf-8')) -class PathDiskTests(PathTests, unittest.TestCase): - data = data01 - +class PathDiskTests(PathTests, util.DiskSetup, unittest.TestCase): def test_natural_path(self): # Guarantee the internal implementation detail that # file-system-backed resources do not get the tempdir diff --git a/Lib/test/test_importlib/resources/test_read.py b/Lib/test/test_importlib/resources/test_read.py index 984feecbb9ed69..59c237d964121e 100644 --- a/Lib/test/test_importlib/resources/test_read.py +++ b/Lib/test/test_importlib/resources/test_read.py @@ -1,7 +1,7 @@ import unittest from importlib import import_module, resources -from . import data01 + from . import util @@ -51,8 +51,8 @@ def test_read_text_with_errors(self): ) -class ReadDiskTests(ReadTests, unittest.TestCase): - data = data01 +class ReadDiskTests(ReadTests, util.DiskSetup, unittest.TestCase): + pass class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): @@ -68,15 +68,12 @@ def test_read_submodule_resource_by_name(self): self.assertEqual(result, bytes(range(4, 8))) -class ReadNamespaceTests(ReadTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - - self.data = namespacedata01 +class ReadNamespaceTests(ReadTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' class ReadNamespaceZipTests(ReadTests, util.ZipSetup, unittest.TestCase): - ZIP_MODULE = 'namespacedata01' + MODULE = 'namespacedata01' def test_read_submodule_resource(self): submodule = import_module('namespacedata01.subdirectory') diff --git a/Lib/test/test_importlib/resources/test_reader.py b/Lib/test/test_importlib/resources/test_reader.py index dac9c2a892ffd2..ed5693ab416798 100644 --- a/Lib/test/test_importlib/resources/test_reader.py +++ b/Lib/test/test_importlib/resources/test_reader.py @@ -1,16 +1,21 @@ import os.path -import sys import pathlib import unittest from importlib import import_module from importlib.readers import MultiplexedPath, NamespaceReader +from . import util -class MultiplexedPathTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.folder = pathlib.Path(__file__).parent / 'namespacedata01' + +class MultiplexedPathTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def setUp(self): + super().setUp() + self.folder = pathlib.Path(self.data.__path__[0]) + self.data01 = pathlib.Path(self.load_fixture('data01').__file__).parent + self.data02 = pathlib.Path(self.load_fixture('data02').__file__).parent def test_init_no_paths(self): with self.assertRaises(FileNotFoundError): @@ -31,9 +36,8 @@ def test_iterdir(self): ) def test_iterdir_duplicate(self): - data01 = pathlib.Path(__file__).parent.joinpath('data01') contents = { - path.name for path in MultiplexedPath(self.folder, data01).iterdir() + path.name for path in MultiplexedPath(self.folder, self.data01).iterdir() } for remove in ('__pycache__', '__init__.pyc'): try: @@ -61,9 +65,8 @@ def test_open_file(self): path.open() def test_join_path(self): - data01 = pathlib.Path(__file__).parent.joinpath('data01') - prefix = str(data01.parent) - path = MultiplexedPath(self.folder, data01) + prefix = str(self.folder.parent) + path = MultiplexedPath(self.folder, self.data01) self.assertEqual( str(path.joinpath('binary.file'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'binary.file'), @@ -83,10 +86,8 @@ def test_join_path_compound(self): assert not path.joinpath('imaginary/foo.py').exists() def test_join_path_common_subdir(self): - data01 = pathlib.Path(__file__).parent.joinpath('data01') - data02 = pathlib.Path(__file__).parent.joinpath('data02') - prefix = str(data01.parent) - path = MultiplexedPath(data01, data02) + prefix = str(self.data02.parent) + path = MultiplexedPath(self.data01, self.data02) self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) self.assertEqual( str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :], @@ -106,16 +107,8 @@ def test_name(self): ) -class NamespaceReaderTest(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) +class NamespaceReaderTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' def test_init_error(self): with self.assertRaises(ValueError): @@ -125,7 +118,7 @@ def test_resource_path(self): namespacedata01 = import_module('namespacedata01') reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + root = self.data.__path__[0] self.assertEqual( reader.resource_path('binary.file'), os.path.join(root, 'binary.file') ) @@ -134,9 +127,8 @@ def test_resource_path(self): ) def test_files(self): - namespacedata01 = import_module('namespacedata01') - reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + reader = NamespaceReader(self.data.__spec__.submodule_search_locations) + root = self.data.__path__[0] self.assertIsInstance(reader.files(), MultiplexedPath) self.assertEqual(repr(reader.files()), f"MultiplexedPath('{root}')") diff --git a/Lib/test/test_importlib/resources/test_resource.py b/Lib/test/test_importlib/resources/test_resource.py index d1d45d9b4617f3..fcede14b891a84 100644 --- a/Lib/test/test_importlib/resources/test_resource.py +++ b/Lib/test/test_importlib/resources/test_resource.py @@ -1,8 +1,5 @@ -import sys import unittest -import pathlib -from . import data01 from . import util from importlib import resources, import_module @@ -24,9 +21,8 @@ def test_is_dir(self): self.assertTrue(target.is_dir()) -class ResourceDiskTests(ResourceTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class ResourceDiskTests(ResourceTests, util.DiskSetup, unittest.TestCase): + pass class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): @@ -37,33 +33,39 @@ def names(traversable): return {item.name for item in traversable.iterdir()} -class ResourceLoaderTests(unittest.TestCase): +class ResourceLoaderTests(util.DiskSetup, unittest.TestCase): def test_resource_contents(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) self.assertEqual(names(resources.files(package)), {'A', 'B', 'C'}) def test_is_file(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertTrue(resources.files(package).joinpath('B').is_file()) def test_is_dir(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertTrue(resources.files(package).joinpath('D').is_dir()) def test_resource_missing(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertFalse(resources.files(package).joinpath('Z').is_file()) -class ResourceCornerCaseTests(unittest.TestCase): +class ResourceCornerCaseTests(util.DiskSetup, unittest.TestCase): def test_package_has_no_reader_fallback(self): """ Test odd ball packages which: @@ -72,7 +74,7 @@ def test_package_has_no_reader_fallback(self): # 3. Are not in a zip file """ module = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) # Give the module a dummy loader. module.__loader__ = object() @@ -83,9 +85,7 @@ def test_package_has_no_reader_fallback(self): self.assertFalse(resources.files(module).joinpath('A').is_file()) -class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = 'data01' - +class ResourceFromZipsTest01(util.ZipSetup, unittest.TestCase): def test_is_submodule_resource(self): submodule = import_module('data01.subdirectory') self.assertTrue(resources.files(submodule).joinpath('binary.file').is_file()) @@ -116,8 +116,8 @@ def test_as_file_directory(self): assert not data.parent.exists() -class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = 'data02' +class ResourceFromZipsTest02(util.ZipSetup, unittest.TestCase): + MODULE = 'data02' def test_unrelated_contents(self): """ @@ -134,7 +134,7 @@ def test_unrelated_contents(self): ) -class DeletingZipsTest(util.ZipSetupBase, unittest.TestCase): +class DeletingZipsTest(util.ZipSetup, unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. """ @@ -216,24 +216,20 @@ def test_submodule_sub_contents_by_name(self): self.assertEqual(contents, {'binary.file'}) -class ResourceFromNamespaceDiskTests(ResourceFromNamespaceTests, unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) +class ResourceFromNamespaceDiskTests( + util.DiskSetup, + ResourceFromNamespaceTests, + unittest.TestCase, +): + MODULE = 'namespacedata01' class ResourceFromNamespaceZipTests( - util.ZipSetupBase, + util.ZipSetup, ResourceFromNamespaceTests, unittest.TestCase, ): - ZIP_MODULE = 'namespacedata01' + MODULE = 'namespacedata01' if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/update-zips.py b/Lib/test/test_importlib/resources/update-zips.py deleted file mode 100755 index 231334aa7e38b4..00000000000000 --- a/Lib/test/test_importlib/resources/update-zips.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Generate the zip test data files. - -Run to build the tests/zipdataNN/ziptestdata.zip files from -files in tests/dataNN. - -Replaces the file with the working copy, but does commit anything -to the source repo. -""" - -import contextlib -import os -import pathlib -import zipfile - - -def main(): - """ - >>> from unittest import mock - >>> monkeypatch = getfixture('monkeypatch') - >>> monkeypatch.setattr(zipfile, 'ZipFile', mock.MagicMock()) - >>> print(); main() # print workaround for bpo-32509 - - ...data01... -> ziptestdata/... - ... - ...data02... -> ziptestdata/... - ... - """ - suffixes = '01', '02' - tuple(map(generate, suffixes)) - - -def generate(suffix): - root = pathlib.Path(__file__).parent.relative_to(os.getcwd()) - zfpath = root / f'zipdata{suffix}/ziptestdata.zip' - with zipfile.ZipFile(zfpath, 'w') as zf: - for src, rel in walk(root / f'data{suffix}'): - dst = 'ziptestdata' / pathlib.PurePosixPath(rel.as_posix()) - print(src, '->', dst) - zf.write(src, dst) - - -def walk(datapath): - for dirpath, dirnames, filenames in os.walk(datapath): - with contextlib.suppress(ValueError): - dirnames.remove('__pycache__') - for filename in filenames: - res = pathlib.Path(dirpath) / filename - rel = res.relative_to(datapath) - yield res, rel - - -__name__ == '__main__' and main() diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index d4bf3e6cc5dfdc..893dda69119192 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -6,10 +6,10 @@ import pathlib import contextlib -from . import data01 from importlib.resources.abc import ResourceReader from test.support import import_helper, os_helper from . import zip as zip_ +from . import _path from importlib.machinery import ModuleSpec @@ -68,7 +68,7 @@ def create_package(file=None, path=None, is_package=True, contents=()): ) -class CommonTests(metaclass=abc.ABCMeta): +class CommonTestsBase(metaclass=abc.ABCMeta): """ Tests shared by test_open, test_path, and test_read. """ @@ -84,34 +84,34 @@ def test_package_name(self): """ Passing in the package name should succeed. """ - self.execute(data01.__name__, 'utf-8.file') + self.execute(self.data.__name__, 'utf-8.file') def test_package_object(self): """ Passing in the package itself should succeed. """ - self.execute(data01, 'utf-8.file') + self.execute(self.data, 'utf-8.file') def test_string_path(self): """ Passing in a string for the path should succeed. """ path = 'utf-8.file' - self.execute(data01, path) + self.execute(self.data, path) def test_pathlib_path(self): """ Passing in a pathlib.PurePath object for the path should succeed. """ path = pathlib.PurePath('utf-8.file') - self.execute(data01, path) + self.execute(self.data, path) def test_importing_module_as_side_effect(self): """ The anchor package can already be imported. """ - del sys.modules[data01.__name__] - self.execute(data01.__name__, 'utf-8.file') + del sys.modules[self.data.__name__] + self.execute(self.data.__name__, 'utf-8.file') def test_missing_path(self): """ @@ -141,24 +141,66 @@ def test_useless_loader(self): self.execute(package, 'utf-8.file') -class ZipSetupBase: - ZIP_MODULE = 'data01' - +fixtures = dict( + data01={ + '__init__.py': '', + 'binary.file': bytes(range(4)), + 'utf-16.file': 'Hello, UTF-16 world!\n'.encode('utf-16'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + '__init__.py': '', + 'binary.file': bytes(range(4, 8)), + }, + }, + data02={ + '__init__.py': '', + 'one': {'__init__.py': '', 'resource1.txt': 'one resource'}, + 'two': {'__init__.py': '', 'resource2.txt': 'two resource'}, + 'subdirectory': {'subsubdir': {'resource.txt': 'a resource'}}, + }, + namespacedata01={ + 'binary.file': bytes(range(4)), + 'utf-16.file': 'Hello, UTF-16 world!\n'.encode('utf-16'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + 'binary.file': bytes(range(12, 16)), + }, + }, +) + + +class ModuleSetup: def setUp(self): self.fixtures = contextlib.ExitStack() self.addCleanup(self.fixtures.close) self.fixtures.enter_context(import_helper.isolated_modules()) + self.data = self.load_fixture(self.MODULE) + + def load_fixture(self, module): + self.tree_on_path({module: fixtures[module]}) + return importlib.import_module(module) + + +class ZipSetup(ModuleSetup): + MODULE = 'data01' + def tree_on_path(self, spec): temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) modules = pathlib.Path(temp_dir) / 'zipped modules.zip' - src_path = pathlib.Path(__file__).parent.joinpath(self.ZIP_MODULE) self.fixtures.enter_context( - import_helper.DirsOnSysPath(str(zip_.make_zip_file(src_path, modules))) + import_helper.DirsOnSysPath(str(zip_.make_zip_file(spec, modules))) ) - self.data = importlib.import_module(self.ZIP_MODULE) + +class DiskSetup(ModuleSetup): + MODULE = 'data01' + + def tree_on_path(self, spec): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + _path.build(spec, pathlib.Path(temp_dir)) + self.fixtures.enter_context(import_helper.DirsOnSysPath(temp_dir)) -class ZipSetup(ZipSetupBase): +class CommonTests(DiskSetup, CommonTestsBase): pass diff --git a/Lib/test/test_importlib/resources/zip.py b/Lib/test/test_importlib/resources/zip.py index 4dcf6facc770cb..fc453f02060a31 100755 --- a/Lib/test/test_importlib/resources/zip.py +++ b/Lib/test/test_importlib/resources/zip.py @@ -2,29 +2,23 @@ Generate zip test data files. """ -import contextlib -import os -import pathlib import zipfile -def make_zip_file(src, dst): +def make_zip_file(tree, dst): """ - Zip the files in src into a new zipfile at dst. + Zip the files in tree into a new zipfile at dst. """ with zipfile.ZipFile(dst, 'w') as zf: - for src_path, rel in walk(src): - dst_name = src.name / pathlib.PurePosixPath(rel.as_posix()) - zf.write(src_path, dst_name) + for name, contents in walk(tree): + zf.writestr(name, contents) zipfile._path.CompleteDirs.inject(zf) return dst -def walk(datapath): - for dirpath, dirnames, filenames in os.walk(datapath): - with contextlib.suppress(ValueError): - dirnames.remove('__pycache__') - for filename in filenames: - res = pathlib.Path(dirpath) / filename - rel = res.relative_to(datapath) - yield res, rel +def walk(tree, prefix=''): + for name, contents in tree.items(): + if isinstance(contents, dict): + yield from walk(contents, prefix=f'{prefix}{name}/') + else: + yield f'{prefix}{name}', contents diff --git a/Lib/test/test_importlib/resources/zipdata01/__init__.py b/Lib/test/test_importlib/resources/zipdata01/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip deleted file mode 100644 index 9a3bb0739f87e97c1084b94d7d153680f6727738..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 876 zcmWIWW@Zs#00HOCX@Q%&m27l?Y!DU);;PJolGNgol*E!m{nC;&T|+ayw9K5;|NlG~ zQWMD z9;rDw`8o=rA#S=B3g!7lIVp-}COK17UPc zNtt;*xhM-3R!jMEPhCreO-3*u>5Df}T7+BJ{639e$2uhfsIs`pJ5Qf}C xGXyDE@VNvOv@o!wQJfLgCAgysx3f@9jKpUmiW^zkK<;1z!tFpk^MROw0RS~O%0&PG diff --git a/Lib/test/test_importlib/resources/zipdata02/__init__.py b/Lib/test/test_importlib/resources/zipdata02/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip b/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip deleted file mode 100644 index d63ff512d2807ef2fd259455283b81b02e0e45fb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 698 zcmWIWW@Zs#00HOCX@Ot{ln@8fRhb1Psl_EJi6x2p@$s2?nI-Y@dIgmMI5kP5Y0A$_ z#jWw|&p#`9ff_(q7K_HB)Z+ZoqU2OVy^@L&ph*fa0WRVlP*R?c+X1opI-R&20MZDv z&j{oIpa8N17@0(vaR(gGH(;=&5k%n(M%;#g0ulz6G@1gL$cA79E2=^00gEsw4~s!C zUxI@ZWaIMqz|BszK;s4KsL2<9jRy!Q2E6`2cTLHjr{wAk1ZCU@!+_ G1_l6Bc%f?m diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 7ceae2b7387b26..e5192c4d074c4b 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -256,17 +256,9 @@ def load_module(self, fullname): def get_resource_reader(self, fullname): - """Return the ResourceReader for a package in a zip file. - - If 'fullname' is a package within the zip file, return the - 'ResourceReader' object for the package. Otherwise return None. - """ - try: - if not self.is_package(fullname): - return None - except ZipImportError: - return None + """Return the ResourceReader for a module in a zip file.""" from importlib.readers import ZipReader + return ZipReader(self, fullname) diff --git a/Makefile.pre.in b/Makefile.pre.in index 77455c0978f71d..579b3fddabc7c3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2489,21 +2489,6 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/namespace_pkgs/project3/parent/child \ test/test_importlib/partial \ test/test_importlib/resources \ - test/test_importlib/resources/data01 \ - test/test_importlib/resources/data01/subdirectory \ - test/test_importlib/resources/data02 \ - test/test_importlib/resources/data02/one \ - test/test_importlib/resources/data02/subdirectory \ - test/test_importlib/resources/data02/subdirectory/subsubdir \ - test/test_importlib/resources/data02/two \ - test/test_importlib/resources/data03 \ - test/test_importlib/resources/data03/namespace \ - test/test_importlib/resources/data03/namespace/portion1 \ - test/test_importlib/resources/data03/namespace/portion2 \ - test/test_importlib/resources/namespacedata01 \ - test/test_importlib/resources/namespacedata01/subdirectory \ - test/test_importlib/resources/zipdata01 \ - test/test_importlib/resources/zipdata02 \ test/test_importlib/source \ test/test_inspect \ test/test_interpreters \ diff --git a/Misc/NEWS.d/next/Library/2024-08-15-09-45-34.gh-issue-121735._1q0qf.rst b/Misc/NEWS.d/next/Library/2024-08-15-09-45-34.gh-issue-121735._1q0qf.rst new file mode 100644 index 00000000000000..e10b2e760bc063 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-15-09-45-34.gh-issue-121735._1q0qf.rst @@ -0,0 +1,3 @@ +When working with zip archives, importlib.resources now properly honors +module-adjacent references (e.g. ``files(pkg.mod)`` and not just +``files(pkg)``). From 43303e362e3a7e2d96747d881021a14c7f7e3d0b Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 12 Sep 2024 06:52:07 +0100 Subject: [PATCH 13/50] gh-116622: Switch test_stress_delivery_simultaneous from SIGUSR1 to SIGUSR2 (#123981) Use SIGUSR1 instead of SIGUSR2 to improve reliability of signal stress test on Android. --- Android/testbed/app/src/main/python/main.py | 22 ++++++++++++++++++--- Lib/test/test_signal.py | 9 ++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/main.py index c7314b500bf821..d6941b14412fcc 100644 --- a/Android/testbed/app/src/main/python/main.py +++ b/Android/testbed/app/src/main/python/main.py @@ -5,9 +5,25 @@ import sys # Some tests use SIGUSR1, but that's blocked by default in an Android app in -# order to make it available to `sigwait` in the "Signal Catcher" thread. That -# thread's functionality is only relevant to the JVM ("forcing GC (no HPROF) and -# profile save"), so disabling it should not weaken the tests. +# order to make it available to `sigwait` in the Signal Catcher thread. +# (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). +# That thread's functionality is only useful for debugging the JVM, so disabling +# it should not weaken the tests. +# +# There's no safe way of stopping the thread completely (#123982), but simply +# unblocking SIGUSR1 is enough to fix most tests. +# +# However, in tests that generate multiple different signals in quick +# succession, it's possible for SIGUSR1 to arrive while the main thread is busy +# running the C-level handler for a different signal. In that case, the SIGUSR1 +# may be sent to the Signal Catcher thread instead, which will generate a log +# message containing the text "reacting to signal". +# +# Such tests may need to be changed in one of the following ways: +# * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in +# test_signal.py). +# * Send the signal to a specific thread rather than the whole process (e.g. +# test_signals in test_threadsignals.py. signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 7f8fe34bb315b2..da36b8576be1ad 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1325,15 +1325,18 @@ def test_stress_delivery_simultaneous(self): def handler(signum, frame): sigs.append(signum) - self.setsig(signal.SIGUSR1, handler) + # On Android, SIGUSR1 is unreliable when used in close proximity to + # another signal – see Android/testbed/app/src/main/python/main.py. + # So we use a different signal. + self.setsig(signal.SIGUSR2, handler) self.setsig(signal.SIGALRM, handler) # for ITIMER_REAL expected_sigs = 0 while expected_sigs < N: # Hopefully the SIGALRM will be received somewhere during - # initial processing of SIGUSR1. + # initial processing of SIGUSR2. signal.setitimer(signal.ITIMER_REAL, 1e-6 + random.random() * 1e-5) - os.kill(os.getpid(), signal.SIGUSR1) + os.kill(os.getpid(), signal.SIGUSR2) expected_sigs += 2 # Wait for handlers to run to avoid signal coalescing From ac918ccad707ab2d7dbb78a4796a7b8a874f334c Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 12 Sep 2024 10:29:06 +0300 Subject: [PATCH 14/50] gh-123935: Fix typo in `_get_slots` in `dataclasses.py` (#123941) --- Lib/dataclasses.py | 2 +- Lib/test/test_dataclasses/__init__.py | 19 +++++++++++++++++++ ...-09-11-13-33-19.gh-issue-123935.fRZ_56.rst | 2 ++ 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-11-13-33-19.gh-issue-123935.fRZ_56.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 141aa41c74d7ed..ac7d40cf2cac2e 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1208,7 +1208,7 @@ def _get_slots(cls): slots = [] if getattr(cls, '__weakrefoffset__', -1) != 0: slots.append('__weakref__') - if getattr(cls, '__dictrefoffset__', -1) != 0: + if getattr(cls, '__dictoffset__', -1) != 0: slots.append('__dict__') yield from slots case str(slot): diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index da696ad961cfd7..6934e88d9d338c 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -3664,6 +3664,25 @@ class A(WithDictSlot): ... self.assertEqual(A().__dict__, {}) A() + @support.cpython_only + def test_dataclass_slot_dict_ctype(self): + # https://github.com/python/cpython/issues/123935 + from test.support import import_helper + # Skips test if `_testcapi` is not present: + _testcapi = import_helper.import_module('_testcapi') + + @dataclass(slots=True) + class HasDictOffset(_testcapi.HeapCTypeWithDict): + __dict__: dict = {} + self.assertNotEqual(_testcapi.HeapCTypeWithDict.__dictoffset__, 0) + self.assertEqual(HasDictOffset.__slots__, ()) + + @dataclass(slots=True) + class DoesNotHaveDictOffset(_testcapi.HeapCTypeWithWeakref): + __dict__: dict = {} + self.assertEqual(_testcapi.HeapCTypeWithWeakref.__dictoffset__, 0) + self.assertEqual(DoesNotHaveDictOffset.__slots__, ('__dict__',)) + @support.cpython_only def test_slots_with_wrong_init_subclass(self): # TODO: This test is for a kinda-buggy behavior. diff --git a/Misc/NEWS.d/next/Library/2024-09-11-13-33-19.gh-issue-123935.fRZ_56.rst b/Misc/NEWS.d/next/Library/2024-09-11-13-33-19.gh-issue-123935.fRZ_56.rst new file mode 100644 index 00000000000000..de720c3714c6ff --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-11-13-33-19.gh-issue-123935.fRZ_56.rst @@ -0,0 +1,2 @@ +Fix parent slots detection for dataclasses that inherit from classes with +``__dictoffset__``. From 77c68b40d58c3af9800f7c20e5fb5640dd2d2afc Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 12 Sep 2024 11:32:57 +0100 Subject: [PATCH 15/50] gh-111997: update what's new in 3.13 re C API for monitoring events (#123822) --- Doc/whatsnew/3.13.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index c60def5ad9623c..61a4399308a3a3 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -159,6 +159,7 @@ C API improvements: * The :doc:`PyTime C API ` has been added, providing access to system clocks. * :c:type:`PyMutex` is a new lightweight mutex that occupies a single byte. +* :doc:`C-API support ` was added for generating :pep:`669` monitoring events. New typing features: @@ -2175,6 +2176,9 @@ New Features if the operation needs to block. (Contributed by Sam Gross in :gh:`108724`.) +* Add C API functions for generating :pep:`669` monitoring events. + (Contributed by Irit Katriel in :gh:`111997`). + Build Changes ============= From f5597dbd0e7938648bfd9244f4a9225825b36b2b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 12 Sep 2024 13:49:19 +0300 Subject: [PATCH 16/50] Docs: Drop letter PDF (#123912) --- Doc/Makefile | 10 ---------- Doc/tools/templates/download.html | 7 +------ 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/Doc/Makefile b/Doc/Makefile index 09fcd0d8291daf..505b8953b50b32 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -222,16 +222,6 @@ dist: cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 @echo "Build finished and archived!" - # archive the letter latex - @echo "Building LaTeX (US paper)..." - rm -rf build/latex - $(MAKE) latex PAPER=letter - -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile - (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) - cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-letter.zip - cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-letter.tar.bz2 - @echo "Build finished and archived!" - # copy the epub build @echo "Building EPUB..." rm -rf build/epub diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index 9f99eea6f3c773..f69adc71f937c4 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -25,12 +25,7 @@

{% trans %}Download Python {{ release }} Documentation{% endtrans %}

{% trans %}Packed as .tar.bz2{% endtrans %} - {% trans %}PDF (US-Letter paper size){% endtrans %} - {% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %} - - - {% trans %}PDF (A4 paper size){% endtrans %} + {% trans %}PDF{% endtrans %} {% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %} {% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %} From 8e99495701737c9d9706622f59581213ef163b23 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Thu, 12 Sep 2024 14:03:32 +0200 Subject: [PATCH 17/50] gh-118577: Clarify that inspect.BoundArguments prefers to put values in args (GH-119936) --- Doc/library/inspect.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index f55824ab5f6cd9..57e5cf7ae023d1 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1043,7 +1043,8 @@ function. .. attribute:: BoundArguments.kwargs A dict of keyword arguments values. Dynamically computed from the - :attr:`arguments` attribute. + :attr:`arguments` attribute. Arguments that can be passed positionally + are included in :attr:`args` instead. .. attribute:: BoundArguments.signature From a362c41bc934fabe6bfef9be1962005b38396860 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Thu, 12 Sep 2024 06:54:18 -0700 Subject: [PATCH 18/50] gh-123968: Fix lower bound for `python -m random --float` (#123971) --- Lib/random.py | 6 +++--- Lib/test/test_random.py | 4 ++-- .../Library/2024-09-11-19-12-23.gh-issue-123968.OwHON_.rst | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-11-19-12-23.gh-issue-123968.OwHON_.rst diff --git a/Lib/random.py b/Lib/random.py index f5a482b28dec78..8b9a270c429e4a 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -1013,7 +1013,7 @@ def _parse_args(arg_list: list[str] | None): help="print a random integer between 1 and N inclusive") group.add_argument( "-f", "--float", type=float, metavar="N", - help="print a random floating-point number between 1 and N inclusive") + help="print a random floating-point number between 0 and N inclusive") group.add_argument( "--test", type=int, const=10_000, nargs="?", help=argparse.SUPPRESS) @@ -1038,7 +1038,7 @@ def main(arg_list: list[str] | None = None) -> int | str: return randint(1, args.integer) if args.float is not None: - return uniform(1, args.float) + return uniform(0, args.float) if args.test: _test(args.test) @@ -1055,7 +1055,7 @@ def main(arg_list: list[str] | None = None) -> int | str: try: # Is it a float? val = float(val) - return uniform(1, val) + return uniform(0, val) except ValueError: # Split in case of space-separated string: "a b c" return choice(val.split()) diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 9a44ab1768656a..51f9193b269eee 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -1433,8 +1433,8 @@ def test_main(self): ("'a a' 'b b' 'c c'", "b b"), ("--integer 5", 4), ("5", 4), - ("--float 2.5", 2.266632777287572), - ("2.5", 2.266632777287572), + ("--float 2.5", 2.1110546288126204), + ("2.5", 2.1110546288126204), ]: random.seed(0) self.assertEqual(random.main(shlex.split(command)), expected) diff --git a/Misc/NEWS.d/next/Library/2024-09-11-19-12-23.gh-issue-123968.OwHON_.rst b/Misc/NEWS.d/next/Library/2024-09-11-19-12-23.gh-issue-123968.OwHON_.rst new file mode 100644 index 00000000000000..4d4894716b7144 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-11-19-12-23.gh-issue-123968.OwHON_.rst @@ -0,0 +1 @@ +Fix the command-line interface for the :mod:`random` module to select floats between 0 and N, not 1 and N. From 3ea51fa2e33797c772af6eaf6ede76d2dc6082ba Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 12 Sep 2024 10:14:46 -0400 Subject: [PATCH 19/50] gh-123994: Generate utf-16 file using little endian and BOM. (#123995) --- Lib/test/test_importlib/resources/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index 893dda69119192..e2d995f596317d 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -145,7 +145,7 @@ def test_useless_loader(self): data01={ '__init__.py': '', 'binary.file': bytes(range(4)), - 'utf-16.file': 'Hello, UTF-16 world!\n'.encode('utf-16'), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), 'subdirectory': { '__init__.py': '', @@ -160,7 +160,7 @@ def test_useless_loader(self): }, namespacedata01={ 'binary.file': bytes(range(4)), - 'utf-16.file': 'Hello, UTF-16 world!\n'.encode('utf-16'), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), 'subdirectory': { 'binary.file': bytes(range(12, 16)), From 4ed7d1d6acc22807bfb5983c98fd59f7cb5061db Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 12 Sep 2024 15:32:45 +0100 Subject: [PATCH 20/50] GH-123996: Explicitly mark 'self_or_null' as an array of size 1 to ensure that it is kept in memory for calls (GH-124003) --- Lib/test/test_generated_cases.py | 38 ++++ Python/bytecodes.c | 143 ++++++------- Python/executor_cases.c.h | 221 ++++++++++--------- Python/generated_cases.c.h | 233 ++++++++++----------- Python/optimizer_cases.c.h | 16 +- Tools/cases_generator/generators_common.py | 9 +- Tools/cases_generator/stack.py | 11 +- 7 files changed, 344 insertions(+), 327 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 7f821810aea00c..5d20e3c30bcf10 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1110,6 +1110,44 @@ def test_push_then_error(self): """ self.run_cases_test(input, output) + def test_scalar_array_inconsistency(self): + + input = """ + op(FIRST, ( -- a)) { + a = 1; + } + + op(SECOND, (a[1] -- b)) { + b = 1; + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_array_size_inconsistency(self): + + input = """ + op(FIRST, ( -- a[2])) { + a[0] = 1; + } + + op(SECOND, (a[1] -- b)) { + b = 1; + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b5a642dccd2aec..e8cf636656ee9d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3207,11 +3207,11 @@ dummy_func( CALL_NON_PY_GENERAL, }; - specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null[1], args[oparg] -- callable, self_or_null[1], args[oparg])) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -3219,31 +3219,27 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- func, maybe_self, args[oparg])) { - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null[1], args[oparg] -- func, maybe_self[1], args[oparg])) { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self = PyStackRef_FromPyObjectNew(self); + maybe_self[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; func = PyStackRef_FromPyObjectNew(method); - /* Make sure that callable and all args are in memory */ - args[-2] = func; - args[-1] = maybe_self; PyStackRef_CLOSE(callable); } else { func = callable; - maybe_self = self_or_null; } } // When calling Python, inline the call using DISPATCH_INLINED(). - op(_DO_CALL, (callable, self_or_null, args[oparg] -- res)) { + op(_DO_CALL, (callable, self_or_null[1], args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3308,12 +3304,12 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_MONITOR_CALL, (func, maybe_self, args[oparg] -- func, maybe_self, args[oparg])) { - int is_meth = !PyStackRef_IsNull(maybe_self); + op(_MONITOR_CALL, (func, maybe_self[1], args[oparg] -- func, maybe_self[1], args[oparg])) { + int is_meth = !PyStackRef_IsNull(maybe_self[0]); PyObject *function = PyStackRef_AsPyObjectBorrow(func); PyObject *arg0; if (is_meth) { - arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); } else if (oparg) { arg0 = PyStackRef_AsPyObjectBorrow(args[0]); @@ -3331,13 +3327,12 @@ dummy_func( macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC; macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC; - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + op(_PY_FRAME_GENERAL, (callable, self_or_null[1], args[oparg] -- new_frame: _PyInterpreterFrame*)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3356,7 +3351,7 @@ dummy_func( } } - op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null[1], unused[oparg] -- callable, self_or_null[1], unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; @@ -3371,22 +3366,22 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { + op(_CHECK_METHOD_VERSION, (func_version/2, callable, null[1], unused[oparg] -- callable, null[1], unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); PyObject *func = ((PyMethodObject *)callable_o)->im_func; EXIT_IF(!PyFunction_Check(func)); EXIT_IF(((PyFunctionObject *)func)->func_version != func_version); - EXIT_IF(!PyStackRef_IsNull(null)); + EXIT_IF(!PyStackRef_IsNull(null[0])); } - op(_EXPAND_METHOD, (callable, null, unused[oparg] -- method, self, unused[oparg])) { + op(_EXPAND_METHOD, (callable, null[1], unused[oparg] -- method, self[1], unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyStackRef_IsNull(null)); + assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(method)); PyStackRef_CLOSE(callable); @@ -3402,21 +3397,20 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused[1], unused[oparg] -- callable, unused[1], unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(PyFunction_Check(callable_o)); EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); } - op(_CALL_NON_PY_GENERAL, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_NON_PY_GENERAL, (callable, self_or_null[1], args[oparg] -- res)) { #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3447,15 +3441,15 @@ dummy_func( _CALL_NON_PY_GENERAL + _CHECK_PERIODIC; - op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - EXIT_IF(!PyStackRef_IsNull(null)); + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null[1], unused[oparg] -- callable, null[1], unused[oparg])) { + EXIT_IF(!PyStackRef_IsNull(null[0])); EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type); } - op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- func, self, unused[oparg])) { + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null[1], unused[oparg] -- func, self[1], unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); - self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); PyStackRef_CLOSE(callable); } @@ -3464,15 +3458,15 @@ dummy_func( DEOPT_IF(tstate->interp->eval_frame); } - op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null[1], unused[oparg] -- callable, self_or_null[1], unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - EXIT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))); + EXIT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))); } - op(_CHECK_STACK_SPACE, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_STACK_SPACE, (callable, self_or_null[1], unused[oparg] -- callable, self_or_null[1], unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; @@ -3480,14 +3474,14 @@ dummy_func( DEOPT_IF(tstate->py_recursion_remaining <= 1); } - replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null[1], args[oparg] -- new_frame: _PyInterpreterFrame*)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int has_self = !PyStackRef_IsNull(self_or_null); + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } @@ -3640,11 +3634,11 @@ dummy_func( } } - op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_BUILTIN_CLASS, (callable, self_or_null[1], args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3674,12 +3668,12 @@ dummy_func( _CALL_BUILTIN_CLASS + _CHECK_PERIODIC; - op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_BUILTIN_O, (callable, self_or_null[1], args[oparg] -- res)) { /* Builtin METH_O functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3708,12 +3702,12 @@ dummy_func( _CALL_BUILTIN_O + _CHECK_PERIODIC; - op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_BUILTIN_FAST, (callable, self_or_null[1], args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3749,12 +3743,12 @@ dummy_func( _CALL_BUILTIN_FAST + _CHECK_PERIODIC; - op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null[1], args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3791,12 +3785,12 @@ dummy_func( _CALL_BUILTIN_FAST_WITH_KEYWORDS + _CHECK_PERIODIC; - inst(CALL_LEN, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + inst(CALL_LEN, (unused/1, unused/2, callable, self_or_null[1], args[oparg] -- res)) { /* len(o) */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3820,12 +3814,12 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - inst(CALL_ISINSTANCE, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + inst(CALL_ISINSTANCE, (unused/1, unused/2, callable, self_or_null[1], args[oparg] -- res)) { /* isinstance(o, o2) */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3869,11 +3863,11 @@ dummy_func( #endif } - op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null[1], args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3910,11 +3904,11 @@ dummy_func( _CALL_METHOD_DESCRIPTOR_O + _CHECK_PERIODIC; - op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null[1], args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3954,12 +3948,12 @@ dummy_func( _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + _CHECK_PERIODIC; - op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null[1], args[oparg] -- res)) { assert(oparg == 0 || oparg == 1); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3991,11 +3985,11 @@ dummy_func( _CALL_METHOD_DESCRIPTOR_NOARGS + _CHECK_PERIODIC; - op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null[1], args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4056,18 +4050,17 @@ dummy_func( GO_TO_INSTRUCTION(CALL_KW); } - op(_DO_CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) { + op(_DO_CALL_KW, (callable, self_or_null[1], args[oparg], kwnames -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } - if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) { + else if (Py_TYPE(callable_o) == &PyMethod_Type) { args--; total_args++; PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -4140,13 +4133,12 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _PyInterpreterFrame*)) { + op(_PY_FRAME_KW, (callable, self_or_null[1], args[oparg], kwnames -- new_frame: _PyInterpreterFrame*)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4168,7 +4160,7 @@ dummy_func( } } - op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, self_or_null, unused[oparg], kwnames -- callable, self_or_null, unused[oparg], kwnames)) { + op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, self_or_null[1], unused[oparg], kwnames -- callable, self_or_null[1], unused[oparg], kwnames)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; @@ -4183,22 +4175,22 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null, unused[oparg], kwnames -- callable, null, unused[oparg], kwnames)) { + op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null[1], unused[oparg], kwnames -- callable, null[1], unused[oparg], kwnames)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); PyObject *func = ((PyMethodObject *)callable_o)->im_func; EXIT_IF(!PyFunction_Check(func)); EXIT_IF(((PyFunctionObject *)func)->func_version != func_version); - EXIT_IF(!PyStackRef_IsNull(null)); + EXIT_IF(!PyStackRef_IsNull(null[0])); } - op(_EXPAND_METHOD_KW, (callable, null, unused[oparg], kwnames -- method, self, unused[oparg], kwnames)) { + op(_EXPAND_METHOD_KW, (callable, null[1], unused[oparg], kwnames -- method, self[1], unused[oparg], kwnames)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyStackRef_IsNull(null)); + assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(method)); PyStackRef_CLOSE(callable); @@ -4214,11 +4206,11 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null, args[oparg], kwnames -- callable, self_or_null, args[oparg], kwnames)) { + specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null[1], args[oparg], kwnames -- callable, self_or_null[1], args[oparg], kwnames)) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -4231,22 +4223,21 @@ dummy_func( unused/2 + _DO_CALL_KW; - op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg], kwnames -- callable, unused, unused[oparg], kwnames)) { + op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused[1], unused[oparg], kwnames -- callable, unused[1], unused[oparg], kwnames)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(PyFunction_Check(callable_o)); EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); } - op(_CALL_KW_NON_PY, (callable, self_or_null, args[oparg], kwnames -- res)) { + op(_CALL_KW_NON_PY, (callable, self_or_null[1], args[oparg], kwnames -- res)) { #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6d687bbb48b0ba..b9f532f9b118a2 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3556,31 +3556,26 @@ case _MAYBE_EXPAND_METHOD: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyStackRef func; - _PyStackRef maybe_self; + _PyStackRef *maybe_self; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - args = &stack_pointer[-oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self = PyStackRef_FromPyObjectNew(self); - stack_pointer[-1 - oparg] = maybe_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; func = PyStackRef_FromPyObjectNew(method); stack_pointer[-2 - oparg] = func; - /* Make sure that callable and all args are in memory */ - args[-2] = func; - args[-1] = maybe_self; PyStackRef_CLOSE(callable); } else { func = callable; - maybe_self = self_or_null; } break; } @@ -3591,18 +3586,17 @@ case _PY_FRAME_GENERAL: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3645,10 +3639,10 @@ } case _CHECK_METHOD_VERSION: { - _PyStackRef null; + _PyStackRef *null; _PyStackRef callable; oparg = CURRENT_OPARG(); - null = stack_pointer[-1 - oparg]; + null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -3665,7 +3659,7 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (!PyStackRef_IsNull(null)) { + if (!PyStackRef_IsNull(null[0])) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3673,18 +3667,18 @@ } case _EXPAND_METHOD: { - _PyStackRef null; + _PyStackRef *null; _PyStackRef callable; _PyStackRef method; - _PyStackRef self; + _PyStackRef *self; oparg = CURRENT_OPARG(); - null = stack_pointer[-1 - oparg]; + null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyStackRef_IsNull(null)); + assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - stack_pointer[-1 - oparg] = self; + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); stack_pointer[-2 - oparg] = method; assert(PyStackRef_FunctionCheck(method)); @@ -3710,20 +3704,19 @@ case _CALL_NON_PY_GENERAL: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -3731,7 +3724,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -3756,12 +3749,12 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - _PyStackRef null; + _PyStackRef *null; _PyStackRef callable; oparg = CURRENT_OPARG(); - null = stack_pointer[-1 - oparg]; + null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - if (!PyStackRef_IsNull(null)) { + if (!PyStackRef_IsNull(null[0])) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3775,13 +3768,13 @@ case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { _PyStackRef callable; _PyStackRef func; - _PyStackRef self; + _PyStackRef *self; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); - self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - stack_pointer[-1 - oparg] = self; + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); stack_pointer[-2 - oparg] = func; PyStackRef_CLOSE(callable); @@ -3797,16 +3790,16 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3833,21 +3826,21 @@ case _INIT_CALL_PY_EXACT_ARGS_0: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = 0; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int has_self = !PyStackRef_IsNull(self_or_null); + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } @@ -3859,21 +3852,21 @@ case _INIT_CALL_PY_EXACT_ARGS_1: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = 1; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int has_self = !PyStackRef_IsNull(self_or_null); + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } @@ -3885,21 +3878,21 @@ case _INIT_CALL_PY_EXACT_ARGS_2: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = 2; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int has_self = !PyStackRef_IsNull(self_or_null); + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } @@ -3911,21 +3904,21 @@ case _INIT_CALL_PY_EXACT_ARGS_3: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = 3; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int has_self = !PyStackRef_IsNull(self_or_null); + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } @@ -3937,21 +3930,21 @@ case _INIT_CALL_PY_EXACT_ARGS_4: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = 4; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int has_self = !PyStackRef_IsNull(self_or_null); + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } @@ -3963,20 +3956,20 @@ case _INIT_CALL_PY_EXACT_ARGS: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int has_self = !PyStackRef_IsNull(self_or_null); + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } @@ -4191,16 +4184,16 @@ case _CALL_BUILTIN_CLASS: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4217,7 +4210,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -4240,17 +4233,17 @@ case _CALL_BUILTIN_O: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4290,17 +4283,17 @@ case _CALL_BUILTIN_FAST: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4318,7 +4311,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -4345,17 +4338,17 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4375,7 +4368,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -4399,17 +4392,17 @@ case _CALL_LEN: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* len(o) */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4445,17 +4438,17 @@ case _CALL_ISINSTANCE: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4525,16 +4518,16 @@ case _CALL_METHOD_DESCRIPTOR_O: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4585,16 +4578,16 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4621,7 +4614,7 @@ STACKREFS_TO_PYOBJECTS(args, nargs, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -4645,17 +4638,17 @@ case _CALL_METHOD_DESCRIPTOR_NOARGS: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 0 || oparg == 1); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4702,16 +4695,16 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4738,7 +4731,7 @@ STACKREFS_TO_PYOBJECTS(args, nargs, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -4767,19 +4760,18 @@ case _PY_FRAME_KW: { _PyStackRef kwnames; _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = CURRENT_OPARG(); kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4825,10 +4817,10 @@ } case _CHECK_METHOD_VERSION_KW: { - _PyStackRef null; + _PyStackRef *null; _PyStackRef callable; oparg = CURRENT_OPARG(); - null = stack_pointer[-2 - oparg]; + null = &stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4845,7 +4837,7 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (!PyStackRef_IsNull(null)) { + if (!PyStackRef_IsNull(null[0])) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -4854,19 +4846,19 @@ case _EXPAND_METHOD_KW: { _PyStackRef kwnames; - _PyStackRef null; + _PyStackRef *null; _PyStackRef callable; _PyStackRef method; - _PyStackRef self; + _PyStackRef *self; oparg = CURRENT_OPARG(); kwnames = stack_pointer[-1]; - null = stack_pointer[-2 - oparg]; + null = &stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; + self = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyStackRef_IsNull(null)); + assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - stack_pointer[-2 - oparg] = self; + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); stack_pointer[-3 - oparg] = method; assert(PyStackRef_FunctionCheck(method)); @@ -4894,21 +4886,20 @@ case _CALL_KW_NON_PY: { _PyStackRef kwnames; _PyStackRef *args; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4916,7 +4907,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 65dfb990cc2820..09a14ee4d3896b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -834,13 +834,13 @@ _Py_CODEUNIT *this_instr = next_instr - 4; (void)this_instr; _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef func; - _PyStackRef maybe_self; + _PyStackRef *maybe_self; _PyStackRef res; // _SPECIALIZE_CALL - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { uint16_t counter = read_u16(&this_instr[1].cache); @@ -848,7 +848,7 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -859,23 +859,18 @@ // _MAYBE_EXPAND_METHOD args = &stack_pointer[-oparg]; { - args = &stack_pointer[-oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self = PyStackRef_FromPyObjectNew(self); - stack_pointer[-1 - oparg] = maybe_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; func = PyStackRef_FromPyObjectNew(method); stack_pointer[-2 - oparg] = func; - /* Make sure that callable and all args are in memory */ - args[-2] = func; - args[-1] = maybe_self; PyStackRef_CLOSE(callable); } else { func = callable; - maybe_self = self_or_null; } } // _DO_CALL @@ -885,7 +880,7 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -1066,10 +1061,10 @@ INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef null; + _PyStackRef *null; _PyStackRef func; - _PyStackRef self; - _PyStackRef self_or_null; + _PyStackRef *self; + _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ @@ -1078,18 +1073,18 @@ DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - null = stack_pointer[-1 - oparg]; + null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { - DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type, CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { + self = &stack_pointer[-1 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); - self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - stack_pointer[-1 - oparg] = self; + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); stack_pointer[-2 - oparg] = func; PyStackRef_CLOSE(callable); @@ -1105,13 +1100,13 @@ DEOPT_IF(func->func_version != func_version, CALL); } // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null)), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); } // _CHECK_STACK_SPACE { @@ -1125,12 +1120,12 @@ args = &stack_pointer[-oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int has_self = !PyStackRef_IsNull(self_or_null); + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } @@ -1169,10 +1164,10 @@ INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef null; + _PyStackRef *null; _PyStackRef method; - _PyStackRef self; - _PyStackRef self_or_null; + _PyStackRef *self; + _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ @@ -1181,7 +1176,7 @@ DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_METHOD_VERSION - null = stack_pointer[-1 - oparg]; + null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { uint32_t func_version = read_u32(&this_instr[2].cache); @@ -1190,15 +1185,15 @@ PyObject *func = ((PyMethodObject *)callable_o)->im_func; DEOPT_IF(!PyFunction_Check(func), CALL); DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); } // _EXPAND_METHOD { + self = &stack_pointer[-1 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyStackRef_IsNull(null)); + assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - stack_pointer[-1 - oparg] = self; + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); stack_pointer[-2 - oparg] = method; assert(PyStackRef_FunctionCheck(method)); @@ -1207,14 +1202,13 @@ // flush // _PY_FRAME_GENERAL args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -1265,19 +1259,19 @@ INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_CLASS args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -1288,7 +1282,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -1338,20 +1332,20 @@ INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_FAST args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { /* Builtin METH_FASTCALL functions, without keywords */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -1363,7 +1357,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -1417,20 +1411,20 @@ INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_FAST_WITH_KEYWORDS args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -1444,7 +1438,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -1495,20 +1489,20 @@ INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_O args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { /* Builtin METH_O functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -1711,18 +1705,18 @@ INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -1755,12 +1749,12 @@ _Py_CODEUNIT *this_instr = next_instr - 4; (void)this_instr; _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef kwnames; _PyStackRef res; // _SPECIALIZE_CALL_KW - self_or_null = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; { uint16_t counter = read_u16(&this_instr[1].cache); @@ -1768,7 +1762,7 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -1781,15 +1775,14 @@ args = &stack_pointer[-1 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } - if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) { + else if (Py_TYPE(callable_o) == &PyMethod_Type) { args--; total_args++; PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -1828,7 +1821,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -1886,11 +1879,11 @@ INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef null; + _PyStackRef *null; _PyStackRef kwnames; _PyStackRef method; - _PyStackRef self; - _PyStackRef self_or_null; + _PyStackRef *self; + _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ @@ -1899,7 +1892,7 @@ DEOPT_IF(tstate->interp->eval_frame, CALL_KW); } // _CHECK_METHOD_VERSION_KW - null = stack_pointer[-2 - oparg]; + null = &stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; { uint32_t func_version = read_u32(&this_instr[2].cache); @@ -1908,16 +1901,16 @@ PyObject *func = ((PyMethodObject *)callable_o)->im_func; DEOPT_IF(!PyFunction_Check(func), CALL_KW); DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null), CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); } // _EXPAND_METHOD_KW kwnames = stack_pointer[-1]; { + self = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyStackRef_IsNull(null)); + assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - stack_pointer[-2 - oparg] = self; + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); stack_pointer[-3 - oparg] = method; assert(PyStackRef_FunctionCheck(method)); @@ -1927,14 +1920,13 @@ // _PY_FRAME_KW kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -1989,7 +1981,7 @@ static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef kwnames; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ @@ -2004,15 +1996,14 @@ // _CALL_KW_NON_PY kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; { #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -2020,7 +2011,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2077,7 +2068,7 @@ INSTRUCTION_STATS(CALL_KW_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef kwnames; _PyStackRef *args; _PyInterpreterFrame *new_frame; @@ -2098,13 +2089,12 @@ // _PY_FRAME_KW kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -2158,18 +2148,18 @@ INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* len(o) */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -2239,19 +2229,19 @@ INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_FAST args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -2269,7 +2259,7 @@ STACKREFS_TO_PYOBJECTS(args, nargs, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2320,19 +2310,19 @@ INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -2350,7 +2340,7 @@ STACKREFS_TO_PYOBJECTS(args, nargs, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2401,20 +2391,20 @@ INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_NOARGS args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { assert(oparg == 0 || oparg == 1); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -2469,19 +2459,19 @@ INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_O args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -2540,7 +2530,7 @@ INSTRUCTION_STATS(CALL_NON_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ @@ -2554,15 +2544,14 @@ } // _CALL_NON_PY_GENERAL args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; { #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -2570,7 +2559,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); + PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2623,7 +2612,7 @@ INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ @@ -2641,13 +2630,13 @@ DEOPT_IF(func->func_version != func_version, CALL); } // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null)), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); } // _CHECK_STACK_SPACE { @@ -2661,12 +2650,12 @@ args = &stack_pointer[-oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int has_self = !PyStackRef_IsNull(self_or_null); + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } @@ -2705,7 +2694,7 @@ INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ @@ -2724,13 +2713,12 @@ } // _PY_FRAME_GENERAL args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } @@ -4050,43 +4038,38 @@ next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); _PyStackRef callable; - _PyStackRef self_or_null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef func; - _PyStackRef maybe_self; + _PyStackRef *maybe_self; _PyStackRef res; /* Skip 3 cache entries */ // _MAYBE_EXPAND_METHOD args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { - args = &stack_pointer[-oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self = PyStackRef_FromPyObjectNew(self); - stack_pointer[-1 - oparg] = maybe_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; func = PyStackRef_FromPyObjectNew(method); stack_pointer[-2 - oparg] = func; - /* Make sure that callable and all args are in memory */ - args[-2] = func; - args[-1] = maybe_self; PyStackRef_CLOSE(callable); } else { func = callable; - maybe_self = self_or_null; } } // _MONITOR_CALL { - int is_meth = !PyStackRef_IsNull(maybe_self); + int is_meth = !PyStackRef_IsNull(maybe_self[0]); PyObject *function = PyStackRef_AsPyObjectBorrow(func); PyObject *arg0; if (is_meth) { - arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); } else if (oparg) { arg0 = PyStackRef_AsPyObjectBorrow(args[0]); @@ -4107,7 +4090,7 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 672fec3946f2fb..e89c969ad0df67 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1686,11 +1686,13 @@ case _EXPAND_METHOD: { _Py_UopsSymbol *method; - _Py_UopsSymbol *self; + _Py_UopsSymbol **self; + self = &stack_pointer[-1 - oparg]; method = sym_new_not_null(ctx); - self = sym_new_not_null(ctx); + for (int _i = 1; --_i >= 0;) { + self[_i] = sym_new_not_null(ctx); + } stack_pointer[-2 - oparg] = method; - stack_pointer[-1 - oparg] = self; break; } @@ -2045,13 +2047,15 @@ case _EXPAND_METHOD_KW: { _Py_UopsSymbol *method; - _Py_UopsSymbol *self; + _Py_UopsSymbol **self; _Py_UopsSymbol *kwnames; + self = &stack_pointer[-2 - oparg]; method = sym_new_not_null(ctx); - self = sym_new_not_null(ctx); + for (int _i = 1; --_i >= 0;) { + self[_i] = sym_new_not_null(ctx); + } kwnames = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = method; - stack_pointer[-2 - oparg] = self; stack_pointer[-1] = kwnames; break; } diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index dd4057c931ca19..2f8fccec2ea409 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -165,9 +165,12 @@ def decref_inputs( if var.name == "unused" or var.name == "null" or var.peek: continue if var.size: - self.out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - self.out.emit(f"PyStackRef_CLOSE({var.name}[_i]);\n") - self.out.emit("}\n") + if var.size == "1": + self.out.emit(f"PyStackRef_CLOSE({var.name}[0]);\n") + else: + self.out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + self.out.emit(f"PyStackRef_CLOSE({var.name}[_i]);\n") + self.out.emit("}\n") elif var.condition: if var.condition == "1": self.out.emit(f"PyStackRef_CLOSE({var.name});\n") diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 34bf597f2f552d..de4d900563ee0b 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -164,6 +164,8 @@ def clear(self) -> None: class StackError(Exception): pass +def array_or_scalar(var: StackItem | Local) -> str: + return "array" if var.is_array() else "scalar" class Stack: def __init__(self) -> None: @@ -177,10 +179,15 @@ def pop(self, var: StackItem, extract_bits: bool = False) -> tuple[str, Local]: indirect = "&" if var.is_array() else "" if self.variables: popped = self.variables.pop() + if var.is_array() ^ popped.is_array(): + raise StackError( + f"Array mismatch when popping '{popped.name}' from stack to assign to '{var.name}'. " + f"Expected {array_or_scalar(var)} got {array_or_scalar(popped)}" + ) if popped.size != var.size: raise StackError( - f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. " - f"Expected {var.size} got {popped.size}" + f"Size mismatch when popping '{popped.name}' from stack to assign to '{var.name}'. " + f"Expected {var_size(var)} got {var_size(popped.item)}" ) if var.name in UNUSED: if popped.name not in UNUSED and popped.name in self.defined: From b2afe2aae487ebf89897e22c01d9095944fd334f Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 12 Sep 2024 12:37:06 -0400 Subject: [PATCH 21/50] gh-123923: Defer refcounting for `f_executable` in `_PyInterpreterFrame` (#123924) Use a `_PyStackRef` and defer the reference to `f_executable` when possible. This avoids some reference count contention in the common case of executing the same code object from multiple threads concurrently in the free-threaded build. --- Include/internal/pycore_frame.h | 13 +- Include/internal/pycore_gc.h | 3 + Include/internal/pycore_stackref.h | 12 ++ Lib/test/test_generators.py | 22 +++ ...-09-10-20-25-00.gh-issue-123923.A7uxqa.rst | 4 + Modules/_testexternalinspection.c | 7 +- Objects/frameobject.c | 7 +- Objects/genobject.c | 16 +- Python/bytecodes.c | 4 +- Python/ceval.c | 6 +- Python/executor_cases.c.h | 4 +- Python/frame.c | 11 +- Python/gc.c | 7 + Python/gc_free_threading.c | 137 ++++++++++-------- Python/generated_cases.c.h | 2 +- Python/legacy_tracing.c | 3 +- Python/tracemalloc.c | 4 +- Tools/gdb/libpython.py | 14 +- 18 files changed, 177 insertions(+), 99 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-10-20-25-00.gh-issue-123923.A7uxqa.rst diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index a6f7c1735b349f..a77658134fae8c 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -60,7 +60,7 @@ enum _frameowner { }; typedef struct _PyInterpreterFrame { - PyObject *f_executable; /* Strong reference (code object or None) */ + _PyStackRef f_executable; /* Deferred or strong reference (code object or None) */ struct _PyInterpreterFrame *previous; PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ @@ -79,8 +79,9 @@ typedef struct _PyInterpreterFrame { ((int)((IF)->instr_ptr - _PyCode_CODE(_PyFrame_GetCode(IF)))) static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { - assert(PyCode_Check(f->f_executable)); - return (PyCodeObject *)f->f_executable; + PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); + assert(PyCode_Check(executable)); + return (PyCodeObject *)executable; } static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { @@ -130,7 +131,7 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * dest->previous = NULL; #ifdef Py_GIL_DISABLED - PyCodeObject *co = (PyCodeObject *)dest->f_executable; + PyCodeObject *co = _PyFrame_GetCode(dest); for (int i = stacktop; i < co->co_nlocalsplus + co->co_stacksize; i++) { dest->localsplus[i] = PyStackRef_NULL; } @@ -148,7 +149,7 @@ _PyFrame_Initialize( { frame->previous = previous; frame->f_funcobj = (PyObject *)func; - frame->f_executable = Py_NewRef(code); + frame->f_executable = PyStackRef_FromPyObjectNew(code); frame->f_builtins = func->func_builtins; frame->f_globals = func->func_globals; frame->f_locals = locals; @@ -321,7 +322,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int assert(tstate->datastack_top < tstate->datastack_limit); frame->previous = previous; frame->f_funcobj = Py_None; - frame->f_executable = Py_NewRef(code); + frame->f_executable = PyStackRef_FromPyObjectNew(code); #ifdef Py_DEBUG frame->f_builtins = NULL; frame->f_globals = NULL; diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 89f6017bacc525..b674313d97ff82 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -381,8 +381,11 @@ extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); +union _PyStackRef; + // GC visit callback for tracked interpreter frames extern int _PyGC_VisitFrameStack(struct _PyInterpreterFrame *frame, visitproc visit, void *arg); +extern int _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg); #ifdef __cplusplus } diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 7c106577671b5f..f23f641a47e25f 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -227,6 +227,13 @@ PyStackRef_DUP(_PyStackRef stackref) # define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref))) #endif +// Convert a possibly deferred reference to a strong reference. +static inline _PyStackRef +PyStackRef_AsStrongReference(_PyStackRef stackref) +{ + return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref)); +} + static inline void _PyObjectStack_FromStackRefStack(PyObject **dst, const _PyStackRef *src, size_t length) { @@ -261,6 +268,11 @@ PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref) return PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(stackref)); } +static inline bool +PyStackRef_CodeCheck(_PyStackRef stackref) +{ + return PyCode_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} static inline bool PyStackRef_FunctionCheck(_PyStackRef stackref) diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 3cb4d51fff1eeb..03a31ec6a05726 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -6,6 +6,7 @@ import unittest import weakref import inspect +import textwrap import types from test import support @@ -112,6 +113,27 @@ def g3(): return (yield from f()) gen.send(2) self.assertEqual(cm.exception.value, 2) + def test_generator_resurrect(self): + # Test that a resurrected generator still has a valid gi_code + resurrected = [] + + # Resurrect a generator in a finalizer + exec(textwrap.dedent(""" + def gen(): + try: + yield + except: + resurrected.append(g) + + g = gen() + next(g) + """), {"resurrected": resurrected}) + + support.gc_collect() + + self.assertEqual(len(resurrected), 1) + self.assertIsInstance(resurrected[0].gi_code, types.CodeType) + class GeneratorTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-10-20-25-00.gh-issue-123923.A7uxqa.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-10-20-25-00.gh-issue-123923.A7uxqa.rst new file mode 100644 index 00000000000000..b7bc965b68b059 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-10-20-25-00.gh-issue-123923.A7uxqa.rst @@ -0,0 +1,4 @@ +The ``f_executable`` field in the internal :c:struct:`_PyInterpreterFrame` +struct now uses a tagged pointer. Profilers and debuggers that uses this +field should clear the least significant bit to recover the +:c:expr:`PyObject*` pointer. diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 2a665affb5e7f8..2476346777c319 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -510,7 +510,7 @@ parse_frame_object( return 0; } - void* address_of_code_object; + uintptr_t address_of_code_object; bytes_read = read_memory( pid, (void*)(address + offsets->interpreter_frame.executable), @@ -520,10 +520,11 @@ parse_frame_object( return -1; } - if (address_of_code_object == NULL) { + if (address_of_code_object == 0) { return 0; } - return parse_code_object(pid, result, offsets, address_of_code_object, previous_frame); + address_of_code_object &= ~Py_TAG_BITS; + return parse_code_object(pid, result, offsets, (void *)address_of_code_object, previous_frame); } static PyObject* diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 85c24550d0b409..b567327f970836 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1625,8 +1625,6 @@ frame_dealloc(PyFrameObject *f) } Py_TRASHCAN_BEGIN(f, frame_dealloc); - PyObject *co = NULL; - /* GH-106092: If f->f_frame was on the stack and we reached the maximum * nesting depth for deallocations, the trashcan may have delayed this * deallocation until after f->f_frame is freed. Avoid dereferencing @@ -1635,9 +1633,7 @@ frame_dealloc(PyFrameObject *f) /* Kill all local variables including specials, if we own them */ if (f->f_frame == frame && frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { - /* Don't clear code object until the end */ - co = frame->f_executable; - frame->f_executable = NULL; + PyStackRef_CLEAR(frame->f_executable); Py_CLEAR(frame->f_funcobj); Py_CLEAR(frame->f_locals); _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); @@ -1652,7 +1648,6 @@ frame_dealloc(PyFrameObject *f) Py_CLEAR(f->f_extra_locals); Py_CLEAR(f->f_locals_cache); PyObject_GC_Del(f); - Py_XDECREF(co); Py_TRASHCAN_END; } diff --git a/Objects/genobject.c b/Objects/genobject.c index b281af8d7e1f4e..5dc8f926557b52 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -55,6 +55,14 @@ gen_traverse(PyGenObject *gen, visitproc visit, void *arg) return err; } } + else { + // We still need to visit the code object when the frame is cleared to + // ensure that it's kept alive if the reference is deferred. + int err = _PyGC_VisitStackRef(&gen->gi_iframe.f_executable, visit, arg); + if (err) { + return err; + } + } /* No need to visit cr_origin, because it's just tuples/str/int, so can't participate in a reference cycle. */ Py_VISIT(gen->gi_exc_state.exc_value); @@ -139,6 +147,9 @@ gen_dealloc(PyGenObject *gen) and GC_Del. */ Py_CLEAR(((PyAsyncGenObject*)gen)->ag_origin_or_finalizer); } + if (PyCoro_CheckExact(gen)) { + Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer); + } if (gen->gi_frame_state != FRAME_CLEARED) { _PyInterpreterFrame *frame = &gen->gi_iframe; gen->gi_frame_state = FRAME_CLEARED; @@ -147,10 +158,7 @@ gen_dealloc(PyGenObject *gen) _PyErr_ClearExcState(&gen->gi_exc_state); } assert(gen->gi_exc_state.exc_value == NULL); - if (_PyGen_GetCode(gen)->co_flags & CO_COROUTINE) { - Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer); - } - Py_DECREF(_PyGen_GetCode(gen)); + PyStackRef_CLEAR(gen->gi_iframe.f_executable); Py_CLEAR(gen->gi_name); Py_CLEAR(gen->gi_qualname); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e8cf636656ee9d..078f06d697cc3c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3598,7 +3598,7 @@ dummy_func( op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _PyInterpreterFrame *)) { _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + assert(_PyCode_CODE(_PyFrame_GetCode(shim))[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self); PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); @@ -4798,7 +4798,7 @@ dummy_func( #endif _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { - PyCodeObject *code = (PyCodeObject *)frame->f_executable; + PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); } diff --git a/Python/ceval.c b/Python/ceval.c index 0ebd5bb58c859c..252833a6243293 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -194,7 +194,7 @@ static void lltrace_resume_frame(_PyInterpreterFrame *frame) { PyObject *fobj = frame->f_funcobj; - if (!PyCode_Check(frame->f_executable) || + if (!PyStackRef_CodeCheck(frame->f_executable) || fobj == NULL || !PyFunction_Check(fobj) ) { @@ -784,7 +784,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry_frame.f_globals = (PyObject*)0xaaa3; entry_frame.f_builtins = (PyObject*)0xaaa4; #endif - entry_frame.f_executable = Py_None; + entry_frame.f_executable = PyStackRef_None; entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; entry_frame.stackpointer = entry_frame.localsplus; entry_frame.owner = FRAME_OWNED_BY_CSTACK; @@ -1681,7 +1681,7 @@ clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) tstate->c_recursion_remaining--; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); _PyFrame_ClearExceptCode(frame); - Py_DECREF(frame->f_executable); + PyStackRef_CLEAR(frame->f_executable); tstate->c_recursion_remaining++; _PyThreadState_PopFrame(tstate, frame); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b9f532f9b118a2..b36fff9961febe 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4143,7 +4143,7 @@ self = stack_pointer[-2 - oparg]; _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + assert(_PyCode_CODE(_PyFrame_GetCode(shim))[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self); PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); @@ -5413,7 +5413,7 @@ #endif _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { - PyCodeObject *code = (PyCodeObject *)frame->f_executable; + PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); } diff --git a/Python/frame.c b/Python/frame.c index 3192968a0fb1b5..d7bb29811bfa50 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -14,7 +14,10 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) Py_VISIT(frame->frame_obj); Py_VISIT(frame->f_locals); Py_VISIT(frame->f_funcobj); - Py_VISIT(_PyFrame_GetCode(frame)); + int err = _PyGC_VisitStackRef(&frame->f_executable, visit, arg); + if (err) { + return err; + } return _PyGC_VisitFrameStack(frame, visit, arg); } @@ -53,10 +56,10 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); assert(frame->owner != FRAME_CLEARED); Py_ssize_t size = ((char*)frame->stackpointer) - (char *)frame; - Py_INCREF(_PyFrame_GetCode(frame)); memcpy((_PyInterpreterFrame *)f->_f_frame_data, frame, size); frame = (_PyInterpreterFrame *)f->_f_frame_data; frame->stackpointer = (_PyStackRef *)(((char *)frame) + size); + frame->f_executable = PyStackRef_DUP(frame->f_executable); f->f_frame = frame; frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; if (_PyFrame_IsIncomplete(frame)) { @@ -131,9 +134,7 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) PyObject * PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame) { - PyObject *code = frame->f_executable; - Py_INCREF(code); - return code; + return PyStackRef_AsPyObjectNew(frame->f_executable); } int diff --git a/Python/gc.c b/Python/gc.c index 3d36792ffb27fc..024d041437be4a 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -534,6 +534,13 @@ visit_decref(PyObject *op, void *parent) return 0; } +int +_PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg) +{ + Py_VISIT(PyStackRef_AsPyObjectBorrow(*ref)); + return 0; +} + int _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) { diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 54de0c2671ae68..e981f87401a066 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -161,39 +161,6 @@ gc_decref(PyObject *op) op->ob_tid -= 1; } -static void -disable_deferred_refcounting(PyObject *op) -{ - if (!_PyObject_HasDeferredRefcount(op)) { - return; - } - - op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED; - op->ob_ref_shared -= _Py_REF_SHARED(_Py_REF_DEFERRED, 0); - - if (PyType_Check(op)) { - // Disable thread-local refcounting for heap types - PyTypeObject *type = (PyTypeObject *)op; - if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { - _PyType_ReleaseId((PyHeapTypeObject *)op); - } - } - else if (PyGen_CheckExact(op) || PyCoro_CheckExact(op) || PyAsyncGen_CheckExact(op)) { - // Ensure any non-refcounted pointers in locals are converted to - // strong references. This ensures that the generator/coroutine is not - // freed before its locals. - PyGenObject *gen = (PyGenObject *)op; - struct _PyInterpreterFrame *frame = &gen->gi_iframe; - assert(frame->stackpointer != NULL); - for (_PyStackRef *ref = frame->localsplus; ref < frame->stackpointer; ref++) { - if (!PyStackRef_IsNull(*ref) && PyStackRef_IsDeferred(*ref)) { - // Convert a deferred reference to a strong reference. - *ref = PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(*ref)); - } - } - } -} - static Py_ssize_t merge_refcount(PyObject *op, Py_ssize_t extra) { @@ -213,6 +180,51 @@ merge_refcount(PyObject *op, Py_ssize_t extra) return refcount; } +static void +frame_disable_deferred_refcounting(_PyInterpreterFrame *frame) +{ + // Convert locals, variables, and the executable object to strong + // references from (possibly) deferred references. + assert(frame->stackpointer != NULL); + frame->f_executable = PyStackRef_AsStrongReference(frame->f_executable); + for (_PyStackRef *ref = frame->localsplus; ref < frame->stackpointer; ref++) { + if (!PyStackRef_IsNull(*ref) && PyStackRef_IsDeferred(*ref)) { + *ref = PyStackRef_AsStrongReference(*ref); + } + } +} + +static void +disable_deferred_refcounting(PyObject *op) +{ + if (_PyObject_HasDeferredRefcount(op)) { + op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED; + op->ob_ref_shared -= _Py_REF_SHARED(_Py_REF_DEFERRED, 0); + merge_refcount(op, 0); + } + + // Heap types also use thread-local refcounting -- disable it here. + if (PyType_Check(op)) { + // Disable thread-local refcounting for heap types + PyTypeObject *type = (PyTypeObject *)op; + if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + _PyType_ReleaseId((PyHeapTypeObject *)op); + } + } + + // Generators and frame objects may contain deferred references to other + // objects. If the pointed-to objects are part of cyclic trash, we may + // have disabled deferred refcounting on them and need to ensure that we + // use strong references, in case the generator or frame object is + // resurrected by a finalizer. + if (PyGen_CheckExact(op) || PyCoro_CheckExact(op) || PyAsyncGen_CheckExact(op)) { + frame_disable_deferred_refcounting(&((PyGenObject *)op)->gi_iframe); + } + else if (PyFrame_Check(op)) { + frame_disable_deferred_refcounting(((PyFrameObject *)op)->f_frame); + } +} + static void gc_restore_tid(PyObject *op) { @@ -349,16 +361,18 @@ gc_visit_thread_stacks(PyInterpreterState *interp) { HEAD_LOCK(&_PyRuntime); for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { - _PyInterpreterFrame *f = p->current_frame; - while (f != NULL) { - if (f->f_executable != NULL && PyCode_Check(f->f_executable)) { - PyCodeObject *co = (PyCodeObject *)f->f_executable; - int max_stack = co->co_nlocalsplus + co->co_stacksize; - for (int i = 0; i < max_stack; i++) { - gc_visit_stackref(f->localsplus[i]); - } + for (_PyInterpreterFrame *f = p->current_frame; f != NULL; f = f->previous) { + PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); + if (executable == NULL || !PyCode_Check(executable)) { + continue; + } + + PyCodeObject *co = (PyCodeObject *)executable; + int max_stack = co->co_nlocalsplus + co->co_stacksize; + gc_visit_stackref(f->f_executable); + for (int i = 0; i < max_stack; i++) { + gc_visit_stackref(f->localsplus[i]); } - f = f->previous; } } HEAD_UNLOCK(&_PyRuntime); @@ -623,23 +637,19 @@ scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, else { worklist_push(&state->unreachable, op); } + return true; } - else if (state->reason == _Py_GC_REASON_SHUTDOWN && - _PyObject_HasDeferredRefcount(op)) - { + + if (state->reason == _Py_GC_REASON_SHUTDOWN) { // Disable deferred refcounting for reachable objects as well during // interpreter shutdown. This ensures that these objects are collected // immediately when their last reference is removed. disable_deferred_refcounting(op); - merge_refcount(op, 0); - state->long_lived_total++; - } - else { - // object is reachable, restore `ob_tid`; we're done with these objects - gc_restore_tid(op); - state->long_lived_total++; } + // object is reachable, restore `ob_tid`; we're done with these objects + gc_restore_tid(op); + state->long_lived_total++; return true; } @@ -951,20 +961,29 @@ visit_decref_unreachable(PyObject *op, void *data) return 0; } +int +_PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg) +{ + // This is a bit tricky! We want to ignore deferred references when + // computing the incoming references, but otherwise treat them like + // regular references. + if (!PyStackRef_IsDeferred(*ref) || + (visit != visit_decref && visit != visit_decref_unreachable)) + { + Py_VISIT(PyStackRef_AsPyObjectBorrow(*ref)); + } + return 0; +} + int _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) { _PyStackRef *ref = _PyFrame_GetLocalsArray(frame); /* locals and stack */ for (; ref < frame->stackpointer; ref++) { - // This is a bit tricky! We want to ignore deferred references when - // computing the incoming references, but otherwise treat them like - // regular references. - if (PyStackRef_IsDeferred(*ref) && - (visit == visit_decref || visit == visit_decref_unreachable)) { - continue; + if (_PyGC_VisitStackRef(ref, visit, arg) < 0) { + return -1; } - Py_VISIT(PyStackRef_AsPyObjectBorrow(*ref)); } return 0; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 09a14ee4d3896b..3e9f0396e495b7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1018,7 +1018,7 @@ { _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + assert(_PyCode_CODE(_PyFrame_GetCode(shim))[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self); PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 9cc3af1f5e162c..1436921a19b768 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -143,9 +143,8 @@ _PyEval_SetOpcodeTrace( bool enable ) { assert(frame != NULL); - assert(PyCode_Check(frame->f_frame->f_executable)); - PyCodeObject *code = (PyCodeObject *)frame->f_frame->f_executable; + PyCodeObject *code = _PyFrame_GetCode(frame->f_frame); _PyMonitoringEventSet events = 0; if (_PyMonitoring_GetLocalEvents(code, PY_MONITORING_SYS_TRACE_ID, &events) < 0) { diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index e58b60ddd5e484..f661d69c0312fa 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -250,7 +250,7 @@ hashtable_compare_traceback(const void *key1, const void *key2) static void tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) { - assert(PyCode_Check(pyframe->f_executable)); + assert(PyStackRef_CodeCheck(pyframe->f_executable)); frame->filename = &_Py_STR(anon_unknown); int lineno = PyUnstable_InterpreterFrame_GetLine(pyframe); if (lineno < 0) { @@ -258,7 +258,7 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) } frame->lineno = (unsigned int)lineno; - PyObject *filename = filename = ((PyCodeObject *)pyframe->f_executable)->co_filename; + PyObject *filename = filename = _PyFrame_GetCode(pyframe)->co_filename; if (filename == NULL) { #ifdef TRACE_DEBUG diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index cf03788d037a81..946af4be1a7589 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -158,8 +158,11 @@ class PyObjectPtr(object): def __init__(self, gdbval, cast_to=None): # Clear the tagged pointer - gdbval = gdb.Value(int(gdbval) & (~USED_TAGS)).cast(gdbval.type) - if cast_to: + if gdbval.type.name == '_PyStackRef': + if cast_to is None: + cast_to = gdb.lookup_type('PyObject').pointer() + self._gdbval = gdb.Value(int(gdbval['bits']) & ~USED_TAGS).cast(cast_to) + elif cast_to: self._gdbval = gdbval.cast(cast_to) else: self._gdbval = gdbval @@ -1052,7 +1055,7 @@ def iter_locals(self): obj_ptr_ptr = gdb.lookup_type("PyObject").pointer().pointer() - localsplus = self._gdbval["localsplus"].cast(obj_ptr_ptr) + localsplus = self._gdbval["localsplus"] for i in safe_range(self.co_nlocals): pyop_value = PyObjectPtr.from_pyobject_ptr(localsplus[i]) @@ -1581,7 +1584,10 @@ def to_string (self): return stringify(proxyval) def pretty_printer_lookup(gdbval): - type = gdbval.type.unqualified() + type = gdbval.type.strip_typedefs().unqualified() + if type.code == gdb.TYPE_CODE_UNION and type.tag == '_PyStackRef': + return PyObjectPtrPrinter(gdbval) + if type.code != gdb.TYPE_CODE_PTR: return None From 8145ebea587284db3be3f152ee0298952977d6f4 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 12 Sep 2024 19:58:32 +0100 Subject: [PATCH 22/50] gh-124013: remove _PyCompile_IsTopLevelAwait (#124014) --- Include/internal/pycore_compile.h | 4 ---- Python/codegen.c | 16 ---------------- Python/compile.c | 9 --------- 3 files changed, 29 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 9219d4d735a576..51db6fc608caf7 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -141,10 +141,6 @@ PyObject *_PyCompile_Qualname(struct _PyCompiler *c); _PyCompile_CodeUnitMetadata *_PyCompile_Metadata(struct _PyCompiler *c); PyObject *_PyCompile_StaticAttributesAsTuple(struct _PyCompiler *c); -#ifndef NDEBUG -int _PyCompile_IsTopLevelAwait(struct _PyCompiler *c); -#endif - struct symtable *_PyCompile_Symtable(struct _PyCompiler *c); PySTEntryObject *_PyCompile_SymtableEntry(struct _PyCompiler *c); diff --git a/Python/codegen.c b/Python/codegen.c index fd2260a538f76d..9ce4e1fb1f6c92 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -65,7 +65,6 @@ struct _PyCompiler; typedef struct _PyCompiler compiler; -#define IS_TOP_LEVEL_AWAIT(C) _PyCompile_IsTopLevelAwait(C) #define INSTR_SEQUENCE(C) _PyCompile_InstrSequence(C) #define FUTURE_FEATURES(C) _PyCompile_FutureFeatures(C) #define SYMTABLE(C) _PyCompile_Symtable(C) @@ -4497,10 +4496,6 @@ codegen_comprehension(compiler *c, expr_ty e, int type, PyCodeObject *co = NULL; _PyCompile_InlinedComprehensionState inline_state = {NULL, NULL, NULL, NO_LABEL}; comprehension_ty outermost; -#ifndef NDEBUG - int scope_type = SCOPE_TYPE(c); - int is_top_level_await = IS_TOP_LEVEL_AWAIT(c); -#endif PySTEntryObject *entry = _PySymtable_Lookup(SYMTABLE(c), (void *)e); if (entry == NULL) { goto error; @@ -4531,12 +4526,6 @@ codegen_comprehension(compiler *c, expr_ty e, int type, } Py_CLEAR(entry); - assert (!is_async_comprehension || - type == COMP_GENEXP || - scope_type == COMPILE_SCOPE_ASYNC_FUNCTION || - scope_type == COMPILE_SCOPE_COMPREHENSION || - is_top_level_await); - if (type != COMP_GENEXP) { int op; switch (type) { @@ -4961,11 +4950,6 @@ codegen_visit_expr(compiler *c, expr_ty e) ADD_YIELD_FROM(c, loc, 0); break; case Await_kind: - assert(IS_TOP_LEVEL_AWAIT(c) || (_PyST_IsFunctionLike(SYMTABLE_ENTRY(c)) && ( - SCOPE_TYPE(c) == COMPILE_SCOPE_ASYNC_FUNCTION || - SCOPE_TYPE(c) == COMPILE_SCOPE_COMPREHENSION - ))); - VISIT(c, expr, e->v.Await.value); ADDOP_I(c, loc, GET_AWAITABLE, 0); ADDOP_LOAD_CONST(c, loc, Py_None); diff --git a/Python/compile.c b/Python/compile.c index 72399c758d51e8..d54c320babc2b7 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1242,15 +1242,6 @@ _PyCompile_Metadata(compiler *c) return &c->u->u_metadata; } -#ifndef NDEBUG -int -_PyCompile_IsTopLevelAwait(compiler *c) -{ - return c->c_flags.cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT && - c->u->u_ste->ste_type == ModuleBlock; -} -#endif - // Merge *obj* with constant cache, without recursion. int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj) From a53812df126b99bca25187441a123c7785ee82a0 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 12 Sep 2024 15:08:06 -0400 Subject: [PATCH 23/50] gh-123085: Fix issue in inferred caller when resources package has no source (#123102) gh-123085: Fix issue in inferred caller when resources package has no source. From importlib_resources 6.4.3 (python/importlib_resources#314). --- Lib/importlib/resources/_common.py | 5 ++- .../test_importlib/resources/test_files.py | 39 ++++++++++++++++++- ...-08-17-08-17-20.gh-issue-123085.7Io2yH.rst | 3 ++ 3 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-08-17-08-17-20.gh-issue-123085.7Io2yH.rst diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index d5381fb80d85e6..c2c92254370f71 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -93,12 +93,13 @@ def _infer_caller(): """ def is_this_file(frame_info): - return frame_info.filename == __file__ + return frame_info.filename == stack[0].filename def is_wrapper(frame_info): return frame_info.function == 'wrapper' - not_this_file = itertools.filterfalse(is_this_file, inspect.stack()) + stack = inspect.stack() + not_this_file = itertools.filterfalse(is_this_file, stack) # also exclude 'wrapper' due to singledispatch in the call stack callers = itertools.filterfalse(is_wrapper, not_this_file) return next(callers).frame diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 08b840834dfd4b..3cdbee302c5e75 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -1,3 +1,7 @@ +import os +import pathlib +import py_compile +import shutil import textwrap import unittest import warnings @@ -7,6 +11,7 @@ from importlib import resources from importlib.resources.abc import Traversable from . import util +from test.support import os_helper, import_helper @contextlib.contextmanager @@ -97,8 +102,8 @@ class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.Test class ImplicitContextFiles: set_val = textwrap.dedent( - """ - import importlib.resources as res + f""" + import {resources.__name__} as res val = res.files().joinpath('res.txt').read_text(encoding='utf-8') """ ) @@ -108,6 +113,10 @@ class ImplicitContextFiles: 'submod.py': set_val, 'res.txt': 'resources are the best', }, + 'frozenpkg': { + '__init__.py': set_val.replace(resources.__name__, 'c_resources'), + 'res.txt': 'resources are the best', + }, } def test_implicit_files_package(self): @@ -122,6 +131,32 @@ def test_implicit_files_submodule(self): """ assert importlib.import_module('somepkg.submod').val == 'resources are the best' + def _compile_importlib(self): + """ + Make a compiled-only copy of the importlib resources package. + """ + bin_site = self.fixtures.enter_context(os_helper.temp_dir()) + c_resources = pathlib.Path(bin_site, 'c_resources') + sources = pathlib.Path(resources.__file__).parent + shutil.copytree(sources, c_resources, ignore=lambda *_: ['__pycache__']) + + for dirpath, _, filenames in os.walk(c_resources): + for filename in filenames: + source_path = pathlib.Path(dirpath) / filename + cfile = source_path.with_suffix('.pyc') + py_compile.compile(source_path, cfile) + pathlib.Path.unlink(source_path) + self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site)) + + def test_implicit_files_with_compiled_importlib(self): + """ + Caller detection works for compiled-only resources module. + + python/cpython#123085 + """ + self._compile_importlib() + assert importlib.import_module('frozenpkg').val == 'resources are the best' + class ImplicitContextFilesDiskTests( DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase diff --git a/Misc/NEWS.d/next/Library/2024-08-17-08-17-20.gh-issue-123085.7Io2yH.rst b/Misc/NEWS.d/next/Library/2024-08-17-08-17-20.gh-issue-123085.7Io2yH.rst new file mode 100644 index 00000000000000..2e09401ceb5b56 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-17-08-17-20.gh-issue-123085.7Io2yH.rst @@ -0,0 +1,3 @@ +In a bare call to :func:`importlib.resources.files`, ensure the caller's +frame is properly detected when ``importlib.resources`` is itself available +as a compiled module only (no source). From 6e06e01881dcffbeef5baac0c112ffb14cfa0b27 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 12 Sep 2024 20:24:15 +0100 Subject: [PATCH 24/50] gh-124019: do not call codegen_annotations_in_scope if there are no annotations (#124020) --- Python/codegen.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/Python/codegen.c b/Python/codegen.c index 9ce4e1fb1f6c92..2ca5db1fc6ad34 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -1002,26 +1002,22 @@ codegen_annotations(compiler *c, location loc, PySTEntryObject *ste; RETURN_IF_ERROR(_PySymtable_LookupOptional(SYMTABLE(c), args, &ste)); assert(ste != NULL); - bool annotations_used = ste->ste_annotations_used; - int err = annotations_used ? - codegen_setup_annotations_scope(c, loc, (void *)args, ste->ste_name) : SUCCESS; - Py_DECREF(ste); - RETURN_IF_ERROR(err); - - if (codegen_annotations_in_scope(c, loc, args, returns, &annotations_len) < 0) { - if (annotations_used) { - _PyCompile_ExitScope(c); - } - return ERROR; - } - - if (annotations_used) { + if (ste->ste_annotations_used) { + int err = codegen_setup_annotations_scope(c, loc, (void *)args, ste->ste_name); + Py_DECREF(ste); + RETURN_IF_ERROR(err); + RETURN_IF_ERROR_IN_SCOPE( + c, codegen_annotations_in_scope(c, loc, args, returns, &annotations_len) + ); RETURN_IF_ERROR( codegen_leave_annotations_scope(c, loc, annotations_len) ); return MAKE_FUNCTION_ANNOTATE; } + else { + Py_DECREF(ste); + } return 0; } From f4e5643df64d0c2a009ed224560044b3409a47c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 13 Sep 2024 03:07:23 +0200 Subject: [PATCH 25/50] gh-124027: Support Del, PgUp, and PgDn on TERM=vt100 (#124028) pyrepl: Support Del, PgUp, and PgDn on TERM=vt100 From Fedora's /etc/inputrc: "\e[5~": history-search-backward "\e[6~": history-search-forward "\e[3~": delete-char Fixes https://github.com/python/cpython/issues/124027 --- Lib/_pyrepl/historical_reader.py | 2 ++ Lib/_pyrepl/reader.py | 1 + .../2024-09-13-02-25-06.gh-issue-124027.to_9DY.rst | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-13-02-25-06.gh-issue-124027.to_9DY.rst diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index f6e14bdffc3352..5d416f336ad5d2 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -266,7 +266,9 @@ def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: (r"\M-r", "restore-history"), (r"\M-.", "yank-arg"), (r"\", "history-search-forward"), + (r"\x1b[6~", "history-search-forward"), (r"\", "history-search-backward"), + (r"\x1b[5~", "history-search-backward"), ) def select_item(self, i: int) -> None: diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 54bd1ea0222a60..4b0700d069c621 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -150,6 +150,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]: (r"\", "right"), (r"\C-\", "forward-word"), (r"\", "delete"), + (r"\x1b[3~", "delete"), (r"\", "backspace"), (r"\M-\", "backward-kill-word"), (r"\", "end-of-line"), # was 'end' diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-13-02-25-06.gh-issue-124027.to_9DY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-13-02-25-06.gh-issue-124027.to_9DY.rst new file mode 100644 index 00000000000000..1834ba0ba08bfb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-13-02-25-06.gh-issue-124027.to_9DY.rst @@ -0,0 +1,2 @@ +Support ````, ````, and ```` keys in the Python +REPL when ``$TERM`` is set to ``vt100``. From a47cd21890fee845d582b41ecf7d029801899ea5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 12 Sep 2024 22:29:08 -0400 Subject: [PATCH 26/50] gh-123976: Refresh docs around custom providers. (#123977) * gh-123976: Refresh docs around custom providers. * Remove excess whitespace. --- Doc/library/importlib.metadata.rst | 50 ++++++++++--------- ...-09-11-16-52-08.gh-issue-123976.jhOfNR.rst | 1 + 2 files changed, 28 insertions(+), 23 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2024-09-11-16-52-08.gh-issue-123976.jhOfNR.rst diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 9c0879f5ca850f..66ba621084c898 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -19,7 +19,7 @@ such as its entry points or its top-level names (`Import Package `_\s, modules, if any). Built in part on Python's import system, this library intends to replace similar functionality in the `entry point -API`_ and `metadata API`_ of ``pkg_resources``. Along with +API`_ and `metadata API`_ of ``pkg_resources``. Along with :mod:`importlib.resources`, this package can eliminate the need to use the older and less efficient ``pkg_resources`` package. @@ -46,7 +46,7 @@ and metadata defined by the `Core metadata specifications `_ you've installed -using ``pip``. We start by creating a virtual environment and installing +using ``pip``. We start by creating a virtual environment and installing something into it: .. code-block:: shell-session @@ -87,7 +87,7 @@ You can get the version string for ``wheel`` by running the following: '0.32.3' You can also get a collection of entry points selectable by properties of the EntryPoint (typically 'group' or 'name'), such as -``console_scripts``, ``distutils.commands`` and others. Each group contains a +``console_scripts``, ``distutils.commands`` and others. Each group contains a collection of :ref:`EntryPoint ` objects. You can get the :ref:`metadata for a distribution `:: @@ -114,7 +114,7 @@ Entry points The ``entry_points()`` function returns a collection of entry points. Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and -a ``.load()`` method to resolve the value. There are also ``.module``, +a ``.load()`` method to resolve the value. There are also ``.module``, ``.attr``, and ``.extras`` attributes for getting the components of the ``.value`` attribute. @@ -167,7 +167,7 @@ Inspect the resolved entry point:: The ``group`` and ``name`` are arbitrary values defined by the package author and usually a client will wish to resolve all entry points for a particular -group. Read `the setuptools docs +group. Read `the setuptools docs `_ for more information on entry points, their definition, and usage. @@ -240,12 +240,12 @@ number, as a string:: Distribution files ------------------ -You can also get the full set of files contained within a distribution. The +You can also get the full set of files contained within a distribution. The ``files()`` function takes a `Distribution Package `_ name and returns all of the -files installed by this distribution. Each file object returned is a +files installed by this distribution. Each file object returned is a ``PackagePath``, a :class:`pathlib.PurePath` derived object with additional ``dist``, -``size``, and ``hash`` properties as indicated by the metadata. For example:: +``size``, and ``hash`` properties as indicated by the metadata. For example:: >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP >>> util # doctest: +SKIP @@ -321,9 +321,9 @@ Distributions ============= While the above API is the most common and convenient usage, you can get all -of that information from the ``Distribution`` class. A ``Distribution`` is an +of that information from the ``Distribution`` class. A ``Distribution`` is an abstract object that represents the metadata for -a Python `Distribution Package `_. You can +a Python `Distribution Package `_. You can get the ``Distribution`` instance:: >>> from importlib.metadata import distribution # doctest: +SKIP @@ -366,21 +366,26 @@ This metadata finder search defaults to ``sys.path``, but varies slightly in how - ``importlib.metadata`` will incidentally honor :py:class:`pathlib.Path` objects on ``sys.path`` even though such values will be ignored for imports. -Extending the search algorithm -============================== +Implementing Custom Providers +============================= + +``importlib.metadata`` address two API surfaces, one for *consumers* +and another for *providers*. Most users are consumers, consuming +metadata provided by the packages. There are other use-cases, however, +where users wish to expose metadata through some other mechanism, +such as alongside a custom importer. Such a use case calls for a +*custom provider*. Because `Distribution Package `_ metadata is not available through :data:`sys.path` searches, or package loaders directly, the metadata for a distribution is found through import -system :ref:`finders `. To find a distribution package's metadata, +system :ref:`finders `. To find a distribution package's metadata, ``importlib.metadata`` queries the list of :term:`meta path finders ` on :data:`sys.meta_path`. -By default ``importlib.metadata`` installs a finder for distribution packages -found on the file system. -This finder doesn't actually find any *distributions*, -but it can find their metadata. +The implementation has hooks integrated into the ``PathFinder``, +serving metadata for distribution packages found on the file system. The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the interface expected of finders by Python's import system. @@ -391,16 +396,16 @@ interface expected of finders by Python's import system. method:: @abc.abstractmethod - def find_distributions(context=DistributionFinder.Context()): + def find_distributions(context=DistributionFinder.Context()) -> Iterable[Distribution]: """Return an iterable of all Distribution instances capable of loading the metadata for packages for the indicated ``context``. """ The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` properties indicating the path to search and name to match and may -supply other relevant context. +supply other relevant context sought by the consumer. -What this means in practice is that to support finding distribution package +In practice, to support finding distribution package metadata in locations other than the file system, subclass ``Distribution`` and implement the abstract methods. Then from a custom finder, return instances of this derived ``Distribution`` in the @@ -409,8 +414,7 @@ a custom finder, return instances of this derived ``Distribution`` in the Example ------- -Consider for example a custom finder that loads Python -modules from a database:: +Imagine a custom finder that loads Python modules from a database:: class DatabaseImporter(importlib.abc.MetaPathFinder): def __init__(self, db): diff --git a/Misc/NEWS.d/next/Documentation/2024-09-11-16-52-08.gh-issue-123976.jhOfNR.rst b/Misc/NEWS.d/next/Documentation/2024-09-11-16-52-08.gh-issue-123976.jhOfNR.rst new file mode 100644 index 00000000000000..7f8e5801ae6597 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-09-11-16-52-08.gh-issue-123976.jhOfNR.rst @@ -0,0 +1 @@ +Refresh docs around custom providers. From 1f9d163850c43ba85193ef853986c5e96b168c8c Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 13 Sep 2024 05:23:54 +0100 Subject: [PATCH 27/50] gh-116622: Android test script improvements (#124012) * Set Android test script stdout to line-buffered * Print warning logcat messages on stderr * Add a -vv option to display high-volume messages which are rarely useful * Documentation and comment improvements --- Android/README.md | 26 ++++++++++++++------------ Android/android.py | 25 ++++++++++++++++++++----- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/Android/README.md b/Android/README.md index bae9150ef057ac..f9e3f290ef46f5 100644 --- a/Android/README.md +++ b/Android/README.md @@ -12,8 +12,12 @@ approachable user experience: ## Prerequisites -Export the `ANDROID_HOME` environment variable to point at your Android SDK. If -you don't already have the SDK, here's how to install it: +First, make sure you have all the usual tools and libraries needed to build +Python for your development machine. + +Second, you'll need an Android SDK. If you already have the SDK installed, +export the `ANDROID_HOME` environment variable to point at its location. +Otherwise, here's how to install it: * Download the "Command line tools" from . * Create a directory `android-sdk/cmdline-tools`, and unzip the command line @@ -37,11 +41,6 @@ development tools, which currently means Linux or macOS. This involves doing a cross-build where you use a "build" Python (for your development machine) to help produce a "host" Python for Android. -First, make sure you have all the usual tools and libraries needed to build -Python for your development machine. The only Android tool you need to install -is the command line tools package above: the build script will download the -rest. - The easiest way to do a build is to use the `android.py` script. You can either have it perform the entire build process from start to finish in one step, or you can do it in discrete steps that mirror running `configure` and `make` for @@ -80,12 +79,15 @@ call. For example, if you want a pydebug build that also caches the results from ## Testing -The tests can be run on Linux, macOS, or Windows, although on Windows you'll -have to build the `cross-build/HOST` subdirectory on one of the other platforms -and copy it over. +The test suite can be run on Linux, macOS, or Windows: + +* On Linux, the emulator needs access to the KVM virtualization interface, and + a DISPLAY environment variable pointing at an X server. +* On Windows, you won't be able to do the build on the same machine, so you'll + have to copy the `cross-build/HOST` directory from somewhere else. -The test suite can usually be run on a device with 2 GB of RAM, though for some -configurations or test orders you may need to increase this. As of Android +The test suite can usually be run on a device with 2 GB of RAM, but this is +borderline, so you may need to increase it to 4 GB. As of Android Studio Koala, 2 GB is the default for all emulators, although the user interface may indicate otherwise. The effective setting is `hw.ramSize` in ~/.android/avd/*.avd/hardware-qemu.ini, whereas Android Studio displays the diff --git a/Android/android.py b/Android/android.py index bfa7832a4a83da..8696d9eaeca19c 100755 --- a/Android/android.py +++ b/Android/android.py @@ -259,8 +259,8 @@ def setup_testbed(): f"{temp_dir}/{outer_jar}", "gradle-wrapper.jar"]) -# run_testbed will build the app automatically, but it hides the Gradle output -# by default, so it's useful to have this as a separate command for the buildbot. +# run_testbed will build the app automatically, but it's useful to have this as +# a separate command to allow running the app outside of this script. def build_testbed(context): setup_sdk() setup_testbed() @@ -376,6 +376,8 @@ async def find_pid(serial): shown_error = False while True: try: + # `pidof` requires API level 24 or higher. The level 23 emulator + # includes it, but it doesn't work (it returns all processes). pid = (await async_check_output( adb, "-s", serial, "shell", "pidof", "-s", APP_ID )).strip() @@ -407,6 +409,7 @@ async def logcat_task(context, initial_devices): serial = await wait_for(find_device(context, initial_devices), startup_timeout) pid = await wait_for(find_pid(serial), startup_timeout) + # `--pid` requires API level 24 or higher. args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] hidden_output = [] async with async_process( @@ -421,11 +424,15 @@ async def logcat_task(context, initial_devices): # such messages, but other components might. level, message = None, line + # Exclude high-volume messages which are rarely useful. + if context.verbose < 2 and "from python test_syslog" in message: + continue + # Put high-level messages on stderr so they're highlighted in the # buildbot logs. This will include Python's own stderr. stream = ( sys.stderr - if level in ["E", "F"] # ERROR and FATAL (aka ASSERT) + if level in ["W", "E", "F"] # WARNING, ERROR, FATAL (aka ASSERT) else sys.stdout ) @@ -573,8 +580,9 @@ def parse_args(): test = subcommands.add_parser( "test", help="Run the test suite") test.add_argument( - "-v", "--verbose", action="store_true", - help="Show Gradle output, and non-Python logcat messages") + "-v", "--verbose", action="count", default=0, + help="Show Gradle output, and non-Python logcat messages. " + "Use twice to include high-volume messages which are rarely useful.") device_group = test.add_mutually_exclusive_group(required=True) device_group.add_argument( "--connected", metavar="SERIAL", help="Run on a connected device. " @@ -591,6 +599,13 @@ def parse_args(): def main(): install_signal_handler() + + # Under the buildbot, stdout is not a TTY, but we must still flush after + # every line to make sure our output appears in the correct order relative + # to the output of our subprocesses. + for stream in [sys.stdout, sys.stderr]: + stream.reconfigure(line_buffering=True) + context = parse_args() dispatch = {"configure-build": configure_build_python, "make-build": make_build_python, From 584cdf8d4140406b3676515332a26c856c02618b Mon Sep 17 00:00:00 2001 From: Yngve Mardal Moe Date: Fri, 13 Sep 2024 06:36:17 +0200 Subject: [PATCH 28/50] gh-123614: Add save function to turtle.py (#123617) --- Doc/library/turtle.rst | 19 ++++++ Lib/test/test_turtle.py | 66 +++++++++++++++++++ Lib/turtle.py | 36 +++++++++- ...-09-02-20-39-10.gh-issue-123614.26TMHp.rst | 2 + 4 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-39-10.gh-issue-123614.26TMHp.rst diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index afda3685d606bb..da801d4dc1f5b3 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -427,6 +427,7 @@ Input methods Methods specific to Screen | :func:`bye` | :func:`exitonclick` + | :func:`save` | :func:`setup` | :func:`title` @@ -2269,6 +2270,24 @@ Methods specific to Screen, not inherited from TurtleScreen client script. +.. function:: save(filename, overwrite=False) + + Save the current turtle drawing (and turtles) as a PostScript file. + + :param filename: the path of the saved PostScript file + :param overwrite: if ``False`` and there already exists a file with the given + filename, then the function will raise a + ``FileExistsError``. If it is ``True``, the file will be + overwritten. + + .. doctest:: + :skipif: _tkinter is None + + >>> screen.save("my_drawing.ps") + >>> screen.save("my_drawing.ps", overwrite=True) + + .. versionadded:: 3.14 + .. function:: setup(width=_CFG["width"], height=_CFG["height"], startx=_CFG["leftright"], starty=_CFG["topbottom"]) Set the size and position of the main window. Default values of arguments diff --git a/Lib/test/test_turtle.py b/Lib/test/test_turtle.py index 14121a590a5026..c75a002a89b4c4 100644 --- a/Lib/test/test_turtle.py +++ b/Lib/test/test_turtle.py @@ -1,5 +1,9 @@ +import os import pickle +import re import unittest +import unittest.mock +import tempfile from test import support from test.support import import_helper from test.support import os_helper @@ -130,6 +134,7 @@ def assertVectorsAlmostEqual(self, vec1, vec2): self.assertAlmostEqual( i, j, msg='values at index {} do not match'.format(idx)) + class Multiplier: def __mul__(self, other): @@ -461,6 +466,67 @@ def test_teleport(self): self.assertTrue(tpen.isdown()) +class TestTurtleScreen(unittest.TestCase): + def test_save_raises_if_wrong_extension(self) -> None: + screen = unittest.mock.Mock() + + msg = "Unknown file extension: '.png', must be one of {'.ps', '.eps'}" + with ( + tempfile.TemporaryDirectory() as tmpdir, + self.assertRaisesRegex(ValueError, re.escape(msg)) + ): + turtle.TurtleScreen.save(screen, os.path.join(tmpdir, "file.png")) + + def test_save_raises_if_parent_not_found(self) -> None: + screen = unittest.mock.Mock() + + with tempfile.TemporaryDirectory() as tmpdir: + parent = os.path.join(tmpdir, "unknown_parent") + msg = f"The directory '{parent}' does not exist. Cannot save to it" + + with self.assertRaisesRegex(FileNotFoundError, re.escape(msg)): + turtle.TurtleScreen.save(screen, os.path.join(parent, "a.ps")) + + def test_save_raises_if_file_found(self) -> None: + screen = unittest.mock.Mock() + + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, "some_file.ps") + with open(file_path, "w") as f: + f.write("some text") + + msg = ( + f"The file '{file_path}' already exists. To overwrite it use" + " the 'overwrite=True' argument of the save function." + ) + with self.assertRaisesRegex(FileExistsError, re.escape(msg)): + turtle.TurtleScreen.save(screen, file_path) + + def test_save_overwrites_if_specified(self) -> None: + screen = unittest.mock.Mock() + screen.cv.postscript.return_value = "postscript" + + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, "some_file.ps") + with open(file_path, "w") as f: + f.write("some text") + + turtle.TurtleScreen.save(screen, file_path, overwrite=True) + with open(file_path) as f: + assert f.read() == "postscript" + + def test_save(self) -> None: + screen = unittest.mock.Mock() + screen.cv.postscript.return_value = "postscript" + + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, "some_file.ps") + + turtle.TurtleScreen.save(screen, file_path) + with open(file_path) as f: + assert f.read() == "postscript" + + class TestModuleLevel(unittest.TestCase): def test_all_signatures(self): import inspect diff --git a/Lib/turtle.py b/Lib/turtle.py index 99850ae5efe348..8a5801f2efe625 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -106,6 +106,7 @@ import sys from os.path import isfile, split, join +from pathlib import Path from copy import deepcopy from tkinter import simpledialog @@ -115,7 +116,7 @@ 'clearscreen', 'colormode', 'delay', 'exitonclick', 'getcanvas', 'getshapes', 'listen', 'mainloop', 'mode', 'numinput', 'onkey', 'onkeypress', 'onkeyrelease', 'onscreenclick', 'ontimer', - 'register_shape', 'resetscreen', 'screensize', 'setup', + 'register_shape', 'resetscreen', 'screensize', 'save', 'setup', 'setworldcoordinates', 'textinput', 'title', 'tracer', 'turtles', 'update', 'window_height', 'window_width'] _tg_turtle_functions = ['back', 'backward', 'begin_fill', 'begin_poly', 'bk', @@ -1492,6 +1493,39 @@ def screensize(self, canvwidth=None, canvheight=None, bg=None): """ return self._resize(canvwidth, canvheight, bg) + def save(self, filename, *, overwrite=False): + """Save the drawing as a PostScript file + + Arguments: + filename -- a string, the path of the created file. + Must end with '.ps' or '.eps'. + + Optional arguments: + overwrite -- boolean, if true, then existing files will be overwritten + + Example (for a TurtleScreen instance named screen): + >>> screen.save('my_drawing.eps') + """ + filename = Path(filename) + if not filename.parent.exists(): + raise FileNotFoundError( + f"The directory '{filename.parent}' does not exist." + " Cannot save to it." + ) + if not overwrite and filename.exists(): + raise FileExistsError( + f"The file '{filename}' already exists. To overwrite it use" + " the 'overwrite=True' argument of the save function." + ) + if (ext := filename.suffix) not in {".ps", ".eps"}: + raise ValueError( + f"Unknown file extension: '{ext}'," + " must be one of {'.ps', '.eps'}" + ) + + postscript = self.cv.postscript() + filename.write_text(postscript) + onscreenclick = onclick resetscreen = reset clearscreen = clear diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-39-10.gh-issue-123614.26TMHp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-39-10.gh-issue-123614.26TMHp.rst new file mode 100644 index 00000000000000..64a5eac9f7840a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-39-10.gh-issue-123614.26TMHp.rst @@ -0,0 +1,2 @@ +Add :func:`turtle.save` to easily save Turtle drawings as PostScript files. +Patch by Marie Roald and Yngve Mardal Moe. From f5548834256414c6a721e9ebfa511e043e73ef03 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 13 Sep 2024 05:58:11 +0100 Subject: [PATCH 29/50] gh-116622: Mock the passage of time in Android logcat rate limit tests (#124015) Mock the passage of time in Android logcat rate limit tests Co-authored-by: Russell Keith-Magee --- Lib/_android_support.py | 5 ++++- Lib/test/test_android.py | 32 ++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/Lib/_android_support.py b/Lib/_android_support.py index 066e54708fb75c..353b34fa36aca4 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -165,7 +165,10 @@ def write(self, prio, tag, message): now = time() self._bucket_level += ( (now - self._prev_write_time) * MAX_BYTES_PER_SECOND) - self._bucket_level = min(self._bucket_level, BUCKET_SIZE) + + # If the bucket level is still below zero, the clock must have gone + # backwards, so reset it to zero and continue. + self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE)) self._prev_write_time = now self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 09fa21cea877c2..2ef9f10fdcc1cc 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -10,7 +10,7 @@ from contextlib import ExitStack, contextmanager from threading import Thread from test.support import LOOPBACK_TIMEOUT -from time import sleep, time +from time import time from unittest.mock import patch @@ -42,7 +42,8 @@ def logcat_thread(): for line in self.logcat_process.stdout: self.logcat_queue.put(line.rstrip("\n")) self.logcat_process.stdout.close() - Thread(target=logcat_thread).start() + self.logcat_thread = Thread(target=logcat_thread) + self.logcat_thread.start() from ctypes import CDLL, c_char_p, c_int android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") @@ -78,6 +79,7 @@ def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5): def tearDown(self): self.logcat_process.terminate() self.logcat_process.wait(LOOPBACK_TIMEOUT) + self.logcat_thread.join(LOOPBACK_TIMEOUT) @contextmanager def unbuffered(self, stream): @@ -369,6 +371,8 @@ def write(b, lines=None, *, write_len=None): ): stream.write(obj) + +class TestAndroidRateLimit(unittest.TestCase): def test_rate_limit(self): # https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 PER_MESSAGE_OVERHEAD = 28 @@ -387,6 +391,19 @@ def test_rate_limit(self): 1024 - PER_MESSAGE_OVERHEAD - len(tag) - len(message.format(0)) ) + "\n" + # To avoid depending on the performance of the test device, we mock the + # passage of time. + mock_now = time() + + def mock_time(): + # Avoid division by zero by simulating a small delay. + mock_sleep(0.0001) + return mock_now + + def mock_sleep(duration): + nonlocal mock_now + mock_now += duration + # See _android_support.py. The default values of these parameters work # well across a wide range of devices, but we'll use smaller values to # ensure a quick and reliable test that doesn't flood the log too much. @@ -395,21 +412,24 @@ def test_rate_limit(self): with ( patch("_android_support.MAX_BYTES_PER_SECOND", MAX_KB_PER_SECOND * 1024), patch("_android_support.BUCKET_SIZE", BUCKET_KB * 1024), + patch("_android_support.sleep", mock_sleep), + patch("_android_support.time", mock_time), ): # Make sure the token bucket is full. - sleep(BUCKET_KB / MAX_KB_PER_SECOND) + stream.write("Initial message to reset _prev_write_time") + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) line_num = 0 # Write BUCKET_KB messages, and return the rate at which they were # accepted in KB per second. def write_bucketful(): nonlocal line_num - start = time() + start = mock_time() max_line_num = line_num + BUCKET_KB while line_num < max_line_num: stream.write(message.format(line_num)) line_num += 1 - return BUCKET_KB / (time() - start) + return BUCKET_KB / (mock_time() - start) # The first bucketful should be written with minimal delay. The # factor of 2 here is not arbitrary: it verifies that the system can @@ -427,5 +447,5 @@ def write_bucketful(): ) # Once the token bucket refills, we should go back to full speed. - sleep(BUCKET_KB / MAX_KB_PER_SECOND) + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) From e5b0185e43c972ce98decd1493cd0b0c3a6b166b Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 12 Sep 2024 23:14:52 -0700 Subject: [PATCH 30/50] GH-101599: Update docs to remove redundant option in argparse tutorial (#124025) --- Doc/howto/argparse.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index ae5bab90bf8131..30d9ac700376e6 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -444,7 +444,7 @@ And the output: options: -h, --help show this help message and exit - -v {0,1,2}, --verbosity {0,1,2} + -v, --verbosity {0,1,2} increase output verbosity Note that the change also reflects both in the error message as well as the From 403f3ddedcab14f6c16ea78a93bb4acf49d06a07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 13 Sep 2024 12:33:13 +0200 Subject: [PATCH 31/50] gh-123961: Remove global variable `ModDict` in `_cursesmodule.c` (#123962) --- Modules/_cursesmodule.c | 79 ++++++++++++--------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 3 - 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 1d36b40d5236b8..be31cb4f54ddcc 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -3255,8 +3255,6 @@ _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) Py_RETURN_NONE; } -static PyObject *ModDict; - /*[clinic input] _curses.initscr @@ -3285,19 +3283,23 @@ _curses_initscr_impl(PyObject *module) initialised = initialised_setupterm = TRUE; -/* This was moved from initcurses() because it core dumped on SGI, - where they're not defined until you've called initscr() */ -#define SetDictInt(NAME, VALUE) \ - do { \ - PyObject *value = PyLong_FromLong((long)(VALUE)); \ - if (value == NULL) { \ - return NULL; \ - } \ - int rc = PyDict_SetItemString(ModDict, (NAME), value); \ - Py_DECREF(value); \ - if (rc < 0) { \ - return NULL; \ - } \ + PyObject *module_dict = PyModule_GetDict(module); // borrowed + if (module_dict == NULL) { + return NULL; + } + /* This was moved from initcurses() because it core dumped on SGI, + where they're not defined until you've called initscr() */ +#define SetDictInt(NAME, VALUE) \ + do { \ + PyObject *value = PyLong_FromLong((long)(VALUE)); \ + if (value == NULL) { \ + return NULL; \ + } \ + int rc = PyDict_SetItemString(module_dict, (NAME), value); \ + Py_DECREF(value); \ + if (rc < 0) { \ + return NULL; \ + } \ } while (0) /* Here are some graphic symbols you can use */ @@ -3976,11 +3978,11 @@ _curses_qiflush_impl(PyObject *module, int flag) Py_RETURN_NONE; } -/* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES - * and _curses.COLS */ #if defined(HAVE_CURSES_RESIZETERM) || defined(HAVE_CURSES_RESIZE_TERM) +/* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES + * and _curses.COLS. Returns 1 on success and 0 on failure. */ static int -update_lines_cols(void) +update_lines_cols(PyObject *private_module) { PyObject *exposed_module = NULL, *o = NULL; @@ -3992,6 +3994,10 @@ update_lines_cols(void) if (exposed_module_dict == NULL) { goto error; } + PyObject *private_module_dict = PyModule_GetDict(private_module); // borrowed + if (private_module_dict == NULL) { + goto error; + } o = PyLong_FromLong(LINES); if (o == NULL) { @@ -4000,7 +4006,7 @@ update_lines_cols(void) if (PyDict_SetItemString(exposed_module_dict, "LINES", o) < 0) { goto error; } - if (PyDict_SetItemString(ModDict, "LINES", o) < 0) { + if (PyDict_SetItemString(private_module_dict, "LINES", o) < 0) { goto error; } Py_DECREF(o); @@ -4012,7 +4018,7 @@ update_lines_cols(void) if (PyDict_SetItemString(exposed_module_dict, "COLS", o) < 0) { goto error; } - if (PyDict_SetItemString(ModDict, "COLS", o) < 0) { + if (PyDict_SetItemString(private_module_dict, "COLS", o) < 0) { goto error; } Py_DECREF(o); @@ -4034,7 +4040,7 @@ static PyObject * _curses_update_lines_cols_impl(PyObject *module) /*[clinic end generated code: output=423f2b1e63ed0f75 input=5f065ab7a28a5d90]*/ { - if (!update_lines_cols()) { + if (!update_lines_cols(module)) { return NULL; } Py_RETURN_NONE; @@ -4121,7 +4127,7 @@ _curses_resizeterm_impl(PyObject *module, int nlines, int ncols) result = PyCursesCheckERR(resizeterm(nlines, ncols), "resizeterm"); if (!result) return NULL; - if (!update_lines_cols()) { + if (!update_lines_cols(module)) { Py_DECREF(result); return NULL; } @@ -4160,7 +4166,7 @@ _curses_resize_term_impl(PyObject *module, int nlines, int ncols) result = PyCursesCheckERR(resize_term(nlines, ncols), "resize_term"); if (!result) return NULL; - if (!update_lines_cols()) { + if (!update_lines_cols(module)) { Py_DECREF(result); return NULL; } @@ -4232,17 +4238,21 @@ _curses_start_color_impl(PyObject *module) initialisedcolors = TRUE; -#define DICT_ADD_INT_VALUE(NAME, VALUE) \ - do { \ - PyObject *value = PyLong_FromLong((long)(VALUE)); \ - if (value == NULL) { \ - return NULL; \ - } \ - int rc = PyDict_SetItemString(ModDict, (NAME), value); \ - Py_DECREF(value); \ - if (rc < 0) { \ - return NULL; \ - } \ + PyObject *module_dict = PyModule_GetDict(module); // borrowed + if (module_dict == NULL) { + return NULL; + } +#define DICT_ADD_INT_VALUE(NAME, VALUE) \ + do { \ + PyObject *value = PyLong_FromLong((long)(VALUE)); \ + if (value == NULL) { \ + return NULL; \ + } \ + int rc = PyDict_SetItemString(module_dict, (NAME), value); \ + Py_DECREF(value); \ + if (rc < 0) { \ + return NULL; \ + } \ } while (0) DICT_ADD_INT_VALUE("COLORS", COLORS); @@ -4779,7 +4789,6 @@ cursesmodule_exec(PyObject *module) if (module_dict == NULL) { return -1; } - ModDict = module_dict; /* For PyCurses_InitScr to use later */ void **PyCurses_API = PyMem_Calloc(PyCurses_API_pointers, sizeof(void *)); if (PyCurses_API == NULL) { diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index cb9750a69a632b..71f2fded77fe07 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -391,9 +391,6 @@ Modules/xxmodule.c - ErrorObject - ##----------------------- ## other -## initialized once -Modules/_cursesmodule.c - ModDict - - ## state Modules/_datetimemodule.c - _datetime_global_state - Modules/_tkinter.c - tcl_lock - From d7e83398c188a0acd19a496ee2eeeeab52d64a11 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 13 Sep 2024 14:13:52 +0300 Subject: [PATCH 32/50] gh-108303: Remove the non-test `Lib/test/reperf.py` (GH-114356) --- Lib/test/_test_embed_structseq.py | 9 ++++++++- Lib/test/reperf.py | 23 ----------------------- Lib/test/test_embed.py | 11 +++++++---- 3 files changed, 15 insertions(+), 28 deletions(-) delete mode 100644 Lib/test/reperf.py diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py index 868f9f83e8be77..154662efce9412 100644 --- a/Lib/test/_test_embed_structseq.py +++ b/Lib/test/_test_embed_structseq.py @@ -48,7 +48,14 @@ def test_sys_funcs(self): try: - unittest.main() + unittest.main( + module=( + '__main__' + if __name__ == '__main__' + # Avoiding a circular import: + else sys.modules['test._test_embed_structseq'] + ) + ) except SystemExit as exc: if exc.args[0] != 0: raise diff --git a/Lib/test/reperf.py b/Lib/test/reperf.py deleted file mode 100644 index e93bacdb5843c6..00000000000000 --- a/Lib/test/reperf.py +++ /dev/null @@ -1,23 +0,0 @@ -import re -import time - -def main(): - s = "\13hello\14 \13world\14 " * 1000 - p = re.compile(r"([\13\14])") - timefunc(10, p.sub, "", s) - timefunc(10, p.split, s) - timefunc(10, p.findall, s) - -def timefunc(n, func, *args, **kw): - t0 = time.perf_counter() - try: - for i in range(n): - result = func(*args, **kw) - return result - finally: - t1 = time.perf_counter() - if n > 1: - print(n, "times", end=' ') - print(func.__name__, "%.3f" % (t1-t0), "CPU seconds") - -main() diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 6790326a2afa47..7c5cb855a397ab 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1971,7 +1971,11 @@ def test_no_memleak(self): @unittest.skipUnless(support.Py_DEBUG, '-X presite requires a Python debug build') def test_presite(self): - cmd = [sys.executable, "-I", "-X", "presite=test.reperf", "-c", "print('cmd')"] + cmd = [ + sys.executable, + "-I", "-X", "presite=test._test_embed_structseq", + "-c", "print('unique-python-message')", + ] proc = subprocess.run( cmd, stdout=subprocess.PIPE, @@ -1980,9 +1984,8 @@ def test_presite(self): ) self.assertEqual(proc.returncode, 0) out = proc.stdout.strip() - self.assertIn("10 times sub", out) - self.assertIn("CPU seconds", out) - self.assertIn("cmd", out) + self.assertIn("Tests passed", out) + self.assertIn("unique-python-message", out) class StdPrinterTests(EmbeddingTestsMixin, unittest.TestCase): From 432bf31327c6b9647acb8bdb0eac2d392fd9f60a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 13 Sep 2024 13:18:49 +0200 Subject: [PATCH 33/50] gh-123909: PyType_From*: Disallow metaclasses with custom tp_new (GH-123947) --- Doc/c-api/type.rst | 24 +++++++++--- Lib/test/test_capi/test_misc.py | 12 ++---- ...-09-10-16-54-27.gh-issue-123909.CTGxDR.rst | 3 ++ Objects/typeobject.c | 38 +++++-------------- 4 files changed, 34 insertions(+), 43 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2024-09-10-16-54-27.gh-issue-123909.CTGxDR.rst diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 0cae5c09505ebe..fa04d6c0ad5da5 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -345,8 +345,12 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) @@ -362,8 +366,12 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) @@ -378,8 +386,12 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. .. raw:: html diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index d50217b695967e..18392c4fce5dc1 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -722,22 +722,16 @@ def test_heaptype_with_custom_metaclass_custom_new(self): with self.assertRaisesRegex(TypeError, msg): t = _testcapi.pytype_fromspec_meta(metaclass) - def test_heaptype_with_custom_metaclass_deprecation(self): + def test_heaptype_base_with_custom_metaclass(self): metaclass = _testcapi.HeapCTypeMetaclassCustomNew - # gh-103968: a metaclass with custom tp_new is deprecated, but still - # allowed for functions that existed in 3.11 - # (PyType_FromSpecWithBases is used here). class Base(metaclass=metaclass): pass # Class creation from C - with warnings_helper.check_warnings( - ('.* _testcapi.Subclass .* custom tp_new.*in Python 3.14.*', DeprecationWarning), - ): + msg = "Metaclasses with custom tp_new are not supported." + with self.assertRaisesRegex(TypeError, msg): sub = _testcapi.make_type_with_base(Base) - self.assertTrue(issubclass(sub, Base)) - self.assertIsInstance(sub, metaclass) def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): for weakref_cls in (_testcapi.HeapCTypeWithWeakref, diff --git a/Misc/NEWS.d/next/C_API/2024-09-10-16-54-27.gh-issue-123909.CTGxDR.rst b/Misc/NEWS.d/next/C_API/2024-09-10-16-54-27.gh-issue-123909.CTGxDR.rst new file mode 100644 index 00000000000000..b7a4913abbcb89 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-09-10-16-54-27.gh-issue-123909.CTGxDR.rst @@ -0,0 +1,3 @@ +:c:func:`PyType_FromSpec`, :c:func:`PyType_FromSpecWithBases` and +:c:func:`PyType_FromModuleAndSpec` will now fail if the metaclass of the new +type has custom :c:member:`~PyTypeObject.tp_new`. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a6483f74b7947d..28edd801284b81 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4668,10 +4668,10 @@ special_offset_from_member( return -1; } -static PyObject * -_PyType_FromMetaclass_impl( +PyObject * +PyType_FromMetaclass( PyTypeObject *metaclass, PyObject *module, - PyType_Spec *spec, PyObject *bases_in, int _allow_tp_new) + PyType_Spec *spec, PyObject *bases_in) { /* Invariant: A non-NULL value in one of these means this function holds * a strong reference or owns allocated memory. @@ -4848,21 +4848,10 @@ _PyType_FromMetaclass_impl( goto finally; } if (metaclass->tp_new && metaclass->tp_new != PyType_Type.tp_new) { - if (_allow_tp_new) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "Type %s uses PyType_Spec with a metaclass that has custom " - "tp_new. This is deprecated and will no longer be allowed in " - "Python 3.14.", spec->name) < 0) { - goto finally; - } - } - else { - PyErr_SetString( - PyExc_TypeError, - "Metaclasses with custom tp_new are not supported."); - goto finally; - } + PyErr_SetString( + PyExc_TypeError, + "Metaclasses with custom tp_new are not supported."); + goto finally; } /* Calculate best base, and check that all bases are type objects */ @@ -5109,29 +5098,22 @@ _PyType_FromMetaclass_impl( return (PyObject*)res; } -PyObject * -PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, - PyType_Spec *spec, PyObject *bases_in) -{ - return _PyType_FromMetaclass_impl(metaclass, module, spec, bases_in, 0); -} - PyObject * PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) { - return _PyType_FromMetaclass_impl(NULL, module, spec, bases, 1); + return PyType_FromMetaclass(NULL, module, spec, bases); } PyObject * PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) { - return _PyType_FromMetaclass_impl(NULL, NULL, spec, bases, 1); + return PyType_FromMetaclass(NULL, NULL, spec, bases); } PyObject * PyType_FromSpec(PyType_Spec *spec) { - return _PyType_FromMetaclass_impl(NULL, NULL, spec, NULL, 1); + return PyType_FromMetaclass(NULL, NULL, spec, NULL); } PyObject * From e49d1b44d3ef2542c0ae165e14a7e5ffbc32b2d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 13 Sep 2024 13:26:19 +0200 Subject: [PATCH 34/50] gh-124044: protect macros expansions in `_cursesmodules.c` using `do { ... } while (0)` (#124045) --- Modules/_cursesmodule.c | 47 ++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index be31cb4f54ddcc..b1fd839d7a5f82 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -182,22 +182,31 @@ static char *screen_encoding = NULL; /* Utility Macros */ #define PyCursesSetupTermCalled \ - if (initialised_setupterm != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call (at least) setupterm() first"); \ - return 0; } + do { \ + if (initialised_setupterm != TRUE) { \ + PyErr_SetString(PyCursesError, \ + "must call (at least) setupterm() first"); \ + return 0; \ + } \ + } while (0) -#define PyCursesInitialised \ - if (initialised != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call initscr() first"); \ - return 0; } +#define PyCursesInitialised \ + do { \ + if (initialised != TRUE) { \ + PyErr_SetString(PyCursesError, \ + "must call initscr() first"); \ + return 0; \ + } \ + } while (0) #define PyCursesInitialisedColor \ - if (initialisedcolors != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call start_color() first"); \ - return 0; } + do { \ + if (initialisedcolors != TRUE) { \ + PyErr_SetString(PyCursesError, \ + "must call start_color() first"); \ + return 0; \ + } \ + } while (0) /* Utility Functions */ @@ -2633,12 +2642,12 @@ PyTypeObject PyCursesWindow_Type = { #define NoArgNoReturnFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesInitialised; \ return PyCursesCheckERR(X(), # X); } #define NoArgOrFlagNoReturnFunctionBody(X, flag) \ { \ - PyCursesInitialised \ + PyCursesInitialised; \ if (flag) \ return PyCursesCheckERR(X(), # X); \ else \ @@ -2647,23 +2656,23 @@ PyTypeObject PyCursesWindow_Type = { #define NoArgReturnIntFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesInitialised; \ return PyLong_FromLong((long) X()); } #define NoArgReturnStringFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesInitialised; \ return PyBytes_FromString(X()); } #define NoArgTrueFalseFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesInitialised; \ return PyBool_FromLong(X()); } #define NoArgNoReturnVoidFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesInitialised; \ X(); \ Py_RETURN_NONE; } From acb3f875fba0552d55989f1f2686c334fecaecca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 13 Sep 2024 14:43:06 +0200 Subject: [PATCH 35/50] gh-123961: Add `curses` prefix to global variables in `_cursesmodule.c` (#124047) Use the `const char*` type instead of a `const *` for the encoding name. --- Modules/_cursesmodule.c | 28 ++++++++++----------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 8 +++--- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index b1fd839d7a5f82..c9ee5687c2b5d9 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -170,20 +170,20 @@ class _curses.window "PyCursesWindowObject *" "&PyCursesWindow_Type" static PyObject *PyCursesError; /* Tells whether setupterm() has been called to initialise terminfo. */ -static int initialised_setupterm = FALSE; +static int curses_setupterm_called = FALSE; /* Tells whether initscr() has been called to initialise curses. */ -static int initialised = FALSE; +static int curses_initscr_called = FALSE; /* Tells whether start_color() has been called to initialise color usage. */ -static int initialisedcolors = FALSE; +static int curses_start_color_called = FALSE; -static char *screen_encoding = NULL; +static const char *curses_screen_encoding = NULL; /* Utility Macros */ #define PyCursesSetupTermCalled \ do { \ - if (initialised_setupterm != TRUE) { \ + if (curses_setupterm_called != TRUE) { \ PyErr_SetString(PyCursesError, \ "must call (at least) setupterm() first"); \ return 0; \ @@ -192,7 +192,7 @@ static char *screen_encoding = NULL; #define PyCursesInitialised \ do { \ - if (initialised != TRUE) { \ + if (curses_initscr_called != TRUE) { \ PyErr_SetString(PyCursesError, \ "must call initscr() first"); \ return 0; \ @@ -201,7 +201,7 @@ static char *screen_encoding = NULL; #define PyCursesInitialisedColor \ do { \ - if (initialisedcolors != TRUE) { \ + if (curses_start_color_called != TRUE) { \ PyErr_SetString(PyCursesError, \ "must call start_color() first"); \ return 0; \ @@ -267,7 +267,7 @@ PyCurses_ConvertToChtype(PyCursesWindowObject *win, PyObject *obj, chtype *ch) if (win) encoding = win->encoding; else - encoding = screen_encoding; + encoding = curses_screen_encoding; bytes = PyUnicode_AsEncodedString(obj, encoding, NULL); if (bytes == NULL) return 0; @@ -3278,7 +3278,7 @@ _curses_initscr_impl(PyObject *module) { WINDOW *win; - if (initialised) { + if (curses_initscr_called) { wrefresh(stdscr); return (PyObject *)PyCursesWindow_New(stdscr, NULL); } @@ -3290,7 +3290,7 @@ _curses_initscr_impl(PyObject *module) return NULL; } - initialised = initialised_setupterm = TRUE; + curses_initscr_called = curses_setupterm_called = TRUE; PyObject *module_dict = PyModule_GetDict(module); // borrowed if (module_dict == NULL) { @@ -3386,7 +3386,7 @@ _curses_initscr_impl(PyObject *module) if (winobj == NULL) { return NULL; } - screen_encoding = winobj->encoding; + curses_screen_encoding = winobj->encoding; return (PyObject *)winobj; } @@ -3428,7 +3428,7 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) } } - if (!initialised_setupterm && setupterm((char *)term, fd, &err) == ERR) { + if (!curses_setupterm_called && setupterm((char *)term, fd, &err) == ERR) { const char* s = "setupterm: unknown error"; if (err == 0) { @@ -3441,7 +3441,7 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) return NULL; } - initialised_setupterm = TRUE; + curses_setupterm_called = TRUE; Py_RETURN_NONE; } @@ -4245,7 +4245,7 @@ _curses_start_color_impl(PyObject *module) return NULL; } - initialisedcolors = TRUE; + curses_start_color_called = TRUE; PyObject *module_dict = PyModule_GetDict(module); // borrowed if (module_dict == NULL) { diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 71f2fded77fe07..e1c07f88b963bc 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -409,10 +409,10 @@ Modules/_tkinter.c - trbInCmd - Include/datetime.h - PyDateTimeAPI - Modules/_ctypes/cfield.c _ctypes_get_fielddesc initialized - Modules/_ctypes/malloc_closure.c - _pagesize - -Modules/_cursesmodule.c - initialised - -Modules/_cursesmodule.c - initialised_setupterm - -Modules/_cursesmodule.c - initialisedcolors - -Modules/_cursesmodule.c - screen_encoding - +Modules/_cursesmodule.c - curses_initscr_called - +Modules/_cursesmodule.c - curses_setupterm_called - +Modules/_cursesmodule.c - curses_start_color_called - +Modules/_cursesmodule.c - curses_screen_encoding - Modules/_elementtree.c - expat_capi - Modules/readline.c - libedit_append_replace_history_offset - Modules/readline.c - using_libedit_emulation - From b46c65ed2b78214cb8914779ac4e8d343ac4775e Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Fri, 13 Sep 2024 15:49:13 +0300 Subject: [PATCH 36/50] gh-123811: Test that round(Decimal) can return signed zero (GH-124007) --- Lib/test/test_decimal.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 46755107de0102..12479e32d0f5db 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -2071,7 +2071,9 @@ def test_tonum_methods(self): #to quantize, which is already extensively tested test_triples = [ ('123.456', -4, '0E+4'), + ('-123.456', -4, '-0E+4'), ('123.456', -3, '0E+3'), + ('-123.456', -3, '-0E+3'), ('123.456', -2, '1E+2'), ('123.456', -1, '1.2E+2'), ('123.456', 0, '123'), From cfe6074d1fa81cf0684fbf8a623616441a1966e7 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Fri, 13 Sep 2024 08:40:04 -0500 Subject: [PATCH 37/50] gh-112301: Enable warning emitting options and ignore warnings in CI (#123020) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- .github/workflows/reusable-macos.yml | 18 +- .github/workflows/reusable-ubuntu.yml | 18 +- ...-08-14-19-43-57.gh-issue-112301.IQUcOy.rst | 1 + Tools/build/.warningignore_macos | 227 +++++++++++++++ Tools/build/.warningignore_ubuntu | 260 ++++++++++++++++++ Tools/build/check_warnings.py | 146 ++++------ configure | 195 +++++++++++++ configure.ac | 5 + 8 files changed, 779 insertions(+), 91 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index d77723ef27c2dc..49613a8ddaebf5 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -35,7 +35,7 @@ jobs: path: config.cache key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies - run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk + run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk make - name: Configure CPython run: | GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ @@ -48,10 +48,22 @@ jobs: --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt + if : ${{ inputs.free-threading || inputs.os != 'macos-13' }} + run: gmake -j8 + - name: Build CPython for compiler warning check + if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo - name: Check compiler warnings - run: python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang + if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + run: >- + python3 Tools/build/check_warnings.py + --compiler-output-file-path=compiler_output_macos.txt + --warning-ignore-file-path=Tools/build/.warningignore_macos + --compiler-output-type=clang + --fail-on-regression + --fail-on-improvement + --path-prefix="./" - name: Tests run: make test diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index b197db814b2743..f2c83798027d10 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -67,20 +67,32 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: >- ../cpython-ro-srcdir/configure - CFLAGS="-fdiagnostics-format=json" --config-cache --with-pydebug --enable-slower-safety --with-openssl=$OPENSSL_DIR ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - name: Build CPython out-of-tree + if: ${{ inputs.free-threading }} working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: set -o pipefail; make -j4 2>&1 | tee compiler_output.txt + run: make -j4 + - name: Build CPython out-of-tree (for compiler warning check) + if: ${{ !inputs.free-threading}} + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: set -o pipefail; make -j4 2>&1 | tee compiler_output_ubuntu.txt - name: Display build info working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo - name: Check compiler warnings - run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json + if: ${{ !inputs.free-threading }} + run: >- + python Tools/build/check_warnings.py + --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt + --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu + --compiler-output-type=gcc + --fail-on-regression + --fail-on-improvement + --path-prefix="../cpython-ro-srcdir/" - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw diff --git a/Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst b/Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst new file mode 100644 index 00000000000000..9750cf203eef86 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst @@ -0,0 +1 @@ +Enable compiler options that warn of potential security vulnerabilities. diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos index 67f50119db7310..fffb828eb4a8ab 100644 --- a/Tools/build/.warningignore_macos +++ b/Tools/build/.warningignore_macos @@ -3,3 +3,230 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) +Include/internal/mimalloc/mimalloc/internal.h 4 +Include/internal/pycore_backoff.h 1 +Include/internal/pycore_dict.h 2 +Include/internal/pycore_gc.h 1 +Include/internal/pycore_long.h 2 +Include/internal/pycore_object.h 4 +Modules/_asynciomodule.c 3 +Modules/_bisectmodule.c 2 +Modules/_bz2module.c 5 +Modules/_collectionsmodule.c 2 +Modules/_csv.c 3 +Modules/_ctypes/_ctypes.c 37 +Modules/_ctypes/_ctypes_test_generated.c.h 141 +Modules/_ctypes/callbacks.c 6 +Modules/_ctypes/callproc.c 15 +Modules/_ctypes/cfield.c 59 +Modules/_ctypes/malloc_closure.c 3 +Modules/_ctypes/stgdict.c 17 +Modules/_cursesmodule.c 24 +Modules/_datetimemodule.c 28 +Modules/_dbmmodule.c 8 +Modules/_decimal/_decimal.c 15 +Modules/_elementtree.c 42 +Modules/_functoolsmodule.c 6 +Modules/_gdbmmodule.c 5 +Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84 +Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84 +Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 24 +Modules/_hashopenssl.c 16 +Modules/_interpchannelsmodule.c 1 +Modules/_interpqueuesmodule.c 1 +Modules/_io/_iomodule.c 1 +Modules/_io/bufferedio.c 4 +Modules/_io/bytesio.c 11 +Modules/_io/fileio.c 9 +Modules/_io/stringio.c 8 +Modules/_io/textio.c 11 +Modules/_json.c 19 +Modules/_localemodule.c 3 +Modules/_lzmamodule.c 10 +Modules/_multiprocessing/semaphore.c 2 +Modules/_operator.c 5 +Modules/_pickle.c 71 +Modules/_posixsubprocess.c 8 +Modules/_queuemodule.c 4 +Modules/_randommodule.c 3 +Modules/_scproxy.c 3 +Modules/_sqlite/connection.c 4 +Modules/_sqlite/cursor.c 3 +Modules/_sqlite/module.c 2 +Modules/_sre/sre.c 18 +Modules/_sre/sre_lib.h 62 +Modules/_ssl.c 29 +Modules/_struct.c 1 +Modules/_testbuffer.c 22 +Modules/_testcapi/heaptype.c 1 +Modules/_testcapi/long.c 2 +Modules/_testcapi/mem.c 2 +Modules/_testcapi/monitoring.c 3 +Modules/_testcapi/pyatomic.c 1 +Modules/_testcapi/unicode.c 2 +Modules/_testcapi/vectorcall.c 3 +Modules/_testcapi/watchers.c 3 +Modules/_testcapimodule.c 3 +Modules/_testclinic.c 14 +Modules/_testexternalinspection.c 8 +Modules/_testinternalcapi.c 8 +Modules/_testinternalcapi/pytime.c 8 +Modules/_testinternalcapi/test_critical_sections.c 1 +Modules/_testinternalcapi/test_lock.c 2 +Modules/_testlimitedcapi/heaptype_relative.c 4 +Modules/_testlimitedcapi/object.c 2 +Modules/_testlimitedcapi/unicode.c 2 +Modules/_threadmodule.c 2 +Modules/_tkinter.c 6 +Modules/_xxtestfuzz/_xxtestfuzz.c 1 +Modules/_xxtestfuzz/fuzzer.c 11 +Modules/_zoneinfo.c 14 +Modules/arraymodule.c 32 +Modules/atexitmodule.c 1 +Modules/binascii.c 206 +Modules/blake2module.c 6 +Modules/cjkcodecs/_codecs_cn.c 1 +Modules/cjkcodecs/_codecs_iso2022.c 2 +Modules/cjkcodecs/_codecs_jp.c 14 +Modules/cjkcodecs/_codecs_kr.c 3 +Modules/cjkcodecs/cjkcodecs.h 1 +Modules/cjkcodecs/multibytecodec.c 2 +Modules/clinic/_testclinic.c.h 1 +Modules/clinic/arraymodule.c.h 1 +Modules/clinic/unicodedata.c.h 10 +Modules/cmathmodule.c 1 +Modules/expat/siphash.h 8 +Modules/expat/xmlparse.c 45 +Modules/expat/xmltok.c 17 +Modules/expat/xmltok_impl.c 34 +Modules/faulthandler.c 3 +Modules/fcntlmodule.c 1 +Modules/getpath.c 7 +Modules/grpmodule.c 4 +Modules/itertoolsmodule.c 7 +Modules/main.c 2 +Modules/mathmodule.c 15 +Modules/mmapmodule.c 20 +Modules/posixmodule.c 67 +Modules/pwdmodule.c 4 +Modules/pyexpat.c 20 +Modules/readline.c 1 +Modules/resource.c 3 +Modules/rotatingtree.c 1 +Modules/selectmodule.c 6 +Modules/sha3module.c 4 +Modules/signalmodule.c 1 +Modules/socketmodule.c 44 +Modules/syslogmodule.c 3 +Modules/timemodule.c 4 +Modules/unicodedata.c 28 +Modules/unicodedata_db.h 1 +Modules/xxsubtype.c 2 +Modules/zlibmodule.c 16 +Objects/abstract.c 2 +Objects/bytearrayobject.c 34 +Objects/bytes_methods.c 9 +Objects/bytesobject.c 35 +Objects/call.c 13 +Objects/classobject.c 4 +Objects/codeobject.c 15 +Objects/descrobject.c 2 +Objects/dictobject.c 28 +Objects/fileobject.c 3 +Objects/floatobject.c 30 +Objects/frameobject.c 19 +Objects/funcobject.c 1 +Objects/genobject.c 5 +Objects/listobject.c 43 +Objects/longobject.c 46 +Objects/memoryobject.c 6 +Objects/methodobject.c 1 +Objects/mimalloc/alloc.c 6 +Objects/mimalloc/arena.c 6 +Objects/mimalloc/heap.c 1 +Objects/mimalloc/init.c 2 +Objects/mimalloc/options.c 1 +Objects/mimalloc/os.c 4 +Objects/mimalloc/page-queue.c 2 +Objects/mimalloc/page.c 1 +Objects/mimalloc/prim/osx/../unix/prim.c 2 +Objects/mimalloc/random.c 1 +Objects/mimalloc/segment.c 11 +Objects/mimalloc/stats.c 1 +Objects/moduleobject.c 2 +Objects/object.c 1 +Objects/obmalloc.c 6 +Objects/odictobject.c 3 +Objects/rangeobject.c 10 +Objects/setobject.c 13 +Objects/sliceobject.c 4 +Objects/stringlib/codecs.h 26 +Objects/stringlib/eq.h 1 +Objects/stringlib/fastsearch.h 14 +Objects/stringlib/join.h 1 +Objects/stringlib/replace.h 4 +Objects/stringlib/repr.h 21 +Objects/stringlib/transmogrify.h 5 +Objects/structseq.c 14 +Objects/tupleobject.c 10 +Objects/typeobject.c 17 +Objects/unicodectype.c 7 +Objects/unicodeobject.c 113 +Parser/action_helpers.c 4 +Parser/lexer/buffer.c 1 +Parser/lexer/lexer.c 12 +Parser/parser.c 116 +Parser/pegen.c 7 +Parser/string_parser.c 7 +Parser/tokenizer/file_tokenizer.c 8 +Parser/tokenizer/helpers.c 7 +Parser/tokenizer/readline_tokenizer.c 3 +Programs/_freeze_module.c 1 +Python/Python-ast.c 15 +Python/asdl.c 3 +Python/assemble.c 7 +Python/ast_opt.c 7 +Python/bltinmodule.c 9 +Python/bootstrap_hash.c 4 +Python/ceval.c 8 +Python/ceval_gil.c 2 +Python/codecs.c 32 +Python/codegen.c 6 +Python/compile.c 2 +Python/context.c 1 +Python/crossinterp.c 2 +Python/crossinterp_data_lookup.h 1 +Python/dtoa.c 34 +Python/errors.c 1 +Python/fileutils.c 7 +Python/flowgraph.c 8 +Python/formatter_unicode.c 7 +Python/frame.c 4 +Python/gc.c 8 +Python/generated_cases.c.h 35 +Python/getargs.c 11 +Python/import.c 5 +Python/initconfig.c 11 +Python/instrumentation.c 31 +Python/intrinsics.c 1 +Python/legacy_tracing.c 3 +Python/lock.c 4 +Python/marshal.c 11 +Python/modsupport.c 3 +Python/mystrtoul.c 4 +Python/pathconfig.c 1 +Python/preconfig.c 2 +Python/pyarena.c 1 +Python/pyhash.c 2 +Python/pylifecycle.c 7 +Python/pystate.c 6 +Python/pystrhex.c 19 +Python/pystrtod.c 3 +Python/qsbr.c 2 +Python/specialize.c 10 +Python/suggestions.c 12 +Python/symtable.c 18 +Python/sysmodule.c 2 +Python/thread_pthread.h 1 +Python/traceback.c 6 +Python/tracemalloc.c 6 diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 469c727abfb11c..e98305e81808d6 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -3,3 +3,263 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) +/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/evp.h 2 +/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/ssl.h 4 +/usr/include/tcl8.6/tclTomMathDecls.h 1 +Include/cpython/bytearrayobject.h 1 +Include/cpython/bytesobject.h 3 +Include/cpython/dictobject.h 2 +Include/cpython/listobject.h 1 +Include/cpython/pyctype.h 2 +Include/cpython/tupleobject.h 1 +Include/cpython/unicodeobject.h 7 +Include/internal/mimalloc/mimalloc/internal.h 4 +Include/internal/mimalloc/mimalloc/types.h 2 +Include/internal/pycore_asdl.h 1 +Include/internal/pycore_backoff.h 3 +Include/internal/pycore_blocks_output_buffer.h 1 +Include/internal/pycore_dict.h 2 +Include/internal/pycore_gc.h 1 +Include/internal/pycore_gc.h 1 +Include/internal/pycore_interp.h 1 +Include/internal/pycore_list.h 1 +Include/internal/pycore_long.h 3 +Include/internal/pycore_object.h 4 +Include/internal/pycore_obmalloc.h 1 +Include/internal/pycore_pymath.h 1 +Include/internal/pycore_runtime_init.h 1 +Include/longobject.h 1 +Include/object.h 4 +Include/opcode_ids.h 1 +Include/pymacro.h 4 +Include/pymath.h 1 +Include/pymem.h 2 +Include/pyport.h 2 +Modules/_asynciomodule.c 3 +Modules/_bisectmodule.c 4 +Modules/_bz2module.c 5 +Modules/_collectionsmodule.c 2 +Modules/_csv.c 2 +Modules/_ctypes/_ctypes.c 53 +Modules/_ctypes/_ctypes_test.c 7 +Modules/_ctypes/_ctypes_test_generated.c.h 2 +Modules/_ctypes/callbacks.c 3 +Modules/_ctypes/callproc.c 13 +Modules/_ctypes/cfield.c 33 +Modules/_ctypes/stgdict.c 17 +Modules/_cursesmodule.c 27 +Modules/_datetimemodule.c 38 +Modules/_datetimemodule.c 38 +Modules/_dbmmodule.c 7 +Modules/_decimal/_decimal.c 19 +Modules/_elementtree.c 37 +Modules/_functoolsmodule.c 6 +Modules/_gdbmmodule.c 4 +Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84 +Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84 +Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84 +Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84 +Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4 +Modules/_hashopenssl.c 13 +Modules/_io/_iomodule.c 1 +Modules/_io/bufferedio.c 15 +Modules/_io/bytesio.c 14 +Modules/_io/fileio.c 9 +Modules/_io/stringio.c 8 +Modules/_io/textio.c 17 +Modules/_json.c 19 +Modules/_localemodule.c 2 +Modules/_lsprof.c 5 +Modules/_lzmamodule.c 6 +Modules/_multiprocessing/posixshmem.c 1 +Modules/_multiprocessing/semaphore.c 1 +Modules/_operator.c 5 +Modules/_pickle.c 73 +Modules/_posixsubprocess.c 11 +Modules/_queuemodule.c 4 +Modules/_randommodule.c 3 +Modules/_sqlite/connection.c 5 +Modules/_sqlite/cursor.c 3 +Modules/_sqlite/module.c 2 +Modules/_sre/sre.c 14 +Modules/_sre/sre_lib.h 25 +Modules/_ssl.c 26 +Modules/_struct.c 3 +Modules/_testbuffer.c 27 +Modules/_testcapi/bytes.c 1 +Modules/_testcapi/heaptype.c 1 +Modules/_testcapi/long.c 2 +Modules/_testcapi/mem.c 2 +Modules/_testcapi/monitoring.c 3 +Modules/_testcapi/pyatomic.c 4 +Modules/_testcapi/pyatomic.c 4 +Modules/_testcapi/unicode.c 1 +Modules/_testcapi/vectorcall.c 3 +Modules/_testcapi/watchers.c 3 +Modules/_testcapimodule.c 1 +Modules/_testclinic.c 14 +Modules/_testclinic.c 14 +Modules/_testexternalinspection.c 7 +Modules/_testinternalcapi.c 10 +Modules/_testinternalcapi/test_critical_sections.c 1 +Modules/_testinternalcapi/test_lock.c 4 +Modules/_testlimitedcapi/heaptype_relative.c 3 +Modules/_testlimitedcapi/object.c 2 +Modules/_testlimitedcapi/unicode.c 1 +Modules/_testmultiphase.c 1 +Modules/_tkinter.c 8 +Modules/_xxtestfuzz/_xxtestfuzz.c 1 +Modules/_xxtestfuzz/fuzzer.c 13 +Modules/_zoneinfo.c 17 +Modules/arraymodule.c 48 +Modules/binascii.c 208 +Modules/blake2module.c 8 +Modules/cjkcodecs/_codecs_iso2022.c 1 +Modules/cjkcodecs/_codecs_jp.c 17 +Modules/cjkcodecs/_codecs_kr.c 7 +Modules/cjkcodecs/alg_jisx0201.h 2 +Modules/cjkcodecs/cjkcodecs.h 1 +Modules/cjkcodecs/multibytecodec.c 12 +Modules/expat/pyexpatns.h 3 +Modules/expat/siphash.h 1 +Modules/expat/xmlparse.c 43 +Modules/expat/xmltok.c 15 +Modules/expat/xmltok.c 15 +Modules/expat/xmltok_impl.c 8 +Modules/faulthandler.c 5 +Modules/fcntlmodule.c 6 +Modules/getpath.c 7 +Modules/grpmodule.c 4 +Modules/itertoolsmodule.c 4 +Modules/main.c 2 +Modules/mathmodule.c 14 +Modules/mmapmodule.c 22 +Modules/mmapmodule.c 22 +Modules/posixmodule.c 79 +Modules/pwdmodule.c 4 +Modules/pyexpat.c 10 +Modules/readline.c 1 +Modules/resource.c 4 +Modules/rotatingtree.c 2 +Modules/selectmodule.c 1 +Modules/sha3module.c 4 +Modules/signalmodule.c 3 +Modules/socketmodule.c 75 +Modules/syslogmodule.c 3 +Modules/termios.c 1 +Modules/timemodule.c 10 +Modules/unicodedata.c 24 +Modules/unicodedata_db.h 1 +Modules/zlibmodule.c 24 +Objects/abstract.c 6 +Objects/bytearrayobject.c 42 +Objects/bytes_methods.c 4 +Objects/bytesobject.c 45 +Objects/call.c 12 +Objects/classobject.c 4 +Objects/codeobject.c 19 +Objects/descrobject.c 2 +Objects/dictobject.c 31 +Objects/fileobject.c 3 +Objects/floatobject.c 10 +Objects/frameobject.c 16 +Objects/funcobject.c 1 +Objects/genobject.c 3 +Objects/listobject.c 38 +Objects/longobject.c 47 +Objects/memoryobject.c 12 +Objects/methodobject.c 1 +Objects/mimalloc/alloc.c 6 +Objects/mimalloc/arena.c 6 +Objects/mimalloc/heap.c 2 +Objects/mimalloc/init.c 2 +Objects/mimalloc/options.c 4 +Objects/mimalloc/os.c 4 +Objects/mimalloc/page-queue.c 2 +Objects/mimalloc/page.c 2 +Objects/mimalloc/prim/unix/prim.c 6 +Objects/mimalloc/random.c 1 +Objects/mimalloc/segment.c 11 +Objects/mimalloc/stats.c 5 +Objects/moduleobject.c 4 +Objects/object.c 1 +Objects/obmalloc.c 6 +Objects/odictobject.c 6 +Objects/rangeobject.c 10 +Objects/setobject.c 13 +Objects/sliceobject.c 2 +Objects/stringlib/codecs.h 12 +Objects/stringlib/eq.h 1 +Objects/stringlib/fastsearch.h 8 +Objects/stringlib/join.h 3 +Objects/stringlib/replace.h 4 +Objects/stringlib/repr.h 21 +Objects/stringlib/transmogrify.h 26 +Objects/structseq.c 10 +Objects/tupleobject.c 8 +Objects/typeobject.c 38 +Objects/unicodectype.c 7 +Objects/unicodeobject.c 135 +Parser/action_helpers.c 3 +Parser/lexer/buffer.c 1 +Parser/lexer/lexer.c 14 +Parser/parser.c 116 +Parser/pegen.c 8 +Parser/string_parser.c 7 +Parser/tokenizer/file_tokenizer.c 9 +Parser/tokenizer/helpers.c 7 +Parser/tokenizer/readline_tokenizer.c 4 +Python/assemble.c 11 +Python/ast_opt.c 5 +Python/bltinmodule.c 8 +Python/bootstrap_hash.c 7 +Python/ceval.c 8 +Python/ceval_gil.c 2 +Python/codecs.c 28 +Python/codegen.c 6 +Python/compile.c 2 +Python/context.c 1 +Python/crossinterp.c 2 +Python/crossinterp_data_lookup.h 1 +Python/dtoa.c 30 +Python/errors.c 1 +Python/fileutils.c 11 +Python/flowgraph.c 7 +Python/formatter_unicode.c 6 +Python/frame.c 3 +Python/gc.c 9 +Python/gc.c 9 +Python/generated_cases.c.h 27 +Python/generated_cases.c.h 27 +Python/getargs.c 7 +Python/hashtable.c 1 +Python/import.c 6 +Python/import.c 7 +Python/initconfig.c 11 +Python/instrumentation.c 43 +Python/intrinsics.c 1 +Python/legacy_tracing.c 3 +Python/lock.c 4 +Python/marshal.c 16 +Python/modsupport.c 3 +Python/mystrtoul.c 4 +Python/pathconfig.c 1 +Python/perf_jit_trampoline.c 32 +Python/perf_trampoline.c 12 +Python/preconfig.c 2 +Python/pyarena.c 1 +Python/pyhash.c 4 +Python/pylifecycle.c 3 +Python/pystate.c 4 +Python/pystrhex.c 15 +Python/pystrtod.c 12 +Python/pytime.c 2 +Python/qsbr.c 2 +Python/specialize.c 9 +Python/suggestions.c 12 +Python/symtable.c 15 +Python/sysmodule.c 2 +Python/thread.c 1 +Python/thread_pthread.h 6 +Python/traceback.c 6 +Python/tracemalloc.c 6 diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 1ed83447b6b668..a9d0c1eeddbc1f 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -1,38 +1,49 @@ """ -Parses compiler output with -fdiagnostics-format=json and checks that warnings +Parses compiler output from Clang or GCC and checks that warnings exist only in files that are expected to have warnings. """ import argparse from collections import defaultdict -import json import re import sys from pathlib import Path from typing import NamedTuple + class FileWarnings(NamedTuple): name: str count: int -def extract_warnings_from_compiler_output_clang( +def extract_warnings_from_compiler_output( compiler_output: str, + compiler_output_type: str, + path_prefix: str = "", ) -> list[dict]: """ - Extracts warnings from the compiler output when using clang + Extracts warnings from the compiler output based on compiler + output type. Removes path prefix from file paths if provided. + Compatible with GCC and Clang compiler output. """ - # Regex to find warnings in the compiler output - clang_warning_regex = re.compile( - r"(?P.*):(?P\d+):(?P\d+): warning: " - r"(?P.*) (?P