From 78634cfa3dd4b542897835d5f097604dbeb0f3fd Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Thu, 6 Jun 2024 17:31:33 +0200
Subject: [PATCH 01/72] gh-120155: Initialize variables in _tkinter.c (#120156)

Initialize variables in _tkinter.c to make static analyzers happy.
---
 Modules/_tkinter.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c
index 24f87c8d34c6b2..a34646aecb3ec8 100644
--- a/Modules/_tkinter.c
+++ b/Modules/_tkinter.c
@@ -1438,7 +1438,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args)
            marshal the parameters to the interpreter thread. */
         Tkapp_CallEvent *ev;
         Tcl_Condition cond = NULL;
-        PyObject *exc;
+        PyObject *exc = NULL;  // init to make static analyzers happy
         if (!WaitForMainloop(self))
             return NULL;
         ev = (Tkapp_CallEvent*)attemptckalloc(sizeof(Tkapp_CallEvent));
@@ -1712,7 +1712,8 @@ var_invoke(EventFunc func, PyObject *selfptr, PyObject *args, int flags)
     TkappObject *self = (TkappObject*)selfptr;
     if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) {
         VarEvent *ev;
-        PyObject *res, *exc;
+        // init 'res' and 'exc' to make static analyzers happy
+        PyObject *res = NULL, *exc = NULL;
         Tcl_Condition cond = NULL;
 
         /* The current thread is not the interpreter thread.  Marshal
@@ -2413,6 +2414,8 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name,
     data->self = self;
     data->func = Py_NewRef(func);
     if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) {
+        err = 0;  // init to make static analyzers happy
+
         Tcl_Condition cond = NULL;
         CommandEvent *ev = (CommandEvent*)attemptckalloc(sizeof(CommandEvent));
         if (ev == NULL) {
@@ -2468,6 +2471,8 @@ _tkinter_tkapp_deletecommand_impl(TkappObject *self, const char *name)
     TRACE(self, ("((sss))", "rename", name, ""));
 
     if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) {
+        err = 0;  // init to make static analyzers happy
+
         Tcl_Condition cond = NULL;
         CommandEvent *ev;
         ev = (CommandEvent*)attemptckalloc(sizeof(CommandEvent));

From d50a7c478feb4037e65fcaea453d9ecc00259dd9 Mon Sep 17 00:00:00 2001
From: Jelle Zijlstra <jelle.zijlstra@gmail.com>
Date: Thu, 6 Jun 2024 09:25:05 -0700
Subject: [PATCH 02/72] CODEOWNERS: Add myself to symtable and AST (#120139)

Co-authored-by: Carl Meyer <carl@oddbird.net>
---
 .github/CODEOWNERS | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index ca5c8aaa3a0bef..c7021b3c2a4c40 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -40,6 +40,7 @@ Python/bytecodes.c            @markshannon
 Python/optimizer*.c           @markshannon
 Python/optimizer_analysis.c   @Fidget-Spinner
 Python/optimizer_bytecodes.c  @Fidget-Spinner
+Python/symtable.c             @JelleZijlstra @carljm 
 Lib/_pyrepl/*                 @pablogsal @lysnikolaou @ambv
 Lib/test/test_patma.py        @brandtbucher
 Lib/test/test_type_*.py       @JelleZijlstra
@@ -153,10 +154,10 @@ Include/internal/pycore_time.h  @pganssle @abalkin
 /Tools/cases_generator/        @markshannon
 
 # AST
-Python/ast.c                  @isidentical
-Parser/asdl.py                @isidentical
-Parser/asdl_c.py              @isidentical
-Lib/ast.py                    @isidentical
+Python/ast.c                  @isidentical @JelleZijlstra
+Parser/asdl.py                @isidentical @JelleZijlstra
+Parser/asdl_c.py              @isidentical @JelleZijlstra
+Lib/ast.py                    @isidentical @JelleZijlstra
 
 # Mock
 /Lib/unittest/mock.py         @cjw296

From 2d7ff6e0e7d4c08ba84079a5c19a4a485626e1de Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka <storchaka@gmail.com>
Date: Thu, 6 Jun 2024 20:12:32 +0300
Subject: [PATCH 03/72] Restore decimal context after decimal doctests
 (GH-120149)

The modified context caused tests failures in several other tests.
---
 Lib/test/test_decimal.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py
index e927e24b582a5d..46755107de0102 100644
--- a/Lib/test/test_decimal.py
+++ b/Lib/test/test_decimal.py
@@ -5892,13 +5892,17 @@ def load_tests(loader, tests, pattern):
 
     if TODO_TESTS is None:
         from doctest import DocTestSuite, IGNORE_EXCEPTION_DETAIL
+        orig_context = orig_sys_decimal.getcontext().copy()
         for mod in C, P:
             if not mod:
                 continue
             def setUp(slf, mod=mod):
                 sys.modules['decimal'] = mod
-            def tearDown(slf):
+                init(mod)
+            def tearDown(slf, mod=mod):
                 sys.modules['decimal'] = orig_sys_decimal
+                mod.setcontext(ORIGINAL_CONTEXT[mod].copy())
+                orig_sys_decimal.setcontext(orig_context.copy())
             optionflags = IGNORE_EXCEPTION_DETAIL if mod is C else 0
             sys.modules['decimal'] = mod
             tests.addTest(DocTestSuite(mod, setUp=setUp, tearDown=tearDown,
@@ -5913,8 +5917,8 @@ def setUpModule():
     TEST_ALL = ARITH if ARITH is not None else is_resource_enabled('decimal')
 
 def tearDownModule():
-    if C: C.setcontext(ORIGINAL_CONTEXT[C])
-    P.setcontext(ORIGINAL_CONTEXT[P])
+    if C: C.setcontext(ORIGINAL_CONTEXT[C].copy())
+    P.setcontext(ORIGINAL_CONTEXT[P].copy())
     if not C:
         warnings.warn('C tests skipped: no module named _decimal.',
                       UserWarning)

From 417bec733c11e63df559ecf898802dbef590142e Mon Sep 17 00:00:00 2001
From: Tian Gao <gaogaotiantian@hotmail.com>
Date: Thu, 6 Jun 2024 10:20:37 -0700
Subject: [PATCH 04/72] Add Tian Gao to CODEOWNERS and ACKS (GH-120166)

---
 .github/CODEOWNERS | 4 ++++
 Misc/ACKS          | 1 +
 2 files changed, 5 insertions(+)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index c7021b3c2a4c40..811b8cfdab17dc 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -174,6 +174,10 @@ Lib/ast.py                    @isidentical @JelleZijlstra
 /Lib/test/test_subprocess.py  @gpshead
 /Modules/*subprocess*         @gpshead
 
+# debugger
+**/*pdb*                      @gaogaotiantian
+**/*bdb*                      @gaogaotiantian
+
 # Limited C API & stable ABI
 Tools/build/stable_abi.py     @encukou
 Misc/stable_abi.toml          @encukou
diff --git a/Misc/ACKS b/Misc/ACKS
index af92d81ff3141a..2f4c0793437fb6 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -610,6 +610,7 @@ Nitin Ganatra
 Soumendra Ganguly (गङ्गोपाध्याय)
 Fred Gansevles
 Paul Ganssle
+Tian Gao
 Lars Marius Garshol
 Jake Garver
 Dan Gass

From e21057b99967eb5323320e6d1121955e0cd2985e Mon Sep 17 00:00:00 2001
From: Sam Gross <colesbury@gmail.com>
Date: Thu, 6 Jun 2024 13:40:58 -0400
Subject: [PATCH 05/72] gh-117657: Fix TSAN race involving import lock
 (#118523)

This adds a `_PyRecursiveMutex` type based on `PyMutex` and uses that
for the import lock. This fixes some data races in the free-threaded
build and generally simplifies the import lock code.
---
 Include/internal/pycore_import.h           | 18 +----
 Include/internal/pycore_lock.h             | 12 ++++
 Modules/_testinternalcapi/test_lock.c      | 25 +++++++
 Modules/posixmodule.c                      | 11 +--
 Python/import.c                            | 83 ++--------------------
 Python/lock.c                              | 42 +++++++++++
 Tools/tsan/suppressions_free_threading.txt |  4 --
 7 files changed, 90 insertions(+), 105 deletions(-)

diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h
index f8329a460d6cbf..290ba95e1a0ad7 100644
--- a/Include/internal/pycore_import.h
+++ b/Include/internal/pycore_import.h
@@ -20,7 +20,7 @@ PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module);
 extern int _PyImport_SetModuleString(const char *name, PyObject* module);
 
 extern void _PyImport_AcquireLock(PyInterpreterState *interp);
-extern int _PyImport_ReleaseLock(PyInterpreterState *interp);
+extern void _PyImport_ReleaseLock(PyInterpreterState *interp);
 
 // This is used exclusively for the sys and builtins modules:
 extern int _PyImport_FixupBuiltin(
@@ -94,11 +94,7 @@ struct _import_state {
 #endif
     PyObject *import_func;
     /* The global import lock. */
-    struct {
-        PyThread_type_lock mutex;
-        unsigned long thread;
-        int level;
-    } lock;
+    _PyRecursiveMutex lock;
     /* diagnostic info in PyImport_ImportModuleLevelObject() */
     struct {
         int import_level;
@@ -123,11 +119,6 @@ struct _import_state {
 #define IMPORTS_INIT \
     { \
         DLOPENFLAGS_INIT \
-        .lock = { \
-            .mutex = NULL, \
-            .thread = PYTHREAD_INVALID_THREAD_ID, \
-            .level = 0, \
-        }, \
         .find_and_load = { \
             .header = 1, \
         }, \
@@ -180,11 +171,6 @@ extern void _PyImport_FiniCore(PyInterpreterState *interp);
 extern void _PyImport_FiniExternal(PyInterpreterState *interp);
 
 
-#ifdef HAVE_FORK
-extern PyStatus _PyImport_ReInitLock(PyInterpreterState *interp);
-#endif
-
-
 extern PyObject* _PyImport_GetBuiltinModuleNames(void);
 
 struct _module_alias {
diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h
index a5b28e4bd4744e..d5853b2c9ff464 100644
--- a/Include/internal/pycore_lock.h
+++ b/Include/internal/pycore_lock.h
@@ -219,6 +219,18 @@ _PyOnceFlag_CallOnce(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg)
     return _PyOnceFlag_CallOnceSlow(flag, fn, arg);
 }
 
+// A recursive mutex. The mutex should zero-initialized.
+typedef struct {
+    PyMutex mutex;
+    unsigned long long thread;  // i.e., PyThread_get_thread_ident_ex()
+    size_t level;
+} _PyRecursiveMutex;
+
+PyAPI_FUNC(int) _PyRecursiveMutex_IsLockedByCurrentThread(_PyRecursiveMutex *m);
+PyAPI_FUNC(void) _PyRecursiveMutex_Lock(_PyRecursiveMutex *m);
+PyAPI_FUNC(void) _PyRecursiveMutex_Unlock(_PyRecursiveMutex *m);
+
+
 // A readers-writer (RW) lock. The lock supports multiple concurrent readers or
 // a single writer. The lock is write-preferring: if a writer is waiting while
 // the lock is read-locked then, new readers will be blocked. This avoids
diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c
index 4900459c689279..1544fe1363c7c5 100644
--- a/Modules/_testinternalcapi/test_lock.c
+++ b/Modules/_testinternalcapi/test_lock.c
@@ -2,6 +2,7 @@
 
 #include "parts.h"
 #include "pycore_lock.h"
+#include "pycore_pythread.h"      // PyThread_get_thread_ident_ex()
 
 #include "clinic/test_lock.c.h"
 
@@ -476,6 +477,29 @@ test_lock_rwlock(PyObject *self, PyObject *obj)
     Py_RETURN_NONE;
 }
 
+static PyObject *
+test_lock_recursive(PyObject *self, PyObject *obj)
+{
+    _PyRecursiveMutex m = (_PyRecursiveMutex){0};
+    assert(!_PyRecursiveMutex_IsLockedByCurrentThread(&m));
+
+    _PyRecursiveMutex_Lock(&m);
+    assert(m.thread == PyThread_get_thread_ident_ex());
+    assert(PyMutex_IsLocked(&m.mutex));
+    assert(m.level == 0);
+
+    _PyRecursiveMutex_Lock(&m);
+    assert(m.level == 1);
+    _PyRecursiveMutex_Unlock(&m);
+
+    _PyRecursiveMutex_Unlock(&m);
+    assert(m.thread == 0);
+    assert(!PyMutex_IsLocked(&m.mutex));
+    assert(m.level == 0);
+
+    Py_RETURN_NONE;
+}
+
 static PyMethodDef test_methods[] = {
     {"test_lock_basic", test_lock_basic, METH_NOARGS},
     {"test_lock_two_threads", test_lock_two_threads, METH_NOARGS},
@@ -485,6 +509,7 @@ static PyMethodDef test_methods[] = {
     {"test_lock_benchmark", test_lock_benchmark, METH_NOARGS},
     {"test_lock_once", test_lock_once, METH_NOARGS},
     {"test_lock_rwlock", test_lock_rwlock, METH_NOARGS},
+    {"test_lock_recursive", test_lock_recursive, METH_NOARGS},
     {NULL, NULL} /* sentinel */
 };
 
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 386e942d53f539..5f943d4b1c8085 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -16,7 +16,6 @@
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 #include "pycore_ceval.h"         // _PyEval_ReInitThreads()
 #include "pycore_fileutils.h"     // _Py_closerange()
-#include "pycore_import.h"        // _PyImport_ReInitLock()
 #include "pycore_initconfig.h"    // _PyStatus_EXCEPTION()
 #include "pycore_long.h"          // _PyLong_IsNegative()
 #include "pycore_moduleobject.h"  // _PyModule_GetState()
@@ -627,10 +626,7 @@ PyOS_AfterFork_Parent(void)
     _PyEval_StartTheWorldAll(&_PyRuntime);
 
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    if (_PyImport_ReleaseLock(interp) <= 0) {
-        Py_FatalError("failed releasing import lock after fork");
-    }
-
+    _PyImport_ReleaseLock(interp);
     run_at_forkers(interp->after_forkers_parent, 0);
 }
 
@@ -675,10 +671,7 @@ PyOS_AfterFork_Child(void)
     _PyEval_StartTheWorldAll(&_PyRuntime);
     _PyThreadState_DeleteList(list);
 
-    status = _PyImport_ReInitLock(tstate->interp);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto fatal_error;
-    }
+    _PyImport_ReleaseLock(tstate->interp);
 
     _PySignal_AfterFork();
 
diff --git a/Python/import.c b/Python/import.c
index 351d463dcab465..2c7a461ac786c8 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -94,11 +94,7 @@ static struct _inittab *inittab_copy = NULL;
     (interp)->imports.import_func
 
 #define IMPORT_LOCK(interp) \
-    (interp)->imports.lock.mutex
-#define IMPORT_LOCK_THREAD(interp) \
-    (interp)->imports.lock.thread
-#define IMPORT_LOCK_LEVEL(interp) \
-    (interp)->imports.lock.level
+    (interp)->imports.lock
 
 #define FIND_AND_LOAD(interp) \
     (interp)->imports.find_and_load
@@ -115,74 +111,14 @@ static struct _inittab *inittab_copy = NULL;
 void
 _PyImport_AcquireLock(PyInterpreterState *interp)
 {
-    unsigned long me = PyThread_get_thread_ident();
-    if (me == PYTHREAD_INVALID_THREAD_ID)
-        return; /* Too bad */
-    if (IMPORT_LOCK(interp) == NULL) {
-        IMPORT_LOCK(interp) = PyThread_allocate_lock();
-        if (IMPORT_LOCK(interp) == NULL)
-            return;  /* Nothing much we can do. */
-    }
-    if (IMPORT_LOCK_THREAD(interp) == me) {
-        IMPORT_LOCK_LEVEL(interp)++;
-        return;
-    }
-    if (IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID ||
-        !PyThread_acquire_lock(IMPORT_LOCK(interp), 0))
-    {
-        PyThreadState *tstate = PyEval_SaveThread();
-        PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK);
-        PyEval_RestoreThread(tstate);
-    }
-    assert(IMPORT_LOCK_LEVEL(interp) == 0);
-    IMPORT_LOCK_THREAD(interp) = me;
-    IMPORT_LOCK_LEVEL(interp) = 1;
+    _PyRecursiveMutex_Lock(&IMPORT_LOCK(interp));
 }
 
-int
+void
 _PyImport_ReleaseLock(PyInterpreterState *interp)
 {
-    unsigned long me = PyThread_get_thread_ident();
-    if (me == PYTHREAD_INVALID_THREAD_ID || IMPORT_LOCK(interp) == NULL)
-        return 0; /* Too bad */
-    if (IMPORT_LOCK_THREAD(interp) != me)
-        return -1;
-    IMPORT_LOCK_LEVEL(interp)--;
-    assert(IMPORT_LOCK_LEVEL(interp) >= 0);
-    if (IMPORT_LOCK_LEVEL(interp) == 0) {
-        IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID;
-        PyThread_release_lock(IMPORT_LOCK(interp));
-    }
-    return 1;
-}
-
-#ifdef HAVE_FORK
-/* This function is called from PyOS_AfterFork_Child() to ensure that newly
-   created child processes do not share locks with the parent.
-   We now acquire the import lock around fork() calls but on some platforms
-   (Solaris 9 and earlier? see isue7242) that still left us with problems. */
-PyStatus
-_PyImport_ReInitLock(PyInterpreterState *interp)
-{
-    if (IMPORT_LOCK(interp) != NULL) {
-        if (_PyThread_at_fork_reinit(&IMPORT_LOCK(interp)) < 0) {
-            return _PyStatus_ERR("failed to create a new lock");
-        }
-    }
-
-    if (IMPORT_LOCK_LEVEL(interp) > 1) {
-        /* Forked as a side effect of import */
-        unsigned long me = PyThread_get_thread_ident();
-        PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK);
-        IMPORT_LOCK_THREAD(interp) = me;
-        IMPORT_LOCK_LEVEL(interp)--;
-    } else {
-        IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID;
-        IMPORT_LOCK_LEVEL(interp) = 0;
-    }
-    return _PyStatus_OK();
+    _PyRecursiveMutex_Unlock(&IMPORT_LOCK(interp));
 }
-#endif
 
 
 /***************/
@@ -4111,11 +4047,6 @@ _PyImport_FiniCore(PyInterpreterState *interp)
         PyErr_FormatUnraisable("Exception ignored on clearing sys.modules");
     }
 
-    if (IMPORT_LOCK(interp) != NULL) {
-        PyThread_free_lock(IMPORT_LOCK(interp));
-        IMPORT_LOCK(interp) = NULL;
-    }
-
     _PyImport_ClearCore(interp);
 }
 
@@ -4248,8 +4179,7 @@ _imp_lock_held_impl(PyObject *module)
 /*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    return PyBool_FromLong(
-            IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID);
+    return PyBool_FromLong(PyMutex_IsLocked(&IMPORT_LOCK(interp).mutex));
 }
 
 /*[clinic input]
@@ -4283,11 +4213,12 @@ _imp_release_lock_impl(PyObject *module)
 /*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    if (_PyImport_ReleaseLock(interp) < 0) {
+    if (!_PyRecursiveMutex_IsLockedByCurrentThread(&IMPORT_LOCK(interp))) {
         PyErr_SetString(PyExc_RuntimeError,
                         "not holding the import lock");
         return NULL;
     }
+    _PyImport_ReleaseLock(interp);
     Py_RETURN_NONE;
 }
 
diff --git a/Python/lock.c b/Python/lock.c
index 239e56ad929ea3..555f4c25b9b214 100644
--- a/Python/lock.c
+++ b/Python/lock.c
@@ -366,6 +366,48 @@ _PyOnceFlag_CallOnceSlow(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg)
     }
 }
 
+static int
+recursive_mutex_is_owned_by(_PyRecursiveMutex *m, PyThread_ident_t tid)
+{
+    return _Py_atomic_load_ullong_relaxed(&m->thread) == tid;
+}
+
+int
+_PyRecursiveMutex_IsLockedByCurrentThread(_PyRecursiveMutex *m)
+{
+    return recursive_mutex_is_owned_by(m, PyThread_get_thread_ident_ex());
+}
+
+void
+_PyRecursiveMutex_Lock(_PyRecursiveMutex *m)
+{
+    PyThread_ident_t thread = PyThread_get_thread_ident_ex();
+    if (recursive_mutex_is_owned_by(m, thread)) {
+        m->level++;
+        return;
+    }
+    PyMutex_Lock(&m->mutex);
+    _Py_atomic_store_ullong_relaxed(&m->thread, thread);
+    assert(m->level == 0);
+}
+
+void
+_PyRecursiveMutex_Unlock(_PyRecursiveMutex *m)
+{
+    PyThread_ident_t thread = PyThread_get_thread_ident_ex();
+    if (!recursive_mutex_is_owned_by(m, thread)) {
+        Py_FatalError("unlocking a recursive mutex that is not owned by the"
+                      " current thread");
+    }
+    if (m->level > 0) {
+        m->level--;
+        return;
+    }
+    assert(m->level == 0);
+    _Py_atomic_store_ullong_relaxed(&m->thread, 0);
+    PyMutex_Unlock(&m->mutex);
+}
+
 #define _Py_WRITE_LOCKED 1
 #define _PyRWMutex_READER_SHIFT 2
 #define _Py_RWMUTEX_MAX_READERS (UINTPTR_MAX >> _PyRWMutex_READER_SHIFT)
diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt
index 8b64d1ff321858..cb48a30751ac7b 100644
--- a/Tools/tsan/suppressions_free_threading.txt
+++ b/Tools/tsan/suppressions_free_threading.txt
@@ -26,8 +26,6 @@ race:free_threadstate
 race_top:_add_to_weak_set
 race_top:_in_weak_set
 race_top:_PyEval_EvalFrameDefault
-race_top:_PyImport_AcquireLock
-race_top:_PyImport_ReleaseLock
 race_top:_PyType_HasFeature
 race_top:assign_version_tag
 race_top:insertdict
@@ -41,9 +39,7 @@ race_top:set_discard_entry
 race_top:set_inheritable
 race_top:Py_SET_TYPE
 race_top:_PyDict_CheckConsistency
-race_top:_PyImport_AcquireLock
 race_top:_Py_dict_lookup_threadsafe
-race_top:_imp_release_lock
 race_top:_multiprocessing_SemLock_acquire_impl
 race_top:dictiter_new
 race_top:dictresize

From 5bdc87b8859c837092e7c5b19583f98488f7a387 Mon Sep 17 00:00:00 2001
From: David Lowry-Duda <david@lowryduda.com>
Date: Thu, 6 Jun 2024 17:35:24 -0400
Subject: [PATCH 06/72] gh-120178: Documentation typo corrections (#120179)

---
 Doc/glossary.rst    | 2 +-
 Doc/library/dis.rst | 2 +-
 Doc/library/pdb.rst | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index ae9949bc2867c4..8685369117fd87 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -594,7 +594,7 @@ Glossary
       therefore it is never deallocated.
 
       Built-in strings and singletons are immortal objects. For example,
-      :const:`True` and :const:`None` singletons are immmortal.
+      :const:`True` and :const:`None` singletons are immortal.
 
       See `PEP 683 – Immortal Objects, Using a Fixed Refcount
       <https://peps.python.org/pep-0683/>`_ for more information.
diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst
index 87d1bcdfaf3f1d..ab46d4554d8773 100644
--- a/Doc/library/dis.rst
+++ b/Doc/library/dis.rst
@@ -1667,7 +1667,7 @@ iterations of the loop.
 
    A no-op. Performs internal tracing, debugging and optimization checks.
 
-   The ``context`` oparand consists of two parts. The lowest two bits
+   The ``context`` operand consists of two parts. The lowest two bits
    indicate where the ``RESUME`` occurs:
 
    * ``0`` The start of a function, which is neither a generator, coroutine
diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst
index cd6496203949ea..f6085171dccb38 100644
--- a/Doc/library/pdb.rst
+++ b/Doc/library/pdb.rst
@@ -341,7 +341,7 @@ can be overridden by the local file.
    With a *lineno* argument, set a break at line *lineno* in the current file.
    The line number may be prefixed with a *filename* and a colon,
    to specify a breakpoint in another file (possibly one that hasn't been loaded
-   yet).  The file is searched on :data:`sys.path`.  Accepatable forms of *filename*
+   yet).  The file is searched on :data:`sys.path`.  Acceptable forms of *filename*
    are ``/abspath/to/file.py``, ``relpath/file.py``, ``module`` and
    ``package.module``.
 

From 14e1506a6d7056c38fbbc0797268dcf783f91243 Mon Sep 17 00:00:00 2001
From: Barney Gale <barney.gale@gmail.com>
Date: Fri, 7 Jun 2024 00:27:39 +0100
Subject: [PATCH 07/72] GH-119054: Add "Reading directories" section to pathlib
 docs (#119956)

Add a dedicated subsection for `Path.iterdir()`-related methods,
specifically `iterdir()`, `glob()`, `rglob()` and `walk()`.

Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
---
 Doc/library/pathlib.rst | 197 +++++++++++++++++++++-------------------
 1 file changed, 102 insertions(+), 95 deletions(-)

diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst
index f37bb33321fa53..b7ab44706a0160 100644
--- a/Doc/library/pathlib.rst
+++ b/Doc/library/pathlib.rst
@@ -820,6 +820,9 @@ bugs or failures in your application)::
        % (cls.__name__,))
    UnsupportedOperation: cannot instantiate 'WindowsPath' on your system
 
+Some concrete path methods can raise an :exc:`OSError` if a system call fails
+(for example because the path doesn't exist).
+
 
 Parsing and generating URIs
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -1151,69 +1154,32 @@ Reading and writing files
    .. versionadded:: 3.5
 
 
-Other methods
-^^^^^^^^^^^^^
-
-Some of these methods can raise an :exc:`OSError` if a system call fails (for
-example because the path doesn't exist).
-
-
-.. classmethod:: Path.cwd()
-
-   Return a new path object representing the current directory (as returned
-   by :func:`os.getcwd`)::
-
-      >>> Path.cwd()
-      PosixPath('/home/antoine/pathlib')
-
-
-.. classmethod:: Path.home()
-
-   Return a new path object representing the user's home directory (as
-   returned by :func:`os.path.expanduser` with ``~`` construct). If the home
-   directory can't be resolved, :exc:`RuntimeError` is raised.
-
-   ::
-
-      >>> Path.home()
-      PosixPath('/home/antoine')
-
-   .. versionadded:: 3.5
-
-
-.. method:: Path.chmod(mode, *, follow_symlinks=True)
-
-   Change the file mode and permissions, like :func:`os.chmod`.
-
-   This method normally follows symlinks. Some Unix flavours support changing
-   permissions on the symlink itself; on these platforms you may add the
-   argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`.
-
-   ::
-
-      >>> p = Path('setup.py')
-      >>> p.stat().st_mode
-      33277
-      >>> p.chmod(0o444)
-      >>> p.stat().st_mode
-      33060
-
-   .. versionchanged:: 3.10
-      The *follow_symlinks* parameter was added.
+Reading directories
+^^^^^^^^^^^^^^^^^^^
 
-.. method:: Path.expanduser()
+.. method:: Path.iterdir()
 
-   Return a new path with expanded ``~`` and ``~user`` constructs,
-   as returned by :meth:`os.path.expanduser`. If a home directory can't be
-   resolved, :exc:`RuntimeError` is raised.
+   When the path points to a directory, yield path objects of the directory
+   contents::
 
-   ::
+      >>> p = Path('docs')
+      >>> for child in p.iterdir(): child
+      ...
+      PosixPath('docs/conf.py')
+      PosixPath('docs/_templates')
+      PosixPath('docs/make.bat')
+      PosixPath('docs/index.rst')
+      PosixPath('docs/_build')
+      PosixPath('docs/_static')
+      PosixPath('docs/Makefile')
 
-      >>> p = PosixPath('~/films/Monty Python')
-      >>> p.expanduser()
-      PosixPath('/home/eric/films/Monty Python')
+   The children are yielded in arbitrary order, and the special entries
+   ``'.'`` and ``'..'`` are not included.  If a file is removed from or added
+   to the directory after creating the iterator, it is unspecified whether
+   a path object for that file is included.
 
-   .. versionadded:: 3.5
+   If the path is not a directory or otherwise inaccessible, :exc:`OSError` is
+   raised.
 
 
 .. method:: Path.glob(pattern, *, case_sensitive=None, recurse_symlinks=False)
@@ -1281,43 +1247,6 @@ example because the path doesn't exist).
       The *pattern* parameter accepts a :term:`path-like object`.
 
 
-.. method:: Path.group(*, follow_symlinks=True)
-
-   Return the name of the group owning the file. :exc:`KeyError` is raised
-   if the file's gid isn't found in the system database.
-
-   This method normally follows symlinks; to get the group of the symlink, add
-   the argument ``follow_symlinks=False``.
-
-   .. versionchanged:: 3.13
-      Raises :exc:`UnsupportedOperation` if the :mod:`grp` module is not
-      available. In previous versions, :exc:`NotImplementedError` was raised.
-
-   .. versionchanged:: 3.13
-      The *follow_symlinks* parameter was added.
-
-
-.. method:: Path.iterdir()
-
-   When the path points to a directory, yield path objects of the directory
-   contents::
-
-      >>> p = Path('docs')
-      >>> for child in p.iterdir(): child
-      ...
-      PosixPath('docs/conf.py')
-      PosixPath('docs/_templates')
-      PosixPath('docs/make.bat')
-      PosixPath('docs/index.rst')
-      PosixPath('docs/_build')
-      PosixPath('docs/_static')
-      PosixPath('docs/Makefile')
-
-   The children are yielded in arbitrary order, and the special entries
-   ``'.'`` and ``'..'`` are not included.  If a file is removed from or added
-   to the directory after creating the iterator, whether a path object for
-   that file be included is unspecified.
-
 .. method:: Path.walk(top_down=True, on_error=None, follow_symlinks=False)
 
    Generate the file names in a directory tree by walking the tree
@@ -1413,6 +1342,84 @@ example because the path doesn't exist).
 
    .. versionadded:: 3.12
 
+
+Other methods
+^^^^^^^^^^^^^
+
+.. classmethod:: Path.cwd()
+
+   Return a new path object representing the current directory (as returned
+   by :func:`os.getcwd`)::
+
+      >>> Path.cwd()
+      PosixPath('/home/antoine/pathlib')
+
+
+.. classmethod:: Path.home()
+
+   Return a new path object representing the user's home directory (as
+   returned by :func:`os.path.expanduser` with ``~`` construct). If the home
+   directory can't be resolved, :exc:`RuntimeError` is raised.
+
+   ::
+
+      >>> Path.home()
+      PosixPath('/home/antoine')
+
+   .. versionadded:: 3.5
+
+
+.. method:: Path.chmod(mode, *, follow_symlinks=True)
+
+   Change the file mode and permissions, like :func:`os.chmod`.
+
+   This method normally follows symlinks. Some Unix flavours support changing
+   permissions on the symlink itself; on these platforms you may add the
+   argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`.
+
+   ::
+
+      >>> p = Path('setup.py')
+      >>> p.stat().st_mode
+      33277
+      >>> p.chmod(0o444)
+      >>> p.stat().st_mode
+      33060
+
+   .. versionchanged:: 3.10
+      The *follow_symlinks* parameter was added.
+
+.. method:: Path.expanduser()
+
+   Return a new path with expanded ``~`` and ``~user`` constructs,
+   as returned by :meth:`os.path.expanduser`. If a home directory can't be
+   resolved, :exc:`RuntimeError` is raised.
+
+   ::
+
+      >>> p = PosixPath('~/films/Monty Python')
+      >>> p.expanduser()
+      PosixPath('/home/eric/films/Monty Python')
+
+   .. versionadded:: 3.5
+
+
+.. method:: Path.group(*, follow_symlinks=True)
+
+   Return the name of the group owning the file. :exc:`KeyError` is raised
+   if the file's gid isn't found in the system database.
+
+   This method normally follows symlinks; to get the group of the symlink, add
+   the argument ``follow_symlinks=False``.
+
+   .. versionchanged:: 3.13
+      Raises :exc:`UnsupportedOperation` if the :mod:`grp` module is not
+      available. In previous versions, :exc:`NotImplementedError` was raised.
+
+   .. versionchanged:: 3.13
+      The *follow_symlinks* parameter was added.
+
+
 .. method:: Path.lchmod(mode)
 
    Like :meth:`Path.chmod` but, if the path points to a symbolic link, the

From 6b606522ca97488aad6fe2f193d4511e7a8f8334 Mon Sep 17 00:00:00 2001
From: Jacob Walls <jacobtylerwalls@gmail.com>
Date: Thu, 6 Jun 2024 23:18:30 -0400
Subject: [PATCH 08/72] gh-119577: Adjust DeprecationWarning when testing
 element truth values in ElementTree (GH-119762)

Adjust DeprecationWarning when testing element truth values in ElementTree, we're planning to go with the more natural True return rather than a disruptive harder to code around exception raise, and are deferring the behavior change for a few more releases.
---
 Doc/library/xml.etree.elementtree.rst                  |  7 ++++---
 Doc/whatsnew/3.12.rst                                  |  7 +++++--
 Doc/whatsnew/3.13.rst                                  | 10 +++++-----
 Lib/test/test_xml_etree.py                             |  2 +-
 Lib/xml/etree/ElementTree.py                           |  2 +-
 .../2024-05-29-21-50-05.gh-issue-119577.S3BlKJ.rst     |  4 ++++
 Modules/_elementtree.c                                 |  2 +-
 7 files changed, 21 insertions(+), 13 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2024-05-29-21-50-05.gh-issue-119577.S3BlKJ.rst

diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst
index a6a9eb87f56d88..e5919029c62c93 100644
--- a/Doc/library/xml.etree.elementtree.rst
+++ b/Doc/library/xml.etree.elementtree.rst
@@ -1058,9 +1058,10 @@ Element Objects
    :meth:`~object.__getitem__`, :meth:`~object.__setitem__`,
    :meth:`~object.__len__`.
 
-   Caution: Elements with no subelements will test as ``False``.  Testing the
-   truth value of an Element is deprecated and will raise an exception in
-   Python 3.14.  Use specific ``len(elem)`` or ``elem is None`` test instead.::
+   Caution: Elements with no subelements will test as ``False``.  In a future
+   release of Python, all elements will test as ``True`` regardless of whether
+   subelements exist.  Instead, prefer explicit ``len(elem)`` or
+   ``elem is not None`` tests.::
 
      element = root.find('foo')
 
diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst
index f99489fb53db74..28b28e9ce50e11 100644
--- a/Doc/whatsnew/3.12.rst
+++ b/Doc/whatsnew/3.12.rst
@@ -1440,8 +1440,6 @@ and will be removed in Python 3.14.
 
 * :mod:`typing`: :class:`!typing.ByteString`
 
-* :mod:`xml.etree.ElementTree`: Testing the truth value of an :class:`xml.etree.ElementTree.Element`.
-
 * The ``__package__`` and ``__cached__`` attributes on module objects.
 
 * The :attr:`~codeobject.co_lnotab` attribute of code objects.
@@ -1467,6 +1465,11 @@ although there is currently no date scheduled for their removal.
 
 * :class:`typing.Text` (:gh:`92332`)
 
+* :mod:`xml.etree.ElementTree`: Testing the truth value of an
+  :class:`xml.etree.ElementTree.Element` is deprecated. In a future release it
+  will always return True. Prefer explicit ``len(elem)`` or
+  ``elem is not None`` tests instead.
+
 * Currently Python accepts numeric literals immediately followed by keywords,
   for example ``0in x``, ``1or x``, ``0if 1else 2``.  It allows confusing
   and ambiguous expressions like ``[0x1for x in y]`` (which can be
diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst
index a1d2a0d84e7581..81daaabdb889d7 100644
--- a/Doc/whatsnew/3.13.rst
+++ b/Doc/whatsnew/3.13.rst
@@ -1728,11 +1728,6 @@ Pending Removal in Python 3.14
   public API.
   (Contributed by Gregory P. Smith in :gh:`88168`.)
 
-* :mod:`xml.etree.ElementTree`: Testing the truth value of an
-  :class:`~xml.etree.ElementTree.Element` is deprecated and will raise an
-  exception in Python 3.14.
-
-
 Pending Removal in Python 3.15
 ------------------------------
 
@@ -1937,6 +1932,11 @@ although there is currently no date scheduled for their removal.
 * :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial
   writes.
 
+* :mod:`xml.etree.ElementTree`: Testing the truth value of an
+  :class:`~xml.etree.ElementTree.Element` is deprecated. In a future release it
+  it will always return ``True``. Prefer explicit ``len(elem)`` or
+  ``elem is not None`` tests instead.
+
 * :meth:`zipimport.zipimporter.load_module` is deprecated:
   use :meth:`~zipimport.zipimporter.exec_module` instead.
 
diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
index bae61f754e75f5..3d9141fea1ef3e 100644
--- a/Lib/test/test_xml_etree.py
+++ b/Lib/test/test_xml_etree.py
@@ -4088,7 +4088,7 @@ class BoolTest(unittest.TestCase):
     def test_warning(self):
         e = ET.fromstring('<a style="new"></a>')
         msg = (
-            r"Testing an element's truth value will raise an exception in "
+            r"Testing an element's truth value will always return True in "
             r"future versions.  "
             r"Use specific 'len\(elem\)' or 'elem is not None' test instead.")
         with self.assertWarnsRegex(DeprecationWarning, msg):
diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py
index 9e15d34d22aa6c..ce67d7d7d54748 100644
--- a/Lib/xml/etree/ElementTree.py
+++ b/Lib/xml/etree/ElementTree.py
@@ -201,7 +201,7 @@ def __len__(self):
 
     def __bool__(self):
         warnings.warn(
-            "Testing an element's truth value will raise an exception in "
+            "Testing an element's truth value will always return True in "
             "future versions.  "
             "Use specific 'len(elem)' or 'elem is not None' test instead.",
             DeprecationWarning, stacklevel=2
diff --git a/Misc/NEWS.d/next/Library/2024-05-29-21-50-05.gh-issue-119577.S3BlKJ.rst b/Misc/NEWS.d/next/Library/2024-05-29-21-50-05.gh-issue-119577.S3BlKJ.rst
new file mode 100644
index 00000000000000..bd2daf3fb5c16d
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-05-29-21-50-05.gh-issue-119577.S3BlKJ.rst
@@ -0,0 +1,4 @@
+The :exc:`DeprecationWarning` emitted when testing the truth value of an
+:class:`xml.etree.ElementTree.Element` now describes unconditionally
+returning ``True`` in a future version rather than raising an exception in
+Python 3.14.
diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
index b11983d2caa2d1..3818e20b4f0f28 100644
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -1502,7 +1502,7 @@ element_bool(PyObject* self_)
 {
     ElementObject* self = (ElementObject*) self_;
     if (PyErr_WarnEx(PyExc_DeprecationWarning,
-                     "Testing an element's truth value will raise an exception "
+                     "Testing an element's truth value will always return True "
                      "in future versions.  Use specific 'len(elem)' or "
                      "'elem is not None' test instead.",
                      1) < 0) {

From 5c115567b1e3aecb7a53cfd5757e25c088398411 Mon Sep 17 00:00:00 2001
From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
Date: Fri, 7 Jun 2024 00:38:31 -0600
Subject: [PATCH 09/72] Add Plausible for docs metrics (#119977)

Co-authored-by: Julien Palard <julien@palard.fr>
---
 Doc/conf.py                     | 3 ++-
 Doc/tools/templates/layout.html | 3 +++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/Doc/conf.py b/Doc/conf.py
index 47fb96fe1de482..8a14646801ebac 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -339,7 +339,8 @@
 html_context = {
     "is_deployment_preview": os.getenv("READTHEDOCS_VERSION_TYPE") == "external",
     "repository_url": repository_url.removesuffix(".git") if repository_url else None,
-    "pr_id": os.getenv("READTHEDOCS_VERSION")
+    "pr_id": os.getenv("READTHEDOCS_VERSION"),
+    "enable_analytics": os.getenv("PYTHON_DOCS_ENABLE_ANALYTICS"),
 }
 
 # This 'Last updated on:' timestamp is inserted at the bottom of every page.
diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html
index 3f88fc8e91faad..b09fd21a8ddcc9 100644
--- a/Doc/tools/templates/layout.html
+++ b/Doc/tools/templates/layout.html
@@ -26,6 +26,9 @@
 {% endblock %}
 
 {% block extrahead %}
+    {% if builder == "html" and enable_analytics %}
+      <script defer data-domain="docs.python.org" src="https://plausible.io/js/script.js"></script>
+    {% endif %}
     <link rel="canonical" href="https://docs.python.org/3/{{pagename}}.html" />
     {% if builder != "htmlhelp" %}
       {% if pagename == 'whatsnew/changelog' and not embedded %}

From bd826b9c77dbf7c789433cb8061c733c08634c0e Mon Sep 17 00:00:00 2001
From: Clinton <pygeek@users.noreply.github.com>
Date: Fri, 7 Jun 2024 03:39:19 -0400
Subject: [PATCH 10/72] gh-120157: Remove unused code in concurrent.future
 (gh-120187)

---
 Lib/concurrent/futures/_base.py                           | 8 --------
 .../2024-06-07-02-00-31.gh-issue-120157.HnWcF9.rst        | 1 +
 2 files changed, 1 insertion(+), 8 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-07-02-00-31.gh-issue-120157.HnWcF9.rst

diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py
index 6742a07753c921..707fcdfde79acd 100644
--- a/Lib/concurrent/futures/_base.py
+++ b/Lib/concurrent/futures/_base.py
@@ -23,14 +23,6 @@
 CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED'
 FINISHED = 'FINISHED'
 
-_FUTURE_STATES = [
-    PENDING,
-    RUNNING,
-    CANCELLED,
-    CANCELLED_AND_NOTIFIED,
-    FINISHED
-]
-
 _STATE_TO_DESCRIPTION_MAP = {
     PENDING: "pending",
     RUNNING: "running",
diff --git a/Misc/NEWS.d/next/Library/2024-06-07-02-00-31.gh-issue-120157.HnWcF9.rst b/Misc/NEWS.d/next/Library/2024-06-07-02-00-31.gh-issue-120157.HnWcF9.rst
new file mode 100644
index 00000000000000..3e905125797af7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-07-02-00-31.gh-issue-120157.HnWcF9.rst
@@ -0,0 +1 @@
+Remove unused constant ``concurrent.futures._base._FUTURE_STATES`` in :mod:`concurrent.futures`. Patch by Clinton Christian (pygeek).

From 57ad769076201c858a768d81047f6ea44925a33b Mon Sep 17 00:00:00 2001
From: Kirill Podoprigora <kirill.bast9@mail.ru>
Date: Fri, 7 Jun 2024 11:03:28 +0300
Subject: [PATCH 11/72] gh-120080: Accept ``None`` as a valid argument for
 direct call of the ``int.__round__`` (#120088)

Co-authored-by: Nikita Sobolev <mail@sobolevn.me>
---
 Lib/test/test_float.py                                      | 6 ++++++
 Lib/test/test_inspect/test_inspect.py                       | 1 -
 Lib/test/test_int.py                                        | 6 ++++++
 .../2024-06-05-08-39-40.gh-issue-120080.DJFK11.rst          | 2 ++
 Objects/clinic/longobject.c.h                               | 6 +++---
 Objects/longobject.c                                        | 6 +++---
 6 files changed, 20 insertions(+), 7 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-05-08-39-40.gh-issue-120080.DJFK11.rst

diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py
index 5bd640617d6874..53695cefb8fded 100644
--- a/Lib/test/test_float.py
+++ b/Lib/test/test_float.py
@@ -949,6 +949,12 @@ def test_None_ndigits(self):
             self.assertEqual(x, 2)
             self.assertIsInstance(x, int)
 
+    def test_round_with_none_arg_direct_call(self):
+        for val in [(1.0).__round__(None),
+                    round(1.0),
+                    round(1.0, None)]:
+            self.assertEqual(val, 1)
+            self.assertIs(type(val), int)
 
 # Beginning with Python 2.6 float has cross platform compatible
 # ways to create and represent inf and nan
diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py
index 011d42f34b6461..65007c16203c6d 100644
--- a/Lib/test/test_inspect/test_inspect.py
+++ b/Lib/test/test_inspect/test_inspect.py
@@ -5412,7 +5412,6 @@ def test_builtins_have_signatures(self):
             'bytearray': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'},
             'bytes': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'},
             'dict': {'pop'},
-            'int': {'__round__'},
             'memoryview': {'cast', 'hex'},
             'str': {'count', 'endswith', 'find', 'index', 'maketrans', 'rfind', 'rindex', 'startswith'},
         }
diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py
index ce9febd741bba2..77221dfb6d5aa2 100644
--- a/Lib/test/test_int.py
+++ b/Lib/test/test_int.py
@@ -517,6 +517,12 @@ def test_issue31619(self):
         self.assertEqual(int('1_2_3_4_5_6_7_8_9', 16), 0x123456789)
         self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807)
 
+    def test_round_with_none_arg_direct_call(self):
+        for val in [(1).__round__(None),
+                    round(1),
+                    round(1, None)]:
+            self.assertEqual(val, 1)
+            self.assertIs(type(val), int)
 
 class IntStrDigitLimitsTests(unittest.TestCase):
 
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-05-08-39-40.gh-issue-120080.DJFK11.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-08-39-40.gh-issue-120080.DJFK11.rst
new file mode 100644
index 00000000000000..8c5602fcdb4ad2
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-08-39-40.gh-issue-120080.DJFK11.rst	
@@ -0,0 +1,2 @@
+Direct call to the :meth:`!int.__round__` now accepts ``None``
+as a valid argument.
diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h
index 56bc3864582dcb..90375b9a082cca 100644
--- a/Objects/clinic/longobject.c.h
+++ b/Objects/clinic/longobject.c.h
@@ -116,7 +116,7 @@ int___format__(PyObject *self, PyObject *arg)
 }
 
 PyDoc_STRVAR(int___round____doc__,
-"__round__($self, ndigits=<unrepresentable>, /)\n"
+"__round__($self, ndigits=None, /)\n"
 "--\n"
 "\n"
 "Rounding an Integral returns itself.\n"
@@ -133,7 +133,7 @@ static PyObject *
 int___round__(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
-    PyObject *o_ndigits = NULL;
+    PyObject *o_ndigits = Py_None;
 
     if (!_PyArg_CheckPositional("__round__", nargs, 0, 1)) {
         goto exit;
@@ -476,4 +476,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored))
 {
     return int_is_integer_impl(self);
 }
-/*[clinic end generated code: output=2ba2d8dcda9b99da input=a9049054013a1b77]*/
+/*[clinic end generated code: output=a53f5ba9a6c16737 input=a9049054013a1b77]*/
diff --git a/Objects/longobject.c b/Objects/longobject.c
index ee0b2a038a2aab..a3a59a20f0bb97 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -6045,7 +6045,7 @@ _PyLong_DivmodNear(PyObject *a, PyObject *b)
 /*[clinic input]
 int.__round__
 
-    ndigits as o_ndigits: object = NULL
+    ndigits as o_ndigits: object = None
     /
 
 Rounding an Integral returns itself.
@@ -6055,7 +6055,7 @@ Rounding with an ndigits argument also returns an integer.
 
 static PyObject *
 int___round___impl(PyObject *self, PyObject *o_ndigits)
-/*[clinic end generated code: output=954fda6b18875998 input=1614cf23ec9e18c3]*/
+/*[clinic end generated code: output=954fda6b18875998 input=30c2aec788263144]*/
 {
     PyObject *temp, *result, *ndigits;
 
@@ -6073,7 +6073,7 @@ int___round___impl(PyObject *self, PyObject *o_ndigits)
      *
      *   m - divmod_near(m, 10**n)[1].
      */
-    if (o_ndigits == NULL)
+    if (o_ndigits == Py_None)
         return long_long(self);
 
     ndigits = _PyNumber_Index(o_ndigits);

From 6a97929a5ad76c55bc6e1cf32898a1c31093334d Mon Sep 17 00:00:00 2001
From: Xie Yanbo <xieyanbo@gmail.com>
Date: Fri, 7 Jun 2024 16:19:41 +0800
Subject: [PATCH 12/72] Fix typos in comments (#120188)

---
 Include/ceval.h                               | 2 +-
 Include/cpython/object.h                      | 2 +-
 Include/cpython/pyframe.h                     | 2 +-
 Include/internal/mimalloc/mimalloc/atomic.h   | 2 +-
 Include/internal/mimalloc/mimalloc/internal.h | 2 +-
 Include/internal/pycore_initconfig.h          | 2 +-
 Include/internal/pycore_instruments.h         | 2 +-
 Include/internal/pycore_lock.h                | 2 +-
 Include/internal/pycore_pythread.h            | 2 +-
 Include/internal/pycore_unicodeobject.h       | 2 +-
 Include/structmember.h                        | 2 +-
 11 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/Include/ceval.h b/Include/ceval.h
index 1ec746c3708220..e9df8684996e23 100644
--- a/Include/ceval.h
+++ b/Include/ceval.h
@@ -42,7 +42,7 @@ PyAPI_FUNC(int) Py_MakePendingCalls(void);
      level exceeds "current recursion limit + 50". By construction, this
      protection can only be triggered when the "overflowed" flag is set. It
      means the cleanup code has itself gone into an infinite loop, or the
-     RecursionError has been mistakingly ignored. When this protection is
+     RecursionError has been mistakenly ignored. When this protection is
      triggered, the interpreter aborts with a Fatal Error.
 
    In addition, the "overflowed" flag is automatically reset when the
diff --git a/Include/cpython/object.h b/Include/cpython/object.h
index e624326693d8e7..0ab94e5e2a15e5 100644
--- a/Include/cpython/object.h
+++ b/Include/cpython/object.h
@@ -313,7 +313,7 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *);
  * triggered as a side-effect of `dst` getting torn down no longer believes
  * `dst` points to a valid object.
  *
- * Temporary variables are used to only evalutate macro arguments once and so
+ * Temporary variables are used to only evaluate macro arguments once and so
  * avoid the duplication of side effects. _Py_TYPEOF() or memcpy() is used to
  * avoid a miscompilation caused by type punning. See Py_CLEAR() comment for
  * implementation details about type punning.
diff --git a/Include/cpython/pyframe.h b/Include/cpython/pyframe.h
index eeafbb17a56bad..51529763923ec3 100644
--- a/Include/cpython/pyframe.h
+++ b/Include/cpython/pyframe.h
@@ -28,7 +28,7 @@ struct _PyInterpreterFrame;
  * Does not raise an exception. */
 PyAPI_FUNC(PyObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame);
 
-/* Returns a byte ofsset into the last executed instruction.
+/* Returns a byte offset into the last executed instruction.
  * Does not raise an exception. */
 PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame);
 
diff --git a/Include/internal/mimalloc/mimalloc/atomic.h b/Include/internal/mimalloc/mimalloc/atomic.h
index eb8478ceed6adf..52f82487685cdb 100644
--- a/Include/internal/mimalloc/mimalloc/atomic.h
+++ b/Include/internal/mimalloc/mimalloc/atomic.h
@@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file
 // --------------------------------------------------------------------------------------------
 // Atomics
 // We need to be portable between C, C++, and MSVC.
-// We base the primitives on the C/C++ atomics and create a mimimal wrapper for MSVC in C compilation mode.
+// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode.
 // This is why we try to use only `uintptr_t` and `<type>*` as atomic types.
 // To gain better insight in the range of used atomics, we use explicitly named memory order operations
 // instead of passing the memory order as a parameter.
diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h
index 94f88fb603af25..d97f51b8eefbe5 100644
--- a/Include/internal/mimalloc/mimalloc/internal.h
+++ b/Include/internal/mimalloc/mimalloc/internal.h
@@ -10,7 +10,7 @@ terms of the MIT license. A copy of the license can be found in the file
 
 
 // --------------------------------------------------------------------------
-// This file contains the interal API's of mimalloc and various utility
+// This file contains the internal API's of mimalloc and various utility
 // functions and macros.
 // --------------------------------------------------------------------------
 
diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h
index 1c68161341860a..6bf1b53bffd3ba 100644
--- a/Include/internal/pycore_initconfig.h
+++ b/Include/internal/pycore_initconfig.h
@@ -157,7 +157,7 @@ typedef enum {
     /* For now, this means the GIL is enabled.
 
        gh-116329: This will eventually change to "the GIL is disabled but can
-       be reenabled by loading an incompatible extension module." */
+       be re-enabled by loading an incompatible extension module." */
     _PyConfig_GIL_DEFAULT = -1,
 
     /* The GIL has been forced off or on, and will not be affected by module loading. */
diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h
index c98e82c8be5546..4e5b374968ea98 100644
--- a/Include/internal/pycore_instruments.h
+++ b/Include/internal/pycore_instruments.h
@@ -23,7 +23,7 @@ typedef uint32_t _PyMonitoringEventSet;
 #define PY_MONITORING_PROFILER_ID 2
 #define PY_MONITORING_OPTIMIZER_ID 5
 
-/* Internal IDs used to suuport sys.setprofile() and sys.settrace() */
+/* Internal IDs used to support sys.setprofile() and sys.settrace() */
 #define PY_MONITORING_SYS_PROFILE_ID 6
 #define PY_MONITORING_SYS_TRACE_ID 7
 
diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h
index d5853b2c9ff464..882c4888e5058c 100644
--- a/Include/internal/pycore_lock.h
+++ b/Include/internal/pycore_lock.h
@@ -116,7 +116,7 @@ typedef enum _PyLockFlags {
 extern PyLockStatus
 _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout_ns, _PyLockFlags flags);
 
-// Lock a mutex with aditional options. See _PyLockFlags for details.
+// Lock a mutex with additional options. See _PyLockFlags for details.
 static inline void
 PyMutex_LockFlags(PyMutex *m, _PyLockFlags flags)
 {
diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h
index 3610c6254db6af..f3f5942444e851 100644
--- a/Include/internal/pycore_pythread.h
+++ b/Include/internal/pycore_pythread.h
@@ -147,7 +147,7 @@ PyAPI_FUNC(int) PyThread_start_joinable_thread(void (*func)(void *),
 PyAPI_FUNC(int) PyThread_join_thread(PyThread_handle_t);
 /*
  * Detach a thread started with `PyThread_start_joinable_thread`, such
- * that its resources are relased as soon as it exits.
+ * that its resources are released as soon as it exits.
  * This function cannot be interrupted. It returns 0 on success,
  * a non-zero value on failure.
  */
diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h
index fea5ceea0954f4..026d6e461f2108 100644
--- a/Include/internal/pycore_unicodeobject.h
+++ b/Include/internal/pycore_unicodeobject.h
@@ -189,7 +189,7 @@ extern PyObject* _PyUnicode_EncodeCharmap(
 
 /* --- Decimal Encoder ---------------------------------------------------- */
 
-// Coverts a Unicode object holding a decimal value to an ASCII string
+// Converts a Unicode object holding a decimal value to an ASCII string
 // for using in int, float and complex parsers.
 // Transforms code points that have decimal digit property to the
 // corresponding ASCII digit code points.  Transforms spaces to ASCII.
diff --git a/Include/structmember.h b/Include/structmember.h
index f6e8fd829892f4..5f29fbcfed99e3 100644
--- a/Include/structmember.h
+++ b/Include/structmember.h
@@ -11,7 +11,7 @@ extern "C" {
  * New definitions are in descrobject.h.
  *
  * However, there's nothing wrong with old code continuing to use it,
- * and there's not much mainenance overhead in maintaining a few aliases.
+ * and there's not much maintenance overhead in maintaining a few aliases.
  * So, don't be too eager to convert old code.
  *
  * It uses names not prefixed with Py_.

From 47816f465e833a5257a82b759b1081e06381e528 Mon Sep 17 00:00:00 2001
From: Michael Allwright <contact@allwright.io>
Date: Fri, 7 Jun 2024 10:38:15 +0200
Subject: [PATCH 13/72] gh-120154: Fix Emscripten/WASI pattern in case
 statement for LDSHARED (#120173)

Fix Emscripten/WASI pattern in case statement for LDSHARED
---
 configure    | 2 +-
 configure.ac | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index 6cfe114fb2104c..8e605d31bb5eca 100755
--- a/configure
+++ b/configure
@@ -12892,7 +12892,7 @@ then
 		LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)'
 		BLDSHARED="$LDSHARED"
 		;;
-	Emscripten|WASI)
+	Emscripten*|WASI*)
 		LDSHARED='$(CC) -shared'
 		LDCXXSHARED='$(CXX) -shared';;
 	Linux*|GNU*|QNX*|VxWorks*|Haiku*)
diff --git a/configure.ac b/configure.ac
index 8657e09c9a7008..41023ab92bad81 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3417,7 +3417,7 @@ then
 		LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)'
 		BLDSHARED="$LDSHARED"
 		;;
-	Emscripten|WASI)
+	Emscripten*|WASI*)
 		LDSHARED='$(CC) -shared'
 		LDCXXSHARED='$(CXX) -shared';;
 	Linux*|GNU*|QNX*|VxWorks*|Haiku*)

From d5ba4fc9bc9b2d9eff2a90893e8d500e0c367237 Mon Sep 17 00:00:00 2001
From: Kirill Podoprigora <kirill.bast9@mail.ru>
Date: Fri, 7 Jun 2024 12:14:13 +0300
Subject: [PATCH 14/72] gh-120164: Fix test_os.test_win32_mkdir_700() (#120177)

Don't compare the path to avoid encoding issues.

Co-authored-by: Eryk Sun <eryksun@gmail.com>
---
 Lib/test/test_os.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index de5a86f676c4d5..2beb9ca8aa6ccb 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -1837,9 +1837,10 @@ def test_win32_mkdir_700(self):
         os.mkdir(path, mode=0o700)
         out = subprocess.check_output(["cacls.exe", path, "/s"], encoding="oem")
         os.rmdir(path)
+        out = out.strip().rsplit(" ", 1)[1]
         self.assertEqual(
-            out.strip(),
-            f'{path} "D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"',
+            out,
+            '"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"',
         )
 
     def tearDown(self):

From 6646a9da26d12fc54263b22dd2916a2f710f1db7 Mon Sep 17 00:00:00 2001
From: Aditya Borikar <adityaborikar2@gmail.com>
Date: Fri, 7 Jun 2024 03:44:42 -0600
Subject: [PATCH 15/72] gh-110383: Clarify "non-integral" wording in pow() docs
 (#119688)

---
 Doc/library/functions.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 4617767a71be18..1d82f92ea67857 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -1561,7 +1561,9 @@ are always available.  They are listed here in alphabetical order.
    returns ``100``, but ``pow(10, -2)`` returns ``0.01``.  For a negative base of
    type :class:`int` or :class:`float` and a non-integral exponent, a complex
    result is delivered.  For example, ``pow(-9, 0.5)`` returns a value close
-   to ``3j``.
+   to ``3j``. Whereas, for a negative base of type :class:`int` or :class:`float`
+   with an integral exponent, a float result is delivered. For example,
+   ``pow(-9, 2.0)`` returns ``81.0``.
 
    For :class:`int` operands *base* and *exp*, if *mod* is present, *mod* must
    also be of integer type and *mod* must be nonzero. If *mod* is present and

From d68a22e7a68ae09f7db61d5a1a3bd9c0360cf3ee Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka <storchaka@gmail.com>
Date: Fri, 7 Jun 2024 13:49:07 +0300
Subject: [PATCH 16/72] gh-120211: Fix tkinter.ttk with Tcl/Tk 9.0 (GH-120213)

* Use new methods for tracing Tcl variable.
* Fix Combobox.current() for empty combobox.
---
 Lib/tkinter/ttk.py                                       | 9 ++++++---
 .../2024-06-07-13-21-11.gh-issue-120211.Rws_gf.rst       | 1 +
 2 files changed, 7 insertions(+), 3 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-07-13-21-11.gh-issue-120211.Rws_gf.rst

diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py
index 5ca938a670831a..073b3ae20797c3 100644
--- a/Lib/tkinter/ttk.py
+++ b/Lib/tkinter/ttk.py
@@ -690,7 +690,10 @@ def current(self, newindex=None):
         returns the index of the current value in the list of values
         or -1 if the current value does not appear in the list."""
         if newindex is None:
-            return self.tk.getint(self.tk.call(self._w, "current"))
+            res = self.tk.call(self._w, "current")
+            if res == '':
+                return -1
+            return self.tk.getint(res)
         return self.tk.call(self._w, "current", newindex)
 
 
@@ -1522,7 +1525,7 @@ def __init__(self, master=None, variable=None, from_=0, to=10, **kw):
         self.label.place(anchor='n' if label_side == 'top' else 's')
 
         # update the label as scale or variable changes
-        self.__tracecb = self._variable.trace_variable('w', self._adjust)
+        self.__tracecb = self._variable.trace_add('write', self._adjust)
         self.bind('<Configure>', self._adjust)
         self.bind('<Map>', self._adjust)
 
@@ -1530,7 +1533,7 @@ def __init__(self, master=None, variable=None, from_=0, to=10, **kw):
     def destroy(self):
         """Destroy this widget and possibly its associated variable."""
         try:
-            self._variable.trace_vdelete('w', self.__tracecb)
+            self._variable.trace_remove('write', self.__tracecb)
         except AttributeError:
             pass
         else:
diff --git a/Misc/NEWS.d/next/Library/2024-06-07-13-21-11.gh-issue-120211.Rws_gf.rst b/Misc/NEWS.d/next/Library/2024-06-07-13-21-11.gh-issue-120211.Rws_gf.rst
new file mode 100644
index 00000000000000..0106f2d93318b4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-07-13-21-11.gh-issue-120211.Rws_gf.rst
@@ -0,0 +1 @@
+Fix :mod:`tkinter.ttk` with Tcl/Tk 9.0.

From eca3f7762c23b22a73a5e0b09520748c88aab4a0 Mon Sep 17 00:00:00 2001
From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com>
Date: Fri, 7 Jun 2024 14:06:24 +0100
Subject: [PATCH 17/72] gh-93691: fix too broad source locations of
 with-statement instructions (#120125)

---
 Lib/test/test_with.py                         | 44 +++++++++++++++++++
 ...4-06-05-18-29-18.gh-issue-93691.6OautB.rst |  1 +
 Python/compile.c                              |  5 +--
 3 files changed, 47 insertions(+), 3 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-05-18-29-18.gh-issue-93691.6OautB.rst

diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py
index d81902327a7e0a..e8c4ddf979e2ee 100644
--- a/Lib/test/test_with.py
+++ b/Lib/test/test_with.py
@@ -5,6 +5,7 @@
 __email__ = "mbland at acm dot org"
 
 import sys
+import traceback
 import unittest
 from collections import deque
 from contextlib import _GeneratorContextManager, contextmanager, nullcontext
@@ -749,5 +750,48 @@ def testEnterReturnsTuple(self):
             self.assertEqual(10, b1)
             self.assertEqual(20, b2)
 
+    def testExceptionLocation(self):
+        # The location of an exception raised from
+        # __init__, __enter__ or __exit__ of a context
+        # manager should be just the context manager expression,
+        # pinpointing the precise context manager in case there
+        # is more than one.
+
+        def init_raises():
+            try:
+                with self.Dummy(), self.InitRaises() as cm, self.Dummy() as d:
+                    pass
+            except Exception as e:
+                return e
+
+        def enter_raises():
+            try:
+                with self.EnterRaises(), self.Dummy() as d:
+                    pass
+            except Exception as e:
+                return e
+
+        def exit_raises():
+            try:
+                with self.ExitRaises(), self.Dummy() as d:
+                    pass
+            except Exception as e:
+                return e
+
+        for func, expected in [(init_raises, "self.InitRaises()"),
+                               (enter_raises, "self.EnterRaises()"),
+                               (exit_raises, "self.ExitRaises()"),
+                              ]:
+            with self.subTest(func):
+                exc = func()
+                f = traceback.extract_tb(exc.__traceback__)[0]
+                indent = 16
+                co = func.__code__
+                self.assertEqual(f.lineno, co.co_firstlineno + 2)
+                self.assertEqual(f.end_lineno, co.co_firstlineno + 2)
+                self.assertEqual(f.line[f.colno - indent : f.end_colno - indent],
+                                 expected)
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-05-18-29-18.gh-issue-93691.6OautB.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-18-29-18.gh-issue-93691.6OautB.rst
new file mode 100644
index 00000000000000..c06d5a276c03eb
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-18-29-18.gh-issue-93691.6OautB.rst	
@@ -0,0 +1 @@
+Fix source locations of instructions generated for with statements.
diff --git a/Python/compile.c b/Python/compile.c
index 7d74096fcdf94e..cb724154206b7e 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -5900,7 +5900,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
 
     /* Evaluate EXPR */
     VISIT(c, expr, item->context_expr);
-
+    loc = LOC(item->context_expr);
     ADDOP(c, loc, BEFORE_ASYNC_WITH);
     ADDOP_I(c, loc, GET_AWAITABLE, 1);
     ADDOP_LOAD_CONST(c, loc, Py_None);
@@ -5998,7 +5998,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos)
     /* Evaluate EXPR */
     VISIT(c, expr, item->context_expr);
     /* Will push bound __exit__ */
-    location loc = LOC(s);
+    location loc = LOC(item->context_expr);
     ADDOP(c, loc, BEFORE_WITH);
     ADDOP_JUMP(c, loc, SETUP_WITH, final);
 
@@ -6031,7 +6031,6 @@ compiler_with(struct compiler *c, stmt_ty s, int pos)
     /* For successful outcome:
      * call __exit__(None, None, None)
      */
-    loc = LOC(s);
     RETURN_IF_ERROR(compiler_call_exit_with_nones(c, loc));
     ADDOP(c, loc, POP_TOP);
     ADDOP_JUMP(c, loc, JUMP, exit);

From 225aab7f70d804174cc3a75bc04a5bb1545e5adb Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend@python.org>
Date: Fri, 7 Jun 2024 15:37:18 +0200
Subject: [PATCH 18/72] gh-110383: Improve 'old string formatting' text in
 tutorial (#120219)

---
 Doc/tutorial/inputoutput.rst | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst
index 857068a51ab843..b93a0e8cec2d38 100644
--- a/Doc/tutorial/inputoutput.rst
+++ b/Doc/tutorial/inputoutput.rst
@@ -279,9 +279,11 @@ left with zeros.  It understands about plus and minus signs::
 Old string formatting
 ---------------------
 
-The % operator (modulo) can also be used for string formatting. Given ``'string'
-% values``, instances of ``%`` in ``string`` are replaced with zero or more
-elements of ``values``. This operation is commonly known as string
+The % operator (modulo) can also be used for string formatting.
+Given ``format % values`` (where *format* is a string),
+``%`` conversion specifications in *format* are replaced with
+zero or more elements of *values*.
+This operation is commonly known as string
 interpolation. For example::
 
    >>> import math

From 9d6604222e9ef4e136ee9ccfa2d4d5ff9feee976 Mon Sep 17 00:00:00 2001
From: Pieter Eendebak <pieter.eendebak@gmail.com>
Date: Fri, 7 Jun 2024 17:42:01 +0200
Subject: [PATCH 19/72] gh-114264: Optimize performance of copy.deepcopy by
 adding a fast path for atomic types (GH-114266)

---
 Lib/copy.py                                   | 31 ++++++-------------
 ...-01-18-21-44-23.gh-issue-114264.DBKn29.rst |  1 +
 2 files changed, 11 insertions(+), 21 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2024-01-18-21-44-23.gh-issue-114264.DBKn29.rst

diff --git a/Lib/copy.py b/Lib/copy.py
index a69bc4e78c20b3..7a1907d75494d7 100644
--- a/Lib/copy.py
+++ b/Lib/copy.py
@@ -121,6 +121,11 @@ def deepcopy(x, memo=None, _nil=[]):
     See the module's __doc__ string for more info.
     """
 
+    cls = type(x)
+
+    if cls in _atomic_types:
+        return x
+
     d = id(x)
     if memo is None:
         memo = {}
@@ -129,14 +134,12 @@ def deepcopy(x, memo=None, _nil=[]):
         if y is not _nil:
             return y
 
-    cls = type(x)
-
     copier = _deepcopy_dispatch.get(cls)
     if copier is not None:
         y = copier(x, memo)
     else:
         if issubclass(cls, type):
-            y = _deepcopy_atomic(x, memo)
+            y = x # atomic copy
         else:
             copier = getattr(x, "__deepcopy__", None)
             if copier is not None:
@@ -167,26 +170,12 @@ def deepcopy(x, memo=None, _nil=[]):
         _keep_alive(x, memo) # Make sure x lives at least as long as d
     return y
 
+_atomic_types =  {types.NoneType, types.EllipsisType, types.NotImplementedType,
+          int, float, bool, complex, bytes, str, types.CodeType, type, range,
+          types.BuiltinFunctionType, types.FunctionType, weakref.ref, property}
+
 _deepcopy_dispatch = d = {}
 
-def _deepcopy_atomic(x, memo):
-    return x
-d[types.NoneType] = _deepcopy_atomic
-d[types.EllipsisType] = _deepcopy_atomic
-d[types.NotImplementedType] = _deepcopy_atomic
-d[int] = _deepcopy_atomic
-d[float] = _deepcopy_atomic
-d[bool] = _deepcopy_atomic
-d[complex] = _deepcopy_atomic
-d[bytes] = _deepcopy_atomic
-d[str] = _deepcopy_atomic
-d[types.CodeType] = _deepcopy_atomic
-d[type] = _deepcopy_atomic
-d[range] = _deepcopy_atomic
-d[types.BuiltinFunctionType] = _deepcopy_atomic
-d[types.FunctionType] = _deepcopy_atomic
-d[weakref.ref] = _deepcopy_atomic
-d[property] = _deepcopy_atomic
 
 def _deepcopy_list(x, memo, deepcopy=deepcopy):
     y = []
diff --git a/Misc/NEWS.d/next/Library/2024-01-18-21-44-23.gh-issue-114264.DBKn29.rst b/Misc/NEWS.d/next/Library/2024-01-18-21-44-23.gh-issue-114264.DBKn29.rst
new file mode 100644
index 00000000000000..069ac68b4f3a95
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-01-18-21-44-23.gh-issue-114264.DBKn29.rst
@@ -0,0 +1 @@
+Improve performance of :func:`copy.deepcopy` by adding a fast path for atomic types.

From 10fb1b8f36ab2fc3d2fe7392d5735dd19c5e2365 Mon Sep 17 00:00:00 2001
From: Nikita Sobolev <mail@sobolevn.me>
Date: Fri, 7 Jun 2024 18:48:31 +0300
Subject: [PATCH 20/72] gh-120200: Fix `inspect.iscoroutinefunction(inspect) is
 True` corner case (#120214)

---
 Lib/inspect.py                        | 6 +++---
 Lib/test/test_inspect/test_inspect.py | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/Lib/inspect.py b/Lib/inspect.py
index e6e49a4ffa673a..2b7f8bec482f8e 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -403,13 +403,13 @@ def isgeneratorfunction(obj):
     return _has_code_flag(obj, CO_GENERATOR)
 
 # A marker for markcoroutinefunction and iscoroutinefunction.
-_is_coroutine_marker = object()
+_is_coroutine_mark = object()
 
 def _has_coroutine_mark(f):
     while ismethod(f):
         f = f.__func__
     f = functools._unwrap_partial(f)
-    return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_marker
+    return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_mark
 
 def markcoroutinefunction(func):
     """
@@ -417,7 +417,7 @@ def markcoroutinefunction(func):
     """
     if hasattr(func, '__func__'):
         func = func.__func__
-    func._is_coroutine_marker = _is_coroutine_marker
+    func._is_coroutine_marker = _is_coroutine_mark
     return func
 
 def iscoroutinefunction(obj):
diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py
index 65007c16203c6d..0a4fa9343f15e0 100644
--- a/Lib/test/test_inspect/test_inspect.py
+++ b/Lib/test/test_inspect/test_inspect.py
@@ -235,6 +235,7 @@ class PMClass:
                     gen_coroutine_function_example))))
         self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmi))
         self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmc))
+        self.assertFalse(inspect.iscoroutinefunction(inspect))
         self.assertFalse(inspect.iscoroutine(gen_coro))
 
         self.assertTrue(

From a7584245661102a5768c643fbd7db8395fd3c90e Mon Sep 17 00:00:00 2001
From: Xi Ruoyao <xry111@xry111.site>
Date: Fri, 7 Jun 2024 23:51:32 +0800
Subject: [PATCH 21/72] gh-120226: Fix
 test_sendfile_close_peer_in_the_middle_of_receiving on Linux >= 6.10
 (#120227)

The worst case is that the kernel buffers 17 pages with a page size of 64k.
---
 Lib/test/test_asyncio/test_sendfile.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py
index d33ff197bbfa1d..2509d4382cdebd 100644
--- a/Lib/test/test_asyncio/test_sendfile.py
+++ b/Lib/test/test_asyncio/test_sendfile.py
@@ -93,13 +93,10 @@ async def wait_closed(self):
 
 class SendfileBase:
 
-    # 256 KiB plus small unaligned to buffer chunk
-    # Newer versions of Windows seems to have increased its internal
-    # buffer and tries to send as much of the data as it can as it
-    # has some form of buffering for this which is less than 256KiB
-    # on newer server versions and Windows 11.
-    # So DATA should be larger than 256 KiB to make this test reliable.
-    DATA = b"x" * (1024 * 256 + 1)
+    # Linux >= 6.10 seems buffering up to 17 pages of data.
+    # So DATA should be large enough to make this test reliable even with a
+    # 64 KiB page configuration.
+    DATA = b"x" * (1024 * 17 * 64 + 1)
     # Reduce socket buffer size to test on relative small data sets.
     BUF_SIZE = 4 * 1024   # 4 KiB
 

From 90b75405260467814c93738a3325645918d4ea51 Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Fri, 7 Jun 2024 17:58:21 +0200
Subject: [PATCH 22/72] gh-120155: Fix copy/paste error in
 HAVE_SUBOFFSETS_IN_LAST_DIM() (#120228)

Don't hardcode 'dest' in HAVE_SUBOFFSETS_IN_LAST_DIM() macro of
memoryobject.c, but use its 'view' parameter instead.

Fix the Coverity issue:

Error: COPY_PASTE_ERROR (CWE-398):
Python-3.12.2/Objects/memoryobject.c:273:14: original: ""dest->suboffsets + (dest->ndim - 1)"" looks like the original copy.
Python-3.12.2/Objects/memoryobject.c:274:14: copy_paste_error: ""dest"" in ""src->suboffsets + (dest->ndim - 1)"" looks like a copy-paste error.
Python-3.12.2/Objects/memoryobject.c:274:14: remediation: Should it say ""src"" instead?
#  272|       assert(dest->ndim > 0 && src->ndim > 0);
#  273|       return (!HAVE_SUBOFFSETS_IN_LAST_DIM(dest) &&
#  274|->             !HAVE_SUBOFFSETS_IN_LAST_DIM(src) &&
#  275|               dest->strides[dest->ndim-1] == dest->itemsize &&
#  276|               src->strides[src->ndim-1] == src->itemsize);
---
 Objects/memoryobject.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c
index 5caa6504272301..226bd6defdec5a 100644
--- a/Objects/memoryobject.c
+++ b/Objects/memoryobject.c
@@ -268,7 +268,7 @@ PyTypeObject _PyManagedBuffer_Type = {
 /* Assumptions: ndim >= 1. The macro tests for a corner case that should
    perhaps be explicitly forbidden in the PEP. */
 #define HAVE_SUBOFFSETS_IN_LAST_DIM(view) \
-    (view->suboffsets && view->suboffsets[dest->ndim-1] >= 0)
+    (view->suboffsets && view->suboffsets[view->ndim-1] >= 0)
 
 static inline int
 last_dim_is_contiguous(const Py_buffer *dest, const Py_buffer *src)

From 242c7498e5a889b47847fb6f0f133ce461fa7e24 Mon Sep 17 00:00:00 2001
From: Barney Gale <barney.gale@gmail.com>
Date: Fri, 7 Jun 2024 17:59:34 +0100
Subject: [PATCH 23/72] GH-116380: Move pathlib-specific code from `glob` to
 `pathlib._abc`. (#120011)

In `glob._Globber`, move pathlib-specific methods to `pathlib._abc.PathGlobber` and replace them with abstract methods. Rename `glob._Globber` to `glob._GlobberBase`. As a result, the `glob` module is no longer befouled by code that can only ever apply to pathlib.

No change of behaviour.
---
 Lib/glob.py         | 34 ++++++++++++++++++++++------------
 Lib/pathlib/_abc.py | 32 ++++++++++++++++++++++++++++++--
 2 files changed, 52 insertions(+), 14 deletions(-)

diff --git a/Lib/glob.py b/Lib/glob.py
index fbb1d35aab71fa..574e5ad51b601d 100644
--- a/Lib/glob.py
+++ b/Lib/glob.py
@@ -328,8 +328,8 @@ def _compile_pattern(pat, sep, case_sensitive, recursive=True):
     return re.compile(regex, flags=flags).match
 
 
-class _Globber:
-    """Class providing shell-style pattern matching and globbing.
+class _GlobberBase:
+    """Abstract class providing shell-style pattern matching and globbing.
     """
 
     def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False):
@@ -338,29 +338,37 @@ def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False):
         self.case_pedantic = case_pedantic
         self.recursive = recursive
 
-    # Low-level methods
+    # Abstract methods
 
-    lexists = operator.methodcaller('exists', follow_symlinks=False)
-    add_slash = operator.methodcaller('joinpath', '')
+    @staticmethod
+    def lexists(path):
+        """Implements os.path.lexists().
+        """
+        raise NotImplementedError
 
     @staticmethod
     def scandir(path):
-        """Emulates os.scandir(), which returns an object that can be used as
-        a context manager. This method is called by walk() and glob().
+        """Implements os.scandir().
+        """
+        raise NotImplementedError
+
+    @staticmethod
+    def add_slash(path):
+        """Returns a path with a trailing slash added.
         """
-        return contextlib.nullcontext(path.iterdir())
+        raise NotImplementedError
 
     @staticmethod
     def concat_path(path, text):
-        """Appends text to the given path.
+        """Implements path concatenation.
         """
-        return path.with_segments(path._raw_path + text)
+        raise NotImplementedError
 
     @staticmethod
     def parse_entry(entry):
         """Returns the path of an entry yielded from scandir().
         """
-        return entry
+        raise NotImplementedError
 
     # High-level methods
 
@@ -520,7 +528,9 @@ def select_exists(self, path, exists=False):
             yield path
 
 
-class _StringGlobber(_Globber):
+class _StringGlobber(_GlobberBase):
+    """Provides shell-style pattern matching and globbing for string paths.
+    """
     lexists = staticmethod(os.path.lexists)
     scandir = staticmethod(os.scandir)
     parse_entry = operator.attrgetter('path')
diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py
index 1a74f457c3f5a7..ecea8e88d1a2e3 100644
--- a/Lib/pathlib/_abc.py
+++ b/Lib/pathlib/_abc.py
@@ -12,8 +12,9 @@
 """
 
 import functools
+import operator
 import posixpath
-from glob import _Globber, _no_recurse_symlinks
+from glob import _GlobberBase, _no_recurse_symlinks
 from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO
 
 
@@ -84,6 +85,33 @@ def isabs(self, path):
         raise UnsupportedOperation(self._unsupported_msg('isabs()'))
 
 
+class PathGlobber(_GlobberBase):
+    """
+    Class providing shell-style globbing for path objects.
+    """
+
+    lexists = operator.methodcaller('exists', follow_symlinks=False)
+    add_slash = operator.methodcaller('joinpath', '')
+
+    @staticmethod
+    def scandir(path):
+        """Emulates os.scandir(), which returns an object that can be used as
+        a context manager. This method is called by walk() and glob().
+        """
+        import contextlib
+        return contextlib.nullcontext(path.iterdir())
+
+    @staticmethod
+    def concat_path(path, text):
+        """Appends text to the given path."""
+        return path.with_segments(path._raw_path + text)
+
+    @staticmethod
+    def parse_entry(entry):
+        """Returns the path of an entry yielded from scandir()."""
+        return entry
+
+
 class PurePathBase:
     """Base class for pure path objects.
 
@@ -104,7 +132,7 @@ class PurePathBase:
         '_resolving',
     )
     parser = ParserBase()
-    _globber = _Globber
+    _globber = PathGlobber
 
     def __init__(self, path, *paths):
         self._raw_path = self.parser.join(path, *paths) if paths else path

From e6076d1e1303c3cc14bc02baf607535af2cf1501 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Fri, 7 Jun 2024 13:44:56 -0400
Subject: [PATCH 24/72] gh-119659: Get the datetime CAPI Tests Running Again
 (gh-120180)

The tests were accidentally disabled by 2da0dc0, which didn't handle classes correctly.

I considered updating no_rerun() to support classes, but the way test_datetime.py works would have made things fairly messy.  Plus, it looks like the refleaks we had encountered before have been resolved.
---
 Lib/test/datetimetester.py   | 3 +--
 Lib/test/support/__init__.py | 1 +
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py
index 3759504b02e550..b80da5697ef865 100644
--- a/Lib/test/datetimetester.py
+++ b/Lib/test/datetimetester.py
@@ -22,7 +22,7 @@
 
 from test import support
 from test.support import is_resource_enabled, ALWAYS_EQ, LARGEST, SMALLEST
-from test.support import warnings_helper, no_rerun
+from test.support import warnings_helper
 
 import datetime as datetime_module
 from datetime import MINYEAR, MAXYEAR
@@ -6385,7 +6385,6 @@ class IranTest(ZoneInfoTest):
 
 
 @unittest.skipIf(_testcapi is None, 'need _testcapi module')
-@no_rerun("the encapsulated datetime C API does not support reloading")
 class CapiTest(unittest.TestCase):
     def setUp(self):
         # Since the C API is not present in the _Pure tests, skip all tests
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 4b320b494bb8dd..9e6100d2b89d6e 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -1197,6 +1197,7 @@ def no_rerun(reason):
     test using the 'reason' parameter.
     """
     def deco(func):
+        assert not isinstance(func, type), func
         _has_run = False
         def wrapper(self):
             nonlocal _has_run

From 4fc82b6d3b99f873179937215833e7a573ca7876 Mon Sep 17 00:00:00 2001
From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com>
Date: Fri, 7 Jun 2024 22:37:35 +0100
Subject: [PATCH 25/72] gh-120225: fix crash in compiler on empty block at end
 of exception handler (#120235)

---
 Lib/test/test_compile.py                               | 10 ++++++++++
 .../2024-06-07-16-09-04.gh-issue-120225.kuYf9t.rst     |  1 +
 Python/flowgraph.c                                     |  8 ++------
 3 files changed, 13 insertions(+), 6 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-07-16-09-04.gh-issue-120225.kuYf9t.rst

diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index ba0bcc9c1ced99..ae23aea08d99bc 100644
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -1409,6 +1409,16 @@ def f():
         for kw in ("except", "except*"):
             exec(code % kw, g, l);
 
+    def test_regression_gh_120225(self):
+        async def name_4():
+            match b'':
+                case True:
+                    pass
+                case name_5 if f'e':
+                    {name_3: name_4 async for name_2 in name_5}
+                case []:
+                    pass
+            [[]]
 
 @requires_debug_ranges()
 class TestSourcePositions(unittest.TestCase):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-07-16-09-04.gh-issue-120225.kuYf9t.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-07-16-09-04.gh-issue-120225.kuYf9t.rst
new file mode 100644
index 00000000000000..d00b9aaa8192e3
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-07-16-09-04.gh-issue-120225.kuYf9t.rst	
@@ -0,0 +1 @@
+Fix crash in compiler on empty block at end of exception handler.
diff --git a/Python/flowgraph.c b/Python/flowgraph.c
index 17617e119fdaa4..aed694aee91f91 100644
--- a/Python/flowgraph.c
+++ b/Python/flowgraph.c
@@ -2304,15 +2304,11 @@ push_cold_blocks_to_end(cfg_builder *g) {
             if (!IS_LABEL(b->b_next->b_label)) {
                 b->b_next->b_label.id = next_lbl++;
             }
-            cfg_instr *prev_instr = basicblock_last_instr(b);
-            // b cannot be empty because at the end of an exception handler
-            // there is always a POP_EXCEPT + RERAISE/RETURN
-            assert(prev_instr);
-
             basicblock_addop(explicit_jump, JUMP_NO_INTERRUPT, b->b_next->b_label.id,
-                             prev_instr->i_loc);
+                             NO_LOCATION);
             explicit_jump->b_cold = 1;
             explicit_jump->b_next = b->b_next;
+            explicit_jump->b_predecessors = 1;
             b->b_next = explicit_jump;
 
             /* set target */

From 95f4db88d5ab7d900f05d0418b2a2e77bf9ff126 Mon Sep 17 00:00:00 2001
From: Nikita Sobolev <mail@sobolevn.me>
Date: Sat, 8 Jun 2024 10:51:09 +0300
Subject: [PATCH 26/72] gh-120242: Fix handling of `[setUp,tearDown]Class` in
 `test_datetime` (#120243)

---
 Lib/test/test_datetime.py | 36 +++++++++++++++++++++---------------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py
index 3859733a4fe65b..005187f13e665f 100644
--- a/Lib/test/test_datetime.py
+++ b/Lib/test/test_datetime.py
@@ -1,5 +1,6 @@
 import unittest
 import sys
+import functools
 
 from test.support.import_helper import import_fresh_module
 
@@ -39,21 +40,26 @@ def load_tests(loader, tests, pattern):
         for cls in test_classes:
             cls.__name__ += suffix
             cls.__qualname__ += suffix
-            @classmethod
-            def setUpClass(cls_, module=module):
-                cls_._save_sys_modules = sys.modules.copy()
-                sys.modules[TESTS] = module
-                sys.modules['datetime'] = module.datetime_module
-                if hasattr(module, '_pydatetime'):
-                    sys.modules['_pydatetime'] = module._pydatetime
-                sys.modules['_strptime'] = module._strptime
-            @classmethod
-            def tearDownClass(cls_):
-                sys.modules.clear()
-                sys.modules.update(cls_._save_sys_modules)
-            cls.setUpClass = setUpClass
-            cls.tearDownClass = tearDownClass
-            tests.addTests(loader.loadTestsFromTestCase(cls))
+
+            @functools.wraps(cls, updated=())
+            class Wrapper(cls):
+                @classmethod
+                def setUpClass(cls_, module=module):
+                    cls_._save_sys_modules = sys.modules.copy()
+                    sys.modules[TESTS] = module
+                    sys.modules['datetime'] = module.datetime_module
+                    if hasattr(module, '_pydatetime'):
+                        sys.modules['_pydatetime'] = module._pydatetime
+                    sys.modules['_strptime'] = module._strptime
+                    super().setUpClass()
+
+                @classmethod
+                def tearDownClass(cls_):
+                    super().tearDownClass()
+                    sys.modules.clear()
+                    sys.modules.update(cls_._save_sys_modules)
+
+            tests.addTests(loader.loadTestsFromTestCase(Wrapper))
     return tests
 
 

From 2080425154d235b4b7dcc9a8a2f58e71769125ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Enrico=20Tr=C3=B6ger?= <enrico.troeger@uvena.de>
Date: Sat, 8 Jun 2024 11:19:13 +0200
Subject: [PATCH 27/72] bpo-37755: Use configured output in pydoc instead of
 pager (GH-15105)

If the Helper() class was initialized with an output, the topics, keywords
and symbols help still use the pager instead of the output.
Change the behavior so  the output is used if available while keeping the
previous behavior if no output was configured.
---
 Lib/pydoc.py                                  |   8 +-
 Lib/test/test_pydoc/test_pydoc.py             | 125 +++++++++++++++---
 ...4-06-02-13-35-11.gh-issue-81936.ETeW9x.rst |   3 +
 3 files changed, 116 insertions(+), 20 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst

diff --git a/Lib/pydoc.py b/Lib/pydoc.py
index 2ba597d01f245e..d7579c1cc3dcd1 100644
--- a/Lib/pydoc.py
+++ b/Lib/pydoc.py
@@ -2034,7 +2034,7 @@ def help(self, request, is_cli=False):
             elif request in self.symbols: self.showsymbol(request)
             elif request in ['True', 'False', 'None']:
                 # special case these keywords since they are objects too
-                doc(eval(request), 'Help on %s:', is_cli=is_cli)
+                doc(eval(request), 'Help on %s:', output=self._output, is_cli=is_cli)
             elif request in self.keywords: self.showtopic(request)
             elif request in self.topics: self.showtopic(request)
             elif request: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli)
@@ -2127,7 +2127,11 @@ def showtopic(self, topic, more_xrefs=''):
             text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n'
             wrapped_text = textwrap.wrap(text, 72)
             doc += '\n%s\n' % '\n'.join(wrapped_text)
-        pager(doc, f'Help on {topic!s}')
+
+        if self._output is None:
+            pager(doc, f'Help on {topic!s}')
+        else:
+            self.output.write(doc)
 
     def _gettopic(self, topic, more_xrefs=''):
         """Return unbuffered tuple of (topic, xrefs).
diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py
index 436fdb38756ddd..57e5b8e8abddfa 100644
--- a/Lib/test/test_pydoc/test_pydoc.py
+++ b/Lib/test/test_pydoc/test_pydoc.py
@@ -17,6 +17,7 @@
 import types
 import typing
 import unittest
+import unittest.mock
 import urllib.parse
 import xml.etree
 import xml.etree.ElementTree
@@ -658,16 +659,13 @@ def test_fail_help_output_redirect(self):
 
     @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
                      'trace function introduces __locals__ unexpectedly')
+    @unittest.mock.patch('pydoc.pager')
     @requires_docstrings
-    def test_help_output_redirect(self):
+    def test_help_output_redirect(self, pager_mock):
         # issue 940286, if output is set in Helper, then all output from
         # Helper.help should be redirected
-        getpager_old = pydoc.getpager
-        getpager_new = lambda: (lambda x: x)
         self.maxDiff = None
 
-        buf = StringIO()
-        helper = pydoc.Helper(output=buf)
         unused, doc_loc = get_pydoc_text(pydoc_mod)
         module = "test.test_pydoc.pydoc_mod"
         help_header = """
@@ -677,21 +675,112 @@ def test_help_output_redirect(self):
         help_header = textwrap.dedent(help_header)
         expected_help_pattern = help_header + expected_text_pattern
 
-        pydoc.getpager = getpager_new
-        try:
+        with captured_output('stdout') as output, \
+             captured_output('stderr') as err, \
+             StringIO() as buf:
+            helper = pydoc.Helper(output=buf)
+            helper.help(module)
+            result = buf.getvalue().strip()
+            expected_text = expected_help_pattern % (
+                            (doc_loc,) +
+                            expected_text_data_docstrings +
+                            (inspect.getabsfile(pydoc_mod),))
+            self.assertEqual('', output.getvalue())
+            self.assertEqual('', err.getvalue())
+            self.assertEqual(expected_text, result)
+
+        pager_mock.assert_not_called()
+
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                     'trace function introduces __locals__ unexpectedly')
+    @requires_docstrings
+    @unittest.mock.patch('pydoc.pager')
+    def test_help_output_redirect_various_requests(self, pager_mock):
+        # issue 940286, if output is set in Helper, then all output from
+        # Helper.help should be redirected
+
+        def run_pydoc_for_request(request, expected_text_part):
+            """Helper function to run pydoc with its output redirected"""
             with captured_output('stdout') as output, \
-                 captured_output('stderr') as err:
-                helper.help(module)
+                 captured_output('stderr') as err, \
+                 StringIO() as buf:
+                helper = pydoc.Helper(output=buf)
+                helper.help(request)
                 result = buf.getvalue().strip()
-                expected_text = expected_help_pattern % (
-                                (doc_loc,) +
-                                expected_text_data_docstrings +
-                                (inspect.getabsfile(pydoc_mod),))
-                self.assertEqual('', output.getvalue())
-                self.assertEqual('', err.getvalue())
-                self.assertEqual(expected_text, result)
-        finally:
-            pydoc.getpager = getpager_old
+                self.assertEqual('', output.getvalue(), msg=f'failed on request "{request}"')
+                self.assertEqual('', err.getvalue(), msg=f'failed on request "{request}"')
+                self.assertIn(expected_text_part, result, msg=f'failed on request "{request}"')
+                pager_mock.assert_not_called()
+
+        self.maxDiff = None
+
+        # test for "keywords"
+        run_pydoc_for_request('keywords', 'Here is a list of the Python keywords.')
+        # test for "symbols"
+        run_pydoc_for_request('symbols', 'Here is a list of the punctuation symbols')
+        # test for "topics"
+        run_pydoc_for_request('topics', 'Here is a list of available topics.')
+        # test for "modules" skipped, see test_modules()
+        # test for symbol "%"
+        run_pydoc_for_request('%', 'The power operator')
+        # test for special True, False, None keywords
+        run_pydoc_for_request('True', 'class bool(int)')
+        run_pydoc_for_request('False', 'class bool(int)')
+        run_pydoc_for_request('None', 'class NoneType(object)')
+        # test for keyword "assert"
+        run_pydoc_for_request('assert', 'The "assert" statement')
+        # test for topic "TYPES"
+        run_pydoc_for_request('TYPES', 'The standard type hierarchy')
+        # test for "pydoc.Helper.help"
+        run_pydoc_for_request('pydoc.Helper.help', 'Help on function help in pydoc.Helper:')
+        # test for pydoc.Helper.help
+        run_pydoc_for_request(pydoc.Helper.help, 'Help on function help in module pydoc:')
+        # test for pydoc.Helper() instance skipped because it is always meant to be interactive
+
+    def test_showtopic(self):
+        with captured_stdout() as showtopic_io:
+            helper = pydoc.Helper()
+            helper.showtopic('with')
+        helptext = showtopic_io.getvalue()
+        self.assertIn('The "with" statement', helptext)
+
+    def test_fail_showtopic(self):
+        with captured_stdout() as showtopic_io:
+            helper = pydoc.Helper()
+            helper.showtopic('abd')
+            expected = "no documentation found for 'abd'"
+            self.assertEqual(expected, showtopic_io.getvalue().strip())
+
+    @unittest.mock.patch('pydoc.pager')
+    def test_fail_showtopic_output_redirect(self, pager_mock):
+        with StringIO() as buf:
+            helper = pydoc.Helper(output=buf)
+            helper.showtopic("abd")
+            expected = "no documentation found for 'abd'"
+            self.assertEqual(expected, buf.getvalue().strip())
+
+        pager_mock.assert_not_called()
+
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                     'trace function introduces __locals__ unexpectedly')
+    @requires_docstrings
+    @unittest.mock.patch('pydoc.pager')
+    def test_showtopic_output_redirect(self, pager_mock):
+        # issue 940286, if output is set in Helper, then all output from
+        # Helper.showtopic should be redirected
+        self.maxDiff = None
+
+        with captured_output('stdout') as output, \
+             captured_output('stderr') as err, \
+             StringIO() as buf:
+            helper = pydoc.Helper(output=buf)
+            helper.showtopic('with')
+            result = buf.getvalue().strip()
+            self.assertEqual('', output.getvalue())
+            self.assertEqual('', err.getvalue())
+            self.assertIn('The "with" statement', result)
+
+        pager_mock.assert_not_called()
 
     def test_lambda_with_return_annotation(self):
         func = lambda a, b, c: 1
diff --git a/Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst b/Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst
new file mode 100644
index 00000000000000..d53cc73e728d54
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst
@@ -0,0 +1,3 @@
+:meth:`!help` and :meth:`!showtopic` methods now respect a
+configured *output* argument to :class:`!pydoc.Helper` and not use the
+pager in such cases. Patch by Enrico Tröger.

From 55402d3232ca400ebafe4fe3bd70f252304ebe07 Mon Sep 17 00:00:00 2001
From: Saul Shanabrook <s.shanabrook@gmail.com>
Date: Sat, 8 Jun 2024 05:41:45 -0400
Subject: [PATCH 28/72] gh-119258: Eliminate Type Guards in Tier 2 Optimizer
 with Watcher (GH-119365)

Co-authored-by: parmeggiani <parmeggiani@spaziodati.eu>
Co-authored-by: dpdani <git@danieleparmeggiani.me>
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
Co-authored-by: Brandt Bucher <brandtbucher@microsoft.com>
Co-authored-by: Ken Jin <kenjin@python.org>
---
 Include/internal/pycore_optimizer.h           |   9 +-
 Include/internal/pycore_typeobject.h          |  11 ++
 Lib/test/test_capi/test_opt.py                | 147 ++++++++++++++++++
 Lib/test/test_capi/test_watchers.py           |   6 +-
 Lib/test/test_type_cache.py                   |   3 +-
 ...-05-23-20-17-37.gh-issue-119258.wZFIpt.rst |   3 +
 Modules/_testcapimodule.c                     |  17 --
 Modules/_testinternalcapi.c                   |  19 +++
 Objects/typeobject.c                          |  78 ++++++++--
 Python/optimizer_analysis.c                   |  16 +-
 Python/optimizer_bytecodes.c                  |  37 ++++-
 Python/optimizer_cases.c.h                    |  33 +++-
 Python/optimizer_symbols.c                    |  46 +++++-
 13 files changed, 366 insertions(+), 59 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst

diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h
index 76123987ac99f5..fd7833fd231299 100644
--- a/Include/internal/pycore_optimizer.h
+++ b/Include/internal/pycore_optimizer.h
@@ -33,6 +33,7 @@ struct _Py_UopsSymbol {
     int flags;  // 0 bits: Top; 2 or more bits: Bottom
     PyTypeObject *typ;  // Borrowed reference
     PyObject *const_val;  // Owned reference (!)
+    unsigned int type_version; // currently stores type version
 };
 
 #define UOP_FORMAT_TARGET 0
@@ -123,9 +124,11 @@ extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *con
 extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx);
 extern bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym);
 extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ);
+extern bool _Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version);
 extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym);
 extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym);
 extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ);
+extern bool _Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version);
 extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val);
 extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym);
 extern int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym);
@@ -138,9 +141,9 @@ extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx);
 extern _Py_UOpsAbstractFrame *_Py_uop_frame_new(
     _Py_UOpsContext *ctx,
     PyCodeObject *co,
-    _Py_UopsSymbol **localsplus_start,
-    int n_locals_already_filled,
-    int curr_stackentries);
+    int curr_stackentries,
+    _Py_UopsSymbol **args,
+    int arg_len);
 extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx);
 
 PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored);
diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h
index 8664ae0e44533f..bc295b1b066bd1 100644
--- a/Include/internal/pycore_typeobject.h
+++ b/Include/internal/pycore_typeobject.h
@@ -63,6 +63,8 @@ typedef struct {
     PyObject *tp_weaklist;
 } managed_static_type_state;
 
+#define TYPE_VERSION_CACHE_SIZE (1<<12)  /* Must be a power of 2 */
+
 struct types_state {
     /* Used to set PyTypeObject.tp_version_tag.
        It starts at _Py_MAX_GLOBAL_TYPE_VERSION_TAG + 1,
@@ -118,6 +120,12 @@ struct types_state {
         managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_EXT_TYPES];
     } for_extensions;
     PyMutex mutex;
+
+    // Borrowed references to type objects whose
+    // tp_version_tag % TYPE_VERSION_CACHE_SIZE
+    // once was equal to the index in the table.
+    // They are cleared when the type object is deallocated.
+    PyTypeObject *type_version_cache[TYPE_VERSION_CACHE_SIZE];
 };
 
 
@@ -230,6 +238,9 @@ extern void _PyType_SetFlags(PyTypeObject *self, unsigned long mask,
 extern void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask,
                                       unsigned long flags);
 
+extern unsigned int _PyType_GetVersionForCurrentState(PyTypeObject *tp);
+PyAPI_FUNC(void) _PyType_SetVersion(PyTypeObject *tp, unsigned int version);
+PyTypeObject *_PyType_LookupByVersion(unsigned int version);
 
 #ifdef __cplusplus
 }
diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py
index 0491ff9b84d486..fc6d8b0a3f01d2 100644
--- a/Lib/test/test_capi/test_opt.py
+++ b/Lib/test/test_capi/test_opt.py
@@ -1333,6 +1333,153 @@ def test_modified_local_is_seen_by_optimized_code(self):
         self.assertIs(type(s), float)
         self.assertEqual(s, 1024.0)
 
+    def test_guard_type_version_removed(self):
+        def thing(a):
+            x = 0
+            for _ in range(100):
+                x += a.attr
+                x += a.attr
+            return x
+
+        class Foo:
+            attr = 1
+
+        res, ex = self._run_with_optimizer(thing, Foo())
+        opnames = list(iter_opnames(ex))
+        self.assertIsNotNone(ex)
+        self.assertEqual(res, 200)
+        guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION")
+        self.assertEqual(guard_type_version_count, 1)
+
+    def test_guard_type_version_removed_inlined(self):
+        """
+        Verify that the guard type version if we have an inlined function
+        """
+
+        def fn():
+            pass
+
+        def thing(a):
+            x = 0
+            for _ in range(100):
+                x += a.attr
+                fn()
+                x += a.attr
+            return x
+
+        class Foo:
+            attr = 1
+
+        res, ex = self._run_with_optimizer(thing, Foo())
+        opnames = list(iter_opnames(ex))
+        self.assertIsNotNone(ex)
+        self.assertEqual(res, 200)
+        guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION")
+        self.assertEqual(guard_type_version_count, 1)
+
+    def test_guard_type_version_not_removed(self):
+        """
+        Verify that the guard type version is not removed if we modify the class
+        """
+
+        def thing(a):
+            x = 0
+            for i in range(100):
+                x += a.attr
+                # for the first 90 iterations we set the attribute on this dummy function which shouldn't
+                # trigger the type watcher
+                # then after 90  it should trigger it and stop optimizing
+                # Note that the code needs to be in this weird form so it's optimized inline without any control flow
+                setattr((Foo, Bar)[i < 90], "attr", 2)
+                x += a.attr
+            return x
+
+        class Foo:
+            attr = 1
+
+        class Bar:
+            pass
+
+        res, ex = self._run_with_optimizer(thing, Foo())
+        opnames = list(iter_opnames(ex))
+
+        self.assertIsNotNone(ex)
+        self.assertEqual(res, 219)
+        guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION")
+        self.assertEqual(guard_type_version_count, 2)
+
+
+    @unittest.expectedFailure
+    def test_guard_type_version_not_removed_escaping(self):
+        """
+        Verify that the guard type version is not removed if have an escaping function
+        """
+
+        def thing(a):
+            x = 0
+            for i in range(100):
+                x += a.attr
+                # eval should be escaping and so should cause optimization to stop and preserve both type versions
+                eval("None")
+                x += a.attr
+            return x
+
+        class Foo:
+            attr = 1
+        res, ex = self._run_with_optimizer(thing, Foo())
+        opnames = list(iter_opnames(ex))
+        self.assertIsNotNone(ex)
+        self.assertEqual(res, 200)
+        guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION")
+        # Note: This will actually be 1 for noe
+        # https://github.com/python/cpython/pull/119365#discussion_r1626220129
+        self.assertEqual(guard_type_version_count, 2)
+
+
+    def test_guard_type_version_executor_invalidated(self):
+        """
+        Verify that the executor is invalided on a type change.
+        """
+
+        def thing(a):
+            x = 0
+            for i in range(100):
+                x += a.attr
+                x += a.attr
+            return x
+
+        class Foo:
+            attr = 1
+
+        res, ex = self._run_with_optimizer(thing, Foo())
+        self.assertEqual(res, 200)
+        self.assertIsNotNone(ex)
+        self.assertEqual(list(iter_opnames(ex)).count("_GUARD_TYPE_VERSION"), 1)
+        self.assertTrue(ex.is_valid())
+        Foo.attr = 0
+        self.assertFalse(ex.is_valid())
+
+    def test_type_version_doesnt_segfault(self):
+        """
+        Tests that setting a type version doesn't cause a segfault when later looking at the stack.
+        """
+
+        # Minimized from mdp.py benchmark
+
+        class A:
+            def __init__(self):
+                self.attr = {}
+
+            def method(self, arg):
+                self.attr[arg] = None
+
+        def fn(a):
+            for _ in range(100):
+                (_ for _ in [])
+                (_ for _ in [a.method(None)])
+
+        fn(A())
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py
index 90665a7561b316..709b5e1c4b716a 100644
--- a/Lib/test/test_capi/test_watchers.py
+++ b/Lib/test/test_capi/test_watchers.py
@@ -282,8 +282,10 @@ class C: pass
             self.watch(wid, C)
             with catch_unraisable_exception() as cm:
                 C.foo = "bar"
-                self.assertEqual(cm.unraisable.err_msg,
-                    f"Exception ignored in type watcher callback #0 for {C!r}")
+                self.assertEqual(
+                    cm.unraisable.err_msg,
+                    f"Exception ignored in type watcher callback #1 for {C!r}",
+                )
                 self.assertIs(cm.unraisable.object, None)
                 self.assertEqual(str(cm.unraisable.exc_value), "boom!")
             self.assert_events([])
diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py
index e90e315c808361..edaf076707ad8b 100644
--- a/Lib/test/test_type_cache.py
+++ b/Lib/test/test_type_cache.py
@@ -10,8 +10,9 @@
 
 # Skip this test if the _testcapi module isn't available.
 _testcapi = import_helper.import_module("_testcapi")
+_testinternalcapi = import_helper.import_module("_testinternalcapi")
 type_get_version = _testcapi.type_get_version
-type_assign_specific_version_unsafe = _testcapi.type_assign_specific_version_unsafe
+type_assign_specific_version_unsafe = _testinternalcapi.type_assign_specific_version_unsafe
 type_assign_version = _testcapi.type_assign_version
 type_modified = _testcapi.type_modified
 
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst
new file mode 100644
index 00000000000000..68f1ec1efa5751
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst	
@@ -0,0 +1,3 @@
+Eliminate type version guards in the tier two interpreter.
+
+Note that setting the ``tp_version_tag`` manually (which has never been supported) may result in crashes.
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index b58c17260626c2..b139b46c826a3f 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -2403,21 +2403,6 @@ type_modified(PyObject *self, PyObject *type)
     Py_RETURN_NONE;
 }
 
-// Circumvents standard version assignment machinery - use with caution and only on
-// short-lived heap types
-static PyObject *
-type_assign_specific_version_unsafe(PyObject *self, PyObject *args)
-{
-    PyTypeObject *type;
-    unsigned int version;
-    if (!PyArg_ParseTuple(args, "Oi:type_assign_specific_version_unsafe", &type, &version)) {
-        return NULL;
-    }
-    assert(!PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE));
-    type->tp_version_tag = version;
-    type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG;
-    Py_RETURN_NONE;
-}
 
 static PyObject *
 type_assign_version(PyObject *self, PyObject *type)
@@ -3427,8 +3412,6 @@ static PyMethodDef TestMethods[] = {
     {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS},
     {"type_get_version", type_get_version, METH_O, PyDoc_STR("type->tp_version_tag")},
     {"type_modified", type_modified, METH_O, PyDoc_STR("PyType_Modified")},
-    {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS,
-     PyDoc_STR("forcefully assign type->tp_version_tag")},
     {"type_assign_version", type_assign_version, METH_O, PyDoc_STR("PyUnstable_Type_AssignVersionTag")},
     {"type_get_tp_bases", type_get_tp_bases, METH_O},
     {"type_get_tp_mro", type_get_tp_mro, METH_O},
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index 6d4a00c06ca9de..139a0509795de9 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -2002,6 +2002,22 @@ has_inline_values(PyObject *self, PyObject *obj)
 }
 
 
+// Circumvents standard version assignment machinery - use with caution and only on
+// short-lived heap types
+static PyObject *
+type_assign_specific_version_unsafe(PyObject *self, PyObject *args)
+{
+    PyTypeObject *type;
+    unsigned int version;
+    if (!PyArg_ParseTuple(args, "Oi:type_assign_specific_version_unsafe", &type, &version)) {
+        return NULL;
+    }
+    assert(!PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE));
+    _PyType_SetVersion(type, version);
+    type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG;
+    Py_RETURN_NONE;
+}
+
 /*[clinic input]
 gh_119213_getargs
 
@@ -2102,6 +2118,9 @@ static PyMethodDef module_functions[] = {
     {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS},
     {"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS},
     {"has_inline_values", has_inline_values, METH_O},
+    {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS,
+     PyDoc_STR("forcefully assign type->tp_version_tag")},
+
 #ifdef Py_GIL_DISABLED
     {"py_thread_id", get_py_thread_id, METH_NOARGS},
 #endif
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 880ac6b9c009fe..cd16bebd1e1cb8 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -853,7 +853,8 @@ PyType_AddWatcher(PyType_WatchCallback callback)
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
 
-    for (int i = 0; i < TYPE_MAX_WATCHERS; i++) {
+    // start at 1, 0 is reserved for cpython optimizer
+    for (int i = 1; i < TYPE_MAX_WATCHERS; i++) {
         if (!interp->type_watchers[i]) {
             interp->type_watchers[i] = callback;
             return i;
@@ -960,7 +961,7 @@ type_modification_starting_unlocked(PyTypeObject *type)
     }
 
     /* 0 is not a valid version tag */
-    _Py_atomic_store_uint32_release(&type->tp_version_tag, 0);
+    _PyType_SetVersion(type, 0);
 }
 
 #endif
@@ -1024,7 +1025,7 @@ type_modified_unlocked(PyTypeObject *type)
     }
 
     type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG;
-    FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, 0); /* 0 is not a valid version tag */
+    _PyType_SetVersion(type, 0); /* 0 is not a valid version tag */
     if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
         // This field *must* be invalidated if the type is modified (see the
         // comment on struct _specialization_cache):
@@ -1101,7 +1102,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) {
  clear:
     assert(!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN));
     type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG;
-    FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, 0); /* 0 is not a valid version tag */
+    _PyType_SetVersion(type, 0); /* 0 is not a valid version tag */
     if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
         // This field *must* be invalidated if the type is modified (see the
         // comment on struct _specialization_cache):
@@ -1109,6 +1110,64 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) {
     }
 }
 
+/*
+The Tier 2 interpreter requires looking up the type object by the type version, so it can install
+watchers to understand when they change.
+
+So we add a global cache from type version to borrowed references of type objects.
+
+This is similar to func_version_cache.
+*/
+
+void
+_PyType_SetVersion(PyTypeObject *tp, unsigned int version)
+{
+#ifndef Py_GIL_DISABLED
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    // lookup the old version and set to null
+    if (tp->tp_version_tag != 0) {
+        PyTypeObject **slot =
+            interp->types.type_version_cache
+            + (tp->tp_version_tag % TYPE_VERSION_CACHE_SIZE);
+        *slot = NULL;
+    }
+#endif
+    FT_ATOMIC_STORE_UINT32_RELAXED(tp->tp_version_tag, version);
+#ifndef Py_GIL_DISABLED
+    if (version != 0) {
+        PyTypeObject **slot =
+            interp->types.type_version_cache
+            + (version % TYPE_VERSION_CACHE_SIZE);
+        *slot = tp;
+    }
+#endif
+}
+
+PyTypeObject *
+_PyType_LookupByVersion(unsigned int version)
+{
+#ifdef Py_GIL_DISABLED
+    return NULL;
+#else
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    PyTypeObject **slot =
+        interp->types.type_version_cache
+        + (version % TYPE_VERSION_CACHE_SIZE);
+    if (*slot && (*slot)->tp_version_tag == version) {
+        return *slot;
+    }
+    return NULL;
+#endif
+}
+
+unsigned int
+_PyType_GetVersionForCurrentState(PyTypeObject *tp)
+{
+    return tp->tp_version_tag;
+}
+
+
+
 #define MAX_VERSIONS_PER_CLASS 1000
 
 static int
@@ -1137,8 +1196,7 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type)
             /* We have run out of version numbers */
             return 0;
         }
-        FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag,
-                                       NEXT_GLOBAL_VERSION_TAG++);
+        _PyType_SetVersion(type, NEXT_GLOBAL_VERSION_TAG++);
         assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG);
     }
     else {
@@ -1147,8 +1205,7 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type)
             /* We have run out of version numbers */
             return 0;
         }
-        FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag,
-                                       NEXT_VERSION_TAG(interp)++);
+        _PyType_SetVersion(type, NEXT_VERSION_TAG(interp)++);
         assert (type->tp_version_tag != 0);
     }
 
@@ -5768,7 +5825,7 @@ fini_static_type(PyInterpreterState *interp, PyTypeObject *type,
     if (final) {
         type->tp_flags &= ~Py_TPFLAGS_READY;
         type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG;
-        type->tp_version_tag = 0;
+        _PyType_SetVersion(type, 0);
     }
 
     _PyStaticType_ClearWeakRefs(interp, type);
@@ -5798,7 +5855,6 @@ type_dealloc(PyObject *self)
     _PyObject_ASSERT((PyObject *)type, type->tp_flags & Py_TPFLAGS_HEAPTYPE);
 
     _PyObject_GC_UNTRACK(type);
-
     type_dealloc_common(type);
 
     // PyObject_ClearWeakRefs() raises an exception if Py_REFCNT() != 0
@@ -8367,7 +8423,7 @@ init_static_type(PyInterpreterState *interp, PyTypeObject *self,
         self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE;
 
         assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG);
-        self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++;
+        _PyType_SetVersion(self, NEXT_GLOBAL_VERSION_TAG++);
         self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG;
     }
     else {
diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c
index e5d3793bd4d204..75d1d9f6b2a794 100644
--- a/Python/optimizer_analysis.c
+++ b/Python/optimizer_analysis.c
@@ -79,6 +79,7 @@ increment_mutations(PyObject* dict) {
  * so we don't need to check that they haven't been used */
 #define BUILTINS_WATCHER_ID 0
 #define GLOBALS_WATCHER_ID  1
+#define TYPE_WATCHER_ID  0
 
 static int
 globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict,
@@ -92,6 +93,14 @@ globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict,
     return 0;
 }
 
+static int
+type_watcher_callback(PyTypeObject* type)
+{
+    _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), type, 1);
+    PyType_Unwatch(TYPE_WATCHER_ID, (PyObject *)type);
+    return 0;
+}
+
 static PyObject *
 convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj)
 {
@@ -167,6 +176,9 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
     if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) {
         interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback;
     }
+    if (interp->type_watchers[TYPE_WATCHER_ID] == NULL) {
+        interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback;
+    }
     for (int pc = 0; pc < buffer_size; pc++) {
         _PyUOpInstruction *inst = &buffer[pc];
         int opcode = inst->opcode;
@@ -310,9 +322,11 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
 #define sym_has_type _Py_uop_sym_has_type
 #define sym_get_type _Py_uop_sym_get_type
 #define sym_matches_type _Py_uop_sym_matches_type
+#define sym_matches_type_version _Py_uop_sym_matches_type_version
 #define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM)
 #define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM)
 #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE)
+#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION)
 #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST)
 #define sym_is_bottom _Py_uop_sym_is_bottom
 #define sym_truthiness _Py_uop_sym_truthiness
@@ -395,7 +409,7 @@ optimize_uops(
     _PyUOpInstruction *corresponding_check_stack = NULL;
 
     _Py_uop_abstractcontext_init(ctx);
-    _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, ctx->n_consumed, 0, curr_stacklen);
+    _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, curr_stacklen, NULL, 0);
     if (frame == NULL) {
         return -1;
     }
diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c
index a2cb4c0b2c5192..e6fb85a90603eb 100644
--- a/Python/optimizer_bytecodes.c
+++ b/Python/optimizer_bytecodes.c
@@ -21,11 +21,13 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame;
 #define sym_new_const _Py_uop_sym_new_const
 #define sym_new_null _Py_uop_sym_new_null
 #define sym_matches_type _Py_uop_sym_matches_type
+#define sym_matches_type_version _Py_uop_sym_matches_type_version
 #define sym_get_type _Py_uop_sym_get_type
 #define sym_has_type _Py_uop_sym_has_type
 #define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM)
 #define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM)
 #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE)
+#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION)
 #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST)
 #define sym_is_bottom _Py_uop_sym_is_bottom
 #define frame_new _Py_uop_frame_new
@@ -113,6 +115,29 @@ dummy_func(void) {
         sym_set_type(right, &PyLong_Type);
     }
 
+    op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) {
+        assert(type_version);
+        if (sym_matches_type_version(owner, type_version)) {
+            REPLACE_OP(this_instr, _NOP, 0, 0);
+        } else {
+            // add watcher so that whenever the type changes we invalidate this
+            PyTypeObject *type = _PyType_LookupByVersion(type_version);
+            // if the type is null, it was not found in the cache (there was a conflict)
+            // with the key, in which case we can't trust the version
+            if (type) {
+                // if the type version was set properly, then add a watcher
+                // if it wasn't this means that the type version was previously set to something else
+                // and we set the owner to bottom, so we don't need to add a watcher because we must have
+                // already added one earlier.
+                if (sym_set_type_version(owner, type_version)) {
+                    PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type);
+                    _Py_BloomFilter_Add(dependencies, type);
+                }
+            }
+
+        }
+    }
+
     op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) {
         if (sym_matches_type(left, &PyFloat_Type)) {
             if (sym_matches_type(right, &PyFloat_Type)) {
@@ -563,16 +588,12 @@ dummy_func(void) {
             argcount++;
         }
 
-        _Py_UopsSymbol **localsplus_start = ctx->n_consumed;
-        int n_locals_already_filled = 0;
-        // Can determine statically, so we interleave the new locals
-        // and make the current stack the new locals.
-        // This also sets up for true call inlining.
         if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) {
-            localsplus_start = args;
-            n_locals_already_filled = argcount;
+            new_frame = frame_new(ctx, co, 0, args, argcount);
+        } else {
+            new_frame = frame_new(ctx, co, 0, NULL, 0);
+
         }
-        new_frame = frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0);
     }
 
     op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) {
diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h
index b3787345ec6714..18f3ca4cb73e5a 100644
--- a/Python/optimizer_cases.c.h
+++ b/Python/optimizer_cases.c.h
@@ -930,6 +930,28 @@
         }
 
         case _GUARD_TYPE_VERSION: {
+            _Py_UopsSymbol *owner;
+            owner = stack_pointer[-1];
+            uint32_t type_version = (uint32_t)this_instr->operand;
+            assert(type_version);
+            if (sym_matches_type_version(owner, type_version)) {
+                REPLACE_OP(this_instr, _NOP, 0, 0);
+            } else {
+                // add watcher so that whenever the type changes we invalidate this
+                PyTypeObject *type = _PyType_LookupByVersion(type_version);
+                // if the type is null, it was not found in the cache (there was a conflict)
+                // with the key, in which case we can't trust the version
+                if (type) {
+                    // if the type version was set properly, then add a watcher
+                    // if it wasn't this means that the type version was previously set to something else
+                    // and we set the owner to bottom, so we don't need to add a watcher because we must have
+                    // already added one earlier.
+                    if (sym_set_type_version(owner, type_version)) {
+                        PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type);
+                        _Py_BloomFilter_Add(dependencies, type);
+                    }
+                }
+            }
             break;
         }
 
@@ -1583,16 +1605,11 @@
                 args--;
                 argcount++;
             }
-            _Py_UopsSymbol **localsplus_start = ctx->n_consumed;
-            int n_locals_already_filled = 0;
-            // Can determine statically, so we interleave the new locals
-            // and make the current stack the new locals.
-            // This also sets up for true call inlining.
             if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) {
-                localsplus_start = args;
-                n_locals_already_filled = argcount;
+                new_frame = frame_new(ctx, co, 0, args, argcount);
+            } else {
+                new_frame = frame_new(ctx, co, 0, NULL, 0);
             }
-            new_frame = frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0);
             stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame;
             stack_pointer += -1 - oparg;
             break;
diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c
index e546eef306eeca..f3d4078bf1a890 100644
--- a/Python/optimizer_symbols.c
+++ b/Python/optimizer_symbols.c
@@ -52,7 +52,8 @@ static inline int get_lltrace(void) {
 static _Py_UopsSymbol NO_SPACE_SYMBOL = {
     .flags = IS_NULL | NOT_NULL | NO_SPACE,
     .typ = NULL,
-    .const_val = NULL
+    .const_val = NULL,
+    .type_version = 0,
 };
 
 _Py_UopsSymbol *
@@ -76,6 +77,7 @@ sym_new(_Py_UOpsContext *ctx)
     self->flags = 0;
     self->typ = NULL;
     self->const_val = NULL;
+    self->type_version = 0;
 
     return self;
 }
@@ -152,6 +154,18 @@ _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *ty
     }
 }
 
+bool
+_Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version)
+{
+    // if the type version was already set, then it must be different and we should set it to bottom
+    if (sym->type_version) {
+        sym_set_bottom(ctx, sym);
+        return false;
+    }
+    sym->type_version = version;
+    return true;
+}
+
 void
 _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val)
 {
@@ -256,6 +270,12 @@ _Py_uop_sym_get_type(_Py_UopsSymbol *sym)
     return sym->typ;
 }
 
+unsigned int
+_Py_uop_sym_get_type_version(_Py_UopsSymbol *sym)
+{
+    return sym->type_version;
+}
+
 bool
 _Py_uop_sym_has_type(_Py_UopsSymbol *sym)
 {
@@ -272,6 +292,13 @@ _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ)
     return _Py_uop_sym_get_type(sym) == typ;
 }
 
+bool
+_Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version)
+{
+    return _Py_uop_sym_get_type_version(sym) == version;
+}
+
+
 int
 _Py_uop_sym_truthiness(_Py_UopsSymbol *sym)
 {
@@ -311,9 +338,9 @@ _Py_UOpsAbstractFrame *
 _Py_uop_frame_new(
     _Py_UOpsContext *ctx,
     PyCodeObject *co,
-    _Py_UopsSymbol **localsplus_start,
-    int n_locals_already_filled,
-    int curr_stackentries)
+    int curr_stackentries,
+    _Py_UopsSymbol **args,
+    int arg_len)
 {
     assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH);
     _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth];
@@ -321,19 +348,22 @@ _Py_uop_frame_new(
     frame->stack_len = co->co_stacksize;
     frame->locals_len = co->co_nlocalsplus;
 
-    frame->locals = localsplus_start;
+    frame->locals = ctx->n_consumed;
     frame->stack = frame->locals + co->co_nlocalsplus;
     frame->stack_pointer = frame->stack + curr_stackentries;
-    ctx->n_consumed = localsplus_start + (co->co_nlocalsplus + co->co_stacksize);
+    ctx->n_consumed = ctx->n_consumed + (co->co_nlocalsplus + co->co_stacksize);
     if (ctx->n_consumed >= ctx->limit) {
         ctx->done = true;
         ctx->out_of_space = true;
         return NULL;
     }
 
-
     // Initialize with the initial state of all local variables
-    for (int i = n_locals_already_filled; i < co->co_nlocalsplus; i++) {
+    for (int i = 0; i < arg_len; i++) {
+        frame->locals[i] = args[i];
+    }
+
+    for (int i = arg_len; i < co->co_nlocalsplus; i++) {
         _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx);
         frame->locals[i] = local;
     }

From 38a25e9560cf0ff0b80d9e90bce793ff24c6e027 Mon Sep 17 00:00:00 2001
From: neonene <53406459+neonene@users.noreply.github.com>
Date: Sat, 8 Jun 2024 19:22:07 +0900
Subject: [PATCH 29/72] gh-120244: Fix re.sub() reference leak (GH-120245)

---
 .../next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst  | 1 +
 Modules/_sre/sre.c                                               | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst

diff --git a/Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst b/Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst
new file mode 100644
index 00000000000000..d21532f22a1d38
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst
@@ -0,0 +1 @@
+Fix memory leak in :func:`re.sub()` when the replacement string contains backreferences.
diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c
index c1eff63d921de9..e33034086481c2 100644
--- a/Modules/_sre/sre.c
+++ b/Modules/_sre/sre.c
@@ -1622,6 +1622,7 @@ _sre_template_impl(PyObject *module, PyObject *pattern, PyObject *template)
         }
         self->items[i].literal = Py_XNewRef(literal);
     }
+    PyObject_GC_Track(self);
     return (PyObject*) self;
 
 bad_template:

From 5d59b870effa0f576acf7264cfcbfca2b36e34e3 Mon Sep 17 00:00:00 2001
From: AN Long <aisk@users.noreply.github.com>
Date: Sun, 9 Jun 2024 00:11:19 +0800
Subject: [PATCH 30/72] gh-120121: Add InvalidStateError to
 concurrent.futures.__all__ (#120123)

Co-authored-by: Nikita Sobolev <mail@sobolevn.me>
---
 Lib/concurrent/futures/__init__.py                               | 1 +
 .../next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst  | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst

diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py
index 292e886d5a88ac..72de617a5b6f61 100644
--- a/Lib/concurrent/futures/__init__.py
+++ b/Lib/concurrent/futures/__init__.py
@@ -23,6 +23,7 @@
     'ALL_COMPLETED',
     'CancelledError',
     'TimeoutError',
+    'InvalidStateError',
     'BrokenExecutor',
     'Future',
     'Executor',
diff --git a/Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst b/Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst
new file mode 100644
index 00000000000000..4f3526477c8cce
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst
@@ -0,0 +1 @@
+Add :exc:`concurrent.futures.InvalidStateError` to module's ``__all__``.

From 7c016deae62308dd1b4e2767fc6abf04857c7843 Mon Sep 17 00:00:00 2001
From: Clinton <pygeek@users.noreply.github.com>
Date: Sat, 8 Jun 2024 13:18:58 -0400
Subject: [PATCH 31/72] gh-120276: Fix incorrect email.header.Header maxlinelen
 default (GH-120277)

---
 Doc/library/email.header.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst
index 6e230d5faf1654..219fad0d2f6745 100644
--- a/Doc/library/email.header.rst
+++ b/Doc/library/email.header.rst
@@ -77,7 +77,7 @@ Here is the :class:`Header` class description:
    The maximum line length can be specified explicitly via *maxlinelen*.  For
    splitting the first line to a shorter value (to account for the field header
    which isn't included in *s*, e.g. :mailheader:`Subject`) pass in the name of the
-   field in *header_name*.  The default *maxlinelen* is 76, and the default value
+   field in *header_name*.  The default *maxlinelen* is 78, and the default value
    for *header_name* is ``None``, meaning it is not taken into account for the
    first line of a long, split header.
 

From 34f5ae69fe9ab0f5b23311d5c396d0cbb5902913 Mon Sep 17 00:00:00 2001
From: Kirill Podoprigora <kirill.bast9@mail.ru>
Date: Sat, 8 Jun 2024 23:45:57 +0300
Subject: [PATCH 32/72] gh-120268: Prohibit passing ``None`` to
 ``_pydatetime.date.fromtimestamp`` (#120269)

This makes the pure Python implementation consistent with the C implementation.
---
 Lib/_pydatetime.py                                           | 2 ++
 Lib/test/datetimetester.py                                   | 5 +++++
 .../Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst   | 2 ++
 3 files changed, 9 insertions(+)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst

diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py
index b7d569cc41740e..34ccb2da13d0f3 100644
--- a/Lib/_pydatetime.py
+++ b/Lib/_pydatetime.py
@@ -966,6 +966,8 @@ def __new__(cls, year, month=None, day=None):
     @classmethod
     def fromtimestamp(cls, t):
         "Construct a date from a POSIX timestamp (like time.time())."
+        if t is None:
+            raise TypeError("'NoneType' object cannot be interpreted as an integer")
         y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t)
         return cls(y, m, d)
 
diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py
index b80da5697ef865..28f75a803b4e04 100644
--- a/Lib/test/datetimetester.py
+++ b/Lib/test/datetimetester.py
@@ -1336,6 +1336,11 @@ def test_insane_fromtimestamp(self):
             self.assertRaises(OverflowError, self.theclass.fromtimestamp,
                               insane)
 
+    def test_fromtimestamp_with_none_arg(self):
+        # See gh-120268 for more details
+        with self.assertRaises(TypeError):
+            self.theclass.fromtimestamp(None)
+
     def test_today(self):
         import time
 
diff --git a/Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst b/Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst
new file mode 100644
index 00000000000000..d48d43cd047f7a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst
@@ -0,0 +1,2 @@
+Prohibit passing ``None`` to pure-Python :meth:`datetime.date.fromtimestamp`
+to achieve consistency with C-extension implementation.

From 0ae8579b85f9b0cd3f287082ad6e194bdb025d88 Mon Sep 17 00:00:00 2001
From: Carl Meyer <carl@oddbird.net>
Date: Sun, 9 Jun 2024 22:23:30 -0400
Subject: [PATCH 33/72] gh-119666: fix multiple class-scope comprehensions
 referencing __class__ (#120295)

---
 Lib/test/test_listcomps.py                    | 25 +++++++++++++++++++
 ...-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst |  1 +
 Python/symtable.c                             | 23 ++++++++---------
 3 files changed, 36 insertions(+), 13 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst

diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py
index ec2aac81682db8..58b076e9ea5d8a 100644
--- a/Lib/test/test_listcomps.py
+++ b/Lib/test/test_listcomps.py
@@ -168,6 +168,31 @@ def test_references___class__(self):
         """
         self._check_in_scopes(code, raises=NameError)
 
+    def test_references___class___defined(self):
+        code = """
+            __class__ = 2
+            res = [__class__ for x in [1]]
+        """
+        self._check_in_scopes(
+                code, outputs={"res": [2]}, scopes=["module", "function"])
+        self._check_in_scopes(code, raises=NameError, scopes=["class"])
+
+    def test_references___class___enclosing(self):
+        code = """
+            __class__ = 2
+            class C:
+                res = [__class__ for x in [1]]
+            res = C.res
+        """
+        self._check_in_scopes(code, raises=NameError)
+
+    def test_super_and_class_cell_in_sibling_comps(self):
+        code = """
+            [super for _ in [1]]
+            [__class__ for _ in [1]]
+        """
+        self._check_in_scopes(code, raises=NameError)
+
     def test_inner_cell_shadows_outer(self):
         code = """
             items = [(lambda: i) for i in range(5)]
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst
new file mode 100644
index 00000000000000..09c1f553c48702
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst	
@@ -0,0 +1 @@
+Fix a compiler crash in the case where two comprehensions in class scope both reference ``__class__``.
diff --git a/Python/symtable.c b/Python/symtable.c
index 0ee8ca36cf8df0..7e452cdb13badf 100644
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -781,22 +781,19 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp,
         if (existing == NULL && PyErr_Occurred()) {
             return 0;
         }
+        // __class__ is never allowed to be free through a class scope (see
+        // drop_class_free)
+        if (scope == FREE && ste->ste_type == ClassBlock &&
+                _PyUnicode_EqualToASCIIString(k, "__class__")) {
+            scope = GLOBAL_IMPLICIT;
+            if (PySet_Discard(comp_free, k) < 0) {
+                return 0;
+            }
+            remove_dunder_class = 1;
+        }
         if (!existing) {
             // name does not exist in scope, copy from comprehension
             assert(scope != FREE || PySet_Contains(comp_free, k) == 1);
-            if (scope == FREE && ste->ste_type == ClassBlock &&
-                _PyUnicode_EqualToASCIIString(k, "__class__")) {
-                // if __class__ is unbound in the enclosing class scope and free
-                // in the comprehension scope, it needs special handling; just
-                // letting it be marked as free in class scope will break due to
-                // drop_class_free
-                scope = GLOBAL_IMPLICIT;
-                only_flags &= ~DEF_FREE;
-                if (PySet_Discard(comp_free, k) < 0) {
-                    return 0;
-                }
-                remove_dunder_class = 1;
-            }
             PyObject *v_flags = PyLong_FromLong(only_flags);
             if (v_flags == NULL) {
                 return 0;

From e5a7bc6f2eb9a3875063423caa67bb0ffcc3a6b8 Mon Sep 17 00:00:00 2001
From: Clinton <pygeek@users.noreply.github.com>
Date: Mon, 10 Jun 2024 04:17:50 -0400
Subject: [PATCH 34/72] gh-120296: Fix format string of fcntl.ioctl() audit
 (#120301)

---
 Modules/fcntlmodule.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c
index 873bdf2ac0657a..0c06c03a6c403e 100644
--- a/Modules/fcntlmodule.c
+++ b/Modules/fcntlmodule.c
@@ -170,7 +170,7 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code,
     Py_ssize_t len;
     char buf[IOCTL_BUFSZ+1];  /* argument plus NUL byte */
 
-    if (PySys_Audit("fcntl.ioctl", "iIO", fd, code,
+    if (PySys_Audit("fcntl.ioctl", "ikO", fd, code,
                     ob_arg ? ob_arg : Py_None) < 0) {
         return NULL;
     }

From 4829522b8d3e1a28930f1cccfcc9635e035a0eb4 Mon Sep 17 00:00:00 2001
From: "E. M. Bray" <erik.bray@lri.fr>
Date: Mon, 10 Jun 2024 10:55:49 +0200
Subject: [PATCH 35/72] bpo-24766: doc= argument to subclasses of property not
 handled correctly (GH-2487)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
---
 Lib/test/test_property.py                     | 34 +++++++++++++++++++
 .../2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst  |  1 +
 Objects/descrobject.c                         | 19 +++--------
 3 files changed, 39 insertions(+), 15 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst

diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py
index 408e64f53142db..b7a2219b96149a 100644
--- a/Lib/test/test_property.py
+++ b/Lib/test/test_property.py
@@ -463,6 +463,40 @@ def getter3(self):
         self.assertEqual(p.__doc__, "user")
         self.assertEqual(p2.__doc__, "user")
 
+    @unittest.skipIf(sys.flags.optimize >= 2,
+                     "Docstrings are omitted with -O2 and above")
+    def test_prefer_explicit_doc(self):
+        # Issue 25757: subclasses of property lose docstring
+        self.assertEqual(property(doc="explicit doc").__doc__, "explicit doc")
+        self.assertEqual(PropertySub(doc="explicit doc").__doc__, "explicit doc")
+
+        class Foo:
+            spam = PropertySub(doc="spam explicit doc")
+
+            @spam.getter
+            def spam(self):
+                """ignored as doc already set"""
+                return 1
+
+            def _stuff_getter(self):
+                """ignored as doc set directly"""
+            stuff = PropertySub(doc="stuff doc argument", fget=_stuff_getter)
+
+        #self.assertEqual(Foo.spam.__doc__, "spam explicit doc")
+        self.assertEqual(Foo.stuff.__doc__, "stuff doc argument")
+
+    def test_property_no_doc_on_getter(self):
+        # If a property's getter has no __doc__ then the property's doc should
+        # be None; test that this is consistent with subclasses as well; see
+        # GH-2487
+        class NoDoc:
+            @property
+            def __doc__(self):
+                raise AttributeError
+
+        self.assertEqual(property(NoDoc()).__doc__, None)
+        self.assertEqual(PropertySub(NoDoc()).__doc__, None)
+
     @unittest.skipIf(sys.flags.optimize >= 2,
                      "Docstrings are omitted with -O2 and above")
     def test_property_setter_copies_getter_docstring(self):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst b/Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst
new file mode 100644
index 00000000000000..93a8562efe6d6f
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst	
@@ -0,0 +1 @@
+Fix handling of ``doc`` argument to subclasses of ``property``.
diff --git a/Objects/descrobject.c b/Objects/descrobject.c
index 1b7e2fde3ceccd..4eccd1704eb95a 100644
--- a/Objects/descrobject.c
+++ b/Objects/descrobject.c
@@ -1859,22 +1859,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset,
     /* if no docstring given and the getter has one, use that one */
     else if (fget != NULL) {
         int rc = PyObject_GetOptionalAttr(fget, &_Py_ID(__doc__), &prop_doc);
-        if (rc <= 0) {
+        if (rc < 0) {
             return rc;
         }
-        if (!Py_IS_TYPE(self, &PyProperty_Type) &&
-            prop_doc != NULL && prop_doc != Py_None) {
-            // This oddity preserves the long existing behavior of surfacing
-            // an AttributeError when using a dict-less (__slots__) property
-            // subclass as a decorator on a getter method with a docstring.
-            // See PropertySubclassTest.test_slots_docstring_copy_exception.
-            int err = PyObject_SetAttr(
-                        (PyObject *)self, &_Py_ID(__doc__), prop_doc);
-            if (err < 0) {
-                Py_DECREF(prop_doc);  // release our new reference.
-                return -1;
-            }
-        }
         if (prop_doc == Py_None) {
             prop_doc = NULL;
             Py_DECREF(Py_None);
@@ -1902,7 +1889,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset,
         Py_DECREF(prop_doc);
         if (err < 0) {
             assert(PyErr_Occurred());
-            if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
+            if (!self->getter_doc &&
+                PyErr_ExceptionMatches(PyExc_AttributeError))
+            {
                 PyErr_Clear();
                 // https://github.com/python/cpython/issues/98963#issuecomment-1574413319
                 // Python silently dropped this doc assignment through 3.11.

From b90bd3e5bbc136f53b24ee791824acd6b17e0d42 Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Mon, 10 Jun 2024 11:54:35 +0200
Subject: [PATCH 36/72] gh-120155: Fix Coverity issue in zoneinfo load_data()
 (#120232)

Declare the 'rv' varaible at the top of the load_data() function to
make sure that it's initialized before the first 'goto error' which
uses 'rv' (return rv).

Fix the Coverity issue:

Error: UNINIT (CWE-457):
Python-3.12.2/Modules/_zoneinfo.c:1233:5: skipped_decl: Jumping over declaration of ""rv"".
Python-3.12.2/Modules/_zoneinfo.c:1284:5: uninit_use: Using uninitialized value ""rv"".
  1282|       }
  1283|
  1284|->     return rv;
  1285|   }
  1286|
---
 Modules/_zoneinfo.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c
index 38c3f0c45d803f..902ece795b575b 100644
--- a/Modules/_zoneinfo.c
+++ b/Modules/_zoneinfo.c
@@ -944,6 +944,7 @@ ttinfo_eq(const _ttinfo *const tti0, const _ttinfo *const tti1)
 static int
 load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj)
 {
+    int rv = 0;
     PyObject *data_tuple = NULL;
 
     long *utcoff = NULL;
@@ -1220,7 +1221,6 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj)
         }
     }
 
-    int rv = 0;
     goto cleanup;
 error:
     // These resources only need to be freed if we have failed, if we succeed

From c3b6dbff2c8886de1edade737febe85dd47ff4d0 Mon Sep 17 00:00:00 2001
From: Pieter Eendebak <pieter.eendebak@gmail.com>
Date: Mon, 10 Jun 2024 13:06:18 +0200
Subject: [PATCH 37/72] gh-115801: Only allow sequence of strings as input for
 difflib.unified_diff (GH-118333)

---
 Lib/difflib.py                                |  6 ++++
 Lib/test/test_difflib.py                      | 30 +++++++++++++++----
 ...-04-27-18-36-46.gh-issue-115801.SVeHSy.rst |  1 +
 3 files changed, 31 insertions(+), 6 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst

diff --git a/Lib/difflib.py b/Lib/difflib.py
index 0443963b4fd697..7f595b6c72e641 100644
--- a/Lib/difflib.py
+++ b/Lib/difflib.py
@@ -1264,6 +1264,12 @@ def _check_types(a, b, *args):
     if b and not isinstance(b[0], str):
         raise TypeError('lines to compare must be str, not %s (%r)' %
                         (type(b[0]).__name__, b[0]))
+    if isinstance(a, str):
+        raise TypeError('input must be a sequence of strings, not %s' %
+                        type(a).__name__)
+    if isinstance(b, str):
+        raise TypeError('input must be a sequence of strings, not %s' %
+                        type(b).__name__)
     for arg in args:
         if not isinstance(arg, str):
             raise TypeError('all arguments must be str, not: %r' % (arg,))
diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py
index bf6e5b1152b4a2..9e217249be7332 100644
--- a/Lib/test/test_difflib.py
+++ b/Lib/test/test_difflib.py
@@ -295,7 +295,7 @@ def test_close_matches_aligned(self):
 
 class TestOutputFormat(unittest.TestCase):
     def test_tab_delimiter(self):
-        args = ['one', 'two', 'Original', 'Current',
+        args = [['one'], ['two'], 'Original', 'Current',
             '2005-01-26 23:30:50', '2010-04-02 10:20:52']
         ud = difflib.unified_diff(*args, lineterm='')
         self.assertEqual(list(ud)[0:2], [
@@ -307,7 +307,7 @@ def test_tab_delimiter(self):
                            "--- Current\t2010-04-02 10:20:52"])
 
     def test_no_trailing_tab_on_empty_filedate(self):
-        args = ['one', 'two', 'Original', 'Current']
+        args = [['one'], ['two'], 'Original', 'Current']
         ud = difflib.unified_diff(*args, lineterm='')
         self.assertEqual(list(ud)[0:2], ["--- Original", "+++ Current"])
 
@@ -447,6 +447,28 @@ def assertDiff(expect, actual):
                                     lineterm=b'')
         assertDiff(expect, actual)
 
+
+class TestInputTypes(unittest.TestCase):
+    def _assert_type_error(self, msg, generator, *args):
+        with self.assertRaises(TypeError) as ctx:
+            list(generator(*args))
+        self.assertEqual(msg, str(ctx.exception))
+
+    def test_input_type_checks(self):
+        unified = difflib.unified_diff
+        context = difflib.context_diff
+
+        expect = "input must be a sequence of strings, not str"
+        self._assert_type_error(expect, unified, 'a', ['b'])
+        self._assert_type_error(expect, context, 'a', ['b'])
+
+        self._assert_type_error(expect, unified, ['a'], 'b')
+        self._assert_type_error(expect, context, ['a'], 'b')
+
+        expect = "lines to compare must be str, not NoneType (None)"
+        self._assert_type_error(expect, unified, ['a'], [None])
+        self._assert_type_error(expect, context, ['a'], [None])
+
     def test_mixed_types_content(self):
         # type of input content must be consistent: all str or all bytes
         a = [b'hello']
@@ -495,10 +517,6 @@ def test_mixed_types_dates(self):
         b = ['bar\n']
         list(difflib.unified_diff(a, b, 'a', 'b', datea, dateb))
 
-    def _assert_type_error(self, msg, generator, *args):
-        with self.assertRaises(TypeError) as ctx:
-            list(generator(*args))
-        self.assertEqual(msg, str(ctx.exception))
 
 class TestJunkAPIs(unittest.TestCase):
     def test_is_line_junk_true(self):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst
new file mode 100644
index 00000000000000..93b176d5767335
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst	
@@ -0,0 +1 @@
+Raise ``TypeError`` when passing a string to :func:`difflib.unified_diff` and :func:`difflib.context_diff`.

From 56c3815ba14c790d2e9a227b4ac0ead5e6b1e570 Mon Sep 17 00:00:00 2001
From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com>
Date: Mon, 10 Jun 2024 16:15:12 +0100
Subject: [PATCH 38/72] gh-119786: copy compiler doc from devguide to
 InternalDocs and convert to markdown (#120134)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* gh-119876: move compiler doc from devguide to InternalDocs

Copy of https://github.com/python/devguide/commit/78fc0d7aa9fd0d6733d10c23b178b2a0e2799afc

Co-Authored-By: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
Co-Authored-By: Adam Turner <9087854+aa-turner@users.noreply.github.com>
Co-Authored-By: Brett Cannon <brett@python.org>
Co-Authored-By: Carol Willing <carolcode@willingconsulting.com>
Co-Authored-By: Daniel Porteous <danielporteous1@gmail.com>
Co-Authored-By: Dennis Sweeney <36520290+sweeneyde@users.noreply.github.com>
Co-Authored-By: Éric Araujo <merwok@netwok.org>
Co-Authored-By: Erlend Egeberg Aasland <erlend.aasland@innova.no>
Co-Authored-By: Ezio Melotti <ezio.melotti@gmail.com>
Co-Authored-By: Georg Brandl <georg@python.org>
Co-Authored-By: Guido van Rossum <guido@python.org>
Co-Authored-By: Hugo van Kemenade <hugovk@users.noreply.github.com>
Co-Authored-By: Irit Katriel <1055913+iritkatriel@users.noreply.github.com>
Co-Authored-By: Jeff Allen <ja.py@farowl.co.uk>
Co-Authored-By: Jim Fasarakis-Hilliard <d.f.hilliard@gmail.com>
Co-Authored-By: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
Co-Authored-By: Lincoln <71312724+Lincoln-developer@users.noreply.github.com>
Co-Authored-By: Mariatta <Mariatta@users.noreply.github.com>
Co-Authored-By: Muhammad Mahad <mahadpy@gmail.com>
Co-Authored-By: Ned Deily <nad@acm.org>
Co-Authored-By: Pablo Galindo Salgado <Pablogsal@gmail.com>
Co-Authored-By: Serhiy Storchaka <storchaka@gmail.com>
Co-Authored-By: Stéphane Wirtel <stephane@wirtel.be>
Co-Authored-By: Suriyaa ✌️️ <isc.suriyaa@gmail.com>
Co-Authored-By: Zachary Ware <zachary.ware@gmail.com>
Co-Authored-By: psyker156 <242220+psyker156@users.noreply.github.com>
Co-Authored-By: slateny <46876382+slateny@users.noreply.github.com>
Co-Authored-By: svelankar <17737361+svelankar@users.noreply.github.com>
Co-Authored-By: zikcheng <surfingbyte@gmail.com>

* convert to markdown

* add to index

* update more of the out of date stuff

---------

Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
Co-authored-by: Brett Cannon <brett@python.org>
Co-authored-by: Carol Willing <carolcode@willingconsulting.com>
Co-authored-by: Daniel Porteous <danielporteous1@gmail.com>
Co-authored-by: Dennis Sweeney <36520290+sweeneyde@users.noreply.github.com>
Co-authored-by: Éric Araujo <merwok@netwok.org>
Co-authored-by: Erlend Egeberg Aasland <erlend.aasland@innova.no>
Co-authored-by: Ezio Melotti <ezio.melotti@gmail.com>
Co-authored-by: Georg Brandl <georg@python.org>
Co-authored-by: Guido van Rossum <guido@python.org>
Co-authored-by: Hugo van Kemenade <hugovk@users.noreply.github.com>
Co-authored-by: Jeff Allen <ja.py@farowl.co.uk>
Co-authored-by: Jim Fasarakis-Hilliard <d.f.hilliard@gmail.com>
Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
Co-authored-by: Lincoln <71312724+Lincoln-developer@users.noreply.github.com>
Co-authored-by: Mariatta <Mariatta@users.noreply.github.com>
Co-authored-by: Muhammad Mahad <mahadpy@gmail.com>
Co-authored-by: Ned Deily <nad@acm.org>
Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
Co-authored-by: Stéphane Wirtel <stephane@wirtel.be>
Co-authored-by: Suriyaa ✌️️ <isc.suriyaa@gmail.com>
Co-authored-by: Zachary Ware <zachary.ware@gmail.com>
Co-authored-by: psyker156 <242220+psyker156@users.noreply.github.com>
Co-authored-by: slateny <46876382+slateny@users.noreply.github.com>
Co-authored-by: svelankar <17737361+svelankar@users.noreply.github.com>
Co-authored-by: zikcheng <surfingbyte@gmail.com>
---
 InternalDocs/README.md   |   2 +
 InternalDocs/compiler.md | 651 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 653 insertions(+)
 create mode 100644 InternalDocs/compiler.md

diff --git a/InternalDocs/README.md b/InternalDocs/README.md
index a2502fbf198735..42f6125794266a 100644
--- a/InternalDocs/README.md
+++ b/InternalDocs/README.md
@@ -12,6 +12,8 @@ it is not, please report that through the
 [issue tracker](https://github.com/python/cpython/issues).
 
 
+[Compiler Design](compiler.md)
+
 [Exception Handling](exception_handling.md)
 
 [Adaptive Instruction Families](adaptive.md)
diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md
new file mode 100644
index 00000000000000..0abc10da6e05c6
--- /dev/null
+++ b/InternalDocs/compiler.md
@@ -0,0 +1,651 @@
+
+Compiler design
+===============
+
+Abstract
+--------
+
+In CPython, the compilation from source code to bytecode involves several steps:
+
+1. Tokenize the source code
+   [Parser/lexer/](https://github.com/python/cpython/blob/main/Parser/lexer/)
+   and [Parser/tokenizer/](https://github.com/python/cpython/blob/main/Parser/tokenizer/).
+2. Parse the stream of tokens into an Abstract Syntax Tree
+   [Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c).
+3. Transform AST into an instruction sequence
+   [Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c).
+4. Construct a Control Flow Graph and apply optimizations to it
+   [Python/flowgraph.c](https://github.com/python/cpython/blob/main/Python/flowgraph.c).
+5. Emit bytecode based on the Control Flow Graph
+   [Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c).
+
+This document outlines how these steps of the process work.
+
+This document only describes parsing in enough depth to explain what is needed
+for understanding compilation.  This document provides a detailed, though not
+exhaustive, view of the how the entire system works.  You will most likely need
+to read some source code to have an exact understanding of all details.
+
+
+Parsing
+=======
+
+As of Python 3.9, Python's parser is a PEG parser of a somewhat
+unusual design. It is unusual in the sense that the parser's input is a stream
+of tokens rather than a stream of characters which is more common with PEG
+parsers.
+
+The grammar file for Python can be found in
+[Grammar/python.gram](https://github.com/python/cpython/blob/main/Grammar/python.gram).
+The definitions for literal tokens (such as ``:``, numbers, etc.) can be found in
+[Grammar/Tokens](https://github.com/python/cpython/blob/main/Grammar/Tokens).
+Various C files, including
+[Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c)
+are generated from these.
+
+See Also:
+
+* [Guide to the parser](https://devguide.python.org/internals/parser/index.html)
+  for a detailed description of the parser.
+
+* [Changing CPython’s grammar](https://devguide.python.org/developer-workflow/grammar/#grammar)
+  for a detailed description of the grammar.
+
+
+Abstract syntax trees (AST)
+===========================
+
+
+The abstract syntax tree (AST) is a high-level representation of the
+program structure without the necessity of containing the source code;
+it can be thought of as an abstract representation of the source code.  The
+specification of the AST nodes is specified using the Zephyr Abstract
+Syntax Definition Language (ASDL) [^1], [^2].
+
+The definition of the AST nodes for Python is found in the file
+[Parser/Python.asdl](https://github.com/python/cpython/blob/main/Parser/Python.asdl).
+
+Each AST node (representing statements, expressions, and several
+specialized types, like list comprehensions and exception handlers) is
+defined by the ASDL.  Most definitions in the AST correspond to a
+particular source construct, such as an 'if' statement or an attribute
+lookup.  The definition is independent of its realization in any
+particular programming language.
+
+The following fragment of the Python ASDL construct demonstrates the
+approach and syntax:
+
+```
+   module Python
+   {
+       stmt = FunctionDef(identifier name, arguments args, stmt* body,
+                          expr* decorators)
+              | Return(expr? value) | Yield(expr? value)
+              attributes (int lineno)
+   }
+```
+
+The preceding example describes two different kinds of statements and an
+expression: function definitions, return statements, and yield expressions.
+All three kinds are considered of type ``stmt`` as shown by ``|`` separating
+the various kinds.  They all take arguments of various kinds and amounts.
+
+Modifiers on the argument type specify the number of values needed; ``?``
+means it is optional, ``*`` means 0 or more, while no modifier means only one
+value for the argument and it is required.  ``FunctionDef``, for instance,
+takes an ``identifier`` for the *name*, ``arguments`` for *args*, zero or more
+``stmt`` arguments for *body*, and zero or more ``expr`` arguments for
+*decorators*.
+
+Do notice that something like 'arguments', which is a node type, is
+represented as a single AST node and not as a sequence of nodes as with
+stmt as one might expect.
+
+All three kinds also have an 'attributes' argument; this is shown by the
+fact that 'attributes' lacks a '|' before it.
+
+The statement definitions above generate the following C structure type:
+
+
+```
+  typedef struct _stmt *stmt_ty;
+
+  struct _stmt {
+        enum { FunctionDef_kind=1, Return_kind=2, Yield_kind=3 } kind;
+        union {
+                struct {
+                        identifier name;
+                        arguments_ty args;
+                        asdl_seq *body;
+                } FunctionDef;
+
+                struct {
+                        expr_ty value;
+                } Return;
+
+                struct {
+                        expr_ty value;
+                } Yield;
+        } v;
+        int lineno;
+   }
+```
+
+Also generated are a series of constructor functions that allocate (in
+this case) a ``stmt_ty`` struct with the appropriate initialization.  The
+``kind`` field specifies which component of the union is initialized.  The
+``FunctionDef()`` constructor function sets 'kind' to ``FunctionDef_kind`` and
+initializes the *name*, *args*, *body*, and *attributes* fields.
+
+See also
+[Green Tree Snakes - The missing Python AST docs](https://greentreesnakes.readthedocs.io/en/latest)
+ by Thomas Kluyver.
+
+Memory management
+=================
+
+Before discussing the actual implementation of the compiler, a discussion of
+how memory is handled is in order.  To make memory management simple, an **arena**
+is used that pools memory in a single location for easy
+allocation and removal.  This enables the removal of explicit memory
+deallocation.  Because memory allocation for all needed memory in the compiler
+registers that memory with the arena, a single call to free the arena is all
+that is needed to completely free all memory used by the compiler.
+
+In general, unless you are working on the critical core of the compiler, memory
+management can be completely ignored.  But if you are working at either the
+very beginning of the compiler or the end, you need to care about how the arena
+works.  All code relating to the arena is in either
+[Include/internal/pycore_pyarena.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_pyarena.h)
+or [Python/pyarena.c](https://github.com/python/cpython/blob/main/Python/pyarena.c).
+
+``PyArena_New()`` will create a new arena.  The returned ``PyArena`` structure
+will store pointers to all memory given to it.  This does the bookkeeping of
+what memory needs to be freed when the compiler is finished with the memory it
+used. That freeing is done with ``PyArena_Free()``.  This only needs to be
+called in strategic areas where the compiler exits.
+
+As stated above, in general you should not have to worry about memory
+management when working on the compiler.  The technical details of memory
+management have been designed to be hidden from you for most cases.
+
+The only exception comes about when managing a PyObject.  Since the rest
+of Python uses reference counting, there is extra support added
+to the arena to cleanup each PyObject that was allocated.  These cases
+are very rare.  However, if you've allocated a PyObject, you must tell
+the arena about it by calling ``PyArena_AddPyObject()``.
+
+
+Source code to AST
+==================
+
+The AST is generated from source code using the function
+``_PyParser_ASTFromString()`` or ``_PyParser_ASTFromFile()``
+[Parser/peg_api.c](https://github.com/python/cpython/blob/main/Parser/peg_api.c).
+
+After some checks, a helper function in
+[Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c)
+begins applying production rules on the source code it receives; converting source
+code to tokens and matching these tokens recursively to their corresponding rule.  The
+production rule's corresponding rule function is called on every match.  These rule
+functions follow the format `xx_rule`.  Where *xx* is the grammar rule
+that the function handles and is automatically derived from
+[Grammar/python.gram](https://github.com/python/cpython/blob/main/Grammar/python.gram) by
+[Tools/peg_generator/pegen/c_generator.py](https://github.com/python/cpython/blob/main/Tools/peg_generator/pegen/c_generator.py).
+
+Each rule function in turn creates an AST node as it goes along.  It does this
+by allocating all the new nodes it needs, calling the proper AST node creation
+functions for any required supporting functions and connecting them as needed.
+This continues until all nonterminal symbols are replaced with terminals.  If an
+error occurs, the rule functions backtrack and try another rule function.  If
+there are no more rules, an error is set and the parsing ends.
+
+The AST node creation helper functions have the name `_PyAST_{xx}`
+where *xx* is the AST node that the function creates.  These are defined by the
+ASDL grammar and contained in
+[Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c)
+(which is generated by
+[Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py)
+from
+[Parser/Python.asdl](https://github.com/python/cpython/blob/main/Parser/Python.asdl)).
+This all leads to a sequence of AST nodes stored in ``asdl_seq`` structs.
+
+To demonstrate everything explained so far, here's the
+rule function responsible for a simple named import statement such as
+``import sys``.  Note that error-checking and debugging code has been
+omitted.  Removed parts are represented by ``...``.
+Furthermore, some comments have been added for explanation.  These comments
+may not be present in the actual code.
+
+
+```
+   // This is the production rule (from python.gram) the rule function
+   // corresponds to:
+   // import_name: 'import' dotted_as_names
+   static stmt_ty
+   import_name_rule(Parser *p)
+   {
+       ...
+       stmt_ty _res = NULL;
+       { // 'import' dotted_as_names
+           ...
+           Token * _keyword;
+           asdl_alias_seq* a;
+           // The tokenizing steps.
+           if (
+               (_keyword = _PyPegen_expect_token(p, 513))  // token='import'
+               &&
+               (a = dotted_as_names_rule(p))  // dotted_as_names
+           )
+           {
+               ...
+               // Generate an AST for the import statement.
+               _res = _PyAST_Import ( a , ...);
+               ...
+               goto done;
+           }
+           ...
+       }
+       _res = NULL;
+     done:
+       ...
+       return _res;
+   }
+```
+
+
+To improve backtracking performance, some rules (chosen by applying a
+``(memo)`` flag in the grammar file) are memoized.  Each rule function checks if
+a memoized version exists and returns that if so, else it continues in the
+manner stated in the previous paragraphs.
+
+There are macros for creating and using ``asdl_xx_seq *`` types, where *xx* is
+a type of the ASDL sequence.  Three main types are defined
+manually -- ``generic``, ``identifier`` and ``int``.  These types are found in
+[Python/asdl.c](https://github.com/python/cpython/blob/main/Python/asdl.c)
+and its corresponding header file
+[Include/internal/pycore_asdl.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_asdl.h).
+Functions and macros for creating ``asdl_xx_seq *`` types are as follows:
+
+``_Py_asdl_generic_seq_new(Py_ssize_t, PyArena *)``
+        Allocate memory for an ``asdl_generic_seq`` of the specified length
+``_Py_asdl_identifier_seq_new(Py_ssize_t, PyArena *)``
+        Allocate memory for an ``asdl_identifier_seq`` of the specified length
+``_Py_asdl_int_seq_new(Py_ssize_t, PyArena *)``
+        Allocate memory for an ``asdl_int_seq`` of the specified length
+
+In addition to the three types mentioned above, some ASDL sequence types are
+automatically generated by
+[Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py)
+and found in
+[Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h).
+Macros for using both manually defined and automatically generated ASDL
+sequence types are as follows:
+
+``asdl_seq_GET(asdl_xx_seq *, int)``
+        Get item held at a specific position in an ``asdl_xx_seq``
+``asdl_seq_SET(asdl_xx_seq *, int, stmt_ty)``
+        Set a specific index in an ``asdl_xx_seq`` to the specified value
+
+Untyped counterparts exist for some of the typed macros.  These are useful
+when a function needs to manipulate a generic ASDL sequence:
+
+``asdl_seq_GET_UNTYPED(asdl_seq *, int)``
+        Get item held at a specific position in an ``asdl_seq``
+``asdl_seq_SET_UNTYPED(asdl_seq *, int, stmt_ty)``
+        Set a specific index in an ``asdl_seq`` to the specified value
+``asdl_seq_LEN(asdl_seq *)``
+        Return the length of an ``asdl_seq`` or ``asdl_xx_seq``
+
+Note that typed macros and functions are recommended over their untyped
+counterparts.  Typed macros carry out checks in debug mode and aid
+debugging errors caused by incorrectly casting from ``void *``.
+
+If you are working with statements, you must also worry about keeping
+track of what line number generated the statement.  Currently the line
+number is passed as the last parameter to each ``stmt_ty`` function.
+
+See also [PEP 617: New PEG parser for CPython](https://peps.python.org/pep-0617/).
+
+
+Control flow graphs
+===================
+
+A **control flow graph** (often referenced by its acronym, **CFG**) is a
+directed graph that models the flow of a program.  A node of a CFG is
+not an individual bytecode instruction, but instead represents a
+sequence of bytecode instructions that always execute sequentially.
+Each node is called a *basic block* and must always execute from
+start to finish, with a single entry point at the beginning and a
+single exit point at the end.  If some bytecode instruction *a* needs
+to jump to some other bytecode instruction *b*, then *a* must occur at
+the end of its basic block, and *b* must occur at the start of its
+basic block.
+
+As an example, consider the following code snippet:
+
+.. code-block:: Python
+
+   if x < 10:
+       f1()
+       f2()
+   else:
+       g()
+   end()
+
+The ``x < 10`` guard is represented by its own basic block that
+compares ``x`` with ``10`` and then ends in a conditional jump based on
+the result of the comparison.  This conditional jump allows the block
+to point to both the body of the ``if`` and the body of the ``else``.  The
+``if`` basic block contains the ``f1()`` and ``f2()`` calls and points to
+the ``end()`` basic block. The ``else`` basic block contains the ``g()``
+call and similarly points to the ``end()`` block.
+
+Note that more complex code in the guard, the ``if`` body, or the ``else``
+body may be represented by multiple basic blocks. For instance,
+short-circuiting boolean logic in a guard like ``if x or y:``
+will produce one basic block that tests the truth value of ``x``
+and then points both (1) to the start of the ``if`` body and (2) to
+a different basic block that tests the truth value of y.
+
+CFGs are useful as an intermediate representation of the code because
+they are a convenient data structure for optimizations.
+
+AST to CFG to bytecode
+======================
+
+The conversion of an ``AST`` to bytecode is initiated by a call to the function
+``_PyAST_Compile()`` in
+[Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c).
+
+The first step is to construct the symbol table. This is implemented by
+``_PySymtable_Build()`` in
+[Python/symtable.c](https://github.com/python/cpython/blob/main/Python/symtable.c).
+This function begins by entering the starting code block for the AST (passed-in)
+and then calling the proper `symtable_visit_{xx}` function (with *xx* being the
+AST node type).  Next, the AST tree is walked with the various code blocks that
+delineate the reach of a local variable as blocks are entered and exited using
+``symtable_enter_block()`` and ``symtable_exit_block()``, respectively.
+
+Once the symbol table is created, the ``AST`` is transformed by ``compiler_codegen()``
+in [Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c)
+into a sequence of pseudo instructions. These are similar to bytecode, but
+in some cases they are more abstract, and are resolved later into actual
+bytecode. The construction of this instruction sequence is handled by several
+functions that break the task down by various AST node types.  The functions are
+all named `compiler_visit_{xx}` where *xx* is the name of the node type (such
+as ``stmt``, ``expr``, etc.).  Each function receives a ``struct compiler *``
+and `{xx}_ty` where *xx* is the AST node type.  Typically these functions
+consist of a large 'switch' statement, branching based on the kind of
+node type passed to it.  Simple things are handled inline in the
+'switch' statement with more complex transformations farmed out to other
+functions named `compiler_{xx}` with *xx* being a descriptive name of what is
+being handled.
+
+When transforming an arbitrary AST node, use the ``VISIT()`` macro.
+The appropriate `compiler_visit_{xx}` function is called, based on the value
+passed in for <node type> (so `VISIT({c}, expr, {node})` calls
+`compiler_visit_expr({c}, {node})`).  The ``VISIT_SEQ()`` macro is very similar,
+but is called on AST node sequences (those values that were created as
+arguments to a node that used the '*' modifier).
+
+Emission of bytecode is handled by the following macros:
+
+* ``ADDOP(struct compiler *, location, int)``
+    add a specified opcode
+* ``ADDOP_IN_SCOPE(struct compiler *, location, int)``
+    like ``ADDOP``, but also exits current scope; used for adding return value
+    opcodes in lambdas and closures
+* ``ADDOP_I(struct compiler *, location, int, Py_ssize_t)``
+    add an opcode that takes an integer argument
+* ``ADDOP_O(struct compiler *, location, int, PyObject *, TYPE)``
+    add an opcode with the proper argument based on the position of the
+    specified PyObject in PyObject sequence object, but with no handling of
+    mangled names; used for when you
+    need to do named lookups of objects such as globals, consts, or
+    parameters where name mangling is not possible and the scope of the
+    name is known; *TYPE* is the name of PyObject sequence
+    (``names`` or ``varnames``)
+* ``ADDOP_N(struct compiler *, location, int, PyObject *, TYPE)``
+    just like ``ADDOP_O``, but steals a reference to PyObject
+* ``ADDOP_NAME(struct compiler *, location, int, PyObject *, TYPE)``
+    just like ``ADDOP_O``, but name mangling is also handled; used for
+    attribute loading or importing based on name
+* ``ADDOP_LOAD_CONST(struct compiler *, location, PyObject *)``
+    add the ``LOAD_CONST`` opcode with the proper argument based on the
+    position of the specified PyObject in the consts table.
+* ``ADDOP_LOAD_CONST_NEW(struct compiler *, location, PyObject *)``
+    just like ``ADDOP_LOAD_CONST_NEW``, but steals a reference to PyObject
+* ``ADDOP_JUMP(struct compiler *, location, int, basicblock *)``
+    create a jump to a basic block
+
+The ``location`` argument is a struct with the source location to be
+associated with this instruction. It is typically extracted from an
+``AST`` node with the ``LOC`` macro. The ``NO_LOCATION`` can be used
+for *synthetic* instructions, which we do not associate with a line
+number at this stage. For example, the implicit ``return None``
+which is added at the end of a function is not associated with any
+line in the source code.
+
+There are several helper functions that will emit pseudo-instructions
+and are named `compiler_{xx}()` where *xx* is what the function helps
+with (``list``, ``boolop``, etc.).  A rather useful one is ``compiler_nameop()``.
+This function looks up the scope of a variable and, based on the
+expression context, emits the proper opcode to load, store, or delete
+the variable.
+
+Once the instruction sequence is created, it is transformed into a CFG
+by ``_PyCfg_FromInstructionSequence()``. Then ``_PyCfg_OptimizeCodeUnit()``
+applies various peephole optimizations, and
+``_PyCfg_OptimizedCfgToInstructionSequence()`` converts the optimized ``CFG``
+back into an instruction sequence. These conversions and optimizations are
+implemented in
+[Python/flowgraph.c](https://github.com/python/cpython/blob/main/Python/flowgraph.c).
+
+Finally, the sequence of pseudo-instructions is converted into actual
+bytecode. This includes transforming pseudo instructions into actual instructions,
+converting jump targets from logical labels to relative offsets, and
+construction of the
+[exception table](exception_handling.md) and
+[locations table](https://github.com/python/cpython/blob/main/Objects/locations.md).
+The bytecode and tables are then wrapped into a ``PyCodeObject`` along with additional
+metadata, including the ``consts`` and ``names`` arrays, information about function
+reference to the source code (filename, etc). All of this is implemented by
+``_PyAssemble_MakeCodeObject()`` in
+[Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c).
+
+
+Code objects
+============
+
+The result of ``PyAST_CompileObject()`` is a ``PyCodeObject`` which is defined in
+[Include/cpython/code.h](https://github.com/python/cpython/blob/main/Include/cpython/code.h).
+And with that you now have executable Python bytecode!
+
+The code objects (byte code) are executed in
+[Python/ceval.c](https://github.com/python/cpython/blob/main/Python/ceval.c).
+This file will also need a new case statement for the new opcode in the big switch
+statement in ``_PyEval_EvalFrameDefault()``.
+
+
+Important files
+===============
+
+* [Parser/](https://github.com/python/cpython/blob/main/Parser/)
+
+  * [Parser/Python.asdl](https://github.com/python/cpython/blob/main/Parser/Python.asdl):
+    ASDL syntax file.
+
+  * [Parser/asdl.py](https://github.com/python/cpython/blob/main/Parser/asdl.py):
+    Parser for ASDL definition files.
+    Reads in an ASDL description and parses it into an AST that describes it.
+
+  * [Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py):
+    Generate C code from an ASDL description.  Generates
+    [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c)
+    and
+    [Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h).
+
+  * [Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c):
+    The new PEG parser introduced in Python 3.9.
+    Generated by
+    [Tools/peg_generator/pegen/c_generator.py](https://github.com/python/cpython/blob/main/Tools/peg_generator/pegen/c_generator.py)
+    from the grammar [Grammar/python.gram](https://github.com/python/cpython/blob/main/Grammar/python.gram).
+    Creates the AST from source code.  Rule functions for their corresponding production
+    rules are found here.
+
+  * [Parser/peg_api.c](https://github.com/python/cpython/blob/main/Parser/peg_api.c):
+    Contains high-level functions which are
+    used by the interpreter to create an AST from source code.
+
+  * [Parser/pegen.c](https://github.com/python/cpython/blob/main/Parser/pegen.c):
+    Contains helper functions which are used by functions in
+    [Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c)
+    to construct the AST.  Also contains helper functions which help raise better error messages
+    when parsing source code.
+
+  * [Parser/pegen.h](https://github.com/python/cpython/blob/main/Parser/pegen.h):
+    Header file for the corresponding
+    [Parser/pegen.c](https://github.com/python/cpython/blob/main/Parser/pegen.c).
+    Also contains definitions of the ``Parser`` and ``Token`` structs.
+
+* [Python/](https://github.com/python/cpython/blob/main/Python)
+
+  * [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c):
+    Creates C structs corresponding to the ASDL types.  Also contains code for
+    marshalling AST nodes (core ASDL types have marshalling code in
+    [Python/asdl.c](https://github.com/python/cpython/blob/main/Python/asdl.c)).
+    "File automatically generated by
+    [Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py).
+    This file must be committed separately after every grammar change
+    is committed since the ``__version__`` value is set to the latest
+    grammar change revision number.
+
+  * [Python/asdl.c](https://github.com/python/cpython/blob/main/Python/asdl.c):
+    Contains code to handle the ASDL sequence type.
+    Also has code to handle marshalling the core ASDL types, such as number
+    and identifier.  Used by
+    [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c)
+    for marshalling AST nodes.
+
+  * [Python/ast.c](https://github.com/python/cpython/blob/main/Python/ast.c):
+    Used for validating the AST.
+
+  * [Python/ast_opt.c](https://github.com/python/cpython/blob/main/Python/ast_opt.c):
+    Optimizes the AST.
+
+  * [Python/ast_unparse.c](https://github.com/python/cpython/blob/main/Python/ast_unparse.c):
+    Converts the AST expression node back into a string (for string annotations).
+
+  * [Python/ceval.c](https://github.com/python/cpython/blob/main/Python/ceval.c):
+    Executes byte code (aka, eval loop).
+
+  * [Python/symtable.c](https://github.com/python/cpython/blob/main/Python/symtable.c):
+    Generates a symbol table from AST.
+
+  * [Python/pyarena.c](https://github.com/python/cpython/blob/main/Python/pyarena.c):
+    Implementation of the arena memory manager.
+
+  * [Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c):
+    Emits pseudo bytecode based on the AST.
+
+  * [Python/flowgraph.c](https://github.com/python/cpython/blob/main/Python/flowgraph.c):
+    Implements peephole optimizations.
+
+  * [Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c):
+    Constructs a code object from a sequence of pseudo instructions.
+
+  * [Python/instruction_sequence.c.c](https://github.com/python/cpython/blob/main/Python/instruction_sequence.c.c):
+    A data structure representing a sequence of bytecode-like pseudo-instructions.
+
+* [Include/](https://github.com/python/cpython/blob/main/Include/)
+
+  * [Include/cpython/code.h](https://github.com/python/cpython/blob/main/Include/cpython/code.h)
+    : Header file for
+    [Objects/codeobject.c](https://github.com/python/cpython/blob/main/Objects/codeobject.c);
+    contains definition of ``PyCodeObject``.
+
+  * [Include/opcode.h](https://github.com/python/cpython/blob/main/Include/opcode.h)
+    : One of the files that must be modified if
+    [Lib/opcode.py](https://github.com/python/cpython/blob/main/Lib/opcode.py) is.
+
+  * [Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h)
+    : Contains the actual definitions of the C structs as generated by
+    [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c)
+    "Automatically generated by
+    [Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py).
+
+  * [Include/internal/pycore_asdl.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_asdl.h)
+    : Header for the corresponding
+    [Python/ast.c](https://github.com/python/cpython/blob/main/Python/ast.c).
+
+  * [Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h)
+    : Declares ``_PyAST_Validate()`` external (from
+    [Python/ast.c](https://github.com/python/cpython/blob/main/Python/ast.c)).
+
+  * [Include/internal/pycore_symtable.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_symtable.h)
+    :  Header for
+    [Python/symtable.c](https://github.com/python/cpython/blob/main/Python/symtable.c).
+    ``struct symtable`` and ``PySTEntryObject`` are defined here.
+
+  * [Include/internal/pycore_parser.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_parser.h)
+    : Header for the corresponding
+    [Parser/peg_api.c](https://github.com/python/cpython/blob/main/Parser/peg_api.c).
+
+  * [Include/internal/pycore_pyarena.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_pyarena.h)
+    : Header file for the corresponding
+    [Python/pyarena.c](https://github.com/python/cpython/blob/main/Python/pyarena.c).
+
+  * [Include/opcode_ids.h](https://github.com/python/cpython/blob/main/Include/opcode_ids.h)
+    : List of opcodes. Generated from
+    [Python/bytecodes.c](https://github.com/python/cpython/blob/main/Python/bytecodes.c)
+    by
+    [Tools/cases_generator/opcode_id_generator.py](https://github.com/python/cpython/blob/main/Tools/cases_generator/opcode_id_generator.py).
+
+* [Objects/](https://github.com/python/cpython/blob/main/Objects/)
+
+  * [Objects/codeobject.c](https://github.com/python/cpython/blob/main/Objects/codeobject.c)
+    : Contains PyCodeObject-related code.
+
+  * [Objects/frameobject.c](https://github.com/python/cpython/blob/main/Objects/frameobject.c)
+    : Contains the ``frame_setlineno()`` function which should determine whether it is allowed
+    to make a jump between two points in a bytecode.
+
+* [Lib/](https://github.com/python/cpython/blob/main/Lib/)
+
+  * [Lib/opcode.py](https://github.com/python/cpython/blob/main/Lib/opcode.py)
+    : opcode utilities exposed to Python.
+
+  * [Lib/importlib/_bootstrap_external.py](https://github.com/python/cpython/blob/main/Lib/importlib/_bootstrap_external.py)
+    : Home of the magic number (named ``MAGIC_NUMBER``) for bytecode versioning.
+
+
+Objects
+=======
+
+* [Objects/locations.md](https://github.com/python/cpython/blob/main/Objects/locations.md): Describes the location table
+* [Objects/frame_layout.md](https://github.com/python/cpython/blob/main/Objects/frame_layout.md): Describes the frame stack
+* [Objects/object_layout.md](https://github.com/python/cpython/blob/main/Objects/object_layout.md): Descibes object layout for 3.11 and later
+* [Exception Handling](exception_handling.md): Describes the exception table
+
+
+Specializing Adaptive Interpreter
+=================================
+
+Adding a specializing, adaptive interpreter to CPython will bring significant
+performance improvements. These documents provide more information:
+
+* [PEP 659: Specializing Adaptive Interpreter](https://peps.python.org/pep-0659/).
+* [Adding or extending a family of adaptive instructions](adaptive.md)
+
+
+References
+==========
+
+[^1]:  Daniel C. Wang, Andrew W. Appel, Jeff L. Korn, and Chris
+       S. Serra.  `The Zephyr Abstract Syntax Description Language.`_
+       In Proceedings of the Conference on Domain-Specific Languages,
+       pp. 213--227, 1997.
+
+[^2]: The Zephyr Abstract Syntax Description Language.:
+      https://www.cs.princeton.edu/research/techreps/TR-554-97

From 7aff2de62bc28eb23888270b698c6b6915f69b21 Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Mon, 10 Jun 2024 18:34:17 +0200
Subject: [PATCH 39/72] gh-120057: Add os.environ.refresh() method (#120059)

---
 Doc/library/os.rst                            | 11 +++++
 Doc/whatsnew/3.14.rst                         |  7 +++
 Lib/os.py                                     | 25 ++++++++--
 Lib/test/test_os.py                           | 46 +++++++++++++++++++
 ...-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst |  4 ++
 Modules/clinic/posixmodule.c.h                | 20 +++++++-
 Modules/posixmodule.c                         | 15 ++++++
 7 files changed, 124 insertions(+), 4 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst

diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index b93b06d4e72afc..360d71e70960c7 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -193,6 +193,10 @@ process and user.
    to the environment made after this time are not reflected in :data:`os.environ`,
    except for changes made by modifying :data:`os.environ` directly.
 
+   The :meth:`!os.environ.refresh()` method updates :data:`os.environ` with
+   changes to the environment made by :func:`os.putenv`, by
+   :func:`os.unsetenv`, or made outside Python in the same process.
+
    This mapping may be used to modify the environment as well as query the
    environment.  :func:`putenv` will be called automatically when the mapping
    is modified.
@@ -225,6 +229,9 @@ process and user.
    .. versionchanged:: 3.9
       Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators.
 
+   .. versionchanged:: 3.14
+      Added the :meth:`!os.environ.refresh()` method.
+
 
 .. data:: environb
 
@@ -561,6 +568,8 @@ process and user.
    of :data:`os.environ`. This also applies to :func:`getenv` and :func:`getenvb`, which
    respectively use :data:`os.environ` and :data:`os.environb` in their implementations.
 
+   See also the :data:`os.environ.refresh() <os.environ>` method.
+
    .. note::
 
       On some platforms, including FreeBSD and macOS, setting ``environ`` may
@@ -809,6 +818,8 @@ process and user.
    don't update :data:`os.environ`, so it is actually preferable to delete items of
    :data:`os.environ`.
 
+   See also the :data:`os.environ.refresh() <os.environ>` method.
+
    .. audit-event:: os.unsetenv key os.unsetenv
 
    .. versionchanged:: 3.9
diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst
index b2dd80b64a691a..b77ff30a8fbbee 100644
--- a/Doc/whatsnew/3.14.rst
+++ b/Doc/whatsnew/3.14.rst
@@ -92,6 +92,13 @@ ast
 Added :func:`ast.compare` for comparing two ASTs.
 (Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`.)
 
+os
+--
+
+* Added the :data:`os.environ.refresh() <os.environ>` method to update
+  :data:`os.environ` with changes to the environment made by :func:`os.putenv`,
+  by :func:`os.unsetenv`, or made outside Python in the same process.
+  (Contributed by Victor Stinner in :gh:`120057`.)
 
 
 Optimizations
diff --git a/Lib/os.py b/Lib/os.py
index 0408e2db79e66e..4b48afb040e565 100644
--- a/Lib/os.py
+++ b/Lib/os.py
@@ -64,6 +64,10 @@ def _get_exports_list(module):
         from posix import _have_functions
     except ImportError:
         pass
+    try:
+        from posix import _create_environ
+    except ImportError:
+        pass
 
     import posix
     __all__.extend(_get_exports_list(posix))
@@ -88,6 +92,10 @@ def _get_exports_list(module):
         from nt import _have_functions
     except ImportError:
         pass
+    try:
+        from nt import _create_environ
+    except ImportError:
+        pass
 
 else:
     raise ImportError('no os specific module found')
@@ -773,7 +781,18 @@ def __ror__(self, other):
         new.update(self)
         return new
 
-def _createenviron():
+    if _exists("_create_environ"):
+        def refresh(self):
+            data = _create_environ()
+            if name == 'nt':
+                data = {self.encodekey(key): value
+                        for key, value in data.items()}
+
+            # modify in-place to keep os.environb in sync
+            self._data.clear()
+            self._data.update(data)
+
+def _create_environ_mapping():
     if name == 'nt':
         # Where Env Var Names Must Be UPPERCASE
         def check_str(value):
@@ -803,8 +822,8 @@ def decode(value):
         encode, decode)
 
 # unicode environ
-environ = _createenviron()
-del _createenviron
+environ = _create_environ_mapping()
+del _create_environ_mapping
 
 
 def getenv(key, default=None):
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 2beb9ca8aa6ccb..f93937fb587386 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -1298,6 +1298,52 @@ def test_ror_operator(self):
         self._test_underlying_process_env('_A_', '')
         self._test_underlying_process_env(overridden_key, original_value)
 
+    def test_refresh(self):
+        # Test os.environ.refresh()
+        has_environb = hasattr(os, 'environb')
+
+        # Test with putenv() which doesn't update os.environ
+        os.environ['test_env'] = 'python_value'
+        os.putenv("test_env", "new_value")
+        self.assertEqual(os.environ['test_env'], 'python_value')
+        if has_environb:
+            self.assertEqual(os.environb[b'test_env'], b'python_value')
+
+        os.environ.refresh()
+        self.assertEqual(os.environ['test_env'], 'new_value')
+        if has_environb:
+            self.assertEqual(os.environb[b'test_env'], b'new_value')
+
+        # Test with unsetenv() which doesn't update os.environ
+        os.unsetenv('test_env')
+        self.assertEqual(os.environ['test_env'], 'new_value')
+        if has_environb:
+            self.assertEqual(os.environb[b'test_env'], b'new_value')
+
+        os.environ.refresh()
+        self.assertNotIn('test_env', os.environ)
+        if has_environb:
+            self.assertNotIn(b'test_env', os.environb)
+
+        if has_environb:
+            # test os.environb.refresh() with putenv()
+            os.environb[b'test_env'] = b'python_value2'
+            os.putenv("test_env", "new_value2")
+            self.assertEqual(os.environb[b'test_env'], b'python_value2')
+            self.assertEqual(os.environ['test_env'], 'python_value2')
+
+            os.environb.refresh()
+            self.assertEqual(os.environb[b'test_env'], b'new_value2')
+            self.assertEqual(os.environ['test_env'], 'new_value2')
+
+            # test os.environb.refresh() with unsetenv()
+            os.unsetenv('test_env')
+            self.assertEqual(os.environb[b'test_env'], b'new_value2')
+            self.assertEqual(os.environ['test_env'], 'new_value2')
+
+            os.environb.refresh()
+            self.assertNotIn(b'test_env', os.environb)
+            self.assertNotIn('test_env', os.environ)
 
 class WalkTests(unittest.TestCase):
     """Tests for os.walk()."""
diff --git a/Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst b/Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst
new file mode 100644
index 00000000000000..955be59821ee0c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst
@@ -0,0 +1,4 @@
+Added the :data:`os.environ.refresh() <os.environ>` method to update
+:data:`os.environ` with changes to the environment made by :func:`os.putenv`,
+by :func:`os.unsetenv`, or made outside Python in the same process.
+Patch by Victor Stinner.
diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h
index 69fc178331c09c..07b28fef3a57ea 100644
--- a/Modules/clinic/posixmodule.c.h
+++ b/Modules/clinic/posixmodule.c.h
@@ -12152,6 +12152,24 @@ os__is_inputhook_installed(PyObject *module, PyObject *Py_UNUSED(ignored))
     return os__is_inputhook_installed_impl(module);
 }
 
+PyDoc_STRVAR(os__create_environ__doc__,
+"_create_environ($module, /)\n"
+"--\n"
+"\n"
+"Create the environment dictionary.");
+
+#define OS__CREATE_ENVIRON_METHODDEF    \
+    {"_create_environ", (PyCFunction)os__create_environ, METH_NOARGS, os__create_environ__doc__},
+
+static PyObject *
+os__create_environ_impl(PyObject *module);
+
+static PyObject *
+os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored))
+{
+    return os__create_environ_impl(module);
+}
+
 #ifndef OS_TTYNAME_METHODDEF
     #define OS_TTYNAME_METHODDEF
 #endif /* !defined(OS_TTYNAME_METHODDEF) */
@@ -12819,4 +12837,4 @@ os__is_inputhook_installed(PyObject *module, PyObject *Py_UNUSED(ignored))
 #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF
     #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF
 #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */
-/*[clinic end generated code: output=faaa5e5ffb7b165d input=a9049054013a1b77]*/
+/*[clinic end generated code: output=5ae2e5ffcd9c8a84 input=a9049054013a1b77]*/
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 5f943d4b1c8085..a8fd5c494769b5 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -16809,6 +16809,20 @@ os__is_inputhook_installed_impl(PyObject *module)
     return PyBool_FromLong(PyOS_InputHook != NULL);
 }
 
+/*[clinic input]
+os._create_environ
+
+Create the environment dictionary.
+[clinic start generated code]*/
+
+static PyObject *
+os__create_environ_impl(PyObject *module)
+/*[clinic end generated code: output=19d9039ab14f8ad4 input=a4c05686b34635e8]*/
+{
+    return convertenviron();
+}
+
+
 static PyMethodDef posix_methods[] = {
 
     OS_STAT_METHODDEF
@@ -17023,6 +17037,7 @@ static PyMethodDef posix_methods[] = {
     OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF
     OS__INPUTHOOK_METHODDEF
     OS__IS_INPUTHOOK_INSTALLED_METHODDEF
+    OS__CREATE_ENVIRON_METHODDEF
     {NULL,              NULL}            /* Sentinel */
 };
 

From 6efe3460693c4f39de198a64cebeeee8b1d4e8b6 Mon Sep 17 00:00:00 2001
From: AN Long <aisk@users.noreply.github.com>
Date: Tue, 11 Jun 2024 00:45:16 +0800
Subject: [PATCH 40/72] Fix the CODEOWNERS for _interpretersmodule.c
 (gh-120288)

---
 .github/CODEOWNERS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 811b8cfdab17dc..8bc40fcb9e8999 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -245,7 +245,7 @@ Doc/howto/clinic.rst          @erlend-aasland
 **/*interpreteridobject.*     @ericsnowcurrently
 **/*crossinterp*              @ericsnowcurrently
 Lib/test/support/interpreters/  @ericsnowcurrently
-Modules/_xx*interp*module.c   @ericsnowcurrently
+Modules/_interp*module.c      @ericsnowcurrently
 Lib/test/test_interpreters/   @ericsnowcurrently
 
 # Android

From 422c4fc855afd18bcc6415902ea1d85a50cb7ce1 Mon Sep 17 00:00:00 2001
From: Robert Collins <robert.collins@cognite.com>
Date: Tue, 11 Jun 2024 07:41:12 +0200
Subject: [PATCH 41/72] gh-119600: mock: do not access attributes of original
 when new_callable is set (#119601)

In order to patch flask.g e.g. as in #84982, that
proxies getattr must not be invoked. For that,
mock must not try to read from the original
object. In some cases that is unavoidable, e.g.
when doing autospec. However, patch("flask.g",
new_callable=MagicMock) should be entirely safe.
---
 Lib/test/test_unittest/testmock/support.py         | 11 +++++++++++
 Lib/test/test_unittest/testmock/testpatch.py       |  7 +++++++
 Lib/unittest/mock.py                               | 14 +++++++++-----
 .../2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst |  2 ++
 4 files changed, 29 insertions(+), 5 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst

diff --git a/Lib/test/test_unittest/testmock/support.py b/Lib/test/test_unittest/testmock/support.py
index 49986d65dc47af..6c535b7944f261 100644
--- a/Lib/test/test_unittest/testmock/support.py
+++ b/Lib/test/test_unittest/testmock/support.py
@@ -14,3 +14,14 @@ def wibble(self): pass
 
 class X(object):
     pass
+
+# A standin for weurkzeug.local.LocalProxy - issue 119600
+def _inaccessible(*args, **kwargs):
+    raise AttributeError
+
+
+class OpaqueProxy:
+    __getattribute__ = _inaccessible
+
+
+g = OpaqueProxy()
diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py
index be75fda7826af1..f26e74ce0bc1ba 100644
--- a/Lib/test/test_unittest/testmock/testpatch.py
+++ b/Lib/test/test_unittest/testmock/testpatch.py
@@ -2045,6 +2045,13 @@ def test(): pass
         with self.assertRaises(TypeError):
             test()
 
+    def test_patch_proxy_object(self):
+        @patch("test.test_unittest.testmock.support.g", new_callable=MagicMock())
+        def test(_):
+            pass
+
+        test()
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 3ef83e263f53b7..edabb4520c13cd 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -1508,13 +1508,12 @@ def __enter__(self):
                 if isinstance(original, type):
                     # If we're patching out a class and there is a spec
                     inherit = True
-            if spec is None and _is_async_obj(original):
-                Klass = AsyncMock
-            else:
-                Klass = MagicMock
-            _kwargs = {}
+
+            # Determine the Klass to use
             if new_callable is not None:
                 Klass = new_callable
+            elif spec is None and _is_async_obj(original):
+                Klass = AsyncMock
             elif spec is not None or spec_set is not None:
                 this_spec = spec
                 if spec_set is not None:
@@ -1527,7 +1526,12 @@ def __enter__(self):
                     Klass = AsyncMock
                 elif not_callable:
                     Klass = NonCallableMagicMock
+                else:
+                    Klass = MagicMock
+            else:
+                Klass = MagicMock
 
+            _kwargs = {}
             if spec is not None:
                 _kwargs['spec'] = spec
             if spec_set is not None:
diff --git a/Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst b/Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst
new file mode 100644
index 00000000000000..04c9ca9c3fd737
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst
@@ -0,0 +1,2 @@
+Fix :func:`unittest.mock.patch` to not read attributes of the target when
+``new_callable`` is set. Patch by Robert Collins.

From 9e9ee50421c857b443e2060274f17fb884d54473 Mon Sep 17 00:00:00 2001
From: blhsing <blhsing@gmail.com>
Date: Tue, 11 Jun 2024 13:42:49 +0800
Subject: [PATCH 42/72] gh-65454: avoid triggering call to a PropertyMock in
 NonCallableMock.__setattr__ (#120019)

---
 Lib/test/test_unittest/testmock/testhelpers.py            | 8 ++++++++
 Lib/unittest/mock.py                                      | 3 +++
 .../Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst | 1 +
 3 files changed, 12 insertions(+)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst

diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py
index 74785a83757a92..c9c20f008ca5a2 100644
--- a/Lib/test/test_unittest/testmock/testhelpers.py
+++ b/Lib/test/test_unittest/testmock/testhelpers.py
@@ -1127,6 +1127,14 @@ def test_propertymock_side_effect(self):
         p.assert_called_once_with()
 
 
+    def test_propertymock_attach(self):
+        m = Mock()
+        p = PropertyMock()
+        type(m).foo = p
+        m.attach_mock(p, 'foo')
+        self.assertEqual(m.mock_calls, [])
+
+
 class TestCallablePredicate(unittest.TestCase):
 
     def test_type(self):
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index edabb4520c13cd..08975e0e1bd132 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -830,6 +830,9 @@ def __setattr__(self, name, value):
             mock_name = f'{self._extract_mock_name()}.{name}'
             raise AttributeError(f'Cannot set {mock_name}')
 
+        if isinstance(value, PropertyMock):
+            self.__dict__[name] = value
+            return
         return object.__setattr__(self, name, value)
 
 
diff --git a/Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst b/Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst
new file mode 100644
index 00000000000000..0b232cf8ca1baf
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst
@@ -0,0 +1 @@
+:func:`unittest.mock.Mock.attach_mock` no longer triggers a call to a ``PropertyMock`` being attached.

From 141babad9b4eceb83371bf19ba3a36b50dd05250 Mon Sep 17 00:00:00 2001
From: Nikita Sobolev <mail@sobolevn.me>
Date: Tue, 11 Jun 2024 10:04:27 +0300
Subject: [PATCH 43/72] gh-120298: Fix use-after-free in
 `list_richcompare_impl` (#120303)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
---
 Lib/test/test_list.py                                 | 11 +++++++++++
 .../2024-06-10-10-42-48.gh-issue-120298.napREA.rst    |  2 ++
 Objects/listobject.c                                  |  9 ++++++++-
 3 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst

diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py
index 0601b33e79ebb6..d21429fae09b37 100644
--- a/Lib/test/test_list.py
+++ b/Lib/test/test_list.py
@@ -234,6 +234,17 @@ def __eq__(self, other):
         list4 = [1]
         self.assertFalse(list3 == list4)
 
+    def test_lt_operator_modifying_operand(self):
+        # See gh-120298
+        class evil:
+            def __lt__(self, other):
+                other.clear()
+                return NotImplemented
+
+        a = [[evil()]]
+        with self.assertRaises(TypeError):
+            a[0] < a
+
     @cpython_only
     def test_preallocation(self):
         iterable = [0] * 10
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst
new file mode 100644
index 00000000000000..531d39517ac423
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst	
@@ -0,0 +1,2 @@
+Fix use-after free in ``list_richcompare_impl`` which can be invoked via
+some specificly tailored evil input.
diff --git a/Objects/listobject.c b/Objects/listobject.c
index d09bb6391034d1..6829d5d28656cf 100644
--- a/Objects/listobject.c
+++ b/Objects/listobject.c
@@ -3382,7 +3382,14 @@ list_richcompare_impl(PyObject *v, PyObject *w, int op)
     }
 
     /* Compare the final item again using the proper operator */
-    return PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op);
+    PyObject *vitem = vl->ob_item[i];
+    PyObject *witem = wl->ob_item[i];
+    Py_INCREF(vitem);
+    Py_INCREF(witem);
+    PyObject *result = PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op);
+    Py_DECREF(vitem);
+    Py_DECREF(witem);
+    return result;
 }
 
 static PyObject *

From 7d2447137e117ea9a6ee1493bce0b071c76b1bd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= <mgorny@gentoo.org>
Date: Tue, 11 Jun 2024 09:11:13 +0200
Subject: [PATCH 44/72] gh-120291: Fix a bashism in python-config.sh.in
 (#120292)

gh-120291: Fix bashisms in python-config.sh.in

Replace the use of bash-specific `[[ ... ]]` with POSIX-compliant
`[ ... ]` to make the `python-config` shell script work with non-bash
shells again.  While at it, use `local` in a safer way, since it is
not in POSIX either (though universally supported).

Fixes #120291
---
 .../Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst | 1 +
 Misc/python-config.sh.in                                 | 9 +++++----
 2 files changed, 6 insertions(+), 4 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst

diff --git a/Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst b/Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst
new file mode 100644
index 00000000000000..d0bb297b51dc6e
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst
@@ -0,0 +1 @@
+Make the ``python-config`` shell script compatible with non-bash shells.
diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in
index c3c0b34fc1451d..9929f5b2653dca 100644
--- a/Misc/python-config.sh.in
+++ b/Misc/python-config.sh.in
@@ -4,11 +4,12 @@
 
 exit_with_usage ()
 {
-    local USAGE="Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed"
-    if [[ "$1" -eq 0 ]]; then
-        echo "$USAGE"
+    local usage
+    usage="Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed"
+    if [ "$1" -eq 0 ]; then
+        echo "$usage"
     else
-        echo "$USAGE" >&2
+        echo "$usage" >&2
     fi
     exit $1
 }

From 02c1dfff073a3dd6ce34a11b038defde291c2203 Mon Sep 17 00:00:00 2001
From: Kirill Podoprigora <kirill.bast9@mail.ru>
Date: Tue, 11 Jun 2024 10:56:38 +0300
Subject: [PATCH 45/72] gh-120080: Mark test_round_with_none_arg_direct_call as
 cpython_only (#120328)

---
 Lib/test/test_float.py | 1 +
 Lib/test/test_int.py   | 1 +
 2 files changed, 2 insertions(+)

diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py
index 53695cefb8fded..756cf9bd7719c0 100644
--- a/Lib/test/test_float.py
+++ b/Lib/test/test_float.py
@@ -949,6 +949,7 @@ def test_None_ndigits(self):
             self.assertEqual(x, 2)
             self.assertIsInstance(x, int)
 
+    @support.cpython_only
     def test_round_with_none_arg_direct_call(self):
         for val in [(1.0).__round__(None),
                     round(1.0),
diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py
index 77221dfb6d5aa2..2747d9219255ac 100644
--- a/Lib/test/test_int.py
+++ b/Lib/test/test_int.py
@@ -517,6 +517,7 @@ def test_issue31619(self):
         self.assertEqual(int('1_2_3_4_5_6_7_8_9', 16), 0x123456789)
         self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807)
 
+    @support.cpython_only
     def test_round_with_none_arg_direct_call(self):
         for val in [(1).__round__(None),
                     round(1),

From 9b8611eeea172cd4aa626ccd1ca333dc4093cd8c Mon Sep 17 00:00:00 2001
From: Jelle Zijlstra <jelle.zijlstra@gmail.com>
Date: Tue, 11 Jun 2024 07:06:49 -0600
Subject: [PATCH 46/72] gh-119180: PEP 649 compiler changes (#119361)

---
 .../pycore_global_objects_fini_generated.h    |   2 +-
 Include/internal/pycore_global_strings.h      |   2 +-
 Include/internal/pycore_opcode_utils.h        |   1 +
 .../internal/pycore_runtime_init_generated.h  |   2 +-
 Include/internal/pycore_symtable.h            |   8 +-
 .../internal/pycore_unicodeobject_generated.h |   3 -
 Lib/inspect.py                                |   8 +-
 Lib/symtable.py                               |   2 +
 Lib/test/test_dis.py                          |  29 +-
 Lib/test/test_grammar.py                      |  69 +---
 Lib/test/test_module/__init__.py              |   2 +
 Lib/test/test_opcodes.py                      |  13 +-
 Lib/test/test_positional_only_arg.py          |   5 +-
 Lib/test/test_pyclbr.py                       |   2 +
 Lib/test/test_pydoc/test_pydoc.py             |  11 +-
 Lib/test/test_pyrepl/test_interact.py         |   2 +-
 Lib/test/test_symtable.py                     |   6 +-
 Lib/test/test_traceback.py                    |   5 +-
 Lib/test/test_type_annotations.py             | 163 +++++++-
 Lib/test/test_typing.py                       |   4 +-
 Lib/test/typinganndata/ann_module.py          |   4 -
 Lib/typing.py                                 |  23 +-
 ...-05-22-06-22-47.gh-issue-119180.vZMiXm.rst |   1 +
 Python/bytecodes.c                            |   5 +
 Python/compile.c                              | 357 ++++++++++--------
 Python/executor_cases.c.h                     |   5 +
 Python/generated_cases.c.h                    |   5 +
 Python/symtable.c                             | 198 +++++++---
 28 files changed, 609 insertions(+), 328 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst

diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h
index b186408931c92e..30851dc2dbec44 100644
--- a/Include/internal/pycore_global_objects_fini_generated.h
+++ b/Include/internal/pycore_global_objects_fini_generated.h
@@ -559,6 +559,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults));
@@ -745,7 +746,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_align_));
-    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_annotation));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_));
diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h
index e1808c85acfb2d..009802c441685c 100644
--- a/Include/internal/pycore_global_strings.h
+++ b/Include/internal/pycore_global_strings.h
@@ -45,6 +45,7 @@ struct _Py_global_strings {
         STRUCT_FOR_STR(dot, ".")
         STRUCT_FOR_STR(dot_locals, ".<locals>")
         STRUCT_FOR_STR(empty, "")
+        STRUCT_FOR_STR(format, ".format")
         STRUCT_FOR_STR(generic_base, ".generic_base")
         STRUCT_FOR_STR(json_decoder, "json.decoder")
         STRUCT_FOR_STR(kwdefaults, ".kwdefaults")
@@ -234,7 +235,6 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(_abstract_)
         STRUCT_FOR_ID(_active)
         STRUCT_FOR_ID(_align_)
-        STRUCT_FOR_ID(_annotation)
         STRUCT_FOR_ID(_anonymous_)
         STRUCT_FOR_ID(_argtypes_)
         STRUCT_FOR_ID(_as_parameter_)
diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h
index b06e469dd5bd91..e76f4840a66891 100644
--- a/Include/internal/pycore_opcode_utils.h
+++ b/Include/internal/pycore_opcode_utils.h
@@ -57,6 +57,7 @@ extern "C" {
 #define MAKE_FUNCTION_KWDEFAULTS  0x02
 #define MAKE_FUNCTION_ANNOTATIONS 0x04
 #define MAKE_FUNCTION_CLOSURE     0x08
+#define MAKE_FUNCTION_ANNOTATE    0x10
 
 /* Values used as the oparg for LOAD_COMMON_CONSTANT */
 #define CONSTANT_ASSERTIONERROR 0
diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h
index 2dde6febc2cae4..ff5b6ee8e0f006 100644
--- a/Include/internal/pycore_runtime_init_generated.h
+++ b/Include/internal/pycore_runtime_init_generated.h
@@ -554,6 +554,7 @@ extern "C" {
     INIT_STR(dot, "."), \
     INIT_STR(dot_locals, ".<locals>"), \
     INIT_STR(empty, ""), \
+    INIT_STR(format, ".format"), \
     INIT_STR(generic_base, ".generic_base"), \
     INIT_STR(json_decoder, "json.decoder"), \
     INIT_STR(kwdefaults, ".kwdefaults"), \
@@ -743,7 +744,6 @@ extern "C" {
     INIT_ID(_abstract_), \
     INIT_ID(_active), \
     INIT_ID(_align_), \
-    INIT_ID(_annotation), \
     INIT_ID(_anonymous_), \
     INIT_ID(_argtypes_), \
     INIT_ID(_as_parameter_), \
diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h
index ac6c499c08264e..5d544765237df5 100644
--- a/Include/internal/pycore_symtable.h
+++ b/Include/internal/pycore_symtable.h
@@ -12,8 +12,9 @@ struct _mod;   // Type defined in pycore_ast.h
 
 typedef enum _block_type {
     FunctionBlock, ClassBlock, ModuleBlock,
-    // Used for annotations if 'from __future__ import annotations' is active.
-    // Annotation blocks cannot bind names and are not evaluated.
+    // Used for annotations. If 'from __future__ import annotations' is active,
+    // annotation blocks cannot bind names and are not evaluated. Otherwise, they
+    // are lazily evaluated (see PEP 649).
     AnnotationBlock,
     // Used for generics and type aliases. These work mostly like functions
     // (see PEP 695 for details). The three different blocks function identically;
@@ -89,6 +90,7 @@ typedef struct _symtable_entry {
                                      including free refs to globals */
     unsigned ste_generator : 1;   /* true if namespace is a generator */
     unsigned ste_coroutine : 1;   /* true if namespace is a coroutine */
+    unsigned ste_annotations_used : 1;  /* true if there are any annotations in this scope */
     _Py_comprehension_ty ste_comprehension;  /* Kind of comprehension (if any) */
     unsigned ste_varargs : 1;     /* true if block has varargs */
     unsigned ste_varkeywords : 1; /* true if block has varkeywords */
@@ -110,6 +112,7 @@ typedef struct _symtable_entry {
     int ste_end_col_offset;  /* end offset of first line of block */
     int ste_opt_lineno;      /* lineno of last exec or import * */
     int ste_opt_col_offset;  /* offset of last exec or import * */
+    struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */
     struct symtable *ste_table;
 } PySTEntryObject;
 
@@ -126,6 +129,7 @@ extern struct symtable* _PySymtable_Build(
     PyObject *filename,
     _PyFutureFeatures *future);
 extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *);
+extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject **);
 
 extern void _PySymtable_Free(struct symtable *);
 
diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h
index b00119a1bad7ff..69d93a9610a2e5 100644
--- a/Include/internal/pycore_unicodeobject_generated.h
+++ b/Include/internal/pycore_unicodeobject_generated.h
@@ -543,9 +543,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(_align_);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
-    string = &_Py_ID(_annotation);
-    assert(_PyUnicode_CheckConsistency(string, 1));
-    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(_anonymous_);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
diff --git a/Lib/inspect.py b/Lib/inspect.py
index 2b7f8bec482f8e..5570a43ebfea19 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -220,13 +220,7 @@ def get_annotations(obj, *, globals=None, locals=None, eval_str=False):
     """
     if isinstance(obj, type):
         # class
-        obj_dict = getattr(obj, '__dict__', None)
-        if obj_dict and hasattr(obj_dict, 'get'):
-            ann = obj_dict.get('__annotations__', None)
-            if isinstance(ann, types.GetSetDescriptorType):
-                ann = None
-        else:
-            ann = None
+        ann = obj.__annotations__
 
         obj_globals = None
         module_name = getattr(obj, '__module__', None)
diff --git a/Lib/symtable.py b/Lib/symtable.py
index ba2f0dafcd0063..af65e93e68eda4 100644
--- a/Lib/symtable.py
+++ b/Lib/symtable.py
@@ -222,6 +222,8 @@ def get_methods(self):
         if self.__methods is None:
             d = {}
             for st in self._table.children:
+                if st.type == _symtable.TYPE_ANNOTATION:
+                    continue
                 d[st.name] = 1
             self.__methods = tuple(d)
         return self.__methods
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
index b1a1b77c53e8cb..b0ae1289224070 100644
--- a/Lib/test/test_dis.py
+++ b/Lib/test/test_dis.py
@@ -352,32 +352,21 @@ def wrap_func_w_kwargs():
 dis_annot_stmt_str = """\
   0           RESUME                   0
 
-  2           SETUP_ANNOTATIONS
-              LOAD_CONST               0 (1)
+  2           LOAD_CONST               0 (1)
               STORE_NAME               0 (x)
-              LOAD_NAME                1 (int)
-              LOAD_NAME                2 (__annotations__)
-              LOAD_CONST               1 ('x')
-              STORE_SUBSCR
-
-  3           LOAD_NAME                3 (fun)
-              PUSH_NULL
-              LOAD_CONST               0 (1)
-              CALL                     1
-              LOAD_NAME                2 (__annotations__)
-              LOAD_CONST               2 ('y')
-              STORE_SUBSCR
 
   4           LOAD_CONST               0 (1)
-              LOAD_NAME                4 (lst)
-              LOAD_NAME                3 (fun)
+              LOAD_NAME                1 (lst)
+              LOAD_NAME                2 (fun)
               PUSH_NULL
-              LOAD_CONST               3 (0)
+              LOAD_CONST               1 (0)
               CALL                     1
               STORE_SUBSCR
-              LOAD_NAME                1 (int)
-              POP_TOP
-              RETURN_CONST             4 (None)
+
+  2           LOAD_CONST               2 (<code object __annotate__ at 0x..., file "<dis>", line 2>)
+              MAKE_FUNCTION
+              STORE_NAME               3 (__annotate__)
+              RETURN_CONST             3 (None)
 """
 
 compound_stmt_str = """\
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index c72f4387108ca8..5b7a639c025a0f 100644
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -306,16 +306,6 @@ def test_eof_error(self):
 
 var_annot_global: int # a global annotated is necessary for test_var_annot
 
-# custom namespace for testing __annotations__
-
-class CNS:
-    def __init__(self):
-        self._dct = {}
-    def __setitem__(self, item, value):
-        self._dct[item.lower()] = value
-    def __getitem__(self, item):
-        return self._dct[item]
-
 
 class GrammarTests(unittest.TestCase):
 
@@ -446,22 +436,12 @@ class F(C, A):
         self.assertEqual(E.__annotations__, {})
         self.assertEqual(F.__annotations__, {})
 
-
-    def test_var_annot_metaclass_semantics(self):
-        class CMeta(type):
-            @classmethod
-            def __prepare__(metacls, name, bases, **kwds):
-                return {'__annotations__': CNS()}
-        class CC(metaclass=CMeta):
-            XX: 'ANNOT'
-        self.assertEqual(CC.__annotations__['xx'], 'ANNOT')
-
     def test_var_annot_module_semantics(self):
         self.assertEqual(test.__annotations__, {})
         self.assertEqual(ann_module.__annotations__,
-                     {1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float})
+                         {'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float})
         self.assertEqual(ann_module.M.__annotations__,
-                              {'123': 123, 'o': type})
+                         {'o': type})
         self.assertEqual(ann_module2.__annotations__, {})
 
     def test_var_annot_in_module(self):
@@ -476,51 +456,12 @@ def test_var_annot_in_module(self):
             ann_module3.D_bad_ann(5)
 
     def test_var_annot_simple_exec(self):
-        gns = {}; lns= {}
+        gns = {}; lns = {}
         exec("'docstring'\n"
-             "__annotations__[1] = 2\n"
              "x: int = 5\n", gns, lns)
-        self.assertEqual(lns["__annotations__"], {1: 2, 'x': int})
-        with self.assertRaises(KeyError):
-            gns['__annotations__']
-
-    def test_var_annot_custom_maps(self):
-        # tests with custom locals() and __annotations__
-        ns = {'__annotations__': CNS()}
-        exec('X: int; Z: str = "Z"; (w): complex = 1j', ns)
-        self.assertEqual(ns['__annotations__']['x'], int)
-        self.assertEqual(ns['__annotations__']['z'], str)
+        self.assertEqual(lns["__annotate__"](1), {'x': int})
         with self.assertRaises(KeyError):
-            ns['__annotations__']['w']
-        nonloc_ns = {}
-        class CNS2:
-            def __init__(self):
-                self._dct = {}
-            def __setitem__(self, item, value):
-                nonlocal nonloc_ns
-                self._dct[item] = value
-                nonloc_ns[item] = value
-            def __getitem__(self, item):
-                return self._dct[item]
-        exec('x: int = 1', {}, CNS2())
-        self.assertEqual(nonloc_ns['__annotations__']['x'], int)
-
-    def test_var_annot_refleak(self):
-        # complex case: custom locals plus custom __annotations__
-        # this was causing refleak
-        cns = CNS()
-        nonloc_ns = {'__annotations__': cns}
-        class CNS2:
-            def __init__(self):
-                self._dct = {'__annotations__': cns}
-            def __setitem__(self, item, value):
-                nonlocal nonloc_ns
-                self._dct[item] = value
-                nonloc_ns[item] = value
-            def __getitem__(self, item):
-                return self._dct[item]
-        exec('X: str', {}, CNS2())
-        self.assertEqual(nonloc_ns['__annotations__']['x'], str)
+            gns['__annotate__']
 
     def test_var_annot_rhs(self):
         ns = {}
diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py
index 952ba43f72504d..56edd0c637f376 100644
--- a/Lib/test/test_module/__init__.py
+++ b/Lib/test/test_module/__init__.py
@@ -360,6 +360,8 @@ def test_annotations_are_created_correctly(self):
         ann_module4 = import_helper.import_fresh_module(
             'test.typinganndata.ann_module4',
         )
+        self.assertFalse("__annotations__" in ann_module4.__dict__)
+        self.assertEqual(ann_module4.__annotations__, {"a": int, "b": str})
         self.assertTrue("__annotations__" in ann_module4.__dict__)
         del ann_module4.__annotations__
         self.assertFalse("__annotations__" in ann_module4.__dict__)
diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py
index 72488b2bb6b4ff..f7cc8331b8d844 100644
--- a/Lib/test/test_opcodes.py
+++ b/Lib/test/test_opcodes.py
@@ -39,16 +39,19 @@ class C: pass
     def test_use_existing_annotations(self):
         ns = {'__annotations__': {1: 2}}
         exec('x: int', ns)
-        self.assertEqual(ns['__annotations__'], {'x': int, 1: 2})
+        self.assertEqual(ns['__annotations__'], {1: 2})
 
     def test_do_not_recreate_annotations(self):
         # Don't rely on the existence of the '__annotations__' global.
         with support.swap_item(globals(), '__annotations__', {}):
-            del globals()['__annotations__']
+            globals().pop('__annotations__', None)
             class C:
-                del __annotations__
-                with self.assertRaises(NameError):
-                    x: int
+                try:
+                    del __annotations__
+                except NameError:
+                    pass
+                x: int
+            self.assertEqual(C.__annotations__, {"x": int})
 
     def test_raise_class_exceptions(self):
 
diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py
index 1a193814d7535d..eea0625012da6d 100644
--- a/Lib/test/test_positional_only_arg.py
+++ b/Lib/test/test_positional_only_arg.py
@@ -2,6 +2,7 @@
 
 import dis
 import pickle
+import types
 import unittest
 
 from test.support import check_syntax_error
@@ -440,7 +441,9 @@ def f(x: not (int is int), /): ...
         # without constant folding we end up with
         # COMPARE_OP(is), IS_OP (0)
         # with constant folding we should expect a IS_OP (1)
-        codes = [(i.opname, i.argval) for i in dis.get_instructions(g)]
+        code_obj = next(const for const in g.__code__.co_consts
+                        if isinstance(const, types.CodeType) and const.co_name == "__annotate__")
+        codes = [(i.opname, i.argval) for i in dis.get_instructions(code_obj)]
         self.assertNotIn(('UNARY_NOT', None), codes)
         self.assertIn(('IS_OP', 1), codes)
 
diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py
index 46206accbafc36..0c12a3085b12af 100644
--- a/Lib/test/test_pyclbr.py
+++ b/Lib/test/test_pyclbr.py
@@ -109,6 +109,8 @@ def ismethod(oclass, obj, name):
 
                 actualMethods = []
                 for m in py_item.__dict__.keys():
+                    if m == "__annotate__":
+                        continue
                     if ismethod(py_item, getattr(py_item, m), m):
                         actualMethods.append(m)
                 foundMethods = []
diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py
index 57e5b8e8abddfa..a17c16cc73cf0e 100644
--- a/Lib/test/test_pydoc/test_pydoc.py
+++ b/Lib/test/test_pydoc/test_pydoc.py
@@ -77,6 +77,11 @@ class A(builtins.object)
      |  __weakref__%s
 
     class B(builtins.object)
+     |  Methods defined here:
+     |
+     |  __annotate__(...)
+     |
+     |  ----------------------------------------------------------------------
      |  Data descriptors defined here:
      |
      |  __dict__%s
@@ -87,8 +92,6 @@ class B(builtins.object)
      |  Data and other attributes defined here:
      |
      |  NO_MEANING = 'eggs'
-     |
-     |  __annotations__ = {'NO_MEANING': <class 'str'>}
 
     class C(builtins.object)
      |  Methods defined here:
@@ -176,6 +179,9 @@ class A(builtins.object)
             list of weak references to the object
 
 class B(builtins.object)
+    Methods defined here:
+        __annotate__(...)
+    ----------------------------------------------------------------------
     Data descriptors defined here:
         __dict__
             dictionary for instance variables
@@ -184,7 +190,6 @@ class B(builtins.object)
     ----------------------------------------------------------------------
     Data and other attributes defined here:
         NO_MEANING = 'eggs'
-        __annotations__ = {'NO_MEANING': <class 'str'>}
 
 
 class C(builtins.object)
diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py
index df97b1354a168e..31f08cdb25e078 100644
--- a/Lib/test/test_pyrepl/test_interact.py
+++ b/Lib/test/test_pyrepl/test_interact.py
@@ -105,7 +105,7 @@ def test_runsource_shows_syntax_error_for_failed_compilation(self):
 
     def test_no_active_future(self):
         console = InteractiveColoredConsole()
-        source = "x: int = 1; print(__annotations__)"
+        source = "x: int = 1; print(__annotate__(1))"
         f = io.StringIO()
         with contextlib.redirect_stdout(f):
             result = console.runsource(source)
diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py
index ef2a228b15ed4e..a4b111e865c86e 100644
--- a/Lib/test/test_symtable.py
+++ b/Lib/test/test_symtable.py
@@ -205,12 +205,14 @@ def test_assigned(self):
 
     def test_annotated(self):
         st1 = symtable.symtable('def f():\n    x: int\n', 'test', 'exec')
-        st2 = st1.get_children()[0]
+        st2 = st1.get_children()[1]
+        self.assertEqual(st2.get_type(), "function")
         self.assertTrue(st2.lookup('x').is_local())
         self.assertTrue(st2.lookup('x').is_annotated())
         self.assertFalse(st2.lookup('x').is_global())
         st3 = symtable.symtable('def f():\n    x = 1\n', 'test', 'exec')
-        st4 = st3.get_children()[0]
+        st4 = st3.get_children()[1]
+        self.assertEqual(st4.get_type(), "function")
         self.assertTrue(st4.lookup('x').is_local())
         self.assertFalse(st4.lookup('x').is_annotated())
 
diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py
index 5035de114b5e9d..1895c88d23b70d 100644
--- a/Lib/test/test_traceback.py
+++ b/Lib/test/test_traceback.py
@@ -622,6 +622,7 @@ def test_caret_in_type_annotation(self):
         def f_with_type():
             def foo(a: THIS_DOES_NOT_EXIST ) -> int:
                 return 0
+            foo.__annotations__
 
         lineno_f = f_with_type.__code__.co_firstlineno
         expected_f = (
@@ -629,7 +630,9 @@ def foo(a: THIS_DOES_NOT_EXIST ) -> int:
             f'  File "{__file__}", line {self.callable_line}, in get_exception\n'
             '    callable()\n'
             '    ~~~~~~~~^^\n'
-            f'  File "{__file__}", line {lineno_f+1}, in f_with_type\n'
+            f'  File "{__file__}", line {lineno_f+3}, in f_with_type\n'
+            '    foo.__annotations__\n'
+            f'  File "{__file__}", line {lineno_f+1}, in __annotate__\n'
             '    def foo(a: THIS_DOES_NOT_EXIST ) -> int:\n'
             '               ^^^^^^^^^^^^^^^^^^^\n'
         )
diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py
index 5e3c3347a41571..a9be1f5aa84681 100644
--- a/Lib/test/test_type_annotations.py
+++ b/Lib/test/test_type_annotations.py
@@ -1,7 +1,12 @@
 import textwrap
 import types
 import unittest
-from test.support import run_code
+from test.support import run_code, check_syntax_error
+
+VALUE = 1
+FORWARDREF = 2
+SOURCE = 3
+
 
 class TypeAnnotationTests(unittest.TestCase):
 
@@ -49,6 +54,7 @@ def test_annotations_are_created_correctly(self):
         class C:
             a:int=3
             b:str=4
+        self.assertEqual(C.__annotations__, {"a": int, "b": str})
         self.assertTrue("__annotations__" in C.__dict__)
         del C.__annotations__
         self.assertFalse("__annotations__" in C.__dict__)
@@ -106,6 +112,13 @@ class D(metaclass=C):
         self.assertEqual(D.__annotations__, {})
 
 
+def build_module(code: str, name: str = "top") -> types.ModuleType:
+    ns = run_code(code)
+    mod = types.ModuleType(name)
+    mod.__dict__.update(ns)
+    return mod
+
+
 class TestSetupAnnotations(unittest.TestCase):
     def check(self, code: str):
         code = textwrap.dedent(code)
@@ -113,11 +126,10 @@ def check(self, code: str):
             with self.subTest(scope=scope):
                 if scope == "class":
                     code = f"class C:\n{textwrap.indent(code, '    ')}"
-                ns = run_code(code)
-                if scope == "class":
+                    ns = run_code(code)
                     annotations = ns["C"].__annotations__
                 else:
-                    annotations = ns["__annotations__"]
+                    annotations = build_module(code).__annotations__
                 self.assertEqual(annotations, {"x": int})
 
     def test_top_level(self):
@@ -256,3 +268,146 @@ def check_annotations(self, f):
         # Setting f.__annotations__ also clears __annotate__
         f.__annotations__ = {"z": 43}
         self.assertIs(f.__annotate__, None)
+
+
+class DeferredEvaluationTests(unittest.TestCase):
+    def test_function(self):
+        def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined:
+            pass
+
+        with self.assertRaises(NameError):
+            func.__annotations__
+
+        undefined = 1
+        self.assertEqual(func.__annotations__, {
+            "x": 1,
+            "y": 1,
+            "args": 1,
+            "z": 1,
+            "kwargs": 1,
+            "return": 1,
+        })
+
+    def test_async_function(self):
+        async def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined:
+            pass
+
+        with self.assertRaises(NameError):
+            func.__annotations__
+
+        undefined = 1
+        self.assertEqual(func.__annotations__, {
+            "x": 1,
+            "y": 1,
+            "args": 1,
+            "z": 1,
+            "kwargs": 1,
+            "return": 1,
+        })
+
+    def test_class(self):
+        class X:
+            a: undefined
+
+        with self.assertRaises(NameError):
+            X.__annotations__
+
+        undefined = 1
+        self.assertEqual(X.__annotations__, {"a": 1})
+
+    def test_module(self):
+        ns = run_code("x: undefined = 1")
+        anno = ns["__annotate__"]
+        with self.assertRaises(NotImplementedError):
+            anno(2)
+
+        with self.assertRaises(NameError):
+            anno(1)
+
+        ns["undefined"] = 1
+        self.assertEqual(anno(1), {"x": 1})
+
+    def test_class_scoping(self):
+        class Outer:
+            def meth(self, x: Nested): ...
+            x: Nested
+            class Nested: ...
+
+        self.assertEqual(Outer.meth.__annotations__, {"x": Outer.Nested})
+        self.assertEqual(Outer.__annotations__, {"x": Outer.Nested})
+
+    def test_no_exotic_expressions(self):
+        check_syntax_error(self, "def func(x: (yield)): ...", "yield expression cannot be used within an annotation")
+        check_syntax_error(self, "def func(x: (yield from x)): ...", "yield expression cannot be used within an annotation")
+        check_syntax_error(self, "def func(x: (y := 3)): ...", "named expression cannot be used within an annotation")
+        check_syntax_error(self, "def func(x: (await 42)): ...", "await expression cannot be used within an annotation")
+
+    def test_no_exotic_expressions_in_unevaluated_annotations(self):
+        preludes = [
+            "",
+            "class X: ",
+            "def f(): ",
+            "async def f(): ",
+        ]
+        for prelude in preludes:
+            with self.subTest(prelude=prelude):
+                check_syntax_error(self, prelude + "(x): (yield)", "yield expression cannot be used within an annotation")
+                check_syntax_error(self, prelude + "(x): (yield from x)", "yield expression cannot be used within an annotation")
+                check_syntax_error(self, prelude + "(x): (y := 3)", "named expression cannot be used within an annotation")
+                check_syntax_error(self, prelude + "(x): (await 42)", "await expression cannot be used within an annotation")
+
+    def test_ignore_non_simple_annotations(self):
+        ns = run_code("class X: (y): int")
+        self.assertEqual(ns["X"].__annotations__, {})
+        ns = run_code("class X: int.b: int")
+        self.assertEqual(ns["X"].__annotations__, {})
+        ns = run_code("class X: int[str]: int")
+        self.assertEqual(ns["X"].__annotations__, {})
+
+    def test_generated_annotate(self):
+        def func(x: int):
+            pass
+        class X:
+            x: int
+        mod = build_module("x: int")
+        for obj in (func, X, mod):
+            with self.subTest(obj=obj):
+                annotate = obj.__annotate__
+                self.assertIsInstance(annotate, types.FunctionType)
+                self.assertEqual(annotate.__name__, "__annotate__")
+                with self.assertRaises(NotImplementedError):
+                    annotate(FORWARDREF)
+                with self.assertRaises(NotImplementedError):
+                    annotate(SOURCE)
+                with self.assertRaises(NotImplementedError):
+                    annotate(None)
+                self.assertEqual(annotate(VALUE), {"x": int})
+
+    def test_comprehension_in_annotation(self):
+        # This crashed in an earlier version of the code
+        ns = run_code("x: [y for y in range(10)]")
+        self.assertEqual(ns["__annotate__"](1), {"x": list(range(10))})
+
+    def test_future_annotations(self):
+        code = """
+        from __future__ import annotations
+
+        def f(x: int) -> int: pass
+        """
+        ns = run_code(code)
+        f = ns["f"]
+        self.assertIsInstance(f.__annotate__, types.FunctionType)
+        annos = {"x": "int", "return": "int"}
+        self.assertEqual(f.__annotate__(VALUE), annos)
+        self.assertEqual(f.__annotations__, annos)
+
+    def test_name_clash_with_format(self):
+        # this test would fail if __annotate__'s parameter was called "format"
+        code = """
+        class format: pass
+
+        def f(x: format): pass
+        """
+        ns = run_code(code)
+        f = ns["f"]
+        self.assertEqual(f.__annotations__, {"x": ns["format"]})
diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index dac55ceb9e99e0..9800b3b6a7da29 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -6634,7 +6634,7 @@ def test_get_type_hints_from_various_objects(self):
             gth(None)
 
     def test_get_type_hints_modules(self):
-        ann_module_type_hints = {1: 2, 'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float}
+        ann_module_type_hints = {'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float}
         self.assertEqual(gth(ann_module), ann_module_type_hints)
         self.assertEqual(gth(ann_module2), {})
         self.assertEqual(gth(ann_module3), {})
@@ -6652,7 +6652,7 @@ def test_get_type_hints_classes(self):
         self.assertEqual(gth(ann_module.C),  # gth will find the right globalns
                          {'y': Optional[ann_module.C]})
         self.assertIsInstance(gth(ann_module.j_class), dict)
-        self.assertEqual(gth(ann_module.M), {'123': 123, 'o': type})
+        self.assertEqual(gth(ann_module.M), {'o': type})
         self.assertEqual(gth(ann_module.D),
                          {'j': str, 'k': str, 'y': Optional[ann_module.C]})
         self.assertEqual(gth(ann_module.Y), {'z': int})
diff --git a/Lib/test/typinganndata/ann_module.py b/Lib/test/typinganndata/ann_module.py
index 5081e6b58345a9..e1a1792cb4a867 100644
--- a/Lib/test/typinganndata/ann_module.py
+++ b/Lib/test/typinganndata/ann_module.py
@@ -8,8 +8,6 @@
 from typing import Optional
 from functools import wraps
 
-__annotations__[1] = 2
-
 class C:
 
     x = 5; y: Optional['C'] = None
@@ -18,8 +16,6 @@ class C:
 x: int = 5; y: str = x; f: Tuple[int, int]
 
 class M(type):
-
-    __annotations__['123'] = 123
     o: type = object
 
 (pars): bool = True
diff --git a/Lib/typing.py b/Lib/typing.py
index be49aa63464f05..7a9149d3f3c2c1 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -2412,7 +2412,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False):
                 base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {})
             else:
                 base_globals = globalns
-            ann = base.__dict__.get('__annotations__', {})
+            ann = getattr(base, '__annotations__', {})
             if isinstance(ann, types.GetSetDescriptorType):
                 ann = {}
             base_locals = dict(vars(base)) if localns is None else localns
@@ -2970,7 +2970,12 @@ def __new__(cls, typename, bases, ns):
                 raise TypeError(
                     'can only inherit from a NamedTuple type and Generic')
         bases = tuple(tuple if base is _NamedTuple else base for base in bases)
-        types = ns.get('__annotations__', {})
+        if "__annotations__" in ns:
+            types = ns["__annotations__"]
+        elif "__annotate__" in ns:
+            types = ns["__annotate__"](1)  # VALUE
+        else:
+            types = {}
         default_names = []
         for field_name in types:
             if field_name in ns:
@@ -3131,7 +3136,12 @@ def __new__(cls, name, bases, ns, total=True):
             tp_dict.__orig_bases__ = bases
 
         annotations = {}
-        own_annotations = ns.get('__annotations__', {})
+        if "__annotations__" in ns:
+            own_annotations = ns["__annotations__"]
+        elif "__annotate__" in ns:
+            own_annotations = ns["__annotate__"](1)  # VALUE
+        else:
+            own_annotations = {}
         msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type"
         own_annotations = {
             n: _type_check(tp, msg, module=tp_dict.__module__)
@@ -3143,7 +3153,12 @@ def __new__(cls, name, bases, ns, total=True):
         mutable_keys = set()
 
         for base in bases:
-            annotations.update(base.__dict__.get('__annotations__', {}))
+            # TODO: Avoid eagerly evaluating annotations in VALUE format.
+            # Instead, evaluate in FORWARDREF format to figure out which
+            # keys have Required/NotRequired/ReadOnly qualifiers, and create
+            # a new __annotate__ function for the resulting TypedDict that
+            # combines the annotations from this class and its parents.
+            annotations.update(base.__annotations__)
 
             base_required = base.__dict__.get('__required_keys__', set())
             required_keys |= base_required
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst
new file mode 100644
index 00000000000000..265ffb32e6a1f9
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst	
@@ -0,0 +1 @@
+Evaluation of annotations is now deferred. See :pep:`649` for details.
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 413ad1105f9428..05c17ac334b69f 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -3975,6 +3975,11 @@ dummy_func(
                     assert(func_obj->func_defaults == NULL);
                     func_obj->func_defaults = attr;
                     break;
+                case MAKE_FUNCTION_ANNOTATE:
+                    assert(PyCallable_Check(attr));
+                    assert(func_obj->func_annotate == NULL);
+                    func_obj->func_annotate = attr;
+                    break;
                 default:
                     Py_UNREACHABLE();
             }
diff --git a/Python/compile.c b/Python/compile.c
index cb724154206b7e..c3372766d0bd50 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -132,7 +132,7 @@ enum {
     COMPILER_SCOPE_ASYNC_FUNCTION,
     COMPILER_SCOPE_LAMBDA,
     COMPILER_SCOPE_COMPREHENSION,
-    COMPILER_SCOPE_TYPEPARAMS,
+    COMPILER_SCOPE_ANNOTATIONS,
 };
 
 
@@ -142,6 +142,15 @@ typedef _PyInstructionSequence instr_sequence;
 #define INITIAL_INSTR_SEQUENCE_SIZE 100
 #define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10
 
+static const int compare_masks[] = {
+    [Py_LT] = COMPARISON_LESS_THAN,
+    [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS,
+    [Py_EQ] = COMPARISON_EQUALS,
+    [Py_NE] = COMPARISON_NOT_EQUALS,
+    [Py_GT] = COMPARISON_GREATER_THAN,
+    [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS,
+};
+
 /*
  * Resize the array if index is out of range.
  *
@@ -208,6 +217,7 @@ struct compiler_unit {
 
     PyObject *u_private;            /* for private name mangling */
     PyObject *u_static_attributes;  /* for class: attributes accessed via self.X */
+    PyObject *u_deferred_annotations; /* AnnAssign nodes deferred to the end of compilation */
 
     instr_sequence *u_instr_sequence; /* codegen output */
 
@@ -330,6 +340,8 @@ static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *);
 static int compiler_match(struct compiler *, stmt_ty);
 static int compiler_pattern_subpattern(struct compiler *,
                                        pattern_ty, pattern_context *);
+static int compiler_make_closure(struct compiler *c, location loc,
+                                 PyCodeObject *co, Py_ssize_t flags);
 
 static PyCodeObject *optimize_and_assemble(struct compiler *, int addNone);
 
@@ -545,6 +557,7 @@ compiler_unit_free(struct compiler_unit *u)
     Py_CLEAR(u->u_metadata.u_fasthidden);
     Py_CLEAR(u->u_private);
     Py_CLEAR(u->u_static_attributes);
+    Py_CLEAR(u->u_deferred_annotations);
     PyMem_Free(u);
 }
 
@@ -582,8 +595,8 @@ compiler_set_qualname(struct compiler *c)
         capsule = PyList_GET_ITEM(c->c_stack, stack_size - 1);
         parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME);
         assert(parent);
-        if (parent->u_scope_type == COMPILER_SCOPE_TYPEPARAMS) {
-            /* The parent is a type parameter scope, so we need to
+        if (parent->u_scope_type == COMPILER_SCOPE_ANNOTATIONS) {
+            /* The parent is an annotation scope, so we need to
                look at the grandparent. */
             if (stack_size == 2) {
                 // If we're immediately within the module, we can skip
@@ -1128,6 +1141,7 @@ compiler_enter_scope(struct compiler *c, identifier name,
     }
 
     u->u_private = NULL;
+    u->u_deferred_annotations = NULL;
     if (scope_type == COMPILER_SCOPE_CLASS) {
         u->u_static_attributes = PySet_New(0);
         if (!u->u_static_attributes) {
@@ -1209,85 +1223,6 @@ compiler_exit_scope(struct compiler *c)
     PyErr_SetRaisedException(exc);
 }
 
-/* Search if variable annotations are present statically in a block. */
-
-static bool
-find_ann(asdl_stmt_seq *stmts)
-{
-    int i, j, res = 0;
-    stmt_ty st;
-
-    for (i = 0; i < asdl_seq_LEN(stmts); i++) {
-        st = (stmt_ty)asdl_seq_GET(stmts, i);
-        switch (st->kind) {
-        case AnnAssign_kind:
-            return true;
-        case For_kind:
-            res = find_ann(st->v.For.body) ||
-                  find_ann(st->v.For.orelse);
-            break;
-        case AsyncFor_kind:
-            res = find_ann(st->v.AsyncFor.body) ||
-                  find_ann(st->v.AsyncFor.orelse);
-            break;
-        case While_kind:
-            res = find_ann(st->v.While.body) ||
-                  find_ann(st->v.While.orelse);
-            break;
-        case If_kind:
-            res = find_ann(st->v.If.body) ||
-                  find_ann(st->v.If.orelse);
-            break;
-        case With_kind:
-            res = find_ann(st->v.With.body);
-            break;
-        case AsyncWith_kind:
-            res = find_ann(st->v.AsyncWith.body);
-            break;
-        case Try_kind:
-            for (j = 0; j < asdl_seq_LEN(st->v.Try.handlers); j++) {
-                excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
-                    st->v.Try.handlers, j);
-                if (find_ann(handler->v.ExceptHandler.body)) {
-                    return true;
-                }
-            }
-            res = find_ann(st->v.Try.body) ||
-                  find_ann(st->v.Try.finalbody) ||
-                  find_ann(st->v.Try.orelse);
-            break;
-        case TryStar_kind:
-            for (j = 0; j < asdl_seq_LEN(st->v.TryStar.handlers); j++) {
-                excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
-                    st->v.TryStar.handlers, j);
-                if (find_ann(handler->v.ExceptHandler.body)) {
-                    return true;
-                }
-            }
-            res = find_ann(st->v.TryStar.body) ||
-                  find_ann(st->v.TryStar.finalbody) ||
-                  find_ann(st->v.TryStar.orelse);
-            break;
-        case Match_kind:
-            for (j = 0; j < asdl_seq_LEN(st->v.Match.cases); j++) {
-                match_case_ty match_case = (match_case_ty)asdl_seq_GET(
-                    st->v.Match.cases, j);
-                if (find_ann(match_case->body)) {
-                    return true;
-                }
-            }
-            break;
-        default:
-            res = false;
-            break;
-        }
-        if (res) {
-            break;
-        }
-    }
-    return res;
-}
-
 /*
  * Frame block handling functions
  */
@@ -1502,6 +1437,47 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc,
     return SUCCESS;
 }
 
+static int
+compiler_setup_annotations_scope(struct compiler *c, location loc,
+                                 void *key, PyObject *name)
+{
+    if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS,
+                             key, loc.lineno) == -1) {
+        return ERROR;
+    }
+    c->u->u_metadata.u_posonlyargcount = 1;
+    // if .format != 1: raise NotImplementedError
+    _Py_DECLARE_STR(format, ".format");
+    ADDOP_I(c, loc, LOAD_FAST, 0);
+    ADDOP_LOAD_CONST(c, loc, _PyLong_GetOne());
+    ADDOP_I(c, loc, COMPARE_OP, (Py_NE << 5) | compare_masks[Py_NE]);
+    NEW_JUMP_TARGET_LABEL(c, body);
+    ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body);
+    ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR);
+    ADDOP_I(c, loc, RAISE_VARARGS, 1);
+    USE_LABEL(c, body);
+    return 0;
+}
+
+static int
+compiler_leave_annotations_scope(struct compiler *c, location loc,
+                                 Py_ssize_t annotations_len)
+{
+    ADDOP_I(c, loc, BUILD_MAP, annotations_len);
+    ADDOP_IN_SCOPE(c, loc, RETURN_VALUE);
+    PyCodeObject *co = optimize_and_assemble(c, 1);
+    compiler_exit_scope(c);
+    if (co == NULL) {
+        return ERROR;
+    }
+    if (compiler_make_closure(c, loc, co, 0) < 0) {
+        Py_DECREF(co);
+        return ERROR;
+    }
+    Py_DECREF(co);
+    return 0;
+}
+
 /* Compile a sequence of statements, checking for a docstring
    and for annotations. */
 
@@ -1517,34 +1493,79 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts)
         stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
         loc = LOC(st);
     }
-    /* Every annotated class and module should have __annotations__. */
-    if (find_ann(stmts)) {
+    /* If from __future__ import annotations is active,
+     * every annotated class and module should have __annotations__.
+     * Else __annotate__ is created when necessary. */
+    if ((c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) && c->u->u_ste->ste_annotations_used) {
         ADDOP(c, loc, SETUP_ANNOTATIONS);
     }
     if (!asdl_seq_LEN(stmts)) {
         return SUCCESS;
     }
     Py_ssize_t first_instr = 0;
-    PyObject *docstring = _PyAST_GetDocString(stmts);
-    if (docstring) {
-        first_instr = 1;
-        /* if not -OO mode, set docstring */
-        if (c->c_optimize < 2) {
-            PyObject *cleandoc = _PyCompile_CleanDoc(docstring);
-            if (cleandoc == NULL) {
-                return ERROR;
+    if (!c->c_interactive) {
+        PyObject *docstring = _PyAST_GetDocString(stmts);
+        if (docstring) {
+            first_instr = 1;
+            /* if not -OO mode, set docstring */
+            if (c->c_optimize < 2) {
+                PyObject *cleandoc = _PyCompile_CleanDoc(docstring);
+                if (cleandoc == NULL) {
+                    return ERROR;
+                }
+                stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
+                assert(st->kind == Expr_kind);
+                location loc = LOC(st->v.Expr.value);
+                ADDOP_LOAD_CONST(c, loc, cleandoc);
+                Py_DECREF(cleandoc);
+                RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store));
             }
-            stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
-            assert(st->kind == Expr_kind);
-            location loc = LOC(st->v.Expr.value);
-            ADDOP_LOAD_CONST(c, loc, cleandoc);
-            Py_DECREF(cleandoc);
-            RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store));
         }
     }
     for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) {
         VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i));
     }
+    // If there are annotations and the future import is not on, we
+    // collect the annotations in a separate pass and generate an
+    // __annotate__ function. See PEP 649.
+    if (!(c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) &&
+         c->u->u_deferred_annotations != NULL) {
+
+        // It's possible that ste_annotations_block is set but
+        // u_deferred_annotations is not, because the former is still
+        // set if there are only non-simple annotations (i.e., annotations
+        // for attributes, subscripts, or parenthesized names). However, the
+        // reverse should not be possible.
+        assert(c->u->u_ste->ste_annotation_block != NULL);
+        PyObject *deferred_anno = Py_NewRef(c->u->u_deferred_annotations);
+        void *key = (void *)((uintptr_t)c->u->u_ste->ste_id + 1);
+        if (compiler_setup_annotations_scope(c, loc, key,
+                                             c->u->u_ste->ste_annotation_block->ste_name) == -1) {
+            Py_DECREF(deferred_anno);
+            return ERROR;
+        }
+        Py_ssize_t annotations_len = PyList_Size(deferred_anno);
+        for (Py_ssize_t i = 0; i < annotations_len; i++) {
+            PyObject *ptr = PyList_GET_ITEM(deferred_anno, i);
+            stmt_ty st = (stmt_ty)PyLong_AsVoidPtr(ptr);
+            if (st == NULL) {
+                compiler_exit_scope(c);
+                Py_DECREF(deferred_anno);
+                return ERROR;
+            }
+            PyObject *mangled = _Py_Mangle(c->u->u_private, st->v.AnnAssign.target->v.Name.id);
+            ADDOP_LOAD_CONST_NEW(c, LOC(st), mangled);
+            VISIT(c, expr, st->v.AnnAssign.annotation);
+        }
+        Py_DECREF(deferred_anno);
+
+        RETURN_IF_ERROR(
+            compiler_leave_annotations_scope(c, loc, annotations_len)
+        );
+        RETURN_IF_ERROR(
+            compiler_nameop(c, loc, &_Py_ID(__annotate__), Store)
+        );
+    }
     return SUCCESS;
 }
 
@@ -1559,11 +1580,10 @@ compiler_codegen(struct compiler *c, mod_ty mod)
         }
         break;
     case Interactive_kind:
-        if (find_ann(mod->v.Interactive.body)) {
-            ADDOP(c, loc, SETUP_ANNOTATIONS);
-        }
         c->c_interactive = 1;
-        VISIT_SEQ(c, stmt, mod->v.Interactive.body);
+        if (compiler_body(c, loc, mod->v.Interactive.body) < 0) {
+            return ERROR;
+        }
         break;
     case Expression_kind:
         VISIT(c, expr, mod->v.Expression.body);
@@ -1702,6 +1722,9 @@ compiler_make_closure(struct compiler *c, location loc,
     if (flags & MAKE_FUNCTION_ANNOTATIONS) {
         ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATIONS);
     }
+    if (flags & MAKE_FUNCTION_ANNOTATE) {
+        ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATE);
+    }
     if (flags & MAKE_FUNCTION_KWDEFAULTS) {
         ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_KWDEFAULTS);
     }
@@ -1833,7 +1856,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id,
             VISIT(c, expr, annotation);
         }
     }
-    *annotations_len += 2;
+    *annotations_len += 1;
     return SUCCESS;
 }
 
@@ -1856,43 +1879,76 @@ compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args,
 }
 
 static int
-compiler_visit_annotations(struct compiler *c, location loc,
-                           arguments_ty args, expr_ty returns)
+compiler_visit_annotations_in_scope(struct compiler *c, location loc,
+                                    arguments_ty args, expr_ty returns,
+                                    Py_ssize_t *annotations_len)
 {
-    /* Push arg annotation names and values.
-       The expressions are evaluated out-of-order wrt the source code.
-
-       Return -1 on error, 0 if no annotations pushed, 1 if a annotations is pushed.
-       */
-    Py_ssize_t annotations_len = 0;
-
     RETURN_IF_ERROR(
-        compiler_visit_argannotations(c, args->args, &annotations_len, loc));
+        compiler_visit_argannotations(c, args->args, annotations_len, loc));
 
     RETURN_IF_ERROR(
-        compiler_visit_argannotations(c, args->posonlyargs, &annotations_len, loc));
+        compiler_visit_argannotations(c, args->posonlyargs, annotations_len, loc));
 
     if (args->vararg && args->vararg->annotation) {
         RETURN_IF_ERROR(
             compiler_visit_argannotation(c, args->vararg->arg,
-                                         args->vararg->annotation, &annotations_len, loc));
+                                         args->vararg->annotation, annotations_len, loc));
     }
 
     RETURN_IF_ERROR(
-        compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len, loc));
+        compiler_visit_argannotations(c, args->kwonlyargs, annotations_len, loc));
 
     if (args->kwarg && args->kwarg->annotation) {
         RETURN_IF_ERROR(
             compiler_visit_argannotation(c, args->kwarg->arg,
-                                         args->kwarg->annotation, &annotations_len, loc));
+                                         args->kwarg->annotation, annotations_len, loc));
     }
 
     RETURN_IF_ERROR(
-        compiler_visit_argannotation(c, &_Py_ID(return), returns, &annotations_len, loc));
+        compiler_visit_argannotation(c, &_Py_ID(return), returns, annotations_len, loc));
 
-    if (annotations_len) {
-        ADDOP_I(c, loc, BUILD_TUPLE, annotations_len);
-        return 1;
+    return 0;
+}
+
+static int
+compiler_visit_annotations(struct compiler *c, location loc,
+                           arguments_ty args, expr_ty returns)
+{
+    /* Push arg annotation names and values.
+       The expressions are evaluated separately from the rest of the source code.
+
+       Return -1 on error, or a combination of flags to add to the function.
+       */
+    Py_ssize_t annotations_len = 0;
+
+    PySTEntryObject *ste;
+    if (_PySymtable_LookupOptional(c->c_st, args, &ste) < 0) {
+        return ERROR;
+    }
+    assert(ste != NULL);
+    bool annotations_used = ste->ste_annotations_used;
+
+    if (annotations_used) {
+        if (compiler_setup_annotations_scope(c, loc, (void *)args,
+                                             ste->ste_name) < 0) {
+            Py_DECREF(ste);
+            return ERROR;
+        }
+    }
+    Py_DECREF(ste);
+
+    if (compiler_visit_annotations_in_scope(c, loc, args, returns, &annotations_len) < 0) {
+        if (annotations_used) {
+            compiler_exit_scope(c);
+        }
+        return ERROR;
+    }
+
+    if (annotations_used) {
+        RETURN_IF_ERROR(
+            compiler_leave_annotations_scope(c, loc, annotations_len)
+        );
+        return MAKE_FUNCTION_ANNOTATE;
     }
 
     return 0;
@@ -2001,7 +2057,7 @@ compiler_type_param_bound_or_default(struct compiler *c, expr_ty e,
                                      identifier name, void *key,
                                      bool allow_starred)
 {
-    if (compiler_enter_scope(c, name, COMPILER_SCOPE_TYPEPARAMS,
+    if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS,
                              key, e->lineno) == -1) {
         return ERROR;
     }
@@ -2220,7 +2276,6 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
     asdl_expr_seq *decos;
     asdl_type_param_seq *type_params;
     Py_ssize_t funcflags;
-    int annotations;
     int firstlineno;
 
     if (is_async) {
@@ -2274,7 +2329,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
         if (!type_params_name) {
             return ERROR;
         }
-        if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS,
+        if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS,
                                  (void *)type_params, firstlineno) == -1) {
             Py_DECREF(type_params_name);
             return ERROR;
@@ -2286,16 +2341,14 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
         }
     }
 
-    annotations = compiler_visit_annotations(c, loc, args, returns);
-    if (annotations < 0) {
+    int annotations_flag = compiler_visit_annotations(c, loc, args, returns);
+    if (annotations_flag < 0) {
         if (is_generic) {
             compiler_exit_scope(c);
         }
         return ERROR;
     }
-    if (annotations > 0) {
-        funcflags |= MAKE_FUNCTION_ANNOTATIONS;
-    }
+    funcflags |= annotations_flag;
 
     if (compiler_function_body(c, s, is_async, funcflags, firstlineno) < 0) {
         if (is_generic) {
@@ -2510,7 +2563,7 @@ compiler_class(struct compiler *c, stmt_ty s)
         if (!type_params_name) {
             return ERROR;
         }
-        if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS,
+        if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS,
                                  (void *)type_params, firstlineno) == -1) {
             Py_DECREF(type_params_name);
             return ERROR;
@@ -2630,7 +2683,7 @@ compiler_typealias(struct compiler *c, stmt_ty s)
         if (!type_params_name) {
             return ERROR;
         }
-        if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS,
+        if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS,
                                  (void *)type_params, loc.lineno) == -1) {
             Py_DECREF(type_params_name);
             return ERROR;
@@ -2719,15 +2772,6 @@ check_compare(struct compiler *c, expr_ty e)
     return SUCCESS;
 }
 
-static const int compare_masks[] = {
-    [Py_LT] = COMPARISON_LESS_THAN,
-    [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS,
-    [Py_EQ] = COMPARISON_EQUALS,
-    [Py_NE] = COMPARISON_NOT_EQUALS,
-    [Py_GT] = COMPARISON_GREATER_THAN,
-    [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS,
-};
-
 static int compiler_addcompare(struct compiler *c, location loc,
                                cmpop_ty op)
 {
@@ -6366,7 +6410,8 @@ compiler_annassign(struct compiler *c, stmt_ty s)
 {
     location loc = LOC(s);
     expr_ty targ = s->v.AnnAssign.target;
-    PyObject* mangled;
+    bool future_annotations = c->c_future.ff_features & CO_FUTURE_ANNOTATIONS;
+    PyObject *mangled;
 
     assert(s->kind == AnnAssign_kind);
 
@@ -6384,16 +6429,30 @@ compiler_annassign(struct compiler *c, stmt_ty s)
         if (s->v.AnnAssign.simple &&
             (c->u->u_scope_type == COMPILER_SCOPE_MODULE ||
              c->u->u_scope_type == COMPILER_SCOPE_CLASS)) {
-            if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) {
-                VISIT(c, annexpr, s->v.AnnAssign.annotation)
+            if (future_annotations) {
+                VISIT(c, annexpr, s->v.AnnAssign.annotation);
+                ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names);
+                mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id);
+                ADDOP_LOAD_CONST_NEW(c, loc, mangled);
+                ADDOP(c, loc, STORE_SUBSCR);
             }
             else {
-                VISIT(c, expr, s->v.AnnAssign.annotation);
+                if (c->u->u_deferred_annotations == NULL) {
+                    c->u->u_deferred_annotations = PyList_New(0);
+                    if (c->u->u_deferred_annotations == NULL) {
+                        return ERROR;
+                    }
+                }
+                PyObject *ptr = PyLong_FromVoidPtr((void *)s);
+                if (ptr == NULL) {
+                    return ERROR;
+                }
+                if (PyList_Append(c->u->u_deferred_annotations, ptr) < 0) {
+                    Py_DECREF(ptr);
+                    return ERROR;
+                }
+                Py_DECREF(ptr);
             }
-            ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names);
-            mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id);
-            ADDOP_LOAD_CONST_NEW(c, loc, mangled);
-            ADDOP(c, loc, STORE_SUBSCR);
         }
         break;
     case Attribute_kind:
@@ -6419,7 +6478,7 @@ compiler_annassign(struct compiler *c, stmt_ty s)
         return ERROR;
     }
     /* Annotation is evaluated last. */
-    if (!s->v.AnnAssign.simple && check_annotation(c, s) < 0) {
+    if (future_annotations && !s->v.AnnAssign.simple && check_annotation(c, s) < 0) {
         return ERROR;
     }
     return SUCCESS;
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index bab629684c53f6..470c82d938ab7c 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -4061,6 +4061,11 @@
                 assert(func_obj->func_defaults == NULL);
                 func_obj->func_defaults = attr;
                 break;
+                case MAKE_FUNCTION_ANNOTATE:
+                assert(PyCallable_Check(attr));
+                assert(func_obj->func_annotate == NULL);
+                func_obj->func_annotate = attr;
+                break;
                 default:
                 Py_UNREACHABLE();
             }
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 355be966cbb84a..0274f8b7a48c3c 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -5450,6 +5450,11 @@
                 assert(func_obj->func_defaults == NULL);
                 func_obj->func_defaults = attr;
                 break;
+                case MAKE_FUNCTION_ANNOTATE:
+                assert(PyCallable_Check(attr));
+                assert(func_obj->func_annotate == NULL);
+                func_obj->func_annotate = attr;
+                break;
                 default:
                 Py_UNREACHABLE();
             }
diff --git a/Python/symtable.c b/Python/symtable.c
index 7e452cdb13badf..287bc2bd58107d 100644
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -112,6 +112,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
     ste->ste_varkeywords = 0;
     ste->ste_opt_lineno = 0;
     ste->ste_opt_col_offset = 0;
+    ste->ste_annotations_used = 0;
     ste->ste_lineno = lineno;
     ste->ste_col_offset = col_offset;
     ste->ste_end_lineno = end_lineno;
@@ -132,6 +133,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
     ste->ste_can_see_class_scope = 0;
     ste->ste_comp_iter_expr = 0;
     ste->ste_needs_classdict = 0;
+    ste->ste_annotation_block = NULL;
 
     ste->ste_symbols = PyDict_New();
     ste->ste_varnames = PyList_New(0);
@@ -167,6 +169,7 @@ ste_dealloc(PySTEntryObject *ste)
     Py_XDECREF(ste->ste_varnames);
     Py_XDECREF(ste->ste_children);
     Py_XDECREF(ste->ste_directives);
+    Py_XDECREF(ste->ste_annotation_block);
     Py_XDECREF(ste->ste_mangled_names);
     PyObject_Free(ste);
 }
@@ -245,10 +248,11 @@ static int symtable_visit_alias(struct symtable *st, alias_ty);
 static int symtable_visit_comprehension(struct symtable *st, comprehension_ty);
 static int symtable_visit_keyword(struct symtable *st, keyword_ty);
 static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args);
-static int symtable_visit_annotation(struct symtable *st, expr_ty annotation);
+static int symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key);
 static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args);
 static int symtable_implicit_arg(struct symtable *st, int pos);
-static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty);
+static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty,
+                                      struct _symtable_entry *parent_ste);
 static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
 static int symtable_visit_match_case(struct symtable *st, match_case_ty m);
 static int symtable_visit_pattern(struct symtable *st, pattern_ty s);
@@ -504,6 +508,21 @@ _PySymtable_Lookup(struct symtable *st, void *key)
     return (PySTEntryObject *)v;
 }
 
+int
+_PySymtable_LookupOptional(struct symtable *st, void *key,
+                           PySTEntryObject **out)
+{
+    PyObject *k = PyLong_FromVoidPtr(key);
+    if (k == NULL) {
+        *out = NULL;
+        return -1;
+    }
+    int result = PyDict_GetItemRef(st->st_blocks, k, (PyObject **)out);
+    Py_DECREF(k);
+    assert(*out == NULL || PySTEntry_Check(*out));
+    return result;
+}
+
 long
 _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name)
 {
@@ -525,6 +544,7 @@ int
 _PyST_IsFunctionLike(PySTEntryObject *ste)
 {
     return ste->ste_type == FunctionBlock
+        || ste->ste_type == AnnotationBlock
         || ste->ste_type == TypeVarBoundBlock
         || ste->ste_type == TypeAliasBlock
         || ste->ste_type == TypeParamBlock;
@@ -1317,20 +1337,12 @@ symtable_exit_block(struct symtable *st)
 }
 
 static int
-symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
-                     void *ast, int lineno, int col_offset,
-                     int end_lineno, int end_col_offset)
+symtable_enter_existing_block(struct symtable *st, PySTEntryObject* ste)
 {
-    PySTEntryObject *prev = NULL, *ste;
-
-    ste = ste_new(st, name, block, ast, lineno, col_offset, end_lineno, end_col_offset);
-    if (ste == NULL)
-        return 0;
     if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) {
-        Py_DECREF(ste);
         return 0;
     }
-    prev = st->st_cur;
+    PySTEntryObject *prev = st->st_cur;
     /* bpo-37757: For now, disallow *all* assignment expressions in the
      * outermost iterator expression of a comprehension, even those inside
      * a nested comprehension or a lambda expression.
@@ -1340,21 +1352,20 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
     }
     /* No need to inherit ste_mangled_names in classes, where all names
      * are mangled. */
-    if (prev && prev->ste_mangled_names != NULL && block != ClassBlock) {
+    if (prev && prev->ste_mangled_names != NULL && ste->ste_type != ClassBlock) {
         ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names);
     }
     /* The entry is owned by the stack. Borrow it for st_cur. */
-    Py_DECREF(ste);
     st->st_cur = ste;
 
-    /* Annotation blocks shouldn't have any affect on the symbol table since in
-     * the compilation stage, they will all be transformed to strings. They are
-     * only created if future 'annotations' feature is activated. */
-    if (block == AnnotationBlock) {
+    /* If "from __future__ import annotations" is active,
+     * annotation blocks shouldn't have any affect on the symbol table since in
+     * the compilation stage, they will all be transformed to strings. */
+    if (st->st_future->ff_features & CO_FUTURE_ANNOTATIONS && ste->ste_type == AnnotationBlock) {
         return 1;
     }
 
-    if (block == ModuleBlock)
+    if (ste->ste_type == ModuleBlock)
         st->st_global = st->st_cur->ste_symbols;
 
     if (prev) {
@@ -1365,6 +1376,20 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
     return 1;
 }
 
+static int
+symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
+                     void *ast, int lineno, int col_offset,
+                     int end_lineno, int end_col_offset)
+{
+    PySTEntryObject *ste = ste_new(st, name, block, ast,
+                                   lineno, col_offset, end_lineno, end_col_offset);
+    if (ste == NULL)
+        return 0;
+    int result = symtable_enter_existing_block(st, ste);
+    Py_DECREF(ste);
+    return result;
+}
+
 static long
 symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name)
 {
@@ -1643,7 +1668,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
         VISIT_QUIT(st, 0);
     }
     switch (s->kind) {
-    case FunctionDef_kind:
+    case FunctionDef_kind: {
         if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s)))
             VISIT_QUIT(st, 0);
         if (s->v.FunctionDef.args->defaults)
@@ -1665,13 +1690,22 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
             }
             VISIT_SEQ(st, type_param, s->v.FunctionDef.type_params);
         }
+        PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s,
+                                           LOCATION(s));
+        if (!new_ste) {
+            VISIT_QUIT(st, 0);
+        }
+
         if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args,
-                                        s->v.FunctionDef.returns))
+                                        s->v.FunctionDef.returns, new_ste)) {
+            Py_DECREF(new_ste);
             VISIT_QUIT(st, 0);
-        if (!symtable_enter_block(st, s->v.FunctionDef.name,
-                                  FunctionBlock, (void *)s,
-                                  LOCATION(s)))
+        }
+        if (!symtable_enter_existing_block(st, new_ste)) {
+            Py_DECREF(new_ste);
             VISIT_QUIT(st, 0);
+        }
+        Py_DECREF(new_ste);
         VISIT(st, arguments, s->v.FunctionDef.args);
         VISIT_SEQ(st, stmt, s->v.FunctionDef.body);
         if (!symtable_exit_block(st))
@@ -1681,6 +1715,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
                 VISIT_QUIT(st, 0);
         }
         break;
+    }
     case ClassDef_kind: {
         PyObject *tmp;
         if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL, LOCATION(s)))
@@ -1776,6 +1811,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
         VISIT(st, expr, s->v.Assign.value);
         break;
     case AnnAssign_kind:
+        st->st_cur->ste_annotations_used = 1;
         if (s->v.AnnAssign.target->kind == Name_kind) {
             expr_ty e_name = s->v.AnnAssign.target;
             long cur = symtable_lookup(st, e_name->v.Name.id);
@@ -1810,7 +1846,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
         else {
             VISIT(st, expr, s->v.AnnAssign.target);
         }
-        if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) {
+        if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation,
+                                       (void *)((uintptr_t)st->st_cur->ste_id + 1))) {
             VISIT_QUIT(st, 0);
         }
 
@@ -1960,7 +1997,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
         VISIT_SEQ(st, withitem, s->v.With.items);
         VISIT_SEQ(st, stmt, s->v.With.body);
         break;
-    case AsyncFunctionDef_kind:
+    case AsyncFunctionDef_kind: {
         if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s)))
             VISIT_QUIT(st, 0);
         if (s->v.AsyncFunctionDef.args->defaults)
@@ -1983,14 +2020,21 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
             }
             VISIT_SEQ(st, type_param, s->v.AsyncFunctionDef.type_params);
         }
+        PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s,
+                                           LOCATION(s));
+        if (!new_ste) {
+            VISIT_QUIT(st, 0);
+        }
+
         if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args,
-                                        s->v.AsyncFunctionDef.returns))
+                                        s->v.AsyncFunctionDef.returns, new_ste))
             VISIT_QUIT(st, 0);
-        if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name,
-                                  FunctionBlock, (void *)s,
-                                  s->lineno, s->col_offset,
-                                  s->end_lineno, s->end_col_offset))
+        if (!symtable_enter_existing_block(st, new_ste)) {
+            Py_DECREF(new_ste);
             VISIT_QUIT(st, 0);
+        }
+        Py_DECREF(new_ste);
+
         st->st_cur->ste_coroutine = 1;
         VISIT(st, arguments, s->v.AsyncFunctionDef.args);
         VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body);
@@ -2001,6 +2045,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
                 VISIT_QUIT(st, 0);
         }
         break;
+    }
     case AsyncWith_kind:
         VISIT_SEQ(st, withitem, s->v.AsyncWith.items);
         VISIT_SEQ(st, stmt, s->v.AsyncWith.body);
@@ -2444,18 +2489,44 @@ symtable_visit_params(struct symtable *st, asdl_arg_seq *args)
 }
 
 static int
-symtable_visit_annotation(struct symtable *st, expr_ty annotation)
+symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key)
 {
-    int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
-    if (future_annotations &&
-        !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock,
-                              (void *)annotation, annotation->lineno,
-                              annotation->col_offset, annotation->end_lineno,
-                              annotation->end_col_offset)) {
-        VISIT_QUIT(st, 0);
+    struct _symtable_entry *parent_ste = st->st_cur;
+    if (parent_ste->ste_annotation_block == NULL) {
+        _Py_block_ty current_type = parent_ste->ste_type;
+        if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock,
+                                    key, LOCATION(annotation))) {
+            VISIT_QUIT(st, 0);
+        }
+        parent_ste->ste_annotation_block =
+            (struct _symtable_entry *)Py_NewRef(st->st_cur);
+        int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
+        if (current_type == ClassBlock && !future_annotations) {
+            st->st_cur->ste_can_see_class_scope = 1;
+            if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(annotation))) {
+                return 0;
+            }
+        }
+
+        _Py_DECLARE_STR(format, ".format");
+        // The generated __annotate__ function takes a single parameter with the
+        // internal name ".format".
+        if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM,
+                                LOCATION(annotation))) {
+            return 0;
+        }
+        if (!symtable_add_def(st, &_Py_STR(format), USE,
+                                LOCATION(annotation))) {
+            return 0;
+        }
+    }
+    else {
+        if (!symtable_enter_existing_block(st, parent_ste->ste_annotation_block)) {
+            VISIT_QUIT(st, 0);
+        }
     }
     VISIT(st, expr, annotation);
-    if (future_annotations && !symtable_exit_block(st)) {
+    if (!symtable_exit_block(st)) {
         VISIT_QUIT(st, 0);
     }
     return 1;
@@ -2471,37 +2542,58 @@ symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args)
 
     for (i = 0; i < asdl_seq_LEN(args); i++) {
         arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
-        if (arg->annotation)
+        if (arg->annotation) {
+            st->st_cur->ste_annotations_used = 1;
             VISIT(st, expr, arg->annotation);
+        }
     }
 
     return 1;
 }
 
 static int
-symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns)
+symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns,
+                           struct _symtable_entry *function_ste)
 {
-    int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
-    if (future_annotations &&
-        !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock,
-                              (void *)o, o->lineno, o->col_offset, o->end_lineno,
-                              o->end_col_offset)) {
+    int is_in_class = st->st_cur->ste_can_see_class_scope;
+    _Py_block_ty current_type = st->st_cur->ste_type;
+    if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock,
+                              (void *)a, LOCATION(o))) {
         VISIT_QUIT(st, 0);
     }
+    if (is_in_class || current_type == ClassBlock) {
+        st->st_cur->ste_can_see_class_scope = 1;
+        if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(o))) {
+            return 0;
+        }
+    }
+    _Py_DECLARE_STR(format, ".format");
+    // We need to insert code that reads this "parameter" to the function.
+    if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, LOCATION(o))) {
+        return 0;
+    }
+    if (!symtable_add_def(st, &_Py_STR(format), USE, LOCATION(o))) {
+        return 0;
+    }
     if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs))
         return 0;
     if (a->args && !symtable_visit_argannotations(st, a->args))
         return 0;
-    if (a->vararg && a->vararg->annotation)
+    if (a->vararg && a->vararg->annotation) {
+        st->st_cur->ste_annotations_used = 1;
         VISIT(st, expr, a->vararg->annotation);
-    if (a->kwarg && a->kwarg->annotation)
+    }
+    if (a->kwarg && a->kwarg->annotation) {
+        st->st_cur->ste_annotations_used = 1;
         VISIT(st, expr, a->kwarg->annotation);
+    }
     if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs))
         return 0;
-    if (future_annotations && !symtable_exit_block(st)) {
-        VISIT_QUIT(st, 0);
+    if (returns) {
+        st->st_cur->ste_annotations_used = 1;
+        VISIT(st, expr, returns);
     }
-    if (returns && !symtable_visit_annotation(st, returns)) {
+    if (!symtable_exit_block(st)) {
         VISIT_QUIT(st, 0);
     }
     return 1;
@@ -2733,7 +2825,7 @@ symtable_visit_dictcomp(struct symtable *st, expr_ty e)
 static int
 symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e)
 {
-    enum _block_type type = st->st_cur->ste_type;
+    _Py_block_ty type = st->st_cur->ste_type;
     if (type == AnnotationBlock)
         PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name);
     else if (type == TypeVarBoundBlock)

From ec3af291fe2f680ab277edde7113e2762754f4aa Mon Sep 17 00:00:00 2001
From: Pablo Galindo Salgado <Pablogsal@gmail.com>
Date: Tue, 11 Jun 2024 17:15:01 +0100
Subject: [PATCH 47/72] gh-120346: Respect PYTHON_BASIC_REPL when running in
 interactive inspect mode (#120349)

---
 .../2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst             | 2 ++
 Modules/main.c                                                 | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst

diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst
new file mode 100644
index 00000000000000..eb2d0f9a705caa
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst	
@@ -0,0 +1,2 @@
+Respect :envvar:`PYTHON_BASIC_REPL` when running in interative inspect mode
+(``python -i``). Patch by Pablo Galindo
diff --git a/Modules/main.c b/Modules/main.c
index 8eded2639ad90a..1a70b300b6ad17 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -542,7 +542,8 @@ pymain_repl(PyConfig *config, int *exitcode)
         return;
     }
 
-    if (!isatty(fileno(stdin))) {
+    if (!isatty(fileno(stdin))
+        || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) {
         PyCompilerFlags cf = _PyCompilerFlags_INIT;
         int run = PyRun_AnyFileExFlags(stdin, "<stdin>", 0, &cf);
         *exitcode = (run != 0);

From 32a0faba439b239d7b0c242c1e3cd2025c52b8cf Mon Sep 17 00:00:00 2001
From: Matt Wozniski <mwozniski@bloomberg.net>
Date: Tue, 11 Jun 2024 12:42:10 -0400
Subject: [PATCH 48/72] gh-119517: Fixes for pasting in pyrepl (#120253)

* Remove pyrepl's optimization for self-insert

This will be replaced by a less specialized optimization.

* Use line-buffering when pyrepl echoes pastes

Previously echoing was totally suppressed until the entire command had
been pasted and the terminal ended paste mode, but this gives the user
no feedback to indicate that an operation is in progress. Drawing
something to the screen once per line strikes a balance between
perceived responsiveness and performance.

* Remove dead code from pyrepl

`msg_at_bottom` is always true.

* Speed up pyrepl's screen rendering computation

The Reader in pyrepl doesn't hold a complete representation of the
screen area being drawn as persistent state. Instead, it recomputes it,
on each keypress. This is fast enough for a few hundred bytes, but
incredibly slow as the input buffer grows into the kilobytes (likely
because of pasting).

Rather than making some expensive and expansive changes to the repl's
internal representation of the screen, add some caching: remember some
data from one refresh to the next about what was drawn to the screen
and, if we don't find anything that has invalidated the results that
were computed last time around, reuse them. To keep this caching as
simple as possible, all we'll do is look for lines in the buffer that
were above the cursor the last time we were asked to update the screen,
and that are still above the cursor now. We assume that nothing can
affect a line that comes before both the old and new cursor location
without us being informed. Based on this assumption, we can reuse old
lines, which drastically speeds up the overwhelmingly common case where
the user is typing near the end of the buffer.

* Speed up pyrepl prompt drawing

Cache the `can_colorize()` call rather than repeatedly recomputing it.
This call looks up an environment variable, and is called once per
character typed at the REPL. The environment variable lookup shows up as
a hot spot when profiling, and we don't expect this to change while the
REPL is running.

* Speed up pasting multiple lines into the REPL

Previously, we were checking whether the command should be accepted each
time a line break was encountered, but that's not the expected behavior.
In bracketed paste mode, we expect everything pasted to be part of
a single block of code, and encountering a newline shouldn't behave like
a user pressing <Enter> to execute a command. The user should always
have a chance to review the pasted command before running it.

* Use a read buffer for input in pyrepl

Previously we were reading one byte at a time, which causes much slower
IO than necessary. Instead, read in chunks, processing previously read
data before asking for more.

* Optimize finding width of a single character

`wlen` finds the width of a multi-character string by adding up the
width of each character, and then subtracting the width of any escape
sequences. It's often called for single character strings, however,
which can't possibly contain escape sequences. Optimize for that case.

* Optimize disp_str for ASCII characters

Since every ASCII character is known to display as single width, we can
avoid not only the Unicode data lookup in `disp_str` but also the one
hidden in `str_width` for them.

* Speed up cursor movements in long pyrepl commands

When the current pyrepl command buffer contains many lines, scrolling up
becomes slow. We have optimizations in place to reuse lines above the
cursor position from one refresh to the next, but don't currently try to
reuse lines below the cursor position in the same way, so we wind up
with quadratic behavior where all lines of the buffer below the cursor
are recomputed each time the cursor moves up another line.

Optimize this by only computing one screen's worth of lines beyond the
cursor position. Any lines beyond that can't possibly be shown by the
console, and bounding this makes scrolling up have linear time
complexity instead.

---------

Signed-off-by: Matt Wozniski <mwozniski@bloomberg.net>
Co-authored-by: Pablo Galindo <pablogsal@gmail.com>
---
 Lib/_pyrepl/commands.py          |   3 -
 Lib/_pyrepl/completing_reader.py |   8 +-
 Lib/_pyrepl/reader.py            | 154 ++++++++++++++++++++++---------
 Lib/_pyrepl/readline.py          |   4 +
 Lib/_pyrepl/unix_console.py      |  20 +++-
 Lib/_pyrepl/utils.py             |   2 +
 6 files changed, 134 insertions(+), 57 deletions(-)

diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py
index 6bffed1bfe9327..c3fce91013b001 100644
--- a/Lib/_pyrepl/commands.py
+++ b/Lib/_pyrepl/commands.py
@@ -368,8 +368,6 @@ def do(self) -> None:
         r = self.reader
         text = self.event * r.get_arg()
         r.insert(text)
-        if len(text) == 1 and r.pos == len(r.buffer):
-            r.calc_screen = r.append_to_screen
 
 
 class insert_nl(EditCommand):
@@ -483,4 +481,3 @@ def do(self) -> None:
         self.reader.paste_mode = False
         self.reader.in_bracketed_paste = False
         self.reader.dirty = True
-        self.reader.calc_screen = self.reader.calc_complete_screen
diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py
index 8df35ccb9117b1..05770aaf5060cc 100644
--- a/Lib/_pyrepl/completing_reader.py
+++ b/Lib/_pyrepl/completing_reader.py
@@ -209,10 +209,6 @@ def do(self) -> None:
         r = self.reader  # type: ignore[assignment]
 
         commands.self_insert.do(self)
-
-        if r.cmpltn_menu_visible or r.cmpltn_message_visible:
-            r.calc_screen = r.calc_complete_screen
-
         if r.cmpltn_menu_visible:
             stem = r.get_stem()
             if len(stem) < 1:
@@ -261,8 +257,8 @@ def after_command(self, cmd: Command) -> None:
         if not isinstance(cmd, (complete, self_insert)):
             self.cmpltn_reset()
 
-    def calc_complete_screen(self) -> list[str]:
-        screen = super().calc_complete_screen()
+    def calc_screen(self) -> list[str]:
+        screen = super().calc_screen()
         if self.cmpltn_menu_visible:
             ly = self.lxy[1]
             screen[ly:ly] = self.cmpltn_menu
diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py
index beee7764e0eb84..63ae661968408e 100644
--- a/Lib/_pyrepl/reader.py
+++ b/Lib/_pyrepl/reader.py
@@ -35,15 +35,13 @@
 # types
 Command = commands.Command
 if False:
-    from typing import Callable
     from .types import Callback, SimpleContextManager, KeySpec, CommandName
-    CalcScreen = Callable[[], list[str]]
 
 
 def disp_str(buffer: str) -> tuple[str, list[int]]:
     """disp_str(buffer:string) -> (string, [int])
 
-    Return the string that should be the printed represenation of
+    Return the string that should be the printed representation of
     |buffer| and a list detailing where the characters of |buffer|
     get used up.  E.g.:
 
@@ -54,11 +52,17 @@ def disp_str(buffer: str) -> tuple[str, list[int]]:
     b: list[int] = []
     s: list[str] = []
     for c in buffer:
-        if ord(c) > 128 and unicodedata.category(c).startswith("C"):
+        if ord(c) < 128:
+            s.append(c)
+            b.append(1)
+        elif unicodedata.category(c).startswith("C"):
             c = r"\u%04x" % ord(c)
-        s.append(c)
-        b.append(wlen(c))
-        b.extend([0] * (len(c) - 1))
+            s.append(c)
+            b.append(str_width(c))
+            b.extend([0] * (len(c) - 1))
+        else:
+            s.append(c)
+            b.append(str_width(c))
     return "".join(s), b
 
 
@@ -230,7 +234,6 @@ class Reader:
     commands: dict[str, type[Command]] = field(default_factory=make_default_commands)
     last_command: type[Command] | None = None
     syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table)
-    msg_at_bottom: bool = True
     keymap: tuple[tuple[str, str], ...] = ()
     input_trans: input.KeymapTranslator = field(init=False)
     input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list)
@@ -238,8 +241,52 @@ class Reader:
     screeninfo: list[tuple[int, list[int]]] = field(init=False)
     cxy: tuple[int, int] = field(init=False)
     lxy: tuple[int, int] = field(init=False)
-    calc_screen: CalcScreen = field(init=False)
     scheduled_commands: list[str] = field(default_factory=list)
+    can_colorize: bool = False
+
+    ## cached metadata to speed up screen refreshes
+    @dataclass
+    class RefreshCache:
+        in_bracketed_paste: bool = False
+        screen: list[str] = field(default_factory=list)
+        screeninfo: list[tuple[int, list[int]]] = field(init=False)
+        line_end_offsets: list[int] = field(default_factory=list)
+        pos: int = field(init=False)
+        cxy: tuple[int, int] = field(init=False)
+        dimensions: tuple[int, int] = field(init=False)
+
+        def update_cache(self,
+                         reader: Reader,
+                         screen: list[str],
+                         screeninfo: list[tuple[int, list[int]]],
+            ) -> None:
+            self.in_bracketed_paste = reader.in_bracketed_paste
+            self.screen = screen.copy()
+            self.screeninfo = screeninfo.copy()
+            self.pos = reader.pos
+            self.cxy = reader.cxy
+            self.dimensions = reader.console.width, reader.console.height
+
+        def valid(self, reader: Reader) -> bool:
+            dimensions = reader.console.width, reader.console.height
+            dimensions_changed = dimensions != self.dimensions
+            paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste
+            return not (dimensions_changed or paste_changed)
+
+        def get_cached_location(self, reader: Reader) -> tuple[int, int]:
+            offset = 0
+            earliest_common_pos = min(reader.pos, self.pos)
+            num_common_lines = len(self.line_end_offsets)
+            while num_common_lines > 0:
+                offset = self.line_end_offsets[num_common_lines - 1]
+                if earliest_common_pos > offset:
+                    break
+                num_common_lines -= 1
+            else:
+                offset = 0
+            return offset, num_common_lines
+
+    last_refresh_cache: RefreshCache = field(default_factory=RefreshCache)
 
     def __post_init__(self) -> None:
         # Enable the use of `insert` without a `prepare` call - necessary to
@@ -252,53 +299,60 @@ def __post_init__(self) -> None:
         self.screeninfo = [(0, [])]
         self.cxy = self.pos2xy()
         self.lxy = (self.pos, 0)
-        self.calc_screen = self.calc_complete_screen
+        self.can_colorize = can_colorize()
+
+        self.last_refresh_cache.screeninfo = self.screeninfo
+        self.last_refresh_cache.pos = self.pos
+        self.last_refresh_cache.cxy = self.cxy
+        self.last_refresh_cache.dimensions = (0, 0)
 
     def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
         return default_keymap
 
-    def append_to_screen(self) -> list[str]:
-        new_screen = self.screen.copy() or ['']
+    def calc_screen(self) -> list[str]:
+        """Translate changes in self.buffer into changes in self.console.screen."""
+        # Since the last call to calc_screen:
+        # screen and screeninfo may differ due to a completion menu being shown
+        # pos and cxy may differ due to edits, cursor movements, or completion menus
 
-        new_character = self.buffer[-1]
-        new_character_len = wlen(new_character)
+        # Lines that are above both the old and new cursor position can't have changed,
+        # unless the terminal has been resized (which might cause reflowing) or we've
+        # entered or left paste mode (which changes prompts, causing reflowing).
+        num_common_lines = 0
+        offset = 0
+        if self.last_refresh_cache.valid(self):
+            offset, num_common_lines = self.last_refresh_cache.get_cached_location(self)
 
-        last_line_len = wlen(new_screen[-1])
-        if last_line_len + new_character_len >= self.console.width:  # We need to wrap here
-            new_screen[-1] += '\\'
-            self.screeninfo[-1][1].append(1)
-            new_screen.append(self.buffer[-1])
-            self.screeninfo.append((0, [new_character_len]))
-        else:
-            new_screen[-1] += self.buffer[-1]
-            self.screeninfo[-1][1].append(new_character_len)
-        self.cxy = self.pos2xy()
+        screen = self.last_refresh_cache.screen
+        del screen[num_common_lines:]
 
-        # Reset the function that is used for completing the screen
-        self.calc_screen = self.calc_complete_screen
-        return new_screen
+        screeninfo = self.last_refresh_cache.screeninfo
+        del screeninfo[num_common_lines:]
+
+        last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets
+        del last_refresh_line_end_offsets[num_common_lines:]
 
-    def calc_complete_screen(self) -> list[str]:
-        """The purpose of this method is to translate changes in
-        self.buffer into changes in self.screen.  Currently it rips
-        everything down and starts from scratch, which whilst not
-        especially efficient is certainly simple(r).
-        """
-        lines = self.get_unicode().split("\n")
-        screen: list[str] = []
-        screeninfo: list[tuple[int, list[int]]] = []
         pos = self.pos
-        for ln, line in enumerate(lines):
+        pos -= offset
+
+        lines = "".join(self.buffer[offset:]).split("\n")
+        cursor_found = False
+        lines_beyond_cursor = 0
+        for ln, line in enumerate(lines, num_common_lines):
             ll = len(line)
             if 0 <= pos <= ll:
-                if self.msg and not self.msg_at_bottom:
-                    for mline in self.msg.split("\n"):
-                        screen.append(mline)
-                        screeninfo.append((0, []))
                 self.lxy = pos, ln
+                cursor_found = True
+            elif cursor_found:
+                lines_beyond_cursor += 1
+                if lines_beyond_cursor > self.console.height:
+                    # No need to keep formatting lines.
+                    # The console can't show them.
+                    break
             prompt = self.get_prompt(ln, ll >= pos >= 0)
             while "\n" in prompt:
                 pre_prompt, _, prompt = prompt.partition("\n")
+                last_refresh_line_end_offsets.append(offset)
                 screen.append(pre_prompt)
                 screeninfo.append((0, []))
             pos -= ll + 1
@@ -306,6 +360,8 @@ def calc_complete_screen(self) -> list[str]:
             l, l2 = disp_str(line)
             wrapcount = (wlen(l) + lp) // self.console.width
             if wrapcount == 0:
+                offset += ll + 1  # Takes all of the line plus the newline
+                last_refresh_line_end_offsets.append(offset)
                 screen.append(prompt + l)
                 screeninfo.append((lp, l2))
             else:
@@ -321,11 +377,14 @@ def calc_complete_screen(self) -> list[str]:
                         column += character_width
                     pre = prompt if i == 0 else ""
                     if len(l) > index_to_wrap_before:
+                        offset += index_to_wrap_before
                         post = "\\"
                         after = [1]
                     else:
+                        offset += index_to_wrap_before + 1  # Takes the newline
                         post = ""
                         after = []
+                    last_refresh_line_end_offsets.append(offset)
                     screen.append(pre + l[:index_to_wrap_before] + post)
                     screeninfo.append((prelen, l2[:index_to_wrap_before] + after))
                     l = l[index_to_wrap_before:]
@@ -333,10 +392,12 @@ def calc_complete_screen(self) -> list[str]:
                     i += 1
         self.screeninfo = screeninfo
         self.cxy = self.pos2xy()
-        if self.msg and self.msg_at_bottom:
+        if self.msg:
             for mline in self.msg.split("\n"):
                 screen.append(mline)
                 screeninfo.append((0, []))
+
+        self.last_refresh_cache.update_cache(self, screen, screeninfo)
         return screen
 
     @staticmethod
@@ -456,7 +517,7 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
         'lineno'."""
         if self.arg is not None and cursor_on_line:
             prompt = f"(arg: {self.arg}) "
-        elif self.paste_mode:
+        elif self.paste_mode and not self.in_bracketed_paste:
             prompt = "(paste) "
         elif "\n" in self.buffer:
             if lineno == 0:
@@ -468,7 +529,7 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
         else:
             prompt = self.ps1
 
-        if can_colorize():
+        if self.can_colorize:
             prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}"
         return prompt
 
@@ -604,6 +665,9 @@ def update_screen(self) -> None:
 
     def refresh(self) -> None:
         """Recalculate and refresh the screen."""
+        if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n":
+            return
+
         # this call sets up self.cxy, so call it first.
         self.screen = self.calc_screen()
         self.console.refresh(self.screen, self.cxy)
@@ -627,7 +691,7 @@ def do_cmd(self, cmd: tuple[str, list[str]]) -> None:
 
         self.after_command(command)
 
-        if self.dirty and not self.in_bracketed_paste:
+        if self.dirty:
             self.refresh()
         else:
             self.update_cursor()
diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py
index 7d811bf41773fe..b10d0c66e4f813 100644
--- a/Lib/_pyrepl/readline.py
+++ b/Lib/_pyrepl/readline.py
@@ -263,6 +263,10 @@ def do(self) -> None:
         r = self.reader  # type: ignore[assignment]
         r.dirty = True  # this is needed to hide the completion menu, if visible
 
+        if self.reader.in_bracketed_paste:
+            r.insert("\n")
+            return
+
         # if there are already several lines and the cursor
         # is not on the last one, always insert a new \n.
         text = r.get_unicode()
diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py
index 2f73a59dd1fced..f1a6b84adfb671 100644
--- a/Lib/_pyrepl/unix_console.py
+++ b/Lib/_pyrepl/unix_console.py
@@ -150,6 +150,8 @@ def __init__(
 
         self.pollob = poll()
         self.pollob.register(self.input_fd, select.POLLIN)
+        self.input_buffer = b""
+        self.input_buffer_pos = 0
         curses.setupterm(term or None, self.output_fd)
         self.term = term
 
@@ -197,6 +199,18 @@ def _my_getstr(cap: str, optional: bool = False) -> bytes | None:
         self.event_queue = EventQueue(self.input_fd, self.encoding)
         self.cursor_visible = 1
 
+    def __read(self, n: int) -> bytes:
+        if not self.input_buffer or self.input_buffer_pos >= len(self.input_buffer):
+            self.input_buffer = os.read(self.input_fd, 10000)
+
+        ret = self.input_buffer[self.input_buffer_pos : self.input_buffer_pos + n]
+        self.input_buffer_pos += len(ret)
+        if self.input_buffer_pos >= len(self.input_buffer):
+            self.input_buffer = b""
+            self.input_buffer_pos = 0
+        return ret
+
+
     def change_encoding(self, encoding: str) -> None:
         """
         Change the encoding used for I/O operations.
@@ -373,7 +387,7 @@ def get_event(self, block: bool = True) -> Event | None:
         while self.event_queue.empty():
             while True:
                 try:
-                    self.push_char(os.read(self.input_fd, 1))
+                    self.push_char(self.__read(1))
                 except OSError as err:
                     if err.errno == errno.EINTR:
                         if not self.event_queue.empty():
@@ -491,7 +505,7 @@ def getpending(self):
                 e.raw += e.raw
 
             amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0]
-            raw = os.read(self.input_fd, amount)
+            raw = self.__read(amount)
             data = str(raw, self.encoding, "replace")
             e.data += data
             e.raw += raw
@@ -514,7 +528,7 @@ def getpending(self):
                 e.raw += e.raw
 
             amount = 10000
-            raw = os.read(self.input_fd, amount)
+            raw = self.__read(amount)
             data = str(raw, self.encoding, "replace")
             e.data += data
             e.raw += raw
diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py
index 96e917e487d91a..20dbb1f7e17229 100644
--- a/Lib/_pyrepl/utils.py
+++ b/Lib/_pyrepl/utils.py
@@ -16,6 +16,8 @@ def str_width(c: str) -> int:
 
 
 def wlen(s: str) -> int:
+    if len(s) == 1:
+        return str_width(s)
     length = sum(str_width(i) for i in s)
     # remove lengths of any escape sequences
     sequence = ANSI_ESCAPE_SEQUENCE.findall(s)

From 1b62bcee941e54244b3ce6476aef8913604987c9 Mon Sep 17 00:00:00 2001
From: Lysandros Nikolaou <lisandrosnik@gmail.com>
Date: Tue, 11 Jun 2024 19:00:53 +0200
Subject: [PATCH 49/72] gh-120343: Do not reset byte_col_offset_diff after
 multiline tokens (#120352)

Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
---
 Lib/test/test_tokenize.py                             | 11 +++++++++++
 .../2024-06-11-16-34-41.gh-issue-120343.hdiXeU.rst    |  1 +
 Python/Python-tokenize.c                              |  7 ++++++-
 3 files changed, 18 insertions(+), 1 deletion(-)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-11-16-34-41.gh-issue-120343.hdiXeU.rst

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 4428e8cea1964c..36dba71766cc20 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1199,6 +1199,17 @@ def test_closing_parenthesis_from_different_line(self):
     NAME       'x'           (1, 3) (1, 4)
     """)
 
+    def test_multiline_non_ascii_fstring(self):
+        self.check_tokenize("""\
+a = f'''
+    Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli'''""", """\
+    NAME       'a'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    FSTRING_START "f\'\'\'"        (1, 4) (1, 8)
+    FSTRING_MIDDLE '\\n    Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli' (1, 8) (2, 68)
+    FSTRING_END "\'\'\'"         (2, 68) (2, 71)
+    """)
+
 class GenerateTokensTest(TokenizeTest):
     def check_tokenize(self, s, expected):
         # Format the tokens in s in a table format.
diff --git a/Misc/NEWS.d/next/Library/2024-06-11-16-34-41.gh-issue-120343.hdiXeU.rst b/Misc/NEWS.d/next/Library/2024-06-11-16-34-41.gh-issue-120343.hdiXeU.rst
new file mode 100644
index 00000000000000..76714b0c394eef
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-11-16-34-41.gh-issue-120343.hdiXeU.rst
@@ -0,0 +1 @@
+Fix column offset reporting for tokens that come after multiline f-strings in the :mod:`tokenize` module.
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index 09fad18b5b4df7..2591dae35736ba 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -36,6 +36,7 @@ typedef struct
     /* Needed to cache line for performance */
     PyObject *last_line;
     Py_ssize_t last_lineno;
+    Py_ssize_t last_end_lineno;
     Py_ssize_t byte_col_offset_diff;
 } tokenizeriterobject;
 
@@ -77,6 +78,7 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline,
     self->last_line = NULL;
     self->byte_col_offset_diff = 0;
     self->last_lineno = 0;
+    self->last_end_lineno = 0;
 
     return (PyObject *)self;
 }
@@ -227,7 +229,9 @@ tokenizeriter_next(tokenizeriterobject *it)
             Py_XDECREF(it->last_line);
             line = PyUnicode_DecodeUTF8(line_start, size, "replace");
             it->last_line = line;
-            it->byte_col_offset_diff = 0;
+            if (it->tok->lineno != it->last_end_lineno) {
+                it->byte_col_offset_diff = 0;
+            }
         } else {
             // Line hasn't changed so we reuse the cached one.
             line = it->last_line;
@@ -241,6 +245,7 @@ tokenizeriter_next(tokenizeriterobject *it)
     Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
     Py_ssize_t end_lineno = it->tok->lineno;
     it->last_lineno = lineno;
+    it->last_end_lineno = end_lineno;
 
     Py_ssize_t col_offset = -1;
     Py_ssize_t end_col_offset = -1;

From 0335662fe1f663fe96e3e4acf0f34c5959d06b00 Mon Sep 17 00:00:00 2001
From: naglis <827324+naglis@users.noreply.github.com>
Date: Tue, 11 Jun 2024 20:01:48 +0300
Subject: [PATCH 50/72] Fix typo in ElementTree docs (#120342)

---
 Doc/library/xml.etree.elementtree.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst
index e5919029c62c93..4c1e7bd7e6734a 100644
--- a/Doc/library/xml.etree.elementtree.rst
+++ b/Doc/library/xml.etree.elementtree.rst
@@ -508,7 +508,7 @@ Functions
    `C14N 2.0 <https://www.w3.org/TR/xml-c14n2/>`_ transformation function.
 
    Canonicalization is a way to normalise XML output in a way that allows
-   byte-by-byte comparisons and digital signatures.  It reduced the freedom
+   byte-by-byte comparisons and digital signatures.  It reduces the freedom
    that XML serializers have and instead generates a more constrained XML
    representation.  The main restrictions regard the placement of namespace
    declarations, the ordering of attributes, and ignorable whitespace.

From 86a8a1c57a386fb3330bee0fa44fc3fd6c3042a3 Mon Sep 17 00:00:00 2001
From: Eugene Triguba <eugenetriguba@gmail.com>
Date: Tue, 11 Jun 2024 13:40:31 -0400
Subject: [PATCH 51/72] gh-118908: Limit exposed globals from internal imports
 and definitions on new REPL startup (#119547)

---
 Lib/_pyrepl/simple_interact.py                | 21 ++++++-
 Lib/test/test_pyrepl/test_pyrepl.py           | 63 ++++++++++++++++++-
 Lib/test/test_repl.py                         |  5 +-
 ...-05-25-10-40-38.gh-issue-118908.XcZiq4.rst |  2 +
 4 files changed, 83 insertions(+), 8 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2024-05-25-10-40-38.gh-issue-118908.XcZiq4.rst

diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py
index 2e5698eb131684..620f87b4867073 100644
--- a/Lib/_pyrepl/simple_interact.py
+++ b/Lib/_pyrepl/simple_interact.py
@@ -27,6 +27,7 @@
 
 import _sitebuiltins
 import linecache
+import builtins
 import sys
 import code
 from types import ModuleType
@@ -34,6 +35,12 @@
 from .console import InteractiveColoredConsole
 from .readline import _get_reader, multiline_input
 
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
 _error: tuple[type[Exception], ...] | type[Exception]
 try:
     from .unix_console import _error
@@ -73,20 +80,28 @@ def _clear_screen():
     "clear": _clear_screen,
 }
 
+DEFAULT_NAMESPACE: dict[str, Any] = {
+    '__name__': '__main__',
+    '__doc__': None,
+    '__package__': None,
+    '__loader__': None,
+    '__spec__': None,
+    '__annotations__': {},
+    '__builtins__': builtins,
+}
 
 def run_multiline_interactive_console(
     mainmodule: ModuleType | None = None,
     future_flags: int = 0,
     console: code.InteractiveConsole | None = None,
 ) -> None:
-    import __main__
     from .readline import _setup
     _setup()
 
-    mainmodule = mainmodule or __main__
+    namespace = mainmodule.__dict__ if mainmodule else DEFAULT_NAMESPACE
     if console is None:
         console = InteractiveColoredConsole(
-            mainmodule.__dict__, filename="<stdin>"
+            namespace, filename="<stdin>"
         )
     if future_flags:
         console.compile.compiler.flags |= future_flags
diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py
index 45114e7315749f..3167b8473bfe20 100644
--- a/Lib/test/test_pyrepl/test_pyrepl.py
+++ b/Lib/test/test_pyrepl/test_pyrepl.py
@@ -1,9 +1,13 @@
-import itertools
 import io
+import itertools
 import os
 import rlcompleter
-from unittest import TestCase
+import select
+import subprocess
+import sys
+from unittest import TestCase, skipUnless
 from unittest.mock import patch
+from test.support import force_not_colorized
 
 from .support import (
     FakeConsole,
@@ -17,6 +21,10 @@
 from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig
 from _pyrepl.readline import multiline_input as readline_multiline_input
 
+try:
+    import pty
+except ImportError:
+    pty = None
 
 class TestCursorPosition(TestCase):
     def prepare_reader(self, events):
@@ -828,3 +836,54 @@ def test_bracketed_paste_single_line(self):
         reader = self.prepare_reader(events)
         output = multiline_input(reader)
         self.assertEqual(output, input_code)
+
+
+@skipUnless(pty, "requires pty")
+class TestMain(TestCase):
+    @force_not_colorized
+    def test_exposed_globals_in_repl(self):
+        expected_output = (
+            "[\'__annotations__\', \'__builtins__\', \'__doc__\', \'__loader__\', "
+            "\'__name__\', \'__package__\', \'__spec__\']"
+        )
+        output, exit_code = self.run_repl(["sorted(dir())", "exit"])
+        if "can\'t use pyrepl" in output:
+            self.skipTest("pyrepl not available")
+        self.assertEqual(exit_code, 0)
+        self.assertIn(expected_output, output)
+
+    def test_dumb_terminal_exits_cleanly(self):
+        env = os.environ.copy()
+        env.update({"TERM": "dumb"})
+        output, exit_code = self.run_repl("exit()\n", env=env)
+        self.assertEqual(exit_code, 0)
+        self.assertIn("warning: can\'t use pyrepl", output)
+        self.assertNotIn("Exception", output)
+        self.assertNotIn("Traceback", output)
+
+    def run_repl(self, repl_input: str | list[str], env: dict | None = None) -> tuple[str, int]:
+        master_fd, slave_fd = pty.openpty()
+        process = subprocess.Popen(
+            [sys.executable, "-i", "-u"],
+            stdin=slave_fd,
+            stdout=slave_fd,
+            stderr=slave_fd,
+            text=True,
+            close_fds=True,
+            env=env if env else os.environ,
+       )
+        if isinstance(repl_input, list):
+            repl_input = "\n".join(repl_input) + "\n"
+        os.write(master_fd, repl_input.encode("utf-8"))
+
+        output = []
+        while select.select([master_fd], [], [], 0.5)[0]:
+            data = os.read(master_fd, 1024).decode("utf-8")
+            if not data:
+                break
+            output.append(data)
+
+        os.close(master_fd)
+        os.close(slave_fd)
+        exit_code = process.wait()
+        return "\n".join(output), exit_code
diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py
index 340178366fc13a..1caf09ceaf10fc 100644
--- a/Lib/test/test_repl.py
+++ b/Lib/test/test_repl.py
@@ -1,9 +1,9 @@
 """Test the interactive interpreter."""
 
-import sys
 import os
-import unittest
 import subprocess
+import sys
+import unittest
 from textwrap import dedent
 from test import support
 from test.support import cpython_only, has_subprocess_support, SuppressCrashReport
@@ -199,7 +199,6 @@ def test_asyncio_repl_is_ok(self):
         assert_python_ok("-m", "asyncio")
 
 
-
 class TestInteractiveModeSyntaxErrors(unittest.TestCase):
 
     def test_interactive_syntax_error_correct_line(self):
diff --git a/Misc/NEWS.d/next/Library/2024-05-25-10-40-38.gh-issue-118908.XcZiq4.rst b/Misc/NEWS.d/next/Library/2024-05-25-10-40-38.gh-issue-118908.XcZiq4.rst
new file mode 100644
index 00000000000000..bf58d7277fcd51
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-05-25-10-40-38.gh-issue-118908.XcZiq4.rst
@@ -0,0 +1,2 @@
+Limit exposed globals from internal imports and definitions on new REPL
+startup. Patch by Eugene Triguba and Pablo Galindo.

From 939c201e00943c6dc2d515185168c30606ae522c Mon Sep 17 00:00:00 2001
From: Kirill Podoprigora <kirill.bast9@mail.ru>
Date: Tue, 11 Jun 2024 20:50:21 +0300
Subject: [PATCH 52/72] gh-120326: Include <intrin.h> on Windows with Free
 Threading (#120329)

---
 Include/Python.h                                              | 4 ++++
 .../next/Build/2024-06-11-00-38-05.gh-issue-120326.JHSDF1.rst | 2 ++
 2 files changed, 6 insertions(+)
 create mode 100644 Misc/NEWS.d/next/Build/2024-06-11-00-38-05.gh-issue-120326.JHSDF1.rst

diff --git a/Include/Python.h b/Include/Python.h
index 502c5ec5aeaa3c..a1b33f6d3c42b2 100644
--- a/Include/Python.h
+++ b/Include/Python.h
@@ -51,6 +51,10 @@
 #  error "The limited API is not currently supported in the free-threaded build"
 #endif
 
+#if defined(Py_GIL_DISABLED) && defined(_MSC_VER)
+#  include <intrin.h>             // __readgsqword()
+#endif
+
 // Include Python header files
 #include "pyport.h"
 #include "pymacro.h"
diff --git a/Misc/NEWS.d/next/Build/2024-06-11-00-38-05.gh-issue-120326.JHSDF1.rst b/Misc/NEWS.d/next/Build/2024-06-11-00-38-05.gh-issue-120326.JHSDF1.rst
new file mode 100644
index 00000000000000..25cbdf6ba50ab8
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2024-06-11-00-38-05.gh-issue-120326.JHSDF1.rst
@@ -0,0 +1,2 @@
+On Windows, fix build error when ``--disable-gil`` and ``--experimental-jit``
+options are combined.

From 203565b2f9c74656ba519780049b46d4e5afcba1 Mon Sep 17 00:00:00 2001
From: Ken Jin <kenjin@python.org>
Date: Wed, 12 Jun 2024 03:10:23 +0800
Subject: [PATCH 53/72] gh-120198: Fix race condition when editing __class__
 with an audit hook active (GH-120195)

---
 Lib/test/test_free_threading/test_type.py     |  1 +
 Lib/test/test_super.py                        | 35 ++++++++++++++++++-
 ...-06-10-15-07-16.gh-issue-120198.WW_pjO.rst |  1 +
 Objects/typeobject.c                          |  3 +-
 4 files changed, 38 insertions(+), 2 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-10-15-07-16.gh-issue-120198.WW_pjO.rst

diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py
index 6eead198deed46..786336fa0cddce 100644
--- a/Lib/test/test_free_threading/test_type.py
+++ b/Lib/test/test_free_threading/test_type.py
@@ -1,3 +1,4 @@
+import threading
 import unittest
 
 from concurrent.futures import ThreadPoolExecutor
diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py
index 256b416caaa584..3ffbe03f0c2f11 100644
--- a/Lib/test/test_super.py
+++ b/Lib/test/test_super.py
@@ -1,9 +1,10 @@
 """Unit tests for zero-argument super() & related machinery."""
 
 import textwrap
+import threading
 import unittest
 from unittest.mock import patch
-from test.support import import_helper
+from test.support import import_helper, threading_helper
 
 
 ADAPTIVE_WARMUP_DELAY = 2
@@ -505,6 +506,38 @@ def some(cls):
         for _ in range(ADAPTIVE_WARMUP_DELAY):
             C.some(C)
 
+    @threading_helper.requires_working_threading()
+    def test___class___modification_multithreaded(self):
+        """ Note: this test isn't actually testing anything on its own.
+        It requires a sys audithook to be set to crash on older Python.
+        This should be the case anyways as our test suite sets
+        an audit hook.
+        """
+        class Foo:
+            pass
+
+        class Bar:
+            pass
+
+        thing = Foo()
+        def work():
+            foo = thing
+            for _ in range(5000):
+                foo.__class__ = Bar
+                type(foo)
+                foo.__class__ = Foo
+                type(foo)
+
+
+        threads = []
+        for _ in range(6):
+            thread = threading.Thread(target=work)
+            thread.start()
+            threads.append(thread)
+
+        for thread in threads:
+            thread.join()
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-15-07-16.gh-issue-120198.WW_pjO.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-15-07-16.gh-issue-120198.WW_pjO.rst
new file mode 100644
index 00000000000000..8dc8aec44d80c4
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-15-07-16.gh-issue-120198.WW_pjO.rst	
@@ -0,0 +1 @@
+Fix a crash when multiple threads read and write to the same ``__class__`` of an object concurrently.
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index cd16bebd1e1cb8..070e3d2f7bf2b4 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -6522,7 +6522,6 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char*
 static int
 object_set_class(PyObject *self, PyObject *value, void *closure)
 {
-    PyTypeObject *oldto = Py_TYPE(self);
 
     if (value == NULL) {
         PyErr_SetString(PyExc_TypeError,
@@ -6542,6 +6541,8 @@ object_set_class(PyObject *self, PyObject *value, void *closure)
         return -1;
     }
 
+    PyTypeObject *oldto = Py_TYPE(self);
+
     /* In versions of CPython prior to 3.5, the code in
        compatible_for_assignment was not set up to correctly check for memory
        layout / slot / etc. compatibility for non-HEAPTYPE classes, so we just

From 34e4d3287e724c065cc07b04a1ee8715817db284 Mon Sep 17 00:00:00 2001
From: Pablo Galindo Salgado <Pablogsal@gmail.com>
Date: Tue, 11 Jun 2024 20:20:25 +0100
Subject: [PATCH 54/72] gh-120221: Deliver real singals on Ctrl-C and Ctrl-Z in
 the new REPL (#120354)

---
 Lib/_pyrepl/unix_console.py                               | 8 ++++----
 .../2024-06-11-17-56-12.gh-issue-120221.si9hM9.rst        | 2 ++
 2 files changed, 6 insertions(+), 4 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-11-17-56-12.gh-issue-120221.si9hM9.rst

diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py
index f1a6b84adfb671..af9290819c2c78 100644
--- a/Lib/_pyrepl/unix_console.py
+++ b/Lib/_pyrepl/unix_console.py
@@ -324,13 +324,13 @@ def prepare(self):
         """
         self.__svtermstate = tcgetattr(self.input_fd)
         raw = self.__svtermstate.copy()
-        raw.iflag &= ~(termios.BRKINT | termios.INPCK | termios.ISTRIP | termios.IXON)
+        raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON)
         raw.oflag &= ~(termios.OPOST)
         raw.cflag &= ~(termios.CSIZE | termios.PARENB)
         raw.cflag |= termios.CS8
-        raw.lflag &= ~(
-            termios.ICANON | termios.ECHO | termios.IEXTEN | (termios.ISIG * 1)
-        )
+        raw.iflag |= termios.BRKINT
+        raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN)
+        raw.lflag |= termios.ISIG
         raw.cc[termios.VMIN] = 1
         raw.cc[termios.VTIME] = 0
         tcsetattr(self.input_fd, termios.TCSADRAIN, raw)
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-11-17-56-12.gh-issue-120221.si9hM9.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-17-56-12.gh-issue-120221.si9hM9.rst
new file mode 100644
index 00000000000000..3781576bc5a257
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-17-56-12.gh-issue-120221.si9hM9.rst	
@@ -0,0 +1,2 @@
+Deliver real signals on Ctrl-C and Ctrl-Z in the new REPL. Patch by Pablo
+Galindo

From f5a9c34f38886c5cf9c2f8d860eee3473447e030 Mon Sep 17 00:00:00 2001
From: Nikita Sobolev <mail@sobolevn.me>
Date: Wed, 12 Jun 2024 04:00:56 +0300
Subject: [PATCH 55/72] gh-120056: Add `IP_RECVERR`, `IP_RECVORIGDSTADDR`,
 `IP_RECVTTL` to `socket` module (#120058)

* gh-120056: Add `IP_RECVERR` and `IP_RECVTTL` to `socket` module

* Fix news

* Address review

* Update NEWS
---
 Doc/library/socket.rst                                   | 4 ++++
 .../2024-06-04-19-49-16.gh-issue-120056.5aqozw.rst       | 3 +++
 Modules/socketmodule.c                                   | 9 +++++++++
 3 files changed, 16 insertions(+)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-04-19-49-16.gh-issue-120056.5aqozw.rst

diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst
index 2df0257d1f24f0..782fb9b27ae1ba 100644
--- a/Doc/library/socket.rst
+++ b/Doc/library/socket.rst
@@ -450,6 +450,10 @@ Constants
       same way that ``SO_BINDTODEVICE`` is used, but with the index of a
       network interface instead of its name.
 
+   .. versionchanged:: 3.14
+      Added missing ``IP_RECVERR``, ``IP_RECVTTL``, and ``IP_RECVORIGDSTADDR``
+      on Linux.
+
 .. data:: AF_CAN
           PF_CAN
           SOL_CAN_*
diff --git a/Misc/NEWS.d/next/Library/2024-06-04-19-49-16.gh-issue-120056.5aqozw.rst b/Misc/NEWS.d/next/Library/2024-06-04-19-49-16.gh-issue-120056.5aqozw.rst
new file mode 100644
index 00000000000000..0adb70f51e8a0c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-04-19-49-16.gh-issue-120056.5aqozw.rst
@@ -0,0 +1,3 @@
+Add :data:`!socket.IP_RECVERR` and :data:`!socket.IP_RECVTTL` constants
+(both available since Linux 2.2).
+And :data:`!socket.IP_RECVORIGDSTADDR` constant (available since Linux 2.6.29).
diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c
index cb7dc25e23fb3d..0626d7934983db 100644
--- a/Modules/socketmodule.c
+++ b/Modules/socketmodule.c
@@ -8412,15 +8412,24 @@ socket_exec(PyObject *m)
 #ifdef  IP_TTL
     ADD_INT_MACRO(m, IP_TTL);
 #endif
+#ifdef  IP_RECVERR
+    ADD_INT_MACRO(m, IP_RECVERR);
+#endif
 #ifdef  IP_RECVOPTS
     ADD_INT_MACRO(m, IP_RECVOPTS);
 #endif
+#ifdef  IP_RECVORIGDSTADDR
+    ADD_INT_MACRO(m, IP_RECVORIGDSTADDR);
+#endif
 #ifdef  IP_RECVRETOPTS
     ADD_INT_MACRO(m, IP_RECVRETOPTS);
 #endif
 #ifdef  IP_RECVTOS
     ADD_INT_MACRO(m, IP_RECVTOS);
 #endif
+#ifdef  IP_RECVTTL
+    ADD_INT_MACRO(m, IP_RECVTTL);
+#endif
 #ifdef  IP_RECVDSTADDR
     ADD_INT_MACRO(m, IP_RECVDSTADDR);
 #endif

From 19435d299a1fae9ad9a6bbe6609e41ddfd7f6cbe Mon Sep 17 00:00:00 2001
From: Kirill Podoprigora <kirill.bast9@mail.ru>
Date: Wed, 12 Jun 2024 10:37:14 +0300
Subject: [PATCH 56/72] gh-120385: Fix reference leak in symtable (#120386)

Decref 'new_ste' if symtable_visit_annotations() fails.
---
 Python/symtable.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Python/symtable.c b/Python/symtable.c
index 287bc2bd58107d..627184da9ef4ed 100644
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -2027,8 +2027,10 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
         }
 
         if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args,
-                                        s->v.AsyncFunctionDef.returns, new_ste))
+                                        s->v.AsyncFunctionDef.returns, new_ste)) {
+            Py_DECREF(new_ste);
             VISIT_QUIT(st, 0);
+        }
         if (!symtable_enter_existing_block(st, new_ste)) {
             Py_DECREF(new_ste);
             VISIT_QUIT(st, 0);

From 02e74c356223feb0771759286d24d1dbac01d4ca Mon Sep 17 00:00:00 2001
From: Lysandros Nikolaou <lisandrosnik@gmail.com>
Date: Wed, 12 Jun 2024 10:21:53 +0200
Subject: [PATCH 57/72] gh-118908: Fix completions after namespace change in
 REPL (#120370)

---
 Lib/_pyrepl/readline.py        | 13 ++++++++++---
 Lib/_pyrepl/simple_interact.py |  4 ++--
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py
index b10d0c66e4f813..28f592d80b1b03 100644
--- a/Lib/_pyrepl/readline.py
+++ b/Lib/_pyrepl/readline.py
@@ -55,6 +55,11 @@
 from collections.abc import Callable, Collection
 from .types import Callback, Completer, KeySpec, CommandName
 
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+    from typing import Any
+
 
 MoreLinesCallable = Callable[[str], bool]
 
@@ -92,7 +97,7 @@
 
 @dataclass
 class ReadlineConfig:
-    readline_completer: Completer | None = RLCompleter().complete
+    readline_completer: Completer | None = None
     completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?")
 
 
@@ -554,7 +559,7 @@ def stub(*args: object, **kwds: object) -> None:
 # ____________________________________________________________
 
 
-def _setup() -> None:
+def _setup(namespace: dict[str, Any]) -> None:
     global raw_input
     if raw_input is not None:
         return  # don't run _setup twice
@@ -570,9 +575,11 @@ def _setup() -> None:
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
+    # set up namespace in rlcompleter
+    _wrapper.config.readline_completer = RLCompleter(namespace).complete
+
     # this is not really what readline.c does.  Better than nothing I guess
     import builtins
-
     raw_input = builtins.input
     builtins.input = _wrapper.input
 
diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py
index 620f87b4867073..2de3b38c37a9da 100644
--- a/Lib/_pyrepl/simple_interact.py
+++ b/Lib/_pyrepl/simple_interact.py
@@ -96,9 +96,9 @@ def run_multiline_interactive_console(
     console: code.InteractiveConsole | None = None,
 ) -> None:
     from .readline import _setup
-    _setup()
-
     namespace = mainmodule.__dict__ if mainmodule else DEFAULT_NAMESPACE
+    _setup(namespace)
+
     if console is None:
         console = InteractiveColoredConsole(
             namespace, filename="<stdin>"

From 7dd8c37a067f9fcb6a2a658d6a93b294cc2e6fb4 Mon Sep 17 00:00:00 2001
From: Owain Davies <116417456+OTheDev@users.noreply.github.com>
Date: Wed, 12 Jun 2024 17:07:25 +0700
Subject: [PATCH 58/72] gh-101575: document Decimal.__round__() (GH-101737)

---
 Doc/library/decimal.rst | 42 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst
index 3e33581f96f16a..db323802a6f68c 100644
--- a/Doc/library/decimal.rst
+++ b/Doc/library/decimal.rst
@@ -897,6 +897,48 @@ Decimal objects
       :const:`Rounded`.  If given, applies *rounding*; otherwise, uses the
       rounding method in either the supplied *context* or the current context.
 
+   Decimal numbers can be rounded using the :func:`.round` function:
+
+   .. describe:: round(number)
+   .. describe:: round(number, ndigits)
+
+      If *ndigits* is not given or ``None``,
+      returns the nearest :class:`int` to *number*,
+      rounding ties to even, and ignoring the rounding mode of the
+      :class:`Decimal` context.  Raises :exc:`OverflowError` if *number* is an
+      infinity or :exc:`ValueError` if it is a (quiet or signaling) NaN.
+
+      If *ndigits* is an :class:`int`, the context's rounding mode is respected
+      and a :class:`Decimal` representing *number* rounded to the nearest
+      multiple of ``Decimal('1E-ndigits')`` is returned; in this case,
+      ``round(number, ndigits)`` is equivalent to
+      ``self.quantize(Decimal('1E-ndigits'))``.  Returns ``Decimal('NaN')`` if
+      *number* is a quiet NaN.  Raises :class:`InvalidOperation` if *number*
+      is an infinity, a signaling NaN, or if the length of the coefficient after
+      the quantize operation would be greater than the current context's
+      precision.  In other words, for the non-corner cases:
+
+      * if *ndigits* is positive, return *number* rounded to *ndigits* decimal
+        places;
+      * if *ndigits* is zero, return *number* rounded to the nearest integer;
+      * if *ndigits* is negative, return *number* rounded to the nearest
+        multiple of ``10**abs(ndigits)``.
+
+      For example::
+
+          >>> from decimal import Decimal, getcontext, ROUND_DOWN
+          >>> getcontext().rounding = ROUND_DOWN
+          >>> round(Decimal('3.75'))     # context rounding ignored
+          4
+          >>> round(Decimal('3.5'))      # round-ties-to-even
+          4
+          >>> round(Decimal('3.75'), 0)  # uses the context rounding
+          Decimal('3')
+          >>> round(Decimal('3.75'), 1)
+          Decimal('3.7')
+          >>> round(Decimal('3.75'), -1)
+          Decimal('0E+1')
+
 
 .. _logical_operands_label:
 

From 755dab719dfc924dd8aef46f67512dabb8f25071 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?=
 <10796600+picnixz@users.noreply.github.com>
Date: Wed, 12 Jun 2024 13:14:50 +0200
Subject: [PATCH 59/72] gh-120029: make `symtable.Symbol.__repr__` correctly
 reflect the compiler's flags, add methods  (#120099)

Expose :class:`symtable.Symbol` methods :meth:`~symtable.Symbol.is_free_class`,
:meth:`~symtable.Symbol.is_comp_iter` and :meth:`~symtable.Symbol.is_comp_cell`.

---------

Co-authored-by: Carl Meyer <carl@oddbird.net>
---
 Doc/library/symtable.rst                      | 34 +++++++++++++++++++
 Doc/whatsnew/3.14.rst                         | 11 ++++++
 Include/internal/pycore_symtable.h            |  2 +-
 Lib/symtable.py                               | 32 ++++++++++++++---
 Lib/test/test_symtable.py                     | 21 ++++++++++++
 ...-06-05-11-03-10.gh-issue-120029.QBsw47.rst |  4 +++
 Modules/symtablemodule.c                      |  2 ++
 7 files changed, 100 insertions(+), 6 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-05-11-03-10.gh-issue-120029.QBsw47.rst

diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst
index e17a33f7feb1ab..050a941d9d0516 100644
--- a/Doc/library/symtable.rst
+++ b/Doc/library/symtable.rst
@@ -155,6 +155,8 @@ Examining Symbol Tables
 
       Return ``True`` if the symbol is a type parameter.
 
+      .. versionadded:: 3.14
+
    .. method:: is_global()
 
       Return ``True`` if the symbol is global.
@@ -182,10 +184,42 @@ Examining Symbol Tables
       Return ``True`` if the symbol is referenced in its block, but not assigned
       to.
 
+   .. method:: is_free_class()
+
+      Return *True* if a class-scoped symbol is free from
+      the perspective of a method.
+
+      Consider the following example::
+
+         def f():
+             x = 1  # function-scoped
+             class C:
+                 x = 2  # class-scoped
+                 def method(self):
+                     return x
+
+      In this example, the class-scoped symbol ``x`` is considered to
+      be free from the perspective of ``C.method``, thereby allowing
+      the latter to return *1* at runtime and not *2*.
+
+      .. versionadded:: 3.14
+
    .. method:: is_assigned()
 
       Return ``True`` if the symbol is assigned to in its block.
 
+   .. method:: is_comp_iter()
+
+      Return ``True`` if the symbol is a comprehension iteration variable.
+
+      .. versionadded:: 3.14
+
+   .. method:: is_comp_cell()
+
+      Return ``True`` if the symbol is a cell in an inlined comprehension.
+
+      .. versionadded:: 3.14
+
    .. method:: is_namespace()
 
       Return ``True`` if name binding introduces new namespace.
diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst
index b77ff30a8fbbee..b357553735e8bb 100644
--- a/Doc/whatsnew/3.14.rst
+++ b/Doc/whatsnew/3.14.rst
@@ -100,6 +100,17 @@ os
   by :func:`os.unsetenv`, or made outside Python in the same process.
   (Contributed by Victor Stinner in :gh:`120057`.)
 
+symtable
+--------
+
+* Expose the following :class:`symtable.Symbol` methods:
+
+  * :meth:`~symtable.Symbol.is_free_class`
+  * :meth:`~symtable.Symbol.is_comp_iter`
+  * :meth:`~symtable.Symbol.is_comp_cell`
+
+  (Contributed by Bénédikt Tran in :gh:`120029`.)
+
 
 Optimizations
 =============
diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h
index 5d544765237df5..1be48edc80c830 100644
--- a/Include/internal/pycore_symtable.h
+++ b/Include/internal/pycore_symtable.h
@@ -154,7 +154,7 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name);
 #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
 
 /* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol
-   table.  GLOBAL is returned from PyST_GetScope() for either of them.
+   table.  GLOBAL is returned from _PyST_GetScope() for either of them.
    It is stored in ste_symbols at bits 13-16.
 */
 #define SCOPE_OFFSET 12
diff --git a/Lib/symtable.py b/Lib/symtable.py
index af65e93e68eda4..d6ac1f527ba8ba 100644
--- a/Lib/symtable.py
+++ b/Lib/symtable.py
@@ -4,7 +4,10 @@
 from _symtable import (
     USE,
     DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL,
-    DEF_PARAM, DEF_TYPE_PARAM, DEF_IMPORT, DEF_BOUND, DEF_ANNOT,
+    DEF_PARAM, DEF_TYPE_PARAM,
+    DEF_FREE_CLASS,
+    DEF_IMPORT, DEF_BOUND, DEF_ANNOT,
+    DEF_COMP_ITER, DEF_COMP_CELL,
     SCOPE_OFF, SCOPE_MASK,
     FREE, LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL
 )
@@ -158,6 +161,10 @@ def get_children(self):
                 for st in self._table.children]
 
 
+def _get_scope(flags):  # like _PyST_GetScope()
+    return (flags >> SCOPE_OFF) & SCOPE_MASK
+
+
 class Function(SymbolTable):
 
     # Default values for instance variables
@@ -183,7 +190,7 @@ def get_locals(self):
         """
         if self.__locals is None:
             locs = (LOCAL, CELL)
-            test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs
+            test = lambda x: _get_scope(x) in locs
             self.__locals = self.__idents_matching(test)
         return self.__locals
 
@@ -192,7 +199,7 @@ def get_globals(self):
         """
         if self.__globals is None:
             glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
-            test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob
+            test = lambda x: _get_scope(x) in glob
             self.__globals = self.__idents_matching(test)
         return self.__globals
 
@@ -207,7 +214,7 @@ def get_frees(self):
         """Return a tuple of free variables in the function.
         """
         if self.__frees is None:
-            is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE
+            is_free = lambda x: _get_scope(x) == FREE
             self.__frees = self.__idents_matching(is_free)
         return self.__frees
 
@@ -234,7 +241,7 @@ class Symbol:
     def __init__(self, name, flags, namespaces=None, *, module_scope=False):
         self.__name = name
         self.__flags = flags
-        self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope()
+        self.__scope = _get_scope(flags)
         self.__namespaces = namespaces or ()
         self.__module_scope = module_scope
 
@@ -303,6 +310,11 @@ def is_free(self):
         """
         return bool(self.__scope == FREE)
 
+    def is_free_class(self):
+        """Return *True* if a class-scoped symbol is free from
+        the perspective of a method."""
+        return bool(self.__flags & DEF_FREE_CLASS)
+
     def is_imported(self):
         """Return *True* if the symbol is created from
         an import statement.
@@ -313,6 +325,16 @@ def is_assigned(self):
         """Return *True* if a symbol is assigned to."""
         return bool(self.__flags & DEF_LOCAL)
 
+    def is_comp_iter(self):
+        """Return *True* if the symbol is a comprehension iteration variable.
+        """
+        return bool(self.__flags & DEF_COMP_ITER)
+
+    def is_comp_cell(self):
+        """Return *True* if the symbol is a cell in an inlined comprehension.
+        """
+        return bool(self.__flags & DEF_COMP_CELL)
+
     def is_namespace(self):
         """Returns *True* if name binding introduces new namespace.
 
diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py
index a4b111e865c86e..903c6d66f50964 100644
--- a/Lib/test/test_symtable.py
+++ b/Lib/test/test_symtable.py
@@ -304,6 +304,27 @@ def test_symbol_repr(self):
         self.assertEqual(repr(self.GenericMine.lookup("T")),
                          "<symbol 'T': LOCAL, DEF_LOCAL|DEF_TYPE_PARAM>")
 
+        st1 = symtable.symtable("[x for x in [1]]", "?", "exec")
+        self.assertEqual(repr(st1.lookup("x")),
+                         "<symbol 'x': LOCAL, USE|DEF_LOCAL|DEF_COMP_ITER>")
+
+        st2 = symtable.symtable("[(lambda: x) for x in [1]]", "?", "exec")
+        self.assertEqual(repr(st2.lookup("x")),
+                         "<symbol 'x': CELL, DEF_LOCAL|DEF_COMP_ITER|DEF_COMP_CELL>")
+
+        st3 = symtable.symtable("def f():\n"
+                                "   x = 1\n"
+                                "   class A:\n"
+                                "       x = 2\n"
+                                "       def method():\n"
+                                "           return x\n",
+                                "?", "exec")
+        # child 0 is for __annotate__
+        func_f = st3.get_children()[1]
+        class_A = func_f.get_children()[0]
+        self.assertEqual(repr(class_A.lookup('x')),
+                         "<symbol 'x': LOCAL, DEF_LOCAL|DEF_FREE_CLASS>")
+
     def test_symtable_entry_repr(self):
         expected = f"<symtable entry top({self.top.get_id()}), line {self.top.get_lineno()}>"
         self.assertEqual(repr(self.top._table), expected)
diff --git a/Misc/NEWS.d/next/Library/2024-06-05-11-03-10.gh-issue-120029.QBsw47.rst b/Misc/NEWS.d/next/Library/2024-06-05-11-03-10.gh-issue-120029.QBsw47.rst
new file mode 100644
index 00000000000000..d1b2c592a113ce
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-05-11-03-10.gh-issue-120029.QBsw47.rst
@@ -0,0 +1,4 @@
+Expose :class:`symtable.Symbol` methods :meth:`~symtable.Symbol.is_free_class`,
+:meth:`~symtable.Symbol.is_comp_iter` and :meth:`~symtable.Symbol.is_comp_cell`.
+Patch by Bénédikt Tran.
+
diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c
index 63c4dd4225298d..b39b59bf7b06bf 100644
--- a/Modules/symtablemodule.c
+++ b/Modules/symtablemodule.c
@@ -81,6 +81,8 @@ symtable_init_constants(PyObject *m)
     if (PyModule_AddIntMacro(m, DEF_IMPORT) < 0) return -1;
     if (PyModule_AddIntMacro(m, DEF_BOUND) < 0) return -1;
     if (PyModule_AddIntMacro(m, DEF_ANNOT) < 0) return -1;
+    if (PyModule_AddIntMacro(m, DEF_COMP_ITER) < 0) return -1;
+    if (PyModule_AddIntMacro(m, DEF_COMP_CELL) < 0) return -1;
 
     if (PyModule_AddIntConstant(m, "TYPE_FUNCTION", FunctionBlock) < 0)
         return -1;

From 97b69db167be28a33688db436551a6c3c3ea4662 Mon Sep 17 00:00:00 2001
From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com>
Date: Wed, 12 Jun 2024 12:53:19 +0100
Subject: [PATCH 60/72] gh-93691: fix too broad source locations of for
 statement iterators (#120330)

---
 Lib/test/test_compiler_codegen.py             |  1 +
 Lib/test/test_iter.py                         | 46 +++++++++++++++++++
 Lib/test/test_sys_settrace.py                 |  6 +--
 ...4-06-10-22-30-26.gh-issue-93691.68WOTS.rst |  2 +
 Programs/test_frozenmain.h                    |  9 ++--
 Python/compile.c                              |  7 +++
 6 files changed, 63 insertions(+), 8 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-10-22-30-26.gh-issue-93691.68WOTS.rst

diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py
index 1088b4aa9e624d..d82fb85ed259ab 100644
--- a/Lib/test/test_compiler_codegen.py
+++ b/Lib/test/test_compiler_codegen.py
@@ -49,6 +49,7 @@ def test_for_loop(self):
             ('GET_ITER', None, 1),
             loop_lbl := self.Label(),
             ('FOR_ITER', exit_lbl := self.Label(), 1),
+            ('NOP', None, 1, 1),
             ('STORE_NAME', 1, 1),
             ('LOAD_NAME', 2, 2),
             ('PUSH_NULL', None, 2),
diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py
index 9606d5beab71cb..ec2b68acb90785 100644
--- a/Lib/test/test_iter.py
+++ b/Lib/test/test_iter.py
@@ -10,6 +10,7 @@
 import functools
 import contextlib
 import builtins
+import traceback
 
 # Test result of triple loop (too big to inline)
 TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2),
@@ -1143,6 +1144,51 @@ def test_error_iter(self):
             self.assertRaises(TypeError, iter, typ())
         self.assertRaises(ZeroDivisionError, iter, BadIterableClass())
 
+    def test_exception_locations(self):
+        # The location of an exception raised from __init__ or
+        # __next__ should should be the iterator expression
+
+        class Iter:
+            def __init__(self, init_raises=False, next_raises=False):
+                if init_raises:
+                    1/0
+                self.next_raises = next_raises
+
+            def __next__(self):
+                if self.next_raises:
+                    1/0
+
+            def __iter__(self):
+                return self
+
+        def init_raises():
+            try:
+                for x in Iter(init_raises=True):
+                    pass
+            except Exception as e:
+                return e
+
+        def next_raises():
+            try:
+                for x in Iter(next_raises=True):
+                    pass
+            except Exception as e:
+                return e
+
+        for func, expected in [(init_raises, "Iter(init_raises=True)"),
+                               (next_raises, "Iter(next_raises=True)"),
+                              ]:
+            with self.subTest(func):
+                exc = func()
+                f = traceback.extract_tb(exc.__traceback__)[0]
+                indent = 16
+                co = func.__code__
+                self.assertEqual(f.lineno, co.co_firstlineno + 2)
+                self.assertEqual(f.end_lineno, co.co_firstlineno + 2)
+                self.assertEqual(f.line[f.colno - indent : f.end_colno - indent],
+                                 expected)
+
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py
index ded1d9224d82d9..c622fd9ce7c466 100644
--- a/Lib/test/test_sys_settrace.py
+++ b/Lib/test/test_sys_settrace.py
@@ -1650,15 +1650,15 @@ def func():
         EXPECTED_EVENTS = [
             (0, 'call'),
             (2, 'line'),
-            (1, 'line'),
             (-3, 'call'),
             (-2, 'line'),
             (-2, 'return'),
-            (4, 'line'),
             (1, 'line'),
+            (4, 'line'),
+            (2, 'line'),
             (-2, 'call'),
             (-2, 'return'),
-            (1, 'return'),
+            (2, 'return'),
         ]
 
         # C level events should be the same as expected and the same as Python level.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-22-30-26.gh-issue-93691.68WOTS.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-22-30-26.gh-issue-93691.68WOTS.rst
new file mode 100644
index 00000000000000..294f8d892b459b
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-22-30-26.gh-issue-93691.68WOTS.rst	
@@ -0,0 +1,2 @@
+Fix source locations of instructions generated for the iterator of a for
+statement.
diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h
index cdc417e48ebec6..f34d7ea0789310 100644
--- a/Programs/test_frozenmain.h
+++ b/Programs/test_frozenmain.h
@@ -27,12 +27,11 @@ unsigned char M_test_frozenmain[] = {
     3,0,0,0,218,3,107,101,121,169,0,243,0,0,0,0,
     218,18,116,101,115,116,95,102,114,111,122,101,110,109,97,105,
     110,46,112,121,218,8,60,109,111,100,117,108,101,62,114,18,
-    0,0,0,1,0,0,0,115,99,0,0,0,240,3,1,1,
+    0,0,0,1,0,0,0,115,94,0,0,0,240,3,1,1,
     1,243,8,0,1,11,219,0,24,225,0,5,208,6,26,212,
     0,27,217,0,5,128,106,144,35,151,40,145,40,212,0,27,
     216,9,26,215,9,38,210,9,38,211,9,40,168,24,209,9,
-    50,128,6,240,2,6,12,2,242,0,7,1,42,128,67,241,
-    14,0,5,10,136,71,144,67,144,53,152,2,152,54,160,35,
-    153,59,152,45,208,10,40,214,4,41,242,15,7,1,42,114,
-    16,0,0,0,
+    50,128,6,243,2,6,12,2,128,67,241,14,0,5,10,136,
+    71,144,67,144,53,152,2,152,54,160,35,153,59,152,45,208,
+    10,40,214,4,41,242,15,6,12,2,114,16,0,0,0,
 };
diff --git a/Python/compile.c b/Python/compile.c
index c3372766d0bd50..749b69f5911386 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -3025,11 +3025,18 @@ compiler_for(struct compiler *c, stmt_ty s)
     RETURN_IF_ERROR(compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL));
 
     VISIT(c, expr, s->v.For.iter);
+
+    loc = LOC(s->v.For.iter);
     ADDOP(c, loc, GET_ITER);
 
     USE_LABEL(c, start);
     ADDOP_JUMP(c, loc, FOR_ITER, cleanup);
 
+    /* Add NOP to ensure correct line tracing of multiline for statements.
+     * It will be removed later if redundant.
+     */
+    ADDOP(c, LOC(s->v.For.target), NOP);
+
     USE_LABEL(c, body);
     VISIT(c, expr, s->v.For.target);
     VISIT_SEQ(c, stmt, s->v.For.body);

From ce3879bd45e068f8e2a5a214acd234ca44cad53b Mon Sep 17 00:00:00 2001
From: Xie Yanbo <xieyanbo@gmail.com>
Date: Wed, 12 Jun 2024 20:24:43 +0800
Subject: [PATCH 61/72] Fix typos in documentation (#120338)

---
 InternalDocs/compiler.md | 2 +-
 Lib/idlelib/HISTORY.txt  | 2 +-
 Lib/idlelib/News3.txt    | 6 +++---
 Lib/idlelib/TODO.txt     | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md
index 0abc10da6e05c6..17fe0df6e1db10 100644
--- a/InternalDocs/compiler.md
+++ b/InternalDocs/compiler.md
@@ -625,7 +625,7 @@ Objects
 
 * [Objects/locations.md](https://github.com/python/cpython/blob/main/Objects/locations.md): Describes the location table
 * [Objects/frame_layout.md](https://github.com/python/cpython/blob/main/Objects/frame_layout.md): Describes the frame stack
-* [Objects/object_layout.md](https://github.com/python/cpython/blob/main/Objects/object_layout.md): Descibes object layout for 3.11 and later
+* [Objects/object_layout.md](https://github.com/python/cpython/blob/main/Objects/object_layout.md): Describes object layout for 3.11 and later
 * [Exception Handling](exception_handling.md): Describes the exception table
 
 
diff --git a/Lib/idlelib/HISTORY.txt b/Lib/idlelib/HISTORY.txt
index 731fabd185fbbf..a601b25b5f838f 100644
--- a/Lib/idlelib/HISTORY.txt
+++ b/Lib/idlelib/HISTORY.txt
@@ -277,7 +277,7 @@ Command to format a paragraph.
 Debug menu:
 
 JIT (Just-In-Time) stack viewer toggle -- if set, the stack viewer
-automaticall pops up when you get a traceback.
+automatically pops up when you get a traceback.
 
 Windows menu:
 
diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt
index fb07d7b3b3fad8..b1b652dc562c8e 100644
--- a/Lib/idlelib/News3.txt
+++ b/Lib/idlelib/News3.txt
@@ -568,14 +568,14 @@ bpo-33679: Enable theme-specific color configuration for Code Context.
 color setting, default or custom, on the extensions tab, that applied
 to all themes.)  For built-in themes, the foreground is the same as
 normal text and the background is a contrasting gray.  Context colors for
-custom themes are set on the Hightlights tab along with other colors.
+custom themes are set on the Highlights tab along with other colors.
 When one starts IDLE from a console and loads a custom theme without
 definitions for 'context', one will see a warning message on the
 console.
 
 bpo-33642: Display up to maxlines non-blank lines for Code Context.
 If there is no current context, show a single blank line.  (Previously,
-the Code Contex had numlines lines, usually with some blank.)  The use
+the Code Context had numlines lines, usually with some blank.)  The use
 of a new option, 'maxlines' (default 15), avoids possible interference
 with user settings of the old option, 'numlines' (default 3).
 
@@ -729,7 +729,7 @@ not affect their keyset-specific customization after 3.6.3. and vice
 versa.
   Initial patch by Charles Wohlganger, revised by Terry Jan Reedy.
 
-bpo-31051:  Rearrange condigdialog General tab.
+bpo-31051:  Rearrange configdialog General tab.
 Sort non-Help options into Window (Shell+Editor) and Editor (only).
 Leave room for the addition of new options.
 Patch by Terry Jan Reedy.
diff --git a/Lib/idlelib/TODO.txt b/Lib/idlelib/TODO.txt
index e2f1ac0f274001..41b86b0c6d5bbd 100644
--- a/Lib/idlelib/TODO.txt
+++ b/Lib/idlelib/TODO.txt
@@ -179,7 +179,7 @@ it -- i.e. you can only edit the current command, and the cursor can't
 escape from the command area.  (Albert Brandl)
 
 - Set X11 class to "idle/Idle", set icon and title to something
-beginning with "idle" -- for window manangers.  (Randall Hopper)
+beginning with "idle" -- for window managers.  (Randall Hopper)
 
 - Config files editable through a preferences dialog.  (me) DONE
 

From e16aed63f64b18a26859eff3de976ded373e66b8 Mon Sep 17 00:00:00 2001
From: Ken Jin <kenjin@python.org>
Date: Wed, 12 Jun 2024 20:41:07 +0800
Subject: [PATCH 62/72] gh-117657: Make Py_TYPE and Py_SET_TYPE thread safe
 (GH-120165)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
Co-authored-by: Nadeshiko Manju <me@manjusaka.me>
---
 Include/internal/pycore_interp.h           |  5 ++++-
 Include/object.h                           |  8 +++++++
 Lib/test/test_free_threading/test_type.py  | 26 ++++++++++++++++++++++
 Objects/typeobject.c                       |  8 ++++++-
 Tools/tsan/suppressions_free_threading.txt |  2 --
 5 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index 86dada5061e7b5..6b5f50b88f7b85 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -401,7 +401,10 @@ PyAPI_FUNC(PyStatus) _PyInterpreterState_New(
 #define RARE_EVENT_INTERP_INC(interp, name) \
     do { \
         /* saturating add */ \
-        if (interp->rare_events.name < UINT8_MAX) interp->rare_events.name++; \
+        int val = FT_ATOMIC_LOAD_UINT8_RELAXED(interp->rare_events.name); \
+        if (val < UINT8_MAX) { \
+            FT_ATOMIC_STORE_UINT8(interp->rare_events.name, val + 1); \
+        } \
         RARE_EVENT_STAT_INC(name); \
     } while (0); \
 
diff --git a/Include/object.h b/Include/object.h
index c8c63b9b2b1450..4a39ada8c7daa4 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -246,7 +246,11 @@ _Py_IsOwnedByCurrentThread(PyObject *ob)
 
 // bpo-39573: The Py_SET_TYPE() function must be used to set an object type.
 static inline PyTypeObject* Py_TYPE(PyObject *ob) {
+#ifdef Py_GIL_DISABLED
+    return (PyTypeObject *)_Py_atomic_load_ptr_relaxed(&ob->ob_type);
+#else
     return ob->ob_type;
+#endif
 }
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
 #  define Py_TYPE(ob) Py_TYPE(_PyObject_CAST(ob))
@@ -274,7 +278,11 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) {
 
 
 static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) {
+#ifdef Py_GIL_DISABLED
+    _Py_atomic_store_ptr(&ob->ob_type, type);
+#else
     ob->ob_type = type;
+#endif
 }
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
 #  define Py_SET_TYPE(ob, type) Py_SET_TYPE(_PyObject_CAST(ob), type)
diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py
index 786336fa0cddce..1e84b2db2d4882 100644
--- a/Lib/test/test_free_threading/test_type.py
+++ b/Lib/test/test_free_threading/test_type.py
@@ -96,6 +96,32 @@ def reader_func():
 
         self.run_one(writer_func, reader_func)
 
+    def test___class___modification(self):
+        class Foo:
+            pass
+
+        class Bar:
+            pass
+
+        thing = Foo()
+        def work():
+            foo = thing
+            for _ in range(10000):
+                foo.__class__ = Bar
+                type(foo)
+                foo.__class__ = Foo
+                type(foo)
+
+
+        threads = []
+        for i in range(NTHREADS):
+            thread = threading.Thread(target=work)
+            thread.start()
+            threads.append(thread)
+
+        for thread in threads:
+            thread.join()
+
     def run_one(self, writer_func, reader_func):
         writer = Thread(target=writer_func)
         readers = []
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 070e3d2f7bf2b4..8ecab555454cdc 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -6633,9 +6633,15 @@ object_set_class(PyObject *self, PyObject *value, void *closure)
         if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) {
             Py_INCREF(newto);
         }
+        Py_BEGIN_CRITICAL_SECTION(self);
+        // The real Py_TYPE(self) (`oldto`) may have changed from
+        // underneath us in another thread, so we re-fetch it here.
+        oldto = Py_TYPE(self);
         Py_SET_TYPE(self, newto);
-        if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE)
+        Py_END_CRITICAL_SECTION();
+        if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) {
             Py_DECREF(oldto);
+        }
 
         RARE_EVENT_INC(set_class);
         return 0;
diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt
index cb48a30751ac7b..b10b297f50da81 100644
--- a/Tools/tsan/suppressions_free_threading.txt
+++ b/Tools/tsan/suppressions_free_threading.txt
@@ -37,7 +37,6 @@ race_top:set_contains_key
 # https://gist.github.com/colesbury/d13d033f413b4ad07929d044bed86c35
 race_top:set_discard_entry
 race_top:set_inheritable
-race_top:Py_SET_TYPE
 race_top:_PyDict_CheckConsistency
 race_top:_Py_dict_lookup_threadsafe
 race_top:_multiprocessing_SemLock_acquire_impl
@@ -58,7 +57,6 @@ race_top:_PyFrame_Initialize
 race_top:PyInterpreterState_ThreadHead
 race_top:_PyObject_TryGetInstanceAttribute
 race_top:PyThreadState_Next
-race_top:Py_TYPE
 race_top:PyUnstable_InterpreterFrame_GetLine
 race_top:sock_close
 race_top:tstate_delete_common

From 32d3e05fe67d43f7285e582a87e65374cf7c2972 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?=
 <10796600+picnixz@users.noreply.github.com>
Date: Wed, 12 Jun 2024 15:23:45 +0200
Subject: [PATCH 63/72] gh-120029: remove unused macros in ``symtable.c``
 (#120222)

Co-authored-by: Carl Meyer <carl@oddbird.net>
Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
---
 Include/internal/pycore_symtable.h | 4 ----
 Modules/symtablemodule.c           | 1 -
 Python/symtable.c                  | 1 -
 3 files changed, 6 deletions(-)

diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h
index 1be48edc80c830..519505c0edf52a 100644
--- a/Include/internal/pycore_symtable.h
+++ b/Include/internal/pycore_symtable.h
@@ -143,7 +143,6 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name);
 #define DEF_PARAM (2<<1)         /* formal parameter */
 #define DEF_NONLOCAL (2<<2)      /* nonlocal stmt */
 #define USE (2<<3)               /* name is used */
-#define DEF_FREE (2<<4)          /* name used but not defined in nested block */
 #define DEF_FREE_CLASS (2<<5)    /* free variable from class's method */
 #define DEF_IMPORT (2<<6)        /* assignment occurred via import */
 #define DEF_ANNOT (2<<7)         /* this name is annotated */
@@ -166,9 +165,6 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name);
 #define FREE 4
 #define CELL 5
 
-#define GENERATOR 1
-#define GENERATOR_EXPRESSION 2
-
 // Used by symtablemodule.c
 extern struct symtable* _Py_SymtableStringObjectFlags(
     const char *str,
diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c
index b39b59bf7b06bf..618465536e7851 100644
--- a/Modules/symtablemodule.c
+++ b/Modules/symtablemodule.c
@@ -76,7 +76,6 @@ symtable_init_constants(PyObject *m)
     if (PyModule_AddIntMacro(m, DEF_LOCAL) < 0) return -1;
     if (PyModule_AddIntMacro(m, DEF_PARAM) < 0) return -1;
     if (PyModule_AddIntMacro(m, DEF_TYPE_PARAM) < 0) return -1;
-    if (PyModule_AddIntMacro(m, DEF_FREE) < 0) return -1;
     if (PyModule_AddIntMacro(m, DEF_FREE_CLASS) < 0) return -1;
     if (PyModule_AddIntMacro(m, DEF_IMPORT) < 0) return -1;
     if (PyModule_AddIntMacro(m, DEF_BOUND) < 0) return -1;
diff --git a/Python/symtable.c b/Python/symtable.c
index 627184da9ef4ed..0490014166e65c 100644
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -327,7 +327,6 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix)
         if (flags & DEF_PARAM) printf(" DEF_PARAM");
         if (flags & DEF_NONLOCAL) printf(" DEF_NONLOCAL");
         if (flags & USE) printf(" USE");
-        if (flags & DEF_FREE) printf(" DEF_FREE");
         if (flags & DEF_FREE_CLASS) printf(" DEF_FREE_CLASS");
         if (flags & DEF_IMPORT) printf(" DEF_IMPORT");
         if (flags & DEF_ANNOT) printf(" DEF_ANNOT");

From 4b1e85bafc5bcb8cb70bb17164e07aebf7ad7e8e Mon Sep 17 00:00:00 2001
From: ixgbe00 <yangwang@iscas.ac.cn>
Date: Wed, 12 Jun 2024 21:24:46 +0800
Subject: [PATCH 64/72] =?UTF-8?q?gh-120400=20=EF=BC=9ASupport=20Linux=20pe?=
 =?UTF-8?q?rf=20profile=20to=20see=20Python=20calls=20on=20RISC-V=20archit?=
 =?UTF-8?q?ecture=20(#120089)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
---
 .../next/Core and Builtins/2024-06-05-06-26-04.gh-issue-  | 1 +
 .../2024-06-12-12-29-45.gh-issue-120400.lZYHVS.rst        | 1 +
 Python/asm_trampoline.S                                   | 8 ++++++++
 configure                                                 | 2 ++
 configure.ac                                              | 1 +
 5 files changed, 13 insertions(+)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-05-06-26-04.gh-issue-
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-12-12-29-45.gh-issue-120400.lZYHVS.rst

diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-05-06-26-04.gh-issue- b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-06-26-04.gh-issue-
new file mode 100644
index 00000000000000..29f06d43c3598c
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-06-26-04.gh-issue-	
@@ -0,0 +1 @@
+Support Linux perf profiler to see Python calls on RISC-V architecture
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-12-12-29-45.gh-issue-120400.lZYHVS.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-12-29-45.gh-issue-120400.lZYHVS.rst
new file mode 100644
index 00000000000000..8c86d4750e39a8
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-12-29-45.gh-issue-120400.lZYHVS.rst	
@@ -0,0 +1 @@
+Support Linux perf profiler to see Python calls on RISC-V architecture.
diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S
index 460707717df003..0a3265dfeee204 100644
--- a/Python/asm_trampoline.S
+++ b/Python/asm_trampoline.S
@@ -22,6 +22,14 @@ _Py_trampoline_func_start:
     blr     x3
     ldp     x29, x30, [sp], 16
     ret
+#endif
+#ifdef __riscv
+    addi    sp,sp,-16
+    sd      ra,8(sp)
+    jalr    a3
+    ld      ra,8(sp)
+    addi    sp,sp,16
+    jr      ra
 #endif
     .globl	_Py_trampoline_func_end
 _Py_trampoline_func_end:
diff --git a/configure b/configure
index 8e605d31bb5eca..4174633b51c30a 100755
--- a/configure
+++ b/configure
@@ -13133,6 +13133,8 @@ case $PLATFORM_TRIPLET in #(
     perf_trampoline=yes ;; #(
   aarch64-linux-gnu) :
     perf_trampoline=yes ;; #(
+  riscv64-linux-gnu) :
+    perf_trampoline=yes ;; #(
   *) :
     perf_trampoline=no
  ;;
diff --git a/configure.ac b/configure.ac
index 41023ab92bad81..d34ade389cf40c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3641,6 +3641,7 @@ AC_MSG_CHECKING([perf trampoline])
 AS_CASE([$PLATFORM_TRIPLET],
   [x86_64-linux-gnu], [perf_trampoline=yes],
   [aarch64-linux-gnu], [perf_trampoline=yes],
+  [riscv64-linux-gnu], [perf_trampoline=yes],
   [perf_trampoline=no]
 )
 AC_MSG_RESULT([$perf_trampoline])

From 42b25dd61ff3593795c4cc2ffe876ab766098b24 Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Wed, 12 Jun 2024 15:27:07 +0200
Subject: [PATCH 65/72] gh-120155: Add assertion to sre.c match_getindex()
 (#120402)

Add an assertion to help static analyzers to detect that i*2 cannot
overflow.
---
 Modules/_sre/sre.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c
index e33034086481c2..0c656b47991c2f 100644
--- a/Modules/_sre/sre.c
+++ b/Modules/_sre/sre.c
@@ -2217,6 +2217,8 @@ match_getindex(MatchObject* self, PyObject* index)
         return -1;
     }
 
+    // Check that i*2 cannot overflow to make static analyzers happy
+    assert(i <= SRE_MAXGROUPS);
     return i;
 }
 

From 92c9c6ae147e1e658bbc8d454f8c7b2c4dea31d1 Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka <storchaka@gmail.com>
Date: Wed, 12 Jun 2024 17:23:03 +0300
Subject: [PATCH 66/72] gh-120345: Fix incorrect use of the :class: role with
 the "()" suffix (GH-120347)

* Remove "()" when refer to a class as a type.
* Use :func: when refer to a callable.
* Fix reference to the datetime.astimezone() method.
---
 Doc/howto/descriptor.rst    | 2 +-
 Doc/library/collections.rst | 2 +-
 Doc/library/datetime.rst    | 2 +-
 Doc/library/fileinput.rst   | 2 +-
 Doc/tutorial/stdlib2.rst    | 4 ++--
 Doc/whatsnew/2.5.rst        | 4 ++--
 Doc/whatsnew/3.12.rst       | 2 +-
 7 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst
index 51f9f4a6556e57..b29488be39a0a3 100644
--- a/Doc/howto/descriptor.rst
+++ b/Doc/howto/descriptor.rst
@@ -787,7 +787,7 @@ Invocation from super
 ---------------------
 
 The logic for super's dotted lookup is in the :meth:`__getattribute__` method for
-object returned by :class:`super()`.
+object returned by :func:`super`.
 
 A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__``
 for the base class ``B`` immediately following ``A`` and then returns
diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
index 2a269712f1814d..ce89101d6b667c 100644
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -99,7 +99,7 @@ The class can be used to simulate nested scopes and is useful in templating.
         :func:`super` function.  A reference to ``d.parents`` is equivalent to:
         ``ChainMap(*d.maps[1:])``.
 
-    Note, the iteration order of a :class:`ChainMap()` is determined by
+    Note, the iteration order of a :class:`ChainMap` is determined by
     scanning the mappings last to first::
 
         >>> baseline = {'music': 'bach', 'art': 'rembrandt'}
diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index 0723d0fe2fceb0..b6d8e6e6df07fa 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -2153,7 +2153,7 @@ There is one more :class:`tzinfo` method that a subclass may wish to override:
 
 .. method:: tzinfo.fromutc(dt)
 
-   This is called from the default :class:`datetime.astimezone()`
+   This is called from the default :meth:`datetime.astimezone`
    implementation. When called from that, ``dt.tzinfo`` is *self*, and *dt*'s
    date and time data are to be viewed as expressing a UTC time. The purpose
    of :meth:`fromutc` is to adjust the date and time data, returning an
diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst
index 94a4139f64c2e4..8f32b11e565365 100644
--- a/Doc/library/fileinput.rst
+++ b/Doc/library/fileinput.rst
@@ -47,7 +47,7 @@ Lines are returned with any newlines intact, which means that the last line in
 a file may not have one.
 
 You can control how files are opened by providing an opening hook via the
-*openhook* parameter to :func:`fileinput.input` or :class:`FileInput()`. The
+*openhook* parameter to :func:`fileinput.input` or :func:`FileInput`. The
 hook must be a function that takes two arguments, *filename* and *mode*, and
 returns an accordingly opened file-like object. If *encoding* and/or *errors*
 are specified, they will be passed to the hook as additional keyword arguments.
diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst
index 4bc810ce36c71b..719f772e687008 100644
--- a/Doc/tutorial/stdlib2.rst
+++ b/Doc/tutorial/stdlib2.rst
@@ -293,7 +293,7 @@ Many data structure needs can be met with the built-in list type. However,
 sometimes there is a need for alternative implementations with different
 performance trade-offs.
 
-The :mod:`array` module provides an :class:`~array.array()` object that is like
+The :mod:`array` module provides an :class:`~array.array` object that is like
 a list that stores only homogeneous data and stores it more compactly.  The
 following example shows an array of numbers stored as two byte unsigned binary
 numbers (typecode ``"H"``) rather than the usual 16 bytes per entry for regular
@@ -306,7 +306,7 @@ lists of Python int objects::
    >>> a[1:3]
    array('H', [10, 700])
 
-The :mod:`collections` module provides a :class:`~collections.deque()` object
+The :mod:`collections` module provides a :class:`~collections.deque` object
 that is like a list with faster appends and pops from the left side but slower
 lookups in the middle. These objects are well suited for implementing queues
 and breadth first tree searches::
diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst
index 2ae26e7a106a0b..3430ac8668e280 100644
--- a/Doc/whatsnew/2.5.rst
+++ b/Doc/whatsnew/2.5.rst
@@ -1724,7 +1724,7 @@ attribute of the function object to  change this::
 :mod:`ctypes` also provides a wrapper for Python's C API  as the
 ``ctypes.pythonapi`` object.  This object does *not*  release the global
 interpreter lock before calling a function, because the lock must be held when
-calling into the interpreter's code.   There's a :class:`py_object()` type
+calling into the interpreter's code.   There's a :class:`~ctypes.py_object` type
 constructor that will create a  :c:expr:`PyObject *` pointer.  A simple usage::
 
    import ctypes
@@ -1734,7 +1734,7 @@ constructor that will create a  :c:expr:`PyObject *` pointer.  A simple usage::
              ctypes.py_object("abc"),  ctypes.py_object(1))
    # d is now {'abc', 1}.
 
-Don't forget to use :class:`py_object()`; if it's omitted you end  up with a
+Don't forget to use :func:`~ctypes.py_object`; if it's omitted you end  up with a
 segmentation fault.
 
 :mod:`ctypes` has been around for a while, but people still write  and
diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst
index 28b28e9ce50e11..93d18ffc76d07c 100644
--- a/Doc/whatsnew/3.12.rst
+++ b/Doc/whatsnew/3.12.rst
@@ -739,7 +739,7 @@ inspect
 itertools
 ---------
 
-* Add :class:`itertools.batched()` for collecting into even-sized
+* Add :func:`itertools.batched` for collecting into even-sized
   tuples where the last batch may be shorter than the rest.
   (Contributed by Raymond Hettinger in :gh:`98363`.)
 

From fabcf6bc8f89f008319442dea614d5cbeb959544 Mon Sep 17 00:00:00 2001
From: Nikita Sobolev <mail@sobolevn.me>
Date: Wed, 12 Jun 2024 17:50:58 +0300
Subject: [PATCH 67/72] gh-120388: Improve deprecation warning message, when
 test returns non-None (#120401)

Co-authored-by: Alex Waygood <alex.waygood@gmail.com>
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
---
 Lib/test/test_unittest/test_async_case.py     |  3 +++
 Lib/test/test_unittest/test_case.py           | 19 +++++++++++++++++++
 Lib/unittest/async_case.py                    | 10 +++++++---
 Lib/unittest/case.py                          | 15 ++++++++++++---
 ...-06-12-15-07-58.gh-issue-120388.VuTQMT.rst |  3 +++
 5 files changed, 44 insertions(+), 6 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-12-15-07-58.gh-issue-120388.VuTQMT.rst

diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py
index ba1ab838cd4a22..00ef55bdf9bc83 100644
--- a/Lib/test/test_unittest/test_async_case.py
+++ b/Lib/test/test_unittest/test_async_case.py
@@ -312,18 +312,21 @@ async def test3(self):
         self.assertIn('It is deprecated to return a value that is not None', str(w.warning))
         self.assertIn('test1', str(w.warning))
         self.assertEqual(w.filename, __file__)
+        self.assertIn("returned 'int'", str(w.warning))
 
         with self.assertWarns(DeprecationWarning) as w:
             Test('test2').run()
         self.assertIn('It is deprecated to return a value that is not None', str(w.warning))
         self.assertIn('test2', str(w.warning))
         self.assertEqual(w.filename, __file__)
+        self.assertIn("returned 'async_generator'", str(w.warning))
 
         with self.assertWarns(DeprecationWarning) as w:
             Test('test3').run()
         self.assertIn('It is deprecated to return a value that is not None', str(w.warning))
         self.assertIn('test3', str(w.warning))
         self.assertEqual(w.filename, __file__)
+        self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning))
 
     def test_cleanups_interleave_order(self):
         events = []
diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py
index ed5eb5609a5dd1..17420909402107 100644
--- a/Lib/test/test_unittest/test_case.py
+++ b/Lib/test/test_unittest/test_case.py
@@ -325,18 +325,37 @@ def test3(self):
         self.assertIn('It is deprecated to return a value that is not None', str(w.warning))
         self.assertIn('test1', str(w.warning))
         self.assertEqual(w.filename, __file__)
+        self.assertIn("returned 'int'", str(w.warning))
 
         with self.assertWarns(DeprecationWarning) as w:
             Foo('test2').run()
         self.assertIn('It is deprecated to return a value that is not None', str(w.warning))
         self.assertIn('test2', str(w.warning))
         self.assertEqual(w.filename, __file__)
+        self.assertIn("returned 'generator'", str(w.warning))
 
         with self.assertWarns(DeprecationWarning) as w:
             Foo('test3').run()
         self.assertIn('It is deprecated to return a value that is not None', str(w.warning))
         self.assertIn('test3', str(w.warning))
         self.assertEqual(w.filename, __file__)
+        self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning))
+
+    def test_deprecation_of_return_val_from_test_async_method(self):
+        class Foo(unittest.TestCase):
+            async def test1(self):
+                return 1
+
+        with self.assertWarns(DeprecationWarning) as w:
+            Foo('test1').run()
+        self.assertIn('It is deprecated to return a value that is not None', str(w.warning))
+        self.assertIn('test1', str(w.warning))
+        self.assertEqual(w.filename, __file__)
+        self.assertIn("returned 'coroutine'", str(w.warning))
+        self.assertIn(
+            'Maybe you forgot to use IsolatedAsyncioTestCase as the base class?',
+            str(w.warning),
+        )
 
     def _check_call_order__subtests(self, result, events, expected_events):
         class Foo(Test.LoggingTestCase):
diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py
index 63ff6a5d1f8b61..bd06eb3207697a 100644
--- a/Lib/unittest/async_case.py
+++ b/Lib/unittest/async_case.py
@@ -90,9 +90,13 @@ def _callSetUp(self):
         self._callAsync(self.asyncSetUp)
 
     def _callTestMethod(self, method):
-        if self._callMaybeAsync(method) is not None:
-            warnings.warn(f'It is deprecated to return a value that is not None from a '
-                          f'test case ({method})', DeprecationWarning, stacklevel=4)
+        result = self._callMaybeAsync(method)
+        if result is not None:
+            msg = (
+                f'It is deprecated to return a value that is not None '
+                f'from a test case ({method} returned {type(result).__name__!r})',
+            )
+            warnings.warn(msg, DeprecationWarning, stacklevel=4)
 
     def _callTearDown(self):
         self._callAsync(self.asyncTearDown)
diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py
index 36daa61fa31adb..55c79d353539ca 100644
--- a/Lib/unittest/case.py
+++ b/Lib/unittest/case.py
@@ -603,9 +603,18 @@ def _callSetUp(self):
         self.setUp()
 
     def _callTestMethod(self, method):
-        if method() is not None:
-            warnings.warn(f'It is deprecated to return a value that is not None from a '
-                          f'test case ({method})', DeprecationWarning, stacklevel=3)
+        result = method()
+        if result is not None:
+            import inspect
+            msg = (
+                f'It is deprecated to return a value that is not None '
+                f'from a test case ({method} returned {type(result).__name__!r})'
+            )
+            if inspect.iscoroutine(result):
+                msg += (
+                    '. Maybe you forgot to use IsolatedAsyncioTestCase as the base class?'
+                )
+            warnings.warn(msg, DeprecationWarning, stacklevel=3)
 
     def _callTearDown(self):
         self.tearDown()
diff --git a/Misc/NEWS.d/next/Library/2024-06-12-15-07-58.gh-issue-120388.VuTQMT.rst b/Misc/NEWS.d/next/Library/2024-06-12-15-07-58.gh-issue-120388.VuTQMT.rst
new file mode 100644
index 00000000000000..d13df7d88b776c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-12-15-07-58.gh-issue-120388.VuTQMT.rst
@@ -0,0 +1,3 @@
+Improve a warning message when a test method in :mod:`unittest` returns
+something other than ``None``. Now we show the returned object type and
+optional asyncio-related tip.

From 127c1d2771749853e287632c086b6054212bf12a Mon Sep 17 00:00:00 2001
From: neonene <53406459+neonene@users.noreply.github.com>
Date: Thu, 13 Jun 2024 01:46:39 +0900
Subject: [PATCH 68/72] gh-71587: Drop local reference cache to `_strptime`
 module in `_datetime` (gh-120224)

The _strptime module object was cached in a static local variable (in the datetime.strptime() implementation).  That's a problem when it crosses isolation boundaries, such as reinitializing the runtme or between interpreters.  This change fixes the problem by dropping the static variable, instead always relying on the normal sys.modules cache (via PyImport_Import()).
---
 .../pycore_global_objects_fini_generated.h         |  1 +
 Include/internal/pycore_global_strings.h           |  1 +
 Include/internal/pycore_runtime_init_generated.h   |  1 +
 Include/internal/pycore_unicodeobject_generated.h  |  3 +++
 Lib/test/test_embed.py                             |  9 +++++++++
 .../2024-06-07-11-23-31.gh-issue-71587.IjFajE.rst  |  2 ++
 Modules/_datetimemodule.c                          | 14 +++++++-------
 Tools/c-analyzer/cpython/globals-to-fix.tsv        |  1 -
 8 files changed, 24 insertions(+), 8 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2024-06-07-11-23-31.gh-issue-71587.IjFajE.rst

diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h
index 30851dc2dbec44..bc94930b85f098 100644
--- a/Include/internal/pycore_global_objects_fini_generated.h
+++ b/Include/internal/pycore_global_objects_fini_generated.h
@@ -777,6 +777,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_showwarnmsg));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_shutdown));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_slotnames));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_swappedbytes_));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_type_));
diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h
index 009802c441685c..998be2ec490dd9 100644
--- a/Include/internal/pycore_global_strings.h
+++ b/Include/internal/pycore_global_strings.h
@@ -266,6 +266,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(_showwarnmsg)
         STRUCT_FOR_ID(_shutdown)
         STRUCT_FOR_ID(_slotnames)
+        STRUCT_FOR_ID(_strptime)
         STRUCT_FOR_ID(_strptime_datetime)
         STRUCT_FOR_ID(_swappedbytes_)
         STRUCT_FOR_ID(_type_)
diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h
index ff5b6ee8e0f006..bd79a7dff42f89 100644
--- a/Include/internal/pycore_runtime_init_generated.h
+++ b/Include/internal/pycore_runtime_init_generated.h
@@ -775,6 +775,7 @@ extern "C" {
     INIT_ID(_showwarnmsg), \
     INIT_ID(_shutdown), \
     INIT_ID(_slotnames), \
+    INIT_ID(_strptime), \
     INIT_ID(_strptime_datetime), \
     INIT_ID(_swappedbytes_), \
     INIT_ID(_type_), \
diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h
index 69d93a9610a2e5..7284aeb592d7ec 100644
--- a/Include/internal/pycore_unicodeobject_generated.h
+++ b/Include/internal/pycore_unicodeobject_generated.h
@@ -636,6 +636,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(_slotnames);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
+    string = &_Py_ID(_strptime);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(_strptime_datetime);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index d94c63a13b8ea4..634513ec7a5812 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -404,6 +404,15 @@ def test_ucnhash_capi_reset(self):
         out, err = self.run_embedded_interpreter("test_repeated_init_exec", code)
         self.assertEqual(out, '9\n' * INIT_LOOPS)
 
+    def test_datetime_reset_strptime(self):
+        code = (
+            "import datetime;"
+            "d = datetime.datetime.strptime('2000-01-01', '%Y-%m-%d');"
+            "print(d.strftime('%Y%m%d'))"
+        )
+        out, err = self.run_embedded_interpreter("test_repeated_init_exec", code)
+        self.assertEqual(out, '20000101\n' * INIT_LOOPS)
+
 
 @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi")
 class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
diff --git a/Misc/NEWS.d/next/Library/2024-06-07-11-23-31.gh-issue-71587.IjFajE.rst b/Misc/NEWS.d/next/Library/2024-06-07-11-23-31.gh-issue-71587.IjFajE.rst
new file mode 100644
index 00000000000000..50a662977993f5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-07-11-23-31.gh-issue-71587.IjFajE.rst
@@ -0,0 +1,2 @@
+Fix crash in C version of :meth:`datetime.datetime.strptime` when called again
+on the restarted interpreter.
diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c
index bea6e9411a75ed..cb4622893375d7 100644
--- a/Modules/_datetimemodule.c
+++ b/Modules/_datetimemodule.c
@@ -5514,19 +5514,19 @@ datetime_utcfromtimestamp(PyObject *cls, PyObject *args)
 static PyObject *
 datetime_strptime(PyObject *cls, PyObject *args)
 {
-    static PyObject *module = NULL;
-    PyObject *string, *format;
+    PyObject *string, *format, *result;
 
     if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format))
         return NULL;
 
+    PyObject *module = PyImport_Import(&_Py_ID(_strptime));
     if (module == NULL) {
-        module = PyImport_ImportModule("_strptime");
-        if (module == NULL)
-            return NULL;
+        return NULL;
     }
-    return PyObject_CallMethodObjArgs(module, &_Py_ID(_strptime_datetime),
-                                         cls, string, format, NULL);
+    result = PyObject_CallMethodObjArgs(module, &_Py_ID(_strptime_datetime),
+                                        cls, string, format, NULL);
+    Py_DECREF(module);
+    return result;
 }
 
 /* Return new datetime from date/datetime and time arguments. */
diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv
index 4586a59f6ac2ef..cb9750a69a632b 100644
--- a/Tools/c-analyzer/cpython/globals-to-fix.tsv
+++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv
@@ -393,7 +393,6 @@ Modules/xxmodule.c	-	ErrorObject	-
 
 ## initialized once
 Modules/_cursesmodule.c	-	ModDict	-
-Modules/_datetimemodule.c	datetime_strptime	module	-
 
 ## state
 Modules/_datetimemodule.c	-	_datetime_global_state	-

From 4b5d3e0e721a952f4ac9d17bee331e6dfe543dcd Mon Sep 17 00:00:00 2001
From: Lysandros Nikolaou <lisandrosnik@gmail.com>
Date: Wed, 12 Jun 2024 20:52:55 +0200
Subject: [PATCH 69/72] gh-120343: Fix column offsets of multiline tokens in
 tokenize (#120391)

---
 Lib/test/test_tokenize.py | 14 ++++++++++++++
 Python/Python-tokenize.c  | 14 ++++++++++----
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 36dba71766cc20..51aeb35f01065a 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1210,6 +1210,20 @@ def test_multiline_non_ascii_fstring(self):
     FSTRING_END "\'\'\'"         (2, 68) (2, 71)
     """)
 
+    def test_multiline_non_ascii_fstring_with_expr(self):
+        self.check_tokenize("""\
+f'''
+    🔗 This is a test {test_arg1}🔗
+🔗'''""", """\
+    FSTRING_START "f\'\'\'"        (1, 0) (1, 4)
+    FSTRING_MIDDLE '\\n    🔗 This is a test ' (1, 4) (2, 21)
+    OP         '{'           (2, 21) (2, 22)
+    NAME       'test_arg1'   (2, 22) (2, 31)
+    OP         '}'           (2, 31) (2, 32)
+    FSTRING_MIDDLE '🔗\\n🔗'        (2, 32) (3, 1)
+    FSTRING_END "\'\'\'"         (3, 1) (3, 4)
+    """)
+
 class GenerateTokensTest(TokenizeTest):
     def check_tokenize(self, s, expected):
         # Format the tokens in s in a table format.
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index 2591dae35736ba..55c821754c2031 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -215,6 +215,7 @@ tokenizeriter_next(tokenizeriterobject *it)
 
     const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start;
     PyObject* line = NULL;
+    int line_changed = 1;
     if (it->tok->tok_extra_tokens && is_trailing_token) {
         line = PyUnicode_FromString("");
     } else {
@@ -229,12 +230,11 @@ tokenizeriter_next(tokenizeriterobject *it)
             Py_XDECREF(it->last_line);
             line = PyUnicode_DecodeUTF8(line_start, size, "replace");
             it->last_line = line;
-            if (it->tok->lineno != it->last_end_lineno) {
-                it->byte_col_offset_diff = 0;
-            }
+            it->byte_col_offset_diff = 0;
         } else {
             // Line hasn't changed so we reuse the cached one.
             line = it->last_line;
+            line_changed = 0;
         }
     }
     if (line == NULL) {
@@ -252,7 +252,13 @@ tokenizeriter_next(tokenizeriterobject *it)
     Py_ssize_t byte_offset = -1;
     if (token.start != NULL && token.start >= line_start) {
         byte_offset = token.start - line_start;
-        col_offset = byte_offset - it->byte_col_offset_diff;
+        if (line_changed) {
+            col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset);
+            it->byte_col_offset_diff = byte_offset - col_offset;
+        }
+        else {
+            col_offset = byte_offset - it->byte_col_offset_diff;
+        }
     }
     if (token.end != NULL && token.end >= it->tok->line_start) {
         Py_ssize_t end_byte_offset = token.end - it->tok->line_start;

From 4c6d4f5cb33e48519922d635894eef356faddba2 Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Wed, 12 Jun 2024 20:56:42 +0200
Subject: [PATCH 70/72] gh-120417: Remove unused imports in the stdlib
 (#120420)

---
 Lib/_pyrepl/historical_reader.py | 2 +-
 Lib/_pyrepl/pager.py             | 2 +-
 Lib/_pyrepl/unix_console.py      | 1 -
 Lib/_pyrepl/windows_console.py   | 3 ---
 Lib/dataclasses.py               | 3 +--
 Lib/dbm/sqlite3.py               | 1 -
 Lib/idlelib/grep.py              | 2 +-
 Lib/importlib/abc.py             | 1 -
 Lib/ntpath.py                    | 1 -
 Lib/pydoc.py                     | 2 +-
 Lib/stat.py                      | 1 -
 11 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py
index 121de33da5052f..dd90912d1d67f8 100644
--- a/Lib/_pyrepl/historical_reader.py
+++ b/Lib/_pyrepl/historical_reader.py
@@ -27,7 +27,7 @@
 
 
 if False:
-    from .types import Callback, SimpleContextManager, KeySpec, CommandName
+    from .types import SimpleContextManager, KeySpec, CommandName
 
 
 isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple(
diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py
index 1ac733ed3573a4..66dcd99111adfc 100644
--- a/Lib/_pyrepl/pager.py
+++ b/Lib/_pyrepl/pager.py
@@ -8,7 +8,7 @@
 
 # types
 if False:
-    from typing import Protocol, Any
+    from typing import Protocol
     class Pager(Protocol):
         def __call__(self, text: str, title: str = "") -> None:
             ...
diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py
index af9290819c2c78..c4dedd97d1e13d 100644
--- a/Lib/_pyrepl/unix_console.py
+++ b/Lib/_pyrepl/unix_console.py
@@ -27,7 +27,6 @@
 import select
 import signal
 import struct
-import sys
 import termios
 import time
 from fcntl import ioctl
diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py
index f691ca3fbb07b8..9e97b1524e29a0 100644
--- a/Lib/_pyrepl/windows_console.py
+++ b/Lib/_pyrepl/windows_console.py
@@ -20,15 +20,12 @@
 from __future__ import annotations
 
 import io
-from multiprocessing import Value
 import os
 import sys
 import time
 import msvcrt
 
-from abc import ABC, abstractmethod
 from collections import deque
-from dataclasses import dataclass, field
 import ctypes
 from ctypes.wintypes import (
     _COORD,
diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py
index aeafbfbbe6e9c4..74011b7e28b9f3 100644
--- a/Lib/dataclasses.py
+++ b/Lib/dataclasses.py
@@ -7,7 +7,6 @@
 import itertools
 import abc
 from reprlib import recursive_repr
-from types import FunctionType, GenericAlias
 
 
 __all__ = ['dataclass',
@@ -333,7 +332,7 @@ def __set_name__(self, owner, name):
             # it.
             func(self.default, owner, name)
 
-    __class_getitem__ = classmethod(GenericAlias)
+    __class_getitem__ = classmethod(types.GenericAlias)
 
 
 class _DataclassParams:
diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py
index 74c9d9b7e2f1d8..7e0ae2a29e3a64 100644
--- a/Lib/dbm/sqlite3.py
+++ b/Lib/dbm/sqlite3.py
@@ -1,6 +1,5 @@
 import os
 import sqlite3
-import sys
 from pathlib import Path
 from contextlib import suppress, closing
 from collections.abc import MutableMapping
diff --git a/Lib/idlelib/grep.py b/Lib/idlelib/grep.py
index ef14349960bfa2..42048ff2395fe1 100644
--- a/Lib/idlelib/grep.py
+++ b/Lib/idlelib/grep.py
@@ -190,7 +190,7 @@ def grep_it(self, prog, path):
 
 
 def _grep_dialog(parent):  # htest #
-    from tkinter import Toplevel, Text, SEL, END
+    from tkinter import Toplevel, Text, SEL
     from tkinter.ttk import Frame, Button
     from idlelib.pyshell import PyShellFileList
 
diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py
index b6b2c791a3b03f..eea6b38af6fa13 100644
--- a/Lib/importlib/abc.py
+++ b/Lib/importlib/abc.py
@@ -13,7 +13,6 @@
     _frozen_importlib_external = _bootstrap_external
 from ._abc import Loader
 import abc
-import warnings
 
 
 __all__ = [
diff --git a/Lib/ntpath.py b/Lib/ntpath.py
index 83e2d3b865757c..1b1873f08b608b 100644
--- a/Lib/ntpath.py
+++ b/Lib/ntpath.py
@@ -19,7 +19,6 @@
 
 import os
 import sys
-import stat
 import genericpath
 from genericpath import *
 
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
index d7579c1cc3dcd1..278e4846ebb71f 100644
--- a/Lib/pydoc.py
+++ b/Lib/pydoc.py
@@ -75,7 +75,7 @@ class or function within a module or module in a package.  If the
 from reprlib import Repr
 from traceback import format_exception_only
 
-from _pyrepl.pager import (get_pager, plain, escape_less, pipe_pager,
+from _pyrepl.pager import (get_pager, plain, pipe_pager,
                            plain_pager, tempfile_pager, tty_pager)
 
 
diff --git a/Lib/stat.py b/Lib/stat.py
index 9167ab185944fb..1b4ed1ebc940ef 100644
--- a/Lib/stat.py
+++ b/Lib/stat.py
@@ -2,7 +2,6 @@
 
 Suggested usage: from stat import *
 """
-import sys
 
 # Indices for stat struct members in the tuple returned by os.stat()
 

From 3453362183f083e37ea866a7ae1b34147ffaf81d Mon Sep 17 00:00:00 2001
From: Pablo Galindo Salgado <Pablogsal@gmail.com>
Date: Wed, 12 Jun 2024 20:09:25 +0100
Subject: [PATCH 71/72] gh-118908: Protect the REPL subprocess with a timeout
 in tests (#120408)

---
 Lib/test/test_pyrepl/test_pyrepl.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py
index 3167b8473bfe20..41ba5959a1ec34 100644
--- a/Lib/test/test_pyrepl/test_pyrepl.py
+++ b/Lib/test/test_pyrepl/test_pyrepl.py
@@ -8,6 +8,7 @@
 from unittest import TestCase, skipUnless
 from unittest.mock import patch
 from test.support import force_not_colorized
+from test.support import SHORT_TIMEOUT
 
 from .support import (
     FakeConsole,
@@ -885,5 +886,9 @@ def run_repl(self, repl_input: str | list[str], env: dict | None = None) -> tupl
 
         os.close(master_fd)
         os.close(slave_fd)
-        exit_code = process.wait()
+        try:
+            exit_code = process.wait(timeout=SHORT_TIMEOUT)
+        except subprocess.TimeoutExpired:
+            process.kill()
+            exit_code = process.returncode
         return "\n".join(output), exit_code

From 030b452e34bbb0096acacb70a31915b9590c8186 Mon Sep 17 00:00:00 2001
From: Stefano Rivera <stefano@rivera.za.net>
Date: Wed, 12 Jun 2024 12:19:36 -0700
Subject: [PATCH 72/72] gh-120418: Don't assume wheeldata is deleted if
 `WHEEL_PKG_DIR` is set (#120419)

Remove wheeldata from both sides of the `assertEqual`, so that we're
*actually* ignoring it from the test set.

This test is only making assertions about the source tree, no code is
being executed that would do anything different based on the value of
`WHEEL_PKG_DIR`.
---
 Lib/test/test_tools/test_makefile.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py
index 48a7c1a773bb83..df95e6d0068516 100644
--- a/Lib/test/test_tools/test_makefile.py
+++ b/Lib/test/test_tools/test_makefile.py
@@ -41,7 +41,7 @@ def test_makefile_test_folders(self):
         idle_test = 'idlelib/idle_test'
         self.assertIn(idle_test, test_dirs)
 
-        used = [idle_test]
+        used = set([idle_test])
         for dirpath, dirs, files in os.walk(support.TEST_HOME_DIR):
             dirname = os.path.basename(dirpath)
             # Skip temporary dirs:
@@ -65,13 +65,14 @@ def test_makefile_test_folders(self):
                         "of test directories to install"
                     )
                 )
-                used.append(relpath)
+                used.add(relpath)
 
         # Don't check the wheel dir when Python is built --with-wheel-pkg-dir
         if sysconfig.get_config_var('WHEEL_PKG_DIR'):
             test_dirs.remove('test/wheeldata')
+            used.discard('test/wheeldata')
 
         # Check that there are no extra entries:
         unique_test_dirs = set(test_dirs)
-        self.assertSetEqual(unique_test_dirs, set(used))
+        self.assertSetEqual(unique_test_dirs, used)
         self.assertEqual(len(test_dirs), len(unique_test_dirs))