Merge remote-tracking branch 'upstream/main' into cpython-enable-gil

.github/workflows/build.yml

@@ -208,7 +208,7 @@ jobs:
     with:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
       # macos-14 is M1, macos-13 is Intel
-      os-matrix: '["macos-14", "macos-13"]'
+      os-matrix: '["macos-14-xlarge", "macos-13-large"]'
 
   build_macos_free_threading:
     name: 'macOS (free-threading)'
@@ -218,8 +218,8 @@ jobs:
     with:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
       free-threading: true
-      # macos-14 is M1
-      os-matrix: '["macos-14"]'
+      # macos-14-large is Intel with 12 cores (most parallelism)
+      os-matrix: '["macos-14-large"]'
 
   build_ubuntu:
     name: 'Ubuntu'

.github/workflows/jit.yml

@@ -23,7 +23,7 @@ jobs:
   jit:
     name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }})
     runs-on: ${{ matrix.runner }}
-    timeout-minutes: 75
+    timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:

.github/workflows/reusable-macos.yml

@@ -50,7 +50,7 @@ jobs:
           --prefix=/opt/python-dev \
           --with-openssl="$(brew --prefix [email protected])"
     - name: Build CPython
-      run: make -j4
+      run: make -j8
     - name: Display build info
       run: make pythoninfo
     - name: Tests

Doc/library/ast.rst

@@ -61,7 +61,7 @@ Node classes
 
    .. attribute:: _fields
 
-      Each concrete class has an attribute :attr:`_fields` which gives the names
+      Each concrete class has an attribute :attr:`!_fields` which gives the names
       of all child nodes.
 
       Each instance of a concrete class has one attribute for each child node,
@@ -74,6 +74,18 @@ Node classes
       as Python lists.  All possible attributes must be present and have valid
       values when compiling an AST with :func:`compile`.
 
+   .. attribute:: _field_types
+
+      The :attr:`!_field_types` attribute on each concrete class is a dictionary
+      mapping field names (as also listed in :attr:`_fields`) to their types.
+
+      .. doctest::
+
+           >>> ast.TypeVar._field_types
+           {'name': <class 'str'>, 'bound': ast.expr | None, 'default_value': ast.expr | None}
+
+      .. versionadded:: 3.13
+
    .. attribute:: lineno
                   col_offset
                   end_lineno

Doc/library/typing.rst

@@ -3648,8 +3648,14 @@ Aliases to asynchronous ABCs in :mod:`collections.abc`
    is no ``ReturnType`` type parameter. As with :class:`Generator`, the
    ``SendType`` behaves contravariantly.
 
-   If your generator will only yield values, set the ``SendType`` to
-   ``None``::
+   The ``SendType`` defaults to :const:`!None`::
+
+      async def infinite_stream(start: int) -> AsyncGenerator[int]:
+          while True:
+              yield start
+              start = await increment(start)
+
+   It is also possible to set this type explicitly::
 
       async def infinite_stream(start: int) -> AsyncGenerator[int, None]:
           while True:
@@ -3671,6 +3677,9 @@ Aliases to asynchronous ABCs in :mod:`collections.abc`
       now supports subscripting (``[]``).
       See :pep:`585` and :ref:`types-genericalias`.
 
+   .. versionchanged:: 3.13
+      The ``SendType`` parameter now has a default.
+
 .. class:: AsyncIterable(Generic[T_co])
 
    Deprecated alias to :class:`collections.abc.AsyncIterable`.
@@ -3754,8 +3763,14 @@ Aliases to other ABCs in :mod:`collections.abc`
    of :class:`Generator` behaves contravariantly, not covariantly or
    invariantly.
 
-   If your generator will only yield values, set the ``SendType`` and
-   ``ReturnType`` to ``None``::
+   The ``SendType`` and ``ReturnType`` parameters default to :const:`!None`::
+
+      def infinite_stream(start: int) -> Generator[int]:
+          while True:
+              yield start
+              start += 1
+
+   It is also possible to set these types explicitly::
 
       def infinite_stream(start: int) -> Generator[int, None, None]:
           while True:
@@ -3774,6 +3789,9 @@ Aliases to other ABCs in :mod:`collections.abc`
       :class:`collections.abc.Generator` now supports subscripting (``[]``).
       See :pep:`585` and :ref:`types-genericalias`.
 
+   .. versionchanged:: 3.13
+      Default values for the send and return types were added.
+
 .. class:: Hashable
 
    Deprecated alias to :class:`collections.abc.Hashable`.

Doc/whatsnew/3.13.rst

@@ -384,6 +384,12 @@ ast
   argument that does not map to a field on the AST node is now deprecated,
   and will raise an exception in Python 3.15.
 
+  These changes do not apply to user-defined subclasses of :class:`ast.AST`,
+  unless the class opts in to the new behavior by setting the attribute
+  :attr:`ast.AST._field_types`.
+
+  (Contributed by Jelle Zijlstra in :gh:`105858` and :gh:`117486`.)
+
 * :func:`ast.parse` now accepts an optional argument *optimize*
   which is passed on to the :func:`compile` built-in. This makes it
   possible to obtain an optimized AST.

Include/cpython/optimizer.h

@@ -141,16 +141,22 @@ void _Py_ExecutorDetach(_PyExecutorObject *);
 void _Py_BloomFilter_Init(_PyBloomFilter *);
 void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj);
 PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj);
-PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation);
-PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation);
-
 /* For testing */
 PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void);
 PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void);
 
 #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3
 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6
 
+#ifdef _Py_TIER2
+PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation);
+PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation);
+#else
+#  define _Py_Executors_InvalidateDependency(A, B, C) ((void)0)
+#  define _Py_Executors_InvalidateAll(A, B) ((void)0)
+#endif
+
+
 #ifdef __cplusplus
 }
 #endif

Include/cpython/pystate.h

@@ -188,6 +188,7 @@ struct _ts {
 
     PyObject *previous_executor;
 
+    uint64_t dict_global_version;
 };
 
 #ifdef Py_DEBUG

Include/internal/pycore_code.h

@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_lock.h"        // PyMutex
+
 
 // We hide some of the newer PyCodeObject fields behind macros.
 // This helps with backporting certain changes to 3.12.
@@ -16,6 +18,14 @@ extern "C" {
 #define _PyCode_HAS_INSTRUMENTATION(CODE) \
     (CODE->_co_instrumentation_version > 0)
 
+struct _py_code_state {
+    PyMutex mutex;
+    // Interned constants from code objects. Used by the free-threaded build.
+    struct _Py_hashtable_t *constants;
+};
+
+extern PyStatus _PyCode_Init(PyInterpreterState *interp);
+extern void _PyCode_Fini(PyInterpreterState *interp);
 
 #define CODE_MAX_WATCHERS 8
 

Include/internal/pycore_dict.h

@@ -105,10 +105,10 @@ PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObjec
 
 /* Consumes references to key and value */
 PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value);
-extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value);
 extern int _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value);
 extern int _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result);
 extern int _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result);
+extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value);
 
 extern int _PyDict_Pop_KnownHash(
     PyDictObject *dict,
@@ -221,8 +221,25 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
 #define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1)
 
 #ifdef Py_GIL_DISABLED
-#define DICT_NEXT_VERSION(INTERP) \
-    (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT)
+
+#define THREAD_LOCAL_DICT_VERSION_COUNT 256
+#define THREAD_LOCAL_DICT_VERSION_BATCH THREAD_LOCAL_DICT_VERSION_COUNT * DICT_VERSION_INCREMENT
+
+static inline uint64_t
+dict_next_version(PyInterpreterState *interp)
+{
+    PyThreadState *tstate = PyThreadState_GET();
+    uint64_t cur_progress = (tstate->dict_global_version &
+                            (THREAD_LOCAL_DICT_VERSION_BATCH - 1));
+    if (cur_progress == 0) {
+        uint64_t next = _Py_atomic_add_uint64(&interp->dict_state.global_version,
+                                              THREAD_LOCAL_DICT_VERSION_BATCH);
+        tstate->dict_global_version = next;
+    }
+    return tstate->dict_global_version += DICT_VERSION_INCREMENT;
+}
+
+#define DICT_NEXT_VERSION(INTERP) dict_next_version(INTERP)
 
 #else
 #define DICT_NEXT_VERSION(INTERP) \

Include/internal/pycore_interp.h

@@ -245,6 +245,7 @@ struct _is {
     struct _Py_long_state long_state;
     struct _dtoa_state dtoa;
     struct _py_func_state func_state;
+    struct _py_code_state code_state;
 
     struct _Py_dict_state dict_state;
     struct _Py_exc_state exc_state;

Include/internal/pycore_setobject.h

@@ -30,6 +30,9 @@ PyAPI_DATA(PyObject *) _PySet_Dummy;
 
 PyAPI_FUNC(int) _PySet_Contains(PySetObject *so, PyObject *key);
 
+// Clears the set without acquiring locks. Used by _PyCode_Fini.
+extern void _PySet_ClearInternal(PySetObject *so);
+
 #ifdef __cplusplus
 }
 #endif

Lib/_pyrepl/pager.py

@@ -23,7 +23,7 @@ def get_pager() -> Pager:
     if not sys.stdin.isatty() or not sys.stdout.isatty():
         return plain_pager
     if sys.platform == "emscripten":
-        return plainpager
+        return plain_pager
     use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER')
     if use_pager:
         if sys.platform == 'win32': # pipes completely broken in Windows