pythongh-111968: Use per-thread freelists for tuple in free-threading (…

…pythongh-113921)

Include/internal/pycore_freelist.h

@@ -8,11 +8,19 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+// PyTuple_MAXSAVESIZE - largest tuple to save on free list
+// PyTuple_MAXFREELIST - maximum number of tuples of each size to save
+
 #ifdef WITH_FREELISTS
 // with freelists
+#  define PyTuple_MAXSAVESIZE 20
+#  define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE
+#  define PyTuple_MAXFREELIST 2000
 #  define PyList_MAXFREELIST 80
 #  define PyFloat_MAXFREELIST 100
 #else
+#  define PyTuple_NFREELISTS 0
+#  define PyTuple_MAXFREELIST 0
 #  define PyList_MAXFREELIST 0
 #  define PyFloat_MAXFREELIST 0
 #endif
@@ -24,6 +32,23 @@ struct _Py_list_state {
 #endif
 };
 
+struct _Py_tuple_state {
+#if WITH_FREELISTS
+    /* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE.
+       The empty tuple is handled separately.
+
+       Each tuple stored in the array is the head of the linked list
+       (and the next available tuple) for that size.  The actual tuple
+       object is used as the linked list node, with its first item
+       (ob_item[0]) pointing to the next node (i.e. the previous head).
+       Each linked list is initially NULL. */
+    PyTupleObject *free_list[PyTuple_NFREELISTS];
+    int numfree[PyTuple_NFREELISTS];
+#else
+    char _unused;  // Empty structs are not allowed.
+#endif
+};
+
 struct _Py_float_state {
 #ifdef WITH_FREELISTS
     /* Special free list
@@ -36,6 +61,7 @@ struct _Py_float_state {
 
 typedef struct _Py_freelist_state {
     struct _Py_float_state float_state;
+    struct _Py_tuple_state tuple_state;
     struct _Py_list_state list_state;
 } _PyFreeListState;
 

Include/internal/pycore_gc.h

@@ -246,7 +246,7 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs);
 // Functions to clear types free lists
 extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp);
 extern void _Py_ClearFreeLists(_PyFreeListState *state, int is_finalization);
-extern void _PyTuple_ClearFreeList(PyInterpreterState *interp);
+extern void _PyTuple_ClearFreeList(_PyFreeListState *state, int is_finalization);
 extern void _PyFloat_ClearFreeList(_PyFreeListState *state, int is_finalization);
 extern void _PyList_ClearFreeList(_PyFreeListState *state, int is_finalization);
 extern void _PyDict_ClearFreeList(PyInterpreterState *interp);

Include/internal/pycore_tuple.h

@@ -14,59 +14,16 @@ extern void _PyTuple_DebugMallocStats(FILE *out);
 /* runtime lifecycle */
 
 extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *);
-extern void _PyTuple_Fini(PyInterpreterState *);
+extern void _PyTuple_Fini(_PyFreeListState *);
 
 
 /* other API */
 
-// PyTuple_MAXSAVESIZE - largest tuple to save on free list
-// PyTuple_MAXFREELIST - maximum number of tuples of each size to save
-
-#if defined(PyTuple_MAXSAVESIZE) && PyTuple_MAXSAVESIZE <= 0
-   // A build indicated that tuple freelists should not be used.
-#  define PyTuple_NFREELISTS 0
-#  undef PyTuple_MAXSAVESIZE
-#  undef PyTuple_MAXFREELIST
-
-#elif !defined(WITH_FREELISTS)
-#  define PyTuple_NFREELISTS 0
-#  undef PyTuple_MAXSAVESIZE
-#  undef PyTuple_MAXFREELIST
-
-#else
-   // We are using a freelist for tuples.
-#  ifndef PyTuple_MAXSAVESIZE
-#    define PyTuple_MAXSAVESIZE 20
-#  endif
-#  define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE
-#  ifndef PyTuple_MAXFREELIST
-#    define PyTuple_MAXFREELIST 2000
-#  endif
-#endif
-
-struct _Py_tuple_state {
-#if PyTuple_NFREELISTS > 0
-    /* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE.
-       The empty tuple is handled separately.
-
-       Each tuple stored in the array is the head of the linked list
-       (and the next available tuple) for that size.  The actual tuple
-       object is used as the linked list node, with its first item
-       (ob_item[0]) pointing to the next node (i.e. the previous head).
-       Each linked list is initially NULL. */
-    PyTupleObject *free_list[PyTuple_NFREELISTS];
-    int numfree[PyTuple_NFREELISTS];
-#else
-    char _unused;  // Empty structs are not allowed.
-#endif
-};
-
 #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item)
 
 extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t);
 extern PyObject *_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t);
 
-
 typedef struct {
     PyObject_HEAD
     Py_ssize_t it_index;

Objects/tupleobject.c

@@ -962,18 +962,18 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize)
 }
 
 
-static void maybe_freelist_clear(PyInterpreterState *, int);
+static void maybe_freelist_clear(_PyFreeListState *, int);
 
 void
-_PyTuple_Fini(PyInterpreterState *interp)
+_PyTuple_Fini(_PyFreeListState *state)
 {
-    maybe_freelist_clear(interp, 1);
+    maybe_freelist_clear(state, 1);
 }
 
 void
-_PyTuple_ClearFreeList(PyInterpreterState *interp)
+_PyTuple_ClearFreeList(_PyFreeListState *state, int is_finalization)
 {
-    maybe_freelist_clear(interp, 0);
+    maybe_freelist_clear(state, is_finalization);
 }
 
 /*********************** Tuple Iterator **************************/
@@ -1125,18 +1125,14 @@ tuple_iter(PyObject *seq)
  * freelists *
  *************/
 
-#define STATE (interp->tuple)
+#define STATE (state->tuple_state)
 #define FREELIST_FINALIZED (STATE.numfree[0] < 0)
 
 static inline PyTupleObject *
 maybe_freelist_pop(Py_ssize_t size)
 {
-#if PyTuple_NFREELISTS > 0
-    PyInterpreterState *interp = _PyInterpreterState_GET();
-#ifdef Py_DEBUG
-    /* maybe_freelist_pop() must not be called after maybe_freelist_fini(). */
-    assert(!FREELIST_FINALIZED);
-#endif
+#ifdef WITH_FREELISTS
+    _PyFreeListState *state = _PyFreeListState_GET();
     if (size == 0) {
         return NULL;
     }
@@ -1169,18 +1165,15 @@ maybe_freelist_pop(Py_ssize_t size)
 static inline int
 maybe_freelist_push(PyTupleObject *op)
 {
-#if PyTuple_NFREELISTS > 0
-    PyInterpreterState *interp = _PyInterpreterState_GET();
-#ifdef Py_DEBUG
-    /* maybe_freelist_push() must not be called after maybe_freelist_fini(). */
-    assert(!FREELIST_FINALIZED);
-#endif
+#ifdef WITH_FREELISTS
+    _PyFreeListState *state = _PyFreeListState_GET();
     if (Py_SIZE(op) == 0) {
         return 0;
     }
     Py_ssize_t index = Py_SIZE(op) - 1;
     if (index < PyTuple_NFREELISTS
         && STATE.numfree[index] < PyTuple_MAXFREELIST
+        && STATE.numfree[index] >= 0
         && Py_IS_TYPE(op, &PyTuple_Type))
     {
         /* op is the head of a linked list, with the first item
@@ -1196,9 +1189,9 @@ maybe_freelist_push(PyTupleObject *op)
 }
 
 static void
-maybe_freelist_clear(PyInterpreterState *interp, int fini)
+maybe_freelist_clear(_PyFreeListState *state, int fini)
 {
-#if PyTuple_NFREELISTS > 0
+#ifdef WITH_FREELISTS
     for (Py_ssize_t i = 0; i < PyTuple_NFREELISTS; i++) {
         PyTupleObject *p = STATE.free_list[i];
         STATE.free_list[i] = NULL;
@@ -1216,8 +1209,8 @@ maybe_freelist_clear(PyInterpreterState *interp, int fini)
 void
 _PyTuple_DebugMallocStats(FILE *out)
 {
-#if PyTuple_NFREELISTS > 0
-    PyInterpreterState *interp = _PyInterpreterState_GET();
+#ifdef WITH_FREELISTS
+    _PyFreeListState *state = _PyFreeListState_GET();
     for (int i = 0; i < PyTuple_NFREELISTS; i++) {
         int len = i + 1;
         char buf[128];

Python/gc_free_threading.c

@@ -14,7 +14,6 @@
 void
 _PyGC_ClearAllFreeLists(PyInterpreterState *interp)
 {
-    _PyTuple_ClearFreeList(interp);
     _PyDict_ClearFreeList(interp);
     _PyAsyncGen_ClearFreeLists(interp);
     _PyContext_ClearFreeList(interp);

Python/gc_gil.c

@@ -11,7 +11,6 @@
 void
 _PyGC_ClearAllFreeLists(PyInterpreterState *interp)
 {
-    _PyTuple_ClearFreeList(interp);
     _PyDict_ClearFreeList(interp);
     _PyAsyncGen_ClearFreeLists(interp);
     _PyContext_ClearFreeList(interp);

Python/pylifecycle.c

@@ -1752,13 +1752,13 @@ finalize_interp_types(PyInterpreterState *interp)
     _PyUnicode_ClearInterned(interp);
 
     _PyDict_Fini(interp);
-    _PyTuple_Fini(interp);
 
     _PySlice_Fini(interp);
 
     _PyUnicode_Fini(interp);
 
     _PyFreeListState *state = _PyFreeListState_GET();
+    _PyTuple_Fini(state);
     _PyList_Fini(state);
     _PyFloat_Fini(state);
 

Python/pystate.c

@@ -1459,6 +1459,7 @@ void
 _Py_ClearFreeLists(_PyFreeListState *state, int is_finalization)
 {
     _PyFloat_ClearFreeList(state, is_finalization);
+    _PyTuple_ClearFreeList(state, is_finalization);
     _PyList_ClearFreeList(state, is_finalization);
 }