Fix enabling the GIL, also support disabling the GIL

More details:
- Fix a race while enabling the GIL by checking if the GIL was enabled between
  a no-op call to `_PyEval_AcquireLock()` and the thread attaching, and trying
  again if it was.
- Enable the GIL before running a module init function, since we can't know if
  it's a single-phase init module that doesn't support free-threading. Look at
  the state of the module after initialization to determine if it's safe to
  disable the GIL again.
- Add `PyModule_SetGIL()`, which can be used by single-phase init modules to
  declare that they support running without the GIL.
- Change `gil->enabled` from a simple on/off switch to a count of active
  requests to enable the GIL. This allows us to support multiple interleaved
  imports that each independently track whether the GIL should remain enabled.

Doc/c-api/module.rst

@@ -634,6 +634,17 @@ state:
 
    .. versionadded:: 3.9
 
+.. c:function:: int PyModule_SetGIL(PyObject *module, void *gil)
+
+   In Python builds not configured with :option:`--disable-gil`, do
+   nothing. Otherwise, indicate that *module* does or does not support running
+   without the global interpreter lock (GIL), using one of the values from
+   :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization
+   function.
+   Return ``-1`` on error, ``0`` on success.
+
+   .. versionadded:: 3.13
+
 
 Module lookup
 ^^^^^^^^^^^^^

Include/internal/pycore_ceval.h

@@ -126,16 +126,41 @@ extern int _PyEval_ThreadsInitialized(void);
 extern void _PyEval_InitGIL(PyThreadState *tstate, int own_gil);
 extern void _PyEval_FiniGIL(PyInterpreterState *interp);
 
-extern void _PyEval_AcquireLock(PyThreadState *tstate);
+// Acquire the GIL and return 1. In free-threaded builds, this function may
+// return 0 to indicate that the GIL was disabled and not acquired.
+extern int _PyEval_AcquireLock(PyThreadState *tstate);
+
 extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *);
 
 #ifdef Py_GIL_DISABLED
-// Enable the GIL for the given thread's interpreter. This may affect other
-// interpreters, if the GIL is shared.
+// Enable or disable the GIL used by the interpreter that owns tstate, which
+// must be the current thread. This may affect other interpreters, if the GIL
+// is shared. All three functions will be no-ops (and return 0) if the
+// interpreter's `enable_gil' config is not _PyConfig_GIL_DEFAULT.
+//
+// Every call to _PyEval_EnableGILTransient() must be paired with exactly one
+// call to either _PyEval_EnableGILPermanent() or
+// _PyEval_DisableGIL(). _PyEval_EnableGILPermanent() and _PyEval_DisableGIL()
+// must only be called while the GIL is enabled from a call to
+// _PyEval_EnableGILTransient().
+//
+// _PyEval_EnableGILTransient() returns 1 if it enabled the GIL, or 0 if the
+// GIL was already enabled, whether transiently or permanently. The caller will
+// hold the GIL upon return.
+//
+// _PyEval_EnableGILPermanent() returns 1 if it permanently enabled the GIL
+// (which must already be enabled), or 0 if it was already permanently
+// enabled. Once _PyEval_EnableGILPermanent() has been called once, all
+// subsequent calls to any of the three functions will be no-ops.
+//
+// _PyEval_DisableGIL() returns 1 if it disabled the GIL, or 0 if the GIL was
+// kept enabled because of another request, whether transient or permanent.
 //
-// Returns 1 if this call enabled the GIL, or 0 if it was already enabled. The
-// caller will hold the GIL upon return.
-extern int _PyEval_EnableGIL(PyThreadState *tstate);
+// All three functions must be called by an attached thread (this implies that
+// if the GIL is enabled, the current thread must hold it).
+extern int _PyEval_EnableGILTransient(PyThreadState *tstate);
+extern int _PyEval_EnableGILPermanent(PyThreadState *tstate);
+extern int _PyEval_DisableGIL(PyThreadState *state);
 #endif
 
 extern void _PyEval_DeactivateOpCache(void);

Include/internal/pycore_gil.h

@@ -21,8 +21,20 @@ extern "C" {
 
 struct _gil_runtime_state {
 #ifdef Py_GIL_DISABLED
-    /* Whether or not this GIL is being used. Can change from 0 to 1 at runtime
-       if, for example, a module that requires the GIL is loaded. */
+    /* If this GIL is inactive, enabled == 0.
+
+       If this GIL is enabled transiently (most likely to initialize a module
+       of unknown safety), enabled indicates the number of active transient
+       requests.
+
+       If this GIL is enabled permanently, enabled == INT_MAX.
+
+       It must not be modified directly; use _PyEval_EnableGILTransiently(),
+       _PyEval_EnableGILPermanently(), and _PyEval_DisableGIL()
+
+       It is always read and written atomically, but a thread can assume its
+       value will be stable as long as that thread is attached or knows that no
+       other threads are attached (e.g., during a stop-the-world.). */
     int enabled;
 #endif
     /* microseconds (the Python API uses seconds, though) */

Include/internal/pycore_import.h

@@ -206,6 +206,10 @@ extern int _PyImport_CheckSubinterpIncompatibleExtensionAllowed(
 // Export for '_testinternalcapi' shared extension
 PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename);
 
+#ifdef Py_GIL_DISABLED
+extern void _PyImport_CheckGILForModule(PyObject *module, PyObject *name_unicode);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_moduleobject.h

@@ -22,6 +22,9 @@ typedef struct {
     PyObject *md_weaklist;
     // for logging purposes after md_dict is cleared
     PyObject *md_name;
+#ifdef Py_GIL_DISABLED
+    void *md_gil;
+#endif
 } PyModuleObject;
 
 static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) {

Include/moduleobject.h

@@ -98,6 +98,7 @@ struct PyModuleDef_Slot {
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000
 #  define Py_MOD_GIL_USED ((void *)0)
 #  define Py_MOD_GIL_NOT_USED ((void *)1)
+PyAPI_FUNC(int) PyModule_SetGIL(PyObject *module, void *gil);
 #endif
 
 struct PyModuleDef {

Objects/moduleobject.c

@@ -239,6 +239,9 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version)
         }
     }
     m->md_def = module;
+#ifdef Py_GIL_DISABLE
+    m->md_gil = Py_MOD_GIL_USED;
+#endif
     return (PyObject*)m;
 }
 
@@ -348,31 +351,6 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
         goto error;
     }
 
-#ifdef Py_GIL_DISABLED
-    // TODO: We should figure out a way to fit this logic somewhere that it
-    // more naturally fits, like import.c or importdl.c.
-    PyThreadState *tstate = _PyThreadState_GET();
-    const PyConfig *config = _PyInterpreterState_GetConfig(interp);
-    if (gil_slot == Py_MOD_GIL_USED && config->enable_gil == _PyConfig_GIL_DEFAULT) {
-        if (_PyEval_EnableGIL(tstate)) {
-            PyErr_WarnFormat(
-                PyExc_RuntimeWarning,
-                1,
-                "The global interpreter lock (GIL) has been enabled to load "
-                "module '%s', which has not declared that it can run safely "
-                "without the GIL. To override this behavior and keep the GIL "
-                "disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0.",
-                name
-            );
-        }
-        if (config->verbose) {
-            PySys_FormatStderr("# loading module '%s', which requires the GIL\n", name);
-        }
-    }
-#else
-    (void)gil_slot;
-#endif
-
     if (create) {
         m = create(spec, def);
         if (m == NULL) {
@@ -434,6 +412,10 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
         }
     }
 
+#ifdef Py_GIL_DISABLED
+    ((PyModuleObject *)m)->md_gil = gil_slot;
+#endif
+
     Py_DECREF(nameobj);
     return m;
 
@@ -443,6 +425,19 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
     return NULL;
 }
 
+int
+PyModule_SetGIL(PyObject *module, void *gil)
+{
+    if (!PyModule_Check(module)) {
+        PyErr_BadInternalCall();
+        return -1;
+    }
+#ifdef Py_GIL_DISABLED
+    ((PyModuleObject *)module)->md_gil = gil;
+#endif
+    return 0;
+}
+
 int
 PyModule_ExecDef(PyObject *module, PyModuleDef *def)
 {