Allow disabling the GIL with PYTHON_GIL=0

In free-threaded builds, running with `PYTHON_GIL=0` will now disable the
GIL. Follow-up issues track work to re-enable the GIL when loading an
incompatible extension, and to disable the GIL by default.

In order to support re-enabling the GIL at runtime, all GIL-related data
structures are initialized as usual, and disabling the GIL simply sets a flag
that causes `take_gil()` and `drop_gil()` to return early.

With `PYTHON_GIL=0` set, I spot-checked a few tests and small programs that
don't use threads. They all seem to run fine, and very basic threaded programs
work, sometimes. Trying to run the full test suite crashes pretty quickly, in
`test_asyncio`.

Include/cpython/initconfig.h

@@ -181,6 +181,9 @@ typedef struct PyConfig {
     int int_max_str_digits;
 
     int cpu_count;
+#ifdef Py_GIL_DISABLED
+    int enable_gil;
+#endif
 
     /* --- Path configuration inputs ------------ */
     int pathconfig_warnings;

Include/internal/pycore_gil.h

@@ -20,6 +20,11 @@ extern "C" {
 #define FORCE_SWITCHING
 
 struct _gil_runtime_state {
+#ifdef Py_GIL_DISABLED
+    /* Whether or not this GIL is being used. Can change from 0 to 1 at runtime
+       if, for example, a module that requires the GIL is loaded. */
+    int enabled;
+#endif
     /* microseconds (the Python API uses seconds, though) */
     unsigned long interval;
     /* Last PyThreadState holding / having held the GIL. This helps us

Include/internal/pycore_initconfig.h

@@ -153,6 +153,18 @@ typedef enum {
     _PyConfig_INIT_ISOLATED = 3
 } _PyConfigInitEnum;
 
+typedef enum {
+    /* For now, this means the GIL is enabled.
+
+       gh-116329: This will eventually change to "the GIL is disabled but can
+       be reenabled by loading an incompatible extension module." */
+    _PyConfig_GIL_DEFAULT,
+
+    /* The GIL has been forced off or on, and will not be affected by module loading. */
+    _PyConfig_GIL_DISABLE,
+    _PyConfig_GIL_ENABLE,
+} _PyConfigGILEnum;
+
 // Export for '_testembed' program
 PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config);
 

Lib/test/test_embed.py

@@ -452,6 +452,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'hash_seed': 0,
         'int_max_str_digits': sys.int_info.default_max_str_digits,
         'cpu_count': -1,
+        'enable_gil': 0,
         'faulthandler': 0,
         'tracemalloc': 0,
         'perf_profiling': 0,

Python/ceval_gil.c

@@ -219,6 +219,11 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate)
     // XXX assert(tstate == NULL || !tstate->_status.cleared);
 
     struct _gil_runtime_state *gil = ceval->gil;
+#ifdef Py_GIL_DISABLED
+    if (!gil->enabled) {
+        return;
+    }
+#endif
     if (!_Py_atomic_load_ptr_relaxed(&gil->locked)) {
         Py_FatalError("drop_gil: GIL is not locked");
     }
@@ -294,6 +299,11 @@ take_gil(PyThreadState *tstate)
     assert(_PyThreadState_CheckConsistency(tstate));
     PyInterpreterState *interp = tstate->interp;
     struct _gil_runtime_state *gil = interp->ceval.gil;
+#ifdef Py_GIL_DISABLED
+    if (!gil->enabled) {
+        return;
+    }
+#endif
 
     /* Check that _PyEval_InitThreads() was called to create the lock */
     assert(gil_created(gil));
@@ -438,6 +448,11 @@ static void
 init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil)
 {
     assert(!gil_created(gil));
+#ifdef Py_GIL_DISABLED
+    // gh-116329: Once it is safe to do so, change this condition to
+    // (enable_gil == _PyConfig_GIL_ENABLE), so the GIL is disabled by default.
+    gil->enabled = _PyInterpreterState_GetConfig(interp)->enable_gil != _PyConfig_GIL_DISABLE;
+#endif
     create_gil(gil);
     assert(gil_created(gil));
     interp->ceval.gil = gil;

Python/initconfig.c

@@ -93,6 +93,9 @@ static const PyConfigSpec PYCONFIG_SPEC[] = {
     SPEC(safe_path, UINT),
     SPEC(int_max_str_digits, INT),
     SPEC(cpu_count, INT),
+#ifdef Py_GIL_DISABLED
+    SPEC(enable_gil, INT),
+#endif
     SPEC(pathconfig_warnings, UINT),
     SPEC(program_name, WSTR),
     SPEC(pythonpath_env, WSTR_OPT),
@@ -276,6 +279,9 @@ static const char usage_envvars[] =
 "PYTHON_CPU_COUNT: Overrides the return value of os.process_cpu_count(),\n"
 "                  os.cpu_count(), and multiprocessing.cpu_count() if set to\n"
 "                  a positive integer.\n"
+#ifdef Py_GIL_DISABLED
+"PYTHON_GIL      : When set to 0, disables the GIL.\n"
+#endif
 "PYTHONDEVMODE   : enable the development mode.\n"
 "PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n"
 "PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'.\n"
@@ -860,6 +866,9 @@ _PyConfig_InitCompatConfig(PyConfig *config)
     config->_is_python_build = 0;
     config->code_debug_ranges = 1;
     config->cpu_count = -1;
+#ifdef Py_GIL_DISABLED
+    config->enable_gil = _PyConfig_GIL_DEFAULT;
+#endif
 }
 
 
@@ -1642,6 +1651,19 @@ config_read_env_vars(PyConfig *config)
         config->safe_path = 1;
     }
 
+    const char *gil = config_get_env(config, "PYTHON_GIL");
+    if (gil != NULL) {
+#ifdef Py_GIL_DISABLED
+        if (strcmp(gil, "0") == 0) {
+            config->enable_gil = _PyConfig_GIL_DISABLE;
+        } else if (strcmp(gil, "1") != 0) {
+            return _PyStatus_ERR("PYTHON_GIL must be \"0\" or \"1\"");
+        }
+#else
+        return _PyStatus_ERR("PYTHON_GIL is not supported by this build");
+#endif
+    }
+
     return _PyStatus_OK();
 }