diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f3995a020e31b..e5f6fd47e7367b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -202,7 +202,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1w, 3.0.13, 3.1.5, 3.2.1] + openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -266,7 +266,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' env: - OPENSSL_VER: 3.0.13 + OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 @@ -380,7 +380,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 3.0.13 + OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 92069fddc31217..b197db814b2743 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-22.04 env: FORCE_COLOR: 1 - OPENSSL_VER: 3.0.13 + OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: diff --git a/Android/android.py b/Android/android.py index b5403b5d2a4a5e..bfa7832a4a83da 100755 --- a/Android/android.py +++ b/Android/android.py @@ -138,7 +138,7 @@ def make_build_python(context): def unpack_deps(host): deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-1", "libffi-3.4.4-2", "openssl-3.0.13-1", + for name_ver in ["bzip2-1.0.8-1", "libffi-3.4.4-2", "openssl-3.0.15-0", "sqlite-3.45.1-0", "xz-5.4.6-0"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index b59be98608a158..0ef7d015be9b93 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -324,7 +324,7 @@ PyPreConfig * Set :c:member:`PyConfig.filesystem_encoding` to ``"mbcs"``, * Set :c:member:`PyConfig.filesystem_errors` to ``"replace"``. - Initialized the from :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + Initialized from the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable value. Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for @@ -1744,6 +1744,26 @@ only implemented when ``Py_InitializeFromInitConfig()`` is called, not by the * Set an error in *config* and return ``-1`` on error. +Module +------ + +.. c:function:: int PyInitConfig_AddModule(PyInitConfig *config, const char *name, PyObject* (*initfunc)(void)) + + Add a built-in extension module to the table of built-in modules. + + The new module can be imported by the name *name*, and uses the function + *initfunc* as the initialization function called on the first attempted + import. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + If Python is initialized multiple times, ``PyInitConfig_AddModule()`` must + be called at each Python initialization. + + Similar to the :c:func:`PyImport_AppendInittab` function. + + Initialize Python ----------------- diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index f9461ab01f6049..d333df397782e0 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -485,7 +485,8 @@ Accessing attributes of extension types ``PyMemberDef`` may contain a definition for the special member ``"__vectorcalloffset__"``, corresponding to :c:member:`~PyTypeObject.tp_vectorcall_offset` in type objects. - These must be defined with ``Py_T_PYSSIZET`` and ``Py_READONLY``, for example:: + This member must be defined with ``Py_T_PYSSIZET``, and either + ``Py_READONLY`` or ``Py_READONLY | Py_RELATIVE_OFFSET``. For example:: static PyMemberDef spam_type_members[] = { {"__vectorcalloffset__", Py_T_PYSSIZET, @@ -506,6 +507,12 @@ Accessing attributes of extension types ``PyMemberDef`` is always available. Previously, it required including ``"structmember.h"``. + .. versionchanged:: 3.14 + + :c:macro:`Py_RELATIVE_OFFSET` is now allowed for + ``"__vectorcalloffset__"``, ``"__dictoffset__"`` and + ``"__weaklistoffset__"``. + .. c:function:: PyObject* PyMember_GetOne(const char *obj_addr, struct PyMemberDef *m) Get an attribute belonging to the object at address *obj_addr*. The diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index cbfe93319ddaa4..3182d5664ab6ec 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -382,6 +382,10 @@ Logging Flow The flow of log event information in loggers and handlers is illustrated in the following diagram. +.. only:: not html + + .. image:: logging_flow.* + .. raw:: html :file: logging_flow.svg diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 6c1063cda6690e..04bb8c51a3b197 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -82,6 +82,8 @@ A small number of constants live in the built-in namespace. They are: :exc:`SyntaxError`), so they can be considered "true" constants. +.. _site-consts: + Constants added by the :mod:`site` module ----------------------------------------- @@ -97,6 +99,13 @@ should not be used in programs. (i.e. EOF) to exit", and when called, raise :exc:`SystemExit` with the specified exit code. +.. data:: help + :noindex: + + Object that when printed, prints the message "Type help() for interactive + help, or help(object) for help about object.", and when called, + acts as described :func:`elsewhere `. + .. data:: copyright credits diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index b8b442508becbb..0e7dc4f262bab4 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -283,17 +283,23 @@ Class attributes: Note that, because of normalization, ``timedelta.max`` is greater than ``-timedelta.min``. ``-timedelta.max`` is not representable as a :class:`timedelta` object. + Instance attributes (read-only): -+------------------+--------------------------------------------+ -| Attribute | Value | -+==================+============================================+ -| ``days`` | Between -999999999 and 999999999 inclusive | -+------------------+--------------------------------------------+ -| ``seconds`` | Between 0 and 86399 inclusive | -+------------------+--------------------------------------------+ -| ``microseconds`` | Between 0 and 999999 inclusive | -+------------------+--------------------------------------------+ +.. attribute:: timedelta.days + + Between -999,999,999 and 999,999,999 inclusive. + + +.. attribute:: timedelta.seconds + + Between 0 and 86,399 inclusive. + + +.. attribute:: timedelta.microseconds + + Between 0 and 999,999 inclusive. + Supported operations: diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 2c55f2b508bac1..b2b0086437f1db 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1965,10 +1965,10 @@ are always available. They are listed here in alphabetical order. ``D -> B -> C -> A -> object`` and the value of *type* is ``B``, then :func:`super` searches ``C -> A -> object``. - The :attr:`~class.__mro__` attribute of the *object_or_type* lists the method - resolution search order used by both :func:`getattr` and :func:`super`. The - attribute is dynamic and can change whenever the inheritance hierarchy is - updated. + The :attr:`~class.__mro__` attribute of the class corresponding to + *object_or_type* lists the method resolution search order used by both + :func:`getattr` and :func:`super`. The attribute is dynamic and can change + whenever the inheritance hierarchy is updated. If the second argument is omitted, the super object returned is unbound. If the second argument is an object, ``isinstance(obj, type)`` must be true. If diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index b6fb6249155c79..e5bdfbb144b65a 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -131,6 +131,10 @@ write code that handles both IP versions correctly. Address objects are The appropriate version number: ``4`` for IPv4, ``6`` for IPv6. + .. versionchanged:: 3.14 + + Made available on the class. + .. attribute:: max_prefixlen The total number of bits in the address representation for this @@ -140,6 +144,10 @@ write code that handles both IP versions correctly. Address objects are are compared to determine whether or not an address is part of a network. + .. versionchanged:: 3.14 + + Made available on the class. + .. attribute:: compressed .. attribute:: exploded diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 26f85b5ddf8d82..f0c37948ff8d9a 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -241,28 +241,28 @@ Basic Usage *object_hook* is an optional function that will be called with the result of any object literal decoded (a :class:`dict`). The return value of - *object_hook* will be used instead of the :class:`dict`. This feature can be used - to implement custom decoders (e.g. `JSON-RPC `_ - class hinting). + *object_hook* will be used instead of the :class:`dict`. This feature can + be used to implement custom decoders (e.g. `JSON-RPC + `_ class hinting). *object_pairs_hook* is an optional function that will be called with the result of any object literal decoded with an ordered list of pairs. The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders. - If *object_hook* is also defined, the *object_pairs_hook* takes priority. + :class:`dict`. This feature can be used to implement custom decoders. If + *object_hook* is also defined, the *object_pairs_hook* takes priority. .. versionchanged:: 3.1 Added support for *object_pairs_hook*. - *parse_float*, if specified, will be called with the string of every JSON - float to be decoded. By default, this is equivalent to ``float(num_str)``. - This can be used to use another datatype or parser for JSON floats - (e.g. :class:`decimal.Decimal`). + *parse_float* is an optional function that will be called with the string of + every JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser for + JSON floats (e.g. :class:`decimal.Decimal`). - *parse_int*, if specified, will be called with the string of every JSON int - to be decoded. By default, this is equivalent to ``int(num_str)``. This can - be used to use another datatype or parser for JSON integers - (e.g. :class:`float`). + *parse_int* is an optional function that will be called with the string of + every JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser for + JSON integers (e.g. :class:`float`). .. versionchanged:: 3.11 The default *parse_int* of :func:`int` now limits the maximum length of @@ -270,10 +270,9 @@ Basic Usage conversion length limitation ` to help avoid denial of service attacks. - *parse_constant*, if specified, will be called with one of the following - strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. - This can be used to raise an exception if invalid JSON numbers - are encountered. + *parse_constant* is an optional function that will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This can be + used to raise an exception if invalid JSON numbers are encountered. .. versionchanged:: 3.1 *parse_constant* doesn't get called on 'null', 'true', 'false' anymore. @@ -345,34 +344,33 @@ Encoders and Decoders It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as their corresponding ``float`` values, which is outside the JSON spec. - *object_hook*, if specified, will be called with the result of every JSON - object decoded and its return value will be used in place of the given - :class:`dict`. This can be used to provide custom deserializations (e.g. to - support `JSON-RPC `_ class hinting). + *object_hook* is an optional function that will be called with the result of + every JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom deserializations + (e.g. to support `JSON-RPC `_ class hinting). - *object_pairs_hook*, if specified will be called with the result of every - JSON object decoded with an ordered list of pairs. The return value of - *object_pairs_hook* will be used instead of the :class:`dict`. This - feature can be used to implement custom decoders. If *object_hook* is also - defined, the *object_pairs_hook* takes priority. + *object_pairs_hook* is an optional function that will be called with the + result of every JSON object decoded with an ordered list of pairs. The + return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders. If + *object_hook* is also defined, the *object_pairs_hook* takes priority. .. versionchanged:: 3.1 Added support for *object_pairs_hook*. - *parse_float*, if specified, will be called with the string of every JSON - float to be decoded. By default, this is equivalent to ``float(num_str)``. - This can be used to use another datatype or parser for JSON floats - (e.g. :class:`decimal.Decimal`). + *parse_float* is an optional function that will be called with the string of + every JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser for + JSON floats (e.g. :class:`decimal.Decimal`). - *parse_int*, if specified, will be called with the string of every JSON int - to be decoded. By default, this is equivalent to ``int(num_str)``. This can - be used to use another datatype or parser for JSON integers - (e.g. :class:`float`). + *parse_int* is an optional function that will be called with the string of + every JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser for + JSON integers (e.g. :class:`float`). - *parse_constant*, if specified, will be called with one of the following - strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. - This can be used to raise an exception if invalid JSON numbers - are encountered. + *parse_constant* is an optional function that will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This can be + used to raise an exception if invalid JSON numbers are encountered. If *strict* is false (``True`` is the default), then control characters will be allowed inside strings. Control characters in this context are diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 871cfefc8de310..4508091f679dc7 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -15,8 +15,9 @@ import can be suppressed using the interpreter's :option:`-S` option. .. index:: triple: module; search; path -Importing this module will append site-specific paths to the module search path -and add a few builtins, unless :option:`-S` was used. In that case, this module +Importing this module normally appends site-specific paths to the module search path +and adds :ref:`callables `, including :func:`help` to the built-in +namespace. However, Python startup option :option:`-S` blocks this and this module can be safely imported with no automatic modifications to the module search path or additions to the builtins. To explicitly trigger the usual site-specific additions, call the :func:`main` function. diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index a81e11549d5c54..8e3b020d8d7a34 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -413,14 +413,14 @@ Constants ``TCP_USER_TIMEOUT``, ``TCP_CONGESTION`` were added. .. versionchanged:: 3.6.5 - On Windows, ``TCP_FASTOPEN``, ``TCP_KEEPCNT`` appear if run-time Windows - supports. + Added support for ``TCP_FASTOPEN``, ``TCP_KEEPCNT`` on Windows platforms + when available. .. versionchanged:: 3.7 ``TCP_NOTSENT_LOWAT`` was added. - On Windows, ``TCP_KEEPIDLE``, ``TCP_KEEPINTVL`` appear if run-time Windows - supports. + Added support for ``TCP_KEEPIDLE``, ``TCP_KEEPINTVL`` on Windows platforms + when available. .. versionchanged:: 3.10 ``IP_RECVTOS`` was added. @@ -454,6 +454,10 @@ Constants Added missing ``IP_RECVERR``, ``IP_RECVTTL``, and ``IP_RECVORIGDSTADDR`` on Linux. + .. versionchanged:: 3.14 + Added support for ``TCP_QUICKACK`` on Windows platforms when available. + + .. data:: AF_CAN PF_CAN SOL_CAN_* diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 66f7d5fa38dbc9..6cf42b27718022 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -622,7 +622,7 @@ Miscellaneous options .. versionadded:: 3.13 * :samp:`-X gil={0,1}` forces the GIL to be disabled or enabled, - respectively. Only available in builds configured with + respectively. Setting to ``0`` is only available in builds configured with :option:`--disable-gil`. See also :envvar:`PYTHON_GIL` and :ref:`whatsnew313-free-threaded-cpython`. @@ -1221,13 +1221,12 @@ conflict. .. envvar:: PYTHON_GIL If this variable is set to ``1``, the global interpreter lock (GIL) will be - forced on. Setting it to ``0`` forces the GIL off. + forced on. Setting it to ``0`` forces the GIL off (needs Python configured with + the :option:`--disable-gil` build option). See also the :option:`-X gil <-X>` command-line option, which takes precedence over this variable, and :ref:`whatsnew313-free-threaded-cpython`. - Needs Python configured with the :option:`--disable-gil` build option. - .. versionadded:: 3.13 Debug-mode variables diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 83c13e6fb64d1d..e1bd52370d776c 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -524,6 +524,7 @@ New Features * :c:func:`PyInitConfig_SetInt` * :c:func:`PyInitConfig_SetStr` * :c:func:`PyInitConfig_SetStrList` + * :c:func:`PyInitConfig_AddModule` * :c:func:`Py_InitializeFromInitConfig` (Contributed by Victor Stinner in :gh:`107954`.) diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 328828a9152916..c2cb4e3cdd92fb 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -313,6 +313,10 @@ PyAPI_FUNC(int) PyInitConfig_SetStrList(PyInitConfig *config, size_t length, char * const *items); +PyAPI_FUNC(int) PyInitConfig_AddModule(PyInitConfig *config, + const char *name, + PyObject* (*initfunc)(void)); + PyAPI_FUNC(int) Py_InitializeFromInitConfig(PyInitConfig *config); diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index d9b46df507dfd7..6e948e16b7dbe8 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -738,7 +738,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abc_impl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_align_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_)); @@ -759,21 +758,18 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_initializing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_io)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_is_text_encoding)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_layout_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_length_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_limbo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_lock_unlock_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_needs_com_addref_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_only_immortal)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_pack_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_restype_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_showwarnmsg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_shutdown)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_slotnames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_swappedbytes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_type_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_uninitialized_submodules)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_warn_unawaited_coroutine)); @@ -787,6 +783,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arg)); @@ -806,6 +803,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(before)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(big)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(binary_form)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bit_size)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(block)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bound)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(buffer)); @@ -934,6 +932,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fd2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fdel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fget)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fields)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(file)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(file_actions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filename)); @@ -950,6 +949,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fold)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(follow_symlinks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format_spec)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(from_param)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromlist)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromtimestamp)); @@ -986,6 +986,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(importlib)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(in_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(incoming)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(index)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(indexgroup)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(infer_variance)); @@ -1006,6 +1007,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(intersection)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(interval)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(is_running)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(is_struct)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isatty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isinstance)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isoformat)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 10773d7a6c7e3f..5c63a6e519b93d 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -227,7 +227,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(_abc_impl) STRUCT_FOR_ID(_abstract_) STRUCT_FOR_ID(_active) - STRUCT_FOR_ID(_align_) STRUCT_FOR_ID(_anonymous_) STRUCT_FOR_ID(_argtypes_) STRUCT_FOR_ID(_as_parameter_) @@ -248,21 +247,18 @@ struct _Py_global_strings { STRUCT_FOR_ID(_initializing) STRUCT_FOR_ID(_io) STRUCT_FOR_ID(_is_text_encoding) - STRUCT_FOR_ID(_layout_) STRUCT_FOR_ID(_length_) STRUCT_FOR_ID(_limbo) STRUCT_FOR_ID(_lock_unlock_module) STRUCT_FOR_ID(_loop) STRUCT_FOR_ID(_needs_com_addref_) STRUCT_FOR_ID(_only_immortal) - STRUCT_FOR_ID(_pack_) STRUCT_FOR_ID(_restype_) STRUCT_FOR_ID(_showwarnmsg) STRUCT_FOR_ID(_shutdown) STRUCT_FOR_ID(_slotnames) STRUCT_FOR_ID(_strptime) STRUCT_FOR_ID(_strptime_datetime) - STRUCT_FOR_ID(_swappedbytes_) STRUCT_FOR_ID(_type_) STRUCT_FOR_ID(_uninitialized_submodules) STRUCT_FOR_ID(_warn_unawaited_coroutine) @@ -276,6 +272,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) STRUCT_FOR_ID(alias) + STRUCT_FOR_ID(align) STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(append) STRUCT_FOR_ID(arg) @@ -295,6 +292,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(before) STRUCT_FOR_ID(big) STRUCT_FOR_ID(binary_form) + STRUCT_FOR_ID(bit_size) STRUCT_FOR_ID(block) STRUCT_FOR_ID(bound) STRUCT_FOR_ID(buffer) @@ -423,6 +421,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fd2) STRUCT_FOR_ID(fdel) STRUCT_FOR_ID(fget) + STRUCT_FOR_ID(fields) STRUCT_FOR_ID(file) STRUCT_FOR_ID(file_actions) STRUCT_FOR_ID(filename) @@ -439,6 +438,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fold) STRUCT_FOR_ID(follow_symlinks) STRUCT_FOR_ID(format) + STRUCT_FOR_ID(format_spec) STRUCT_FOR_ID(from_param) STRUCT_FOR_ID(fromlist) STRUCT_FOR_ID(fromtimestamp) @@ -475,6 +475,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(importlib) STRUCT_FOR_ID(in_fd) STRUCT_FOR_ID(incoming) + STRUCT_FOR_ID(index) STRUCT_FOR_ID(indexgroup) STRUCT_FOR_ID(inf) STRUCT_FOR_ID(infer_variance) @@ -495,6 +496,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(intersection) STRUCT_FOR_ID(interval) STRUCT_FOR_ID(is_running) + STRUCT_FOR_ID(is_struct) STRUCT_FOR_ID(isatty) STRUCT_FOR_ID(isinstance) STRUCT_FOR_ID(isoformat) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 618f8d0a36b6c3..bac6b5b8fcfd9d 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -736,7 +736,6 @@ extern "C" { INIT_ID(_abc_impl), \ INIT_ID(_abstract_), \ INIT_ID(_active), \ - INIT_ID(_align_), \ INIT_ID(_anonymous_), \ INIT_ID(_argtypes_), \ INIT_ID(_as_parameter_), \ @@ -757,21 +756,18 @@ extern "C" { INIT_ID(_initializing), \ INIT_ID(_io), \ INIT_ID(_is_text_encoding), \ - INIT_ID(_layout_), \ INIT_ID(_length_), \ INIT_ID(_limbo), \ INIT_ID(_lock_unlock_module), \ INIT_ID(_loop), \ INIT_ID(_needs_com_addref_), \ INIT_ID(_only_immortal), \ - INIT_ID(_pack_), \ INIT_ID(_restype_), \ INIT_ID(_showwarnmsg), \ INIT_ID(_shutdown), \ INIT_ID(_slotnames), \ INIT_ID(_strptime), \ INIT_ID(_strptime_datetime), \ - INIT_ID(_swappedbytes_), \ INIT_ID(_type_), \ INIT_ID(_uninitialized_submodules), \ INIT_ID(_warn_unawaited_coroutine), \ @@ -785,6 +781,7 @@ extern "C" { INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ INIT_ID(alias), \ + INIT_ID(align), \ INIT_ID(allow_code), \ INIT_ID(append), \ INIT_ID(arg), \ @@ -804,6 +801,7 @@ extern "C" { INIT_ID(before), \ INIT_ID(big), \ INIT_ID(binary_form), \ + INIT_ID(bit_size), \ INIT_ID(block), \ INIT_ID(bound), \ INIT_ID(buffer), \ @@ -932,6 +930,7 @@ extern "C" { INIT_ID(fd2), \ INIT_ID(fdel), \ INIT_ID(fget), \ + INIT_ID(fields), \ INIT_ID(file), \ INIT_ID(file_actions), \ INIT_ID(filename), \ @@ -948,6 +947,7 @@ extern "C" { INIT_ID(fold), \ INIT_ID(follow_symlinks), \ INIT_ID(format), \ + INIT_ID(format_spec), \ INIT_ID(from_param), \ INIT_ID(fromlist), \ INIT_ID(fromtimestamp), \ @@ -984,6 +984,7 @@ extern "C" { INIT_ID(importlib), \ INIT_ID(in_fd), \ INIT_ID(incoming), \ + INIT_ID(index), \ INIT_ID(indexgroup), \ INIT_ID(inf), \ INIT_ID(infer_variance), \ @@ -1004,6 +1005,7 @@ extern "C" { INIT_ID(intersection), \ INIT_ID(interval), \ INIT_ID(is_running), \ + INIT_ID(is_struct), \ INIT_ID(isatty), \ INIT_ID(isinstance), \ INIT_ID(isoformat), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index f848a002c3b5d1..efdbde4c8ea3c6 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -708,10 +708,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(_align_); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_anonymous_); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -792,10 +788,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(_layout_); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_length_); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -820,10 +812,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(_pack_); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_restype_); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -848,10 +836,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(_swappedbytes_); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_type_); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -904,6 +888,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(align); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(allow_code); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -980,6 +968,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(bit_size); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(block); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1492,6 +1484,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(fields); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(file); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1556,6 +1552,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(format_spec); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(from_param); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1700,6 +1700,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(index); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(indexgroup); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1780,6 +1784,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(is_struct); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isatty); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index 7f4d0672d02094..f6e14bdffc3352 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -71,6 +71,18 @@ def do(self) -> None: r.select_item(r.historyi - 1) +class history_search_backward(commands.Command): + def do(self) -> None: + r = self.reader + r.search_next(forwards=False) + + +class history_search_forward(commands.Command): + def do(self) -> None: + r = self.reader + r.search_next(forwards=True) + + class restore_history(commands.Command): def do(self) -> None: r = self.reader @@ -234,6 +246,8 @@ def __post_init__(self) -> None: isearch_forwards, isearch_backwards, operate_and_get_next, + history_search_backward, + history_search_forward, ]: self.commands[c.__name__] = c self.commands[c.__name__.replace("_", "-")] = c @@ -251,8 +265,8 @@ def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: (r"\C-s", "forward-history-isearch"), (r"\M-r", "restore-history"), (r"\M-.", "yank-arg"), - (r"\", "last-history"), - (r"\", "first-history"), + (r"\", "history-search-forward"), + (r"\", "history-search-backward"), ) def select_item(self, i: int) -> None: @@ -305,6 +319,59 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: else: return super().get_prompt(lineno, cursor_on_line) + def search_next(self, *, forwards: bool) -> None: + """Search history for the current line contents up to the cursor. + + Selects the first item found. If nothing is under the cursor, any next + item in history is selected. + """ + pos = self.pos + s = self.get_unicode() + history_index = self.historyi + + # In multiline contexts, we're only interested in the current line. + nl_index = s.rfind('\n', 0, pos) + prefix = s[nl_index + 1:pos] + pos = len(prefix) + + match_prefix = len(prefix) + len_item = 0 + if history_index < len(self.history): + len_item = len(self.get_item(history_index)) + if len_item and pos == len_item: + match_prefix = False + elif not pos: + match_prefix = False + + while 1: + if forwards: + out_of_bounds = history_index >= len(self.history) - 1 + else: + out_of_bounds = history_index == 0 + if out_of_bounds: + if forwards and not match_prefix: + self.pos = 0 + self.buffer = [] + self.dirty = True + else: + self.error("not found") + return + + history_index += 1 if forwards else -1 + s = self.get_item(history_index) + + if not match_prefix: + self.select_item(history_index) + return + + len_acc = 0 + for i, line in enumerate(s.splitlines(keepends=True)): + if line.startswith(prefix): + self.select_item(history_index) + self.pos = pos + len_acc + return + len_acc += len(line) + def isearch_next(self) -> None: st = self.isearch_term p = self.pos diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 483eb1039fa062..5e1d3085874380 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -249,7 +249,7 @@ def _should_auto_indent(buffer: list[str], pos: int) -> bool: while pos > 0: pos -= 1 if last_char is None: - if buffer[pos] not in " \t\n": # ignore whitespaces + if buffer[pos] not in " \t\n#": # ignore whitespaces and comments last_char = buffer[pos] else: # even if we found a non-whitespace character before @@ -365,8 +365,12 @@ def input(self, prompt: object = "") -> str: except _error: assert raw_input is not None return raw_input(prompt) - reader.ps1 = str(prompt) - return reader.readline(startup_hook=self.startup_hook) + prompt_str = str(prompt) + reader.ps1 = prompt_str + sys.audit("builtins.input", prompt_str) + result = reader.readline(startup_hook=self.startup_hook) + sys.audit("builtins.input/result", result) + return result def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> str: """Read an input on possibly multiple lines, asking for more @@ -423,18 +427,22 @@ def read_history_file(self, filename: str = gethistoryfile()) -> None: history = self.get_reader().history with open(os.path.expanduser(filename), 'rb') as f: - lines = [line.decode('utf-8', errors='replace') for line in f.read().split(b'\n')] + is_editline = f.readline().startswith(b"_HiStOrY_V2_") + if is_editline: + encoding = "unicode-escape" + else: + f.seek(0) + encoding = "utf-8" + + lines = [line.decode(encoding, errors='replace') for line in f.read().split(b'\n')] buffer = [] for line in lines: - # Ignore readline history file header - if line.startswith("_HiStOrY_V2_"): - continue if line.endswith("\r"): buffer.append(line+'\n') else: line = self._histline(line) if buffer: - line = "".join(buffer).replace("\r", "") + line + line = self._histline("".join(buffer).replace("\r", "") + line) del buffer[:] if line: history.append(line) diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 91aef5e01eb867..3c79cf61d04051 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -163,7 +163,8 @@ def maybe_run_command(statement: str) -> bool: r.isearch_direction = '' r.console.forgetinput() r.pop_input_trans() - r.dirty = True + r.pos = len(r.get_unicode()) + r.dirty = True r.refresh() r.in_bracketed_paste = False console.write("\nKeyboardInterrupt\n") diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py new file mode 100644 index 00000000000000..e30db598ab22e1 --- /dev/null +++ b/Lib/ctypes/_layout.py @@ -0,0 +1,337 @@ +"""Python implementation of computing the layout of a struct/union + +This code is internal and tightly coupled to the C part. The interface +may change at any time. +""" + +import sys +import warnings +import struct + +from _ctypes import CField, buffer_info +import ctypes + +def round_down(n, multiple): + assert n >= 0 + assert multiple > 0 + return (n // multiple) * multiple + +def round_up(n, multiple): + assert n >= 0 + assert multiple > 0 + return ((n + multiple - 1) // multiple) * multiple + +def LOW_BIT(offset): + return offset & 0xFFFF + +def NUM_BITS(bitsize): + return bitsize >> 16 + +def BUILD_SIZE(bitsize, offset): + assert 0 <= offset, offset + assert offset <= 0xFFFF, offset + # We don't support zero length bitfields. + # And GET_BITFIELD uses NUM_BITS(size) == 0, + # to figure out whether we are handling a bitfield. + assert bitsize > 0, bitsize + result = (bitsize << 16) + offset + assert bitsize == NUM_BITS(result), (bitsize, result) + assert offset == LOW_BIT(result), (offset, result) + return result + +def build_size(bit_size, bit_offset, big_endian, type_size): + if big_endian: + return BUILD_SIZE(bit_size, 8 * type_size - bit_offset - bit_size) + return BUILD_SIZE(bit_size, bit_offset) + +_INT_MAX = (1 << (ctypes.sizeof(ctypes.c_int) * 8) - 1) - 1 + + +class StructUnionLayout: + def __init__(self, fields, size, align, format_spec): + # sequence of CField objects + self.fields = fields + + # total size of the aggregate (rounded up to alignment) + self.size = size + + # total alignment requirement of the aggregate + self.align = align + + # buffer format specification (as a string, UTF-8 but bes + # kept ASCII-only) + self.format_spec = format_spec + + +def get_layout(cls, input_fields, is_struct, base): + """Return a StructUnionLayout for the given class. + + Called by PyCStructUnionType_update_stginfo when _fields_ is assigned + to a class. + """ + # Currently there are two modes, selectable using the '_layout_' attribute: + # + # 'gcc-sysv' mode places fields one after another, bit by bit. + # But "each bit field must fit within a single object of its specified + # type" (GCC manual, section 15.8 "Bit Field Packing"). When it doesn't, + # we insert a few bits of padding to avoid that. + # + # 'ms' mode works similar except for bitfield packing. Adjacent + # bit-fields are packed into the same 1-, 2-, or 4-byte allocation unit + # if the integral types are the same size and if the next bit-field fits + # into the current allocation unit without crossing the boundary imposed + # by the common alignment requirements of the bit-fields. + # + # See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields + # for details. + + # We do not support zero length bitfields (we use bitsize != 0 + # elsewhere to indicate a bitfield). Here, non-bitfields have bit_size + # set to size*8. + + # For clarity, variables that count bits have `bit` in their names. + + layout = getattr(cls, '_layout_', None) + if layout is None: + if sys.platform == 'win32' or getattr(cls, '_pack_', None): + gcc_layout = False + else: + gcc_layout = True + elif layout == 'ms': + gcc_layout = False + elif layout == 'gcc-sysv': + gcc_layout = True + else: + raise ValueError(f'unknown _layout_: {layout!r}') + + align = getattr(cls, '_align_', 1) + if align < 0: + raise ValueError('_align_ must be a non-negative integer') + elif align == 0: + # Setting `_align_ = 0` amounts to using the default alignment + align == 1 + + if base: + align = max(ctypes.alignment(base), align) + + swapped_bytes = hasattr(cls, '_swappedbytes_') + if swapped_bytes: + big_endian = sys.byteorder == 'little' + else: + big_endian = sys.byteorder == 'big' + + pack = getattr(cls, '_pack_', None) + if pack is not None: + try: + pack = int(pack) + except (TypeError, ValueError): + raise ValueError("_pack_ must be an integer") + if pack < 0: + raise ValueError("_pack_ must be a non-negative integer") + if pack > _INT_MAX: + raise ValueError("_pack_ too big") + if gcc_layout: + raise ValueError('_pack_ is not compatible with gcc-sysv layout') + + result_fields = [] + + if is_struct: + format_spec_parts = ["T{"] + else: + format_spec_parts = ["B"] + + last_field_bit_size = 0 # used in MS layout only + + # `8 * next_byte_offset + next_bit_offset` points to where the + # next field would start. + next_bit_offset = 0 + next_byte_offset = 0 + + # size if this was a struct (sum of field sizes, plus padding) + struct_size = 0 + # max of field sizes; only meaningful for unions + union_size = 0 + + if base: + struct_size = ctypes.sizeof(base) + if gcc_layout: + next_bit_offset = struct_size * 8 + else: + next_byte_offset = struct_size + + last_size = struct_size + for i, field in enumerate(input_fields): + if not is_struct: + # Unions start fresh each time + last_field_bit_size = 0 + next_bit_offset = 0 + next_byte_offset = 0 + + # Unpack the field + field = tuple(field) + try: + name, ctype = field + except (ValueError, TypeError): + try: + name, ctype, bit_size = field + except (ValueError, TypeError) as exc: + raise ValueError( + '_fields_ must be a sequence of (name, C type) pairs ' + + 'or (name, C type, bit size) triples') from exc + is_bitfield = True + if bit_size <= 0: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + type_size = ctypes.sizeof(ctype) + if bit_size > type_size * 8: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + else: + is_bitfield = False + type_size = ctypes.sizeof(ctype) + bit_size = type_size * 8 + + type_bit_size = type_size * 8 + type_align = ctypes.alignment(ctype) or 1 + type_bit_align = type_align * 8 + + if gcc_layout: + # We don't use next_byte_offset here + assert pack is None + assert next_byte_offset == 0 + + # Determine whether the bit field, if placed at the next + # free bit, fits within a single object of its specified type. + # That is: determine a "slot", sized & aligned for the + # specified type, which contains the bitfield's beginning: + slot_start_bit = round_down(next_bit_offset, type_bit_align) + slot_end_bit = slot_start_bit + type_bit_size + # And see if it also contains the bitfield's last bit: + field_end_bit = next_bit_offset + bit_size + if field_end_bit > slot_end_bit: + # It doesn't: add padding (bump up to the next + # alignment boundary) + next_bit_offset = round_up(next_bit_offset, type_bit_align) + + offset = round_down(next_bit_offset, type_bit_align) // 8 + if is_bitfield: + effective_bit_offset = next_bit_offset - 8 * offset + size = build_size(bit_size, effective_bit_offset, + big_endian, type_size) + assert effective_bit_offset <= type_bit_size + else: + assert offset == next_bit_offset / 8 + size = type_size + + next_bit_offset += bit_size + struct_size = round_up(next_bit_offset, 8) // 8 + else: + if pack: + type_align = min(pack, type_align) + + # next_byte_offset points to end of current bitfield. + # next_bit_offset is generally non-positive, + # and 8 * next_byte_offset + next_bit_offset points just behind + # the end of the last field we placed. + if ( + (0 < next_bit_offset + bit_size) + or (type_bit_size != last_field_bit_size) + ): + # Close the previous bitfield (if any) + # and start a new bitfield + next_byte_offset = round_up(next_byte_offset, type_align) + + next_byte_offset += type_size + + last_field_bit_size = type_bit_size + # Reminder: 8 * (next_byte_offset) + next_bit_offset + # points to where we would start a new field, namely + # just behind where we placed the last field plus an + # allowance for alignment. + next_bit_offset = -last_field_bit_size + + assert type_bit_size == last_field_bit_size + + offset = next_byte_offset - last_field_bit_size // 8 + if is_bitfield: + assert 0 <= (last_field_bit_size + next_bit_offset) + size = build_size(bit_size, + last_field_bit_size + next_bit_offset, + big_endian, type_size) + else: + size = type_size + if type_bit_size: + assert (last_field_bit_size + next_bit_offset) < type_bit_size + + next_bit_offset += bit_size + struct_size = next_byte_offset + + assert (not is_bitfield) or (LOW_BIT(size) <= size * 8) + + # Add the format spec parts + if is_struct: + padding = offset - last_size + format_spec_parts.append(padding_spec(padding)) + + fieldfmt, bf_ndim, bf_shape = buffer_info(ctype) + + if bf_shape: + format_spec_parts.extend(( + "(", + ','.join(str(n) for n in bf_shape), + ")", + )) + + if fieldfmt is None: + fieldfmt = "B" + if isinstance(name, bytes): + # a bytes name would be rejected later, but we check early + # to avoid a BytesWarning with `python -bb` + raise TypeError( + "field {name!r}: name must be a string, not bytes") + format_spec_parts.append(f"{fieldfmt}:{name}:") + + result_fields.append(CField( + name=name, + type=ctype, + size=size, + offset=offset, + bit_size=bit_size if is_bitfield else None, + index=i, + )) + if is_bitfield and not gcc_layout: + assert type_bit_size > 0 + + align = max(align, type_align) + last_size = struct_size + if not is_struct: + union_size = max(struct_size, union_size) + + if is_struct: + total_size = struct_size + else: + total_size = union_size + + # Adjust the size according to the alignment requirements + aligned_size = round_up(total_size, align) + + # Finish up the format spec + if is_struct: + padding = aligned_size - total_size + format_spec_parts.append(padding_spec(padding)) + format_spec_parts.append("}") + + return StructUnionLayout( + fields=result_fields, + size=aligned_size, + align=align, + format_spec="".join(format_spec_parts), + ) + + +def padding_spec(padding): + if padding <= 0: + return "" + if padding == 1: + return "x" + return f"{padding}x" diff --git a/Lib/email/generator.py b/Lib/email/generator.py index 42c84aa4da1044..205caf0fe9e81d 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -227,7 +227,7 @@ def _write_headers(self, msg): folded = self.policy.fold(h, v) if self.policy.verify_generated_headers: linesep = self.policy.linesep - if not folded.endswith(self.policy.linesep): + if not folded.endswith(linesep): raise HeaderWriteError( f'folded header does not end with {linesep!r}: {folded!r}') if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)): diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index c165505a53330c..c5a737d76de245 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -239,7 +239,7 @@ def summarize_address_range(first, last): else: raise ValueError('unknown IP version') - ip_bits = first._max_prefixlen + ip_bits = first.max_prefixlen first_int = first._ip last_int = last._ip while first_int <= last_int: @@ -326,12 +326,12 @@ def collapse_addresses(addresses): # split IP addresses and networks for ip in addresses: if isinstance(ip, _BaseAddress): - if ips and ips[-1]._version != ip._version: + if ips and ips[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, ips[-1])) ips.append(ip) - elif ip._prefixlen == ip._max_prefixlen: - if ips and ips[-1]._version != ip._version: + elif ip._prefixlen == ip.max_prefixlen: + if ips and ips[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, ips[-1])) try: @@ -339,7 +339,7 @@ def collapse_addresses(addresses): except AttributeError: ips.append(ip.network_address) else: - if nets and nets[-1]._version != ip._version: + if nets and nets[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, nets[-1])) nets.append(ip) @@ -407,26 +407,21 @@ def reverse_pointer(self): """ return self._reverse_pointer() - @property - def version(self): - msg = '%200s has no version specified' % (type(self),) - raise NotImplementedError(msg) - def _check_int_address(self, address): if address < 0: msg = "%d (< 0) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, self._version)) + raise AddressValueError(msg % (address, self.version)) if address > self._ALL_ONES: msg = "%d (>= 2**%d) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, self._max_prefixlen, - self._version)) + raise AddressValueError(msg % (address, self.max_prefixlen, + self.version)) def _check_packed_address(self, address, expected_len): address_len = len(address) if address_len != expected_len: msg = "%r (len %d != %d) is not permitted as an IPv%d address" raise AddressValueError(msg % (address, address_len, - expected_len, self._version)) + expected_len, self.version)) @classmethod def _ip_int_from_prefix(cls, prefixlen): @@ -455,12 +450,12 @@ def _prefix_from_ip_int(cls, ip_int): ValueError: If the input intermingles zeroes & ones """ trailing_zeroes = _count_righthand_zero_bits(ip_int, - cls._max_prefixlen) - prefixlen = cls._max_prefixlen - trailing_zeroes + cls.max_prefixlen) + prefixlen = cls.max_prefixlen - trailing_zeroes leading_ones = ip_int >> trailing_zeroes all_ones = (1 << prefixlen) - 1 if leading_ones != all_ones: - byteslen = cls._max_prefixlen // 8 + byteslen = cls.max_prefixlen // 8 details = ip_int.to_bytes(byteslen, 'big') msg = 'Netmask pattern %r mixes zeroes & ones' raise ValueError(msg % details) @@ -492,7 +487,7 @@ def _prefix_from_prefix_string(cls, prefixlen_str): prefixlen = int(prefixlen_str) except ValueError: cls._report_invalid_netmask(prefixlen_str) - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen_str) return prefixlen @@ -542,7 +537,7 @@ def _split_addr_prefix(cls, address): """ # a packed address or integer if isinstance(address, (bytes, int)): - return address, cls._max_prefixlen + return address, cls.max_prefixlen if not isinstance(address, tuple): # Assume input argument to be string or any object representation @@ -552,7 +547,7 @@ def _split_addr_prefix(cls, address): # Constructing from a tuple (addr, [mask]) if len(address) > 1: return address - return address[0], cls._max_prefixlen + return address[0], cls.max_prefixlen def __reduce__(self): return self.__class__, (str(self),) @@ -577,14 +572,14 @@ def __int__(self): def __eq__(self, other): try: return (self._ip == other._ip - and self._version == other._version) + and self.version == other.version) except AttributeError: return NotImplemented def __lt__(self, other): if not isinstance(other, _BaseAddress): return NotImplemented - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same version' % ( self, other)) if self._ip != other._ip: @@ -613,7 +608,7 @@ def __hash__(self): return hash(hex(int(self._ip))) def _get_address_key(self): - return (self._version, self) + return (self.version, self) def __reduce__(self): return self.__class__, (self._ip,) @@ -649,15 +644,15 @@ def __format__(self, fmt): # Set some defaults if fmt_base == 'n': - if self._version == 4: + if self.version == 4: fmt_base = 'b' # Binary is default for ipv4 else: fmt_base = 'x' # Hex is default for ipv6 if fmt_base == 'b': - padlen = self._max_prefixlen + padlen = self.max_prefixlen else: - padlen = self._max_prefixlen // 4 + padlen = self.max_prefixlen // 4 if grouping: padlen += padlen // 4 - 1 @@ -716,7 +711,7 @@ def __getitem__(self, n): def __lt__(self, other): if not isinstance(other, _BaseNetwork): return NotImplemented - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same version' % ( self, other)) if self.network_address != other.network_address: @@ -727,7 +722,7 @@ def __lt__(self, other): def __eq__(self, other): try: - return (self._version == other._version and + return (self.version == other.version and self.network_address == other.network_address and int(self.netmask) == int(other.netmask)) except AttributeError: @@ -738,7 +733,7 @@ def __hash__(self): def __contains__(self, other): # always false if one is v4 and the other is v6. - if self._version != other._version: + if self.version != other.version: return False # dealing with another network. if isinstance(other, _BaseNetwork): @@ -829,7 +824,7 @@ def address_exclude(self, other): ValueError: If other is not completely contained by self. """ - if not self._version == other._version: + if not self.version == other.version: raise TypeError("%s and %s are not of the same version" % ( self, other)) @@ -901,10 +896,10 @@ def compare_networks(self, other): """ # does this need to raise a ValueError? - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same type' % ( self, other)) - # self._version == other._version below here: + # self.version == other.version below here: if self.network_address < other.network_address: return -1 if self.network_address > other.network_address: @@ -924,7 +919,7 @@ def _get_networks_key(self): and list.sort(). """ - return (self._version, self.network_address, self.netmask) + return (self.version, self.network_address, self.netmask) def subnets(self, prefixlen_diff=1, new_prefix=None): """The subnets which join to make the current subnet. @@ -952,7 +947,7 @@ def subnets(self, prefixlen_diff=1, new_prefix=None): number means a larger network) """ - if self._prefixlen == self._max_prefixlen: + if self._prefixlen == self.max_prefixlen: yield self return @@ -967,7 +962,7 @@ def subnets(self, prefixlen_diff=1, new_prefix=None): raise ValueError('prefix length diff must be > 0') new_prefixlen = self._prefixlen + prefixlen_diff - if new_prefixlen > self._max_prefixlen: + if new_prefixlen > self.max_prefixlen: raise ValueError( 'prefix length diff %d is invalid for netblock %s' % ( new_prefixlen, self)) @@ -1036,7 +1031,7 @@ def is_multicast(self): def _is_subnet_of(a, b): try: # Always false if one is v4 and the other is v6. - if a._version != b._version: + if a.version != b.version: raise TypeError(f"{a} and {b} are not of the same version") return (b.network_address <= a.network_address and b.broadcast_address >= a.broadcast_address) @@ -1146,11 +1141,11 @@ class _BaseV4: """ __slots__ = () - _version = 4 + version = 4 # Equivalent to 255.255.255.255 or 32 bits of 1's. _ALL_ONES = (2**IPV4LENGTH) - 1 - _max_prefixlen = IPV4LENGTH + max_prefixlen = IPV4LENGTH # There are only a handful of valid v4 netmasks, so we cache them all # when constructed (see _make_netmask()). _netmask_cache = {} @@ -1170,7 +1165,7 @@ def _make_netmask(cls, arg): if arg not in cls._netmask_cache: if isinstance(arg, int): prefixlen = arg - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen) else: try: @@ -1268,15 +1263,6 @@ def _reverse_pointer(self): reverse_octets = str(self).split('.')[::-1] return '.'.join(reverse_octets) + '.in-addr.arpa' - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - class IPv4Address(_BaseV4, _BaseAddress): """Represent and manipulate single IPv4 Addresses.""" @@ -1556,9 +1542,9 @@ def __init__(self, address, strict=True): self.network_address = IPv4Address(packed & int(self.netmask)) - if self._prefixlen == (self._max_prefixlen - 1): + if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ - elif self._prefixlen == (self._max_prefixlen): + elif self._prefixlen == (self.max_prefixlen): self.hosts = lambda: [IPv4Address(addr)] @property @@ -1628,11 +1614,11 @@ class _BaseV6: """ __slots__ = () - _version = 6 + version = 6 _ALL_ONES = (2**IPV6LENGTH) - 1 _HEXTET_COUNT = 8 _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef') - _max_prefixlen = IPV6LENGTH + max_prefixlen = IPV6LENGTH # There are only a bunch of valid v6 netmasks, so we cache them all # when constructed (see _make_netmask()). @@ -1650,7 +1636,7 @@ def _make_netmask(cls, arg): if arg not in cls._netmask_cache: if isinstance(arg, int): prefixlen = arg - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen) else: prefixlen = cls._prefix_from_prefix_string(arg) @@ -1912,15 +1898,6 @@ def _split_scope_id(ip_str): raise AddressValueError('Invalid IPv6 address: "%r"' % ip_str) return addr, scope_id - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - class IPv6Address(_BaseV6, _BaseAddress): """Represent and manipulate single IPv6 Addresses.""" @@ -2332,9 +2309,9 @@ def __init__(self, address, strict=True): self.network_address = IPv6Address(packed & int(self.netmask)) - if self._prefixlen == (self._max_prefixlen - 1): + if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ - elif self._prefixlen == self._max_prefixlen: + elif self._prefixlen == self.max_prefixlen: self.hosts = lambda: [IPv6Address(addr)] def hosts(self): diff --git a/Lib/socket.py b/Lib/socket.py index 9207101dcf9d58..be37c24d6174a2 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -52,7 +52,9 @@ import _socket from _socket import * -import os, sys, io, selectors +import io +import os +import sys from enum import IntEnum, IntFlag try: @@ -110,102 +112,103 @@ def _intenum_converter(value, enum_klass): # WSA error codes if sys.platform.lower().startswith("win"): - errorTab = {} - errorTab[6] = "Specified event object handle is invalid." - errorTab[8] = "Insufficient memory available." - errorTab[87] = "One or more parameters are invalid." - errorTab[995] = "Overlapped operation aborted." - errorTab[996] = "Overlapped I/O event object not in signaled state." - errorTab[997] = "Overlapped operation will complete later." - errorTab[10004] = "The operation was interrupted." - errorTab[10009] = "A bad file handle was passed." - errorTab[10013] = "Permission denied." - errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT - errorTab[10022] = "An invalid operation was attempted." - errorTab[10024] = "Too many open files." - errorTab[10035] = "The socket operation would block." - errorTab[10036] = "A blocking operation is already in progress." - errorTab[10037] = "Operation already in progress." - errorTab[10038] = "Socket operation on nonsocket." - errorTab[10039] = "Destination address required." - errorTab[10040] = "Message too long." - errorTab[10041] = "Protocol wrong type for socket." - errorTab[10042] = "Bad protocol option." - errorTab[10043] = "Protocol not supported." - errorTab[10044] = "Socket type not supported." - errorTab[10045] = "Operation not supported." - errorTab[10046] = "Protocol family not supported." - errorTab[10047] = "Address family not supported by protocol family." - errorTab[10048] = "The network address is in use." - errorTab[10049] = "Cannot assign requested address." - errorTab[10050] = "Network is down." - errorTab[10051] = "Network is unreachable." - errorTab[10052] = "Network dropped connection on reset." - errorTab[10053] = "Software caused connection abort." - errorTab[10054] = "The connection has been reset." - errorTab[10055] = "No buffer space available." - errorTab[10056] = "Socket is already connected." - errorTab[10057] = "Socket is not connected." - errorTab[10058] = "The network has been shut down." - errorTab[10059] = "Too many references." - errorTab[10060] = "The operation timed out." - errorTab[10061] = "Connection refused." - errorTab[10062] = "Cannot translate name." - errorTab[10063] = "The name is too long." - errorTab[10064] = "The host is down." - errorTab[10065] = "The host is unreachable." - errorTab[10066] = "Directory not empty." - errorTab[10067] = "Too many processes." - errorTab[10068] = "User quota exceeded." - errorTab[10069] = "Disk quota exceeded." - errorTab[10070] = "Stale file handle reference." - errorTab[10071] = "Item is remote." - errorTab[10091] = "Network subsystem is unavailable." - errorTab[10092] = "Winsock.dll version out of range." - errorTab[10093] = "Successful WSAStartup not yet performed." - errorTab[10101] = "Graceful shutdown in progress." - errorTab[10102] = "No more results from WSALookupServiceNext." - errorTab[10103] = "Call has been canceled." - errorTab[10104] = "Procedure call table is invalid." - errorTab[10105] = "Service provider is invalid." - errorTab[10106] = "Service provider failed to initialize." - errorTab[10107] = "System call failure." - errorTab[10108] = "Service not found." - errorTab[10109] = "Class type not found." - errorTab[10110] = "No more results from WSALookupServiceNext." - errorTab[10111] = "Call was canceled." - errorTab[10112] = "Database query was refused." - errorTab[11001] = "Host not found." - errorTab[11002] = "Nonauthoritative host not found." - errorTab[11003] = "This is a nonrecoverable error." - errorTab[11004] = "Valid name, no data record requested type." - errorTab[11005] = "QoS receivers." - errorTab[11006] = "QoS senders." - errorTab[11007] = "No QoS senders." - errorTab[11008] = "QoS no receivers." - errorTab[11009] = "QoS request confirmed." - errorTab[11010] = "QoS admission error." - errorTab[11011] = "QoS policy failure." - errorTab[11012] = "QoS bad style." - errorTab[11013] = "QoS bad object." - errorTab[11014] = "QoS traffic control error." - errorTab[11015] = "QoS generic error." - errorTab[11016] = "QoS service type error." - errorTab[11017] = "QoS flowspec error." - errorTab[11018] = "Invalid QoS provider buffer." - errorTab[11019] = "Invalid QoS filter style." - errorTab[11020] = "Invalid QoS filter style." - errorTab[11021] = "Incorrect QoS filter count." - errorTab[11022] = "Invalid QoS object length." - errorTab[11023] = "Incorrect QoS flow count." - errorTab[11024] = "Unrecognized QoS object." - errorTab[11025] = "Invalid QoS policy object." - errorTab[11026] = "Invalid QoS flow descriptor." - errorTab[11027] = "Invalid QoS provider-specific flowspec." - errorTab[11028] = "Invalid QoS provider-specific filterspec." - errorTab[11029] = "Invalid QoS shape discard mode object." - errorTab[11030] = "Invalid QoS shaping rate object." - errorTab[11031] = "Reserved policy QoS element type." + errorTab = { + 6: "Specified event object handle is invalid.", + 8: "Insufficient memory available.", + 87: "One or more parameters are invalid.", + 995: "Overlapped operation aborted.", + 996: "Overlapped I/O event object not in signaled state.", + 997: "Overlapped operation will complete later.", + 10004: "The operation was interrupted.", + 10009: "A bad file handle was passed.", + 10013: "Permission denied.", + 10014: "A fault occurred on the network??", + 10022: "An invalid operation was attempted.", + 10024: "Too many open files.", + 10035: "The socket operation would block.", + 10036: "A blocking operation is already in progress.", + 10037: "Operation already in progress.", + 10038: "Socket operation on nonsocket.", + 10039: "Destination address required.", + 10040: "Message too long.", + 10041: "Protocol wrong type for socket.", + 10042: "Bad protocol option.", + 10043: "Protocol not supported.", + 10044: "Socket type not supported.", + 10045: "Operation not supported.", + 10046: "Protocol family not supported.", + 10047: "Address family not supported by protocol family.", + 10048: "The network address is in use.", + 10049: "Cannot assign requested address.", + 10050: "Network is down.", + 10051: "Network is unreachable.", + 10052: "Network dropped connection on reset.", + 10053: "Software caused connection abort.", + 10054: "The connection has been reset.", + 10055: "No buffer space available.", + 10056: "Socket is already connected.", + 10057: "Socket is not connected.", + 10058: "The network has been shut down.", + 10059: "Too many references.", + 10060: "The operation timed out.", + 10061: "Connection refused.", + 10062: "Cannot translate name.", + 10063: "The name is too long.", + 10064: "The host is down.", + 10065: "The host is unreachable.", + 10066: "Directory not empty.", + 10067: "Too many processes.", + 10068: "User quota exceeded.", + 10069: "Disk quota exceeded.", + 10070: "Stale file handle reference.", + 10071: "Item is remote.", + 10091: "Network subsystem is unavailable.", + 10092: "Winsock.dll version out of range.", + 10093: "Successful WSAStartup not yet performed.", + 10101: "Graceful shutdown in progress.", + 10102: "No more results from WSALookupServiceNext.", + 10103: "Call has been canceled.", + 10104: "Procedure call table is invalid.", + 10105: "Service provider is invalid.", + 10106: "Service provider failed to initialize.", + 10107: "System call failure.", + 10108: "Service not found.", + 10109: "Class type not found.", + 10110: "No more results from WSALookupServiceNext.", + 10111: "Call was canceled.", + 10112: "Database query was refused.", + 11001: "Host not found.", + 11002: "Nonauthoritative host not found.", + 11003: "This is a nonrecoverable error.", + 11004: "Valid name, no data record requested type.", + 11005: "QoS receivers.", + 11006: "QoS senders.", + 11007: "No QoS senders.", + 11008: "QoS no receivers.", + 11009: "QoS request confirmed.", + 11010: "QoS admission error.", + 11011: "QoS policy failure.", + 11012: "QoS bad style.", + 11013: "QoS bad object.", + 11014: "QoS traffic control error.", + 11015: "QoS generic error.", + 11016: "QoS service type error.", + 11017: "QoS flowspec error.", + 11018: "Invalid QoS provider buffer.", + 11019: "Invalid QoS filter style.", + 11020: "Invalid QoS filter style.", + 11021: "Incorrect QoS filter count.", + 11022: "Invalid QoS object length.", + 11023: "Incorrect QoS flow count.", + 11024: "Unrecognized QoS object.", + 11025: "Invalid QoS policy object.", + 11026: "Invalid QoS flow descriptor.", + 11027: "Invalid QoS provider-specific flowspec.", + 11028: "Invalid QoS provider-specific filterspec.", + 11029: "Invalid QoS shape discard mode object.", + 11030: "Invalid QoS shaping rate object.", + 11031: "Reserved policy QoS element type." + } __all__.append("errorTab") @@ -348,6 +351,9 @@ def makefile(self, mode="r", buffering=None, *, if hasattr(os, 'sendfile'): def _sendfile_use_sendfile(self, file, offset=0, count=None): + # Lazy import to improve module import time + import selectors + self._check_sendfile_params(file, offset, count) sockno = self.fileno() try: @@ -549,20 +555,18 @@ def fromfd(fd, family, type, proto=0): return socket(family, type, proto, nfd) if hasattr(_socket.socket, "sendmsg"): - import array - def send_fds(sock, buffers, fds, flags=0, address=None): """ send_fds(sock, buffers, fds[, flags[, address]]) -> integer Send the list of file descriptors fds over an AF_UNIX socket. """ + import array + return sock.sendmsg(buffers, [(_socket.SOL_SOCKET, _socket.SCM_RIGHTS, array.array("i", fds))]) __all__.append("send_fds") if hasattr(_socket.socket, "recvmsg"): - import array - def recv_fds(sock, bufsize, maxfds, flags=0): """ recv_fds(sock, bufsize, maxfds[, flags]) -> (data, list of file descriptors, msg_flags, address) @@ -570,6 +574,8 @@ def recv_fds(sock, bufsize, maxfds, flags=0): Receive up to maxfds file descriptors returning the message data and a list containing the descriptors. """ + import array + # Array of ints fds = array.array("i") msg, ancdata, flags, addr = sock.recvmsg(bufsize, diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 504f8800a00aa5..68e3b2a0d4d932 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -851,8 +851,13 @@ def get_a(x): @requires_limited_api def test_vectorcall_limited_incoming(self): from _testcapi import pyobject_vectorcall - obj = _testlimitedcapi.LimitedVectorCallClass() - self.assertEqual(pyobject_vectorcall(obj, (), ()), "vectorcall called") + for cls in (_testlimitedcapi.LimitedVectorCallClass, + _testlimitedcapi.LimitedRelativeVectorCallClass): + with self.subTest(cls=cls): + obj = cls() + self.assertEqual( + pyobject_vectorcall(obj, (), ()), + "vectorcall called") @requires_limited_api def test_vectorcall_limited_outgoing(self): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index b103bf2450bde0..d50217b695967e 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -541,14 +541,19 @@ def __del__(self): self.assertEqual(new_type_refcnt, sys.getrefcount(A)) def test_heaptype_with_dict(self): - inst = _testcapi.HeapCTypeWithDict() - inst.foo = 42 - self.assertEqual(inst.foo, 42) - self.assertEqual(inst.dictobj, inst.__dict__) - self.assertEqual(inst.dictobj, {"foo": 42}) + for cls in ( + _testcapi.HeapCTypeWithDict, + _testlimitedcapi.HeapCTypeWithRelativeDict, + ): + with self.subTest(cls=cls): + inst = cls() + inst.foo = 42 + self.assertEqual(inst.foo, 42) + self.assertEqual(inst.dictobj, inst.__dict__) + self.assertEqual(inst.dictobj, {"foo": 42}) - inst = _testcapi.HeapCTypeWithDict() - self.assertEqual({}, inst.__dict__) + inst = cls() + self.assertEqual({}, inst.__dict__) def test_heaptype_with_managed_dict(self): inst = _testcapi.HeapCTypeWithManagedDict() @@ -585,10 +590,15 @@ def test_heaptype_with_negative_dict(self): self.assertEqual({}, inst.__dict__) def test_heaptype_with_weakref(self): - inst = _testcapi.HeapCTypeWithWeakref() - ref = weakref.ref(inst) - self.assertEqual(ref(), inst) - self.assertEqual(inst.weakreflist, ref) + for cls in ( + _testcapi.HeapCTypeWithWeakref, + _testlimitedcapi.HeapCTypeWithRelativeWeakref, + ): + with self.subTest(cls=cls): + inst = cls() + ref = weakref.ref(inst) + self.assertEqual(ref(), inst) + self.assertEqual(inst.weakreflist, ref) def test_heaptype_with_managed_weakref(self): inst = _testcapi.HeapCTypeWithManagedWeakref() @@ -730,45 +740,56 @@ class Base(metaclass=metaclass): self.assertIsInstance(sub, metaclass) def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): + for weakref_cls in (_testcapi.HeapCTypeWithWeakref, + _testlimitedcapi.HeapCTypeWithRelativeWeakref): + for dict_cls in (_testcapi.HeapCTypeWithDict, + _testlimitedcapi.HeapCTypeWithRelativeDict): + with self.subTest(weakref_cls=weakref_cls, dict_cls=dict_cls): - with self.assertRaises(TypeError): - class Both1(_testcapi.HeapCTypeWithWeakref, _testcapi.HeapCTypeWithDict): - pass - with self.assertRaises(TypeError): - class Both2(_testcapi.HeapCTypeWithDict, _testcapi.HeapCTypeWithWeakref): - pass + with self.assertRaises(TypeError): + class Both1(weakref_cls, dict_cls): + pass + with self.assertRaises(TypeError): + class Both2(dict_cls, weakref_cls): + pass def test_multiple_inheritance_ctypes_with_weakref_or_dict_and_other_builtin(self): + for dict_cls in (_testcapi.HeapCTypeWithDict, + _testlimitedcapi.HeapCTypeWithRelativeDict): + for weakref_cls in (_testcapi.HeapCTypeWithWeakref, + _testlimitedcapi.HeapCTypeWithRelativeWeakref): + with self.subTest(dict_cls=dict_cls, weakref_cls=weakref_cls): - with self.assertRaises(TypeError): - class C1(_testcapi.HeapCTypeWithDict, list): - pass + with self.assertRaises(TypeError): + class C1(dict_cls, list): + pass - with self.assertRaises(TypeError): - class C2(_testcapi.HeapCTypeWithWeakref, list): - pass + with self.assertRaises(TypeError): + class C2(weakref_cls, list): + pass - class C3(_testcapi.HeapCTypeWithManagedDict, list): - pass - class C4(_testcapi.HeapCTypeWithManagedWeakref, list): - pass + class C3(_testcapi.HeapCTypeWithManagedDict, list): + pass + class C4(_testcapi.HeapCTypeWithManagedWeakref, list): + pass - inst = C3() - inst.append(0) - str(inst.__dict__) + inst = C3() + inst.append(0) + str(inst.__dict__) - inst = C4() - inst.append(0) - str(inst.__weakref__) + inst = C4() + inst.append(0) + str(inst.__weakref__) - for cls in (_testcapi.HeapCTypeWithManagedDict, _testcapi.HeapCTypeWithManagedWeakref): - for cls2 in (_testcapi.HeapCTypeWithDict, _testcapi.HeapCTypeWithWeakref): - class S(cls, cls2): - pass - class B1(C3, cls): - pass - class B2(C4, cls): - pass + for cls in (_testcapi.HeapCTypeWithManagedDict, + _testcapi.HeapCTypeWithManagedWeakref): + for cls2 in (dict_cls, weakref_cls): + class S(cls, cls2): + pass + class B1(C3, cls): + pass + class B2(C4, cls): + pass def test_pytype_fromspec_with_repeated_slots(self): for variant in range(2): @@ -1272,6 +1293,53 @@ def test_heaptype_relative_members_errors(self): SystemError, r"PyMember_SetOne used with Py_RELATIVE_OFFSET"): instance.set_memb_relative(0) + def test_heaptype_relative_special_members_errors(self): + for member_name in "__vectorcalloffset__", "__dictoffset__", "__weaklistoffset__": + with self.subTest(member_name=member_name): + with self.assertRaisesRegex( + SystemError, + r"With Py_RELATIVE_OFFSET, basicsize must be negative."): + _testlimitedcapi.make_heaptype_with_member( + basicsize=sys.getsizeof(object()) + 100, + add_relative_flag=True, + member_name=member_name, + member_offset=0, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=_testlimitedcapi.Py_READONLY, + ) + with self.assertRaisesRegex( + SystemError, + r"Member offset out of range \(0\.\.-basicsize\)"): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-8, + add_relative_flag=True, + member_name=member_name, + member_offset=-1, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=_testlimitedcapi.Py_READONLY, + ) + with self.assertRaisesRegex( + SystemError, + r"type of %s must be Py_T_PYSSIZET" % member_name): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-100, + add_relative_flag=True, + member_name=member_name, + member_offset=0, + member_flags=_testlimitedcapi.Py_READONLY, + ) + with self.assertRaisesRegex( + SystemError, + r"flags for %s must be " % member_name): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-100, + add_relative_flag=True, + member_name=member_name, + member_offset=0, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=0, + ) + def test_pyobject_getitemdata_error(self): """Test PyObject_GetItemData fails on unsupported types""" with self.assertRaises(TypeError): diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index ac99dc4f915f7d..35725718152c56 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -880,19 +880,29 @@ def test_pythondevmode_env(self): self.assertEqual(proc.stdout.rstrip(), 'True') self.assertEqual(proc.returncode, 0, proc) - @unittest.skipUnless(support.Py_GIL_DISABLED, - "PYTHON_GIL and -X gil only supported in Py_GIL_DISABLED builds") def test_python_gil(self): cases = [ # (env, opt, expected, msg) - (None, None, 'None', "no options set"), - ('0', None, '0', "PYTHON_GIL=0"), ('1', None, '1', "PYTHON_GIL=1"), - ('1', '0', '0', "-X gil=0 overrides PYTHON_GIL=1"), - (None, '0', '0', "-X gil=0"), (None, '1', '1', "-X gil=1"), ] + if support.Py_GIL_DISABLED: + cases.extend( + [ + (None, None, 'None', "no options set"), + ('0', None, '0', "PYTHON_GIL=0"), + ('1', '0', '0', "-X gil=0 overrides PYTHON_GIL=1"), + (None, '0', '0', "-X gil=0"), + ] + ) + else: + cases.extend( + [ + (None, None, '1', '-X gil=0 (unsupported by this build)'), + ('1', None, '1', 'PYTHON_GIL=0 (unsupported by this build)'), + ] + ) code = "import sys; print(sys.flags.gil)" environ = dict(os.environ) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index e6509e6bf89e1d..19ba2f4484e7da 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -5,7 +5,9 @@ LittleEndianStructure, BigEndianStructure, c_byte, c_ubyte, c_char, c_char_p, c_void_p, c_wchar, c_uint8, c_uint16, c_uint32, c_uint64, - c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong) + c_short, c_ushort, c_int, c_uint, c_long, c_ulong, + c_longlong, c_ulonglong, + Union) from test import support from test.support import import_helper _ctypes_test = import_helper.import_module("_ctypes_test") @@ -186,8 +188,10 @@ class X(Structure): self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) def fail_fields(self, *fields): - return self.get_except(type(Structure), "X", (), - {"_fields_": fields}) + for layout in "ms", "gcc-sysv": + with self.subTest(layout=layout): + return self.get_except(type(Structure), "X", (), + {"_fields_": fields, "layout": layout}) def test_nonint_types(self): # bit fields are not allowed on non-integer types. @@ -204,9 +208,15 @@ def test_nonint_types(self): result = self.fail_fields(("a", c_char, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char')) - class Dummy(Structure): + class Empty(Structure): _fields_ = [] + result = self.fail_fields(("a", Empty, 1)) + self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) + + class Dummy(Structure): + _fields_ = [("x", c_int)] + result = self.fail_fields(("a", Dummy, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy')) @@ -518,6 +528,21 @@ class Big(BigEndianStructure): x.c = 2 self.assertEqual(b, b'\xab\xcd\xef\x12') + def test_union_bitfield(self): + class BitfieldUnion(Union): + _fields_ = [("a", c_uint32, 1), + ("b", c_uint32, 2), + ("c", c_uint32, 3)] + self.assertEqual(sizeof(BitfieldUnion), 4) + b = bytearray(4) + x = BitfieldUnion.from_buffer(b) + x.a = 1 + self.assertEqual(int.from_bytes(b).bit_count(), 1) + x.b = 3 + self.assertEqual(int.from_bytes(b).bit_count(), 2) + x.c = 7 + self.assertEqual(int.from_bytes(b).bit_count(), 3) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_struct_fields.py b/Lib/test/test_ctypes/test_struct_fields.py index 7adab794809def..7d7a518c0138f9 100644 --- a/Lib/test/test_ctypes/test_struct_fields.py +++ b/Lib/test/test_ctypes/test_struct_fields.py @@ -60,7 +60,6 @@ def test_6(self): self.assertRaises(TypeError, CField) def test_cfield_type_flags(self): - self.assertTrue(CField.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) self.assertTrue(CField.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) def test_cfield_inheritance_hierarchy(self): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 17656d7b3d7fb4..4962586379a223 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1766,7 +1766,6 @@ def test_initconfig_api(self): 'use_hash_seed': True, } config_dev_mode(preconfig, config) - config['faulthandler'] = 0 self.check_all_configs("test_initconfig_api", config, preconfig, api=API_ISOLATED) @@ -1776,6 +1775,9 @@ def test_initconfig_get_api(self): def test_initconfig_exit(self): self.run_embedded_interpreter("test_initconfig_exit") + def test_initconfig_module(self): + self.run_embedded_interpreter("test_initconfig_module") + def test_get_argc_argv(self): self.run_embedded_interpreter("test_get_argc_argv") # ignore output diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index bd8d5d220e9131..7364d6fc7202c4 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -2189,11 +2189,17 @@ def testIPv6AddressTooLarge(self): ipaddress.ip_address('FFFF::c000:201%scope')) def testIPVersion(self): + self.assertEqual(ipaddress.IPv4Address.version, 4) + self.assertEqual(ipaddress.IPv6Address.version, 6) + self.assertEqual(self.ipv4_address.version, 4) self.assertEqual(self.ipv6_address.version, 6) self.assertEqual(self.ipv6_scoped_address.version, 6) def testMaxPrefixLength(self): + self.assertEqual(ipaddress.IPv4Address.max_prefixlen, 32) + self.assertEqual(ipaddress.IPv6Address.max_prefixlen, 128) + self.assertEqual(self.ipv4_interface.max_prefixlen, 32) self.assertEqual(self.ipv6_interface.max_prefixlen, 128) self.assertEqual(self.ipv6_scoped_interface.max_prefixlen, 128) diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 20fba87e4ec120..ca6927554b053c 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -624,8 +624,11 @@ def test_get_loader_handles_missing_spec_attribute(self): mod = type(sys)(name) del mod.__spec__ with CleanImport(name): - sys.modules[name] = mod - loader = pkgutil.get_loader(name) + try: + sys.modules[name] = mod + loader = pkgutil.get_loader(name) + finally: + sys.modules.pop(name, None) self.assertIsNone(loader) @ignore_warnings(category=DeprecationWarning) @@ -634,8 +637,11 @@ def test_get_loader_handles_spec_attribute_none(self): mod = type(sys)(name) mod.__spec__ = None with CleanImport(name): - sys.modules[name] = mod - loader = pkgutil.get_loader(name) + try: + sys.modules[name] = mod + loader = pkgutil.get_loader(name) + finally: + sys.modules.pop(name, None) self.assertIsNone(loader) @ignore_warnings(category=DeprecationWarning) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 012ce7c5a6ba19..e816de3720670f 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -466,6 +466,24 @@ def test_auto_indent_with_comment(self): output = multiline_input(reader) self.assertEqual(output, output_code) + def test_auto_indent_with_multicomment(self): + # fmt: off + events = code_to_events( + "def f(): ## foo\n" + "pass\n\n" + ) + + output_code = ( + "def f(): ## foo\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + def test_auto_indent_ignore_comments(self): # fmt: off events = code_to_events( @@ -658,6 +676,45 @@ def test_control_character(self): self.assertEqual(output, "c\x1d") self.assertEqual(clean_screen(reader.screen), "c") + def test_history_search_backward(self): + # Test history search backward with "imp" input + events = itertools.chain( + code_to_events("import os\n"), + code_to_events("imp"), + [ + Event(evt='key', data='page up', raw=bytearray(b'\x1b[5~')), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + # fill the history + reader = self.prepare_reader(events) + multiline_input(reader) + + # search for "imp" in history + output = multiline_input(reader) + self.assertEqual(output, "import os") + self.assertEqual(clean_screen(reader.screen), "import os") + + def test_history_search_backward_empty(self): + # Test history search backward with an empty input + events = itertools.chain( + code_to_events("import os\n"), + [ + Event(evt='key', data='page up', raw=bytearray(b'\x1b[5~')), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + # fill the history + reader = self.prepare_reader(events) + multiline_input(reader) + + # search backward in history + output = multiline_input(reader) + self.assertEqual(output, "import os") + self.assertEqual(clean_screen(reader.screen), "import os") + class TestPyReplCompleter(TestCase): def prepare_reader(self, events, namespace): @@ -1159,12 +1216,34 @@ def run_repl( *, cmdline_args: list[str] | None = None, cwd: str | None = None, + ) -> tuple[str, int]: + temp_dir = None + if cwd is None: + temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + cwd = temp_dir.name + try: + return self._run_repl( + repl_input, env=env, cmdline_args=cmdline_args, cwd=cwd + ) + finally: + if temp_dir is not None: + temp_dir.cleanup() + + def _run_repl( + self, + repl_input: str | list[str], + *, + env: dict | None, + cmdline_args: list[str] | None, + cwd: str, ) -> tuple[str, int]: assert pty master_fd, slave_fd = pty.openpty() cmd = [sys.executable, "-i", "-u"] if env is None: cmd.append("-I") + elif "PYTHON_HISTORY" not in env: + env["PYTHON_HISTORY"] = os.path.join(cwd, ".regrtest_history") if cmdline_args is not None: cmd.extend(cmdline_args) process = subprocess.Popen( @@ -1203,3 +1282,26 @@ def run_repl( process.kill() exit_code = process.wait() return "".join(output), exit_code + + def test_readline_history_file(self): + # skip, if readline module is not available + readline = import_module('readline') + if readline.backend != "editline": + self.skipTest("GNU readline is not affected by this issue") + + hfile = tempfile.NamedTemporaryFile() + self.addCleanup(unlink, hfile.name) + env = os.environ.copy() + env["PYTHON_HISTORY"] = hfile.name + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl("spam \nexit()\n", env=env) + self.assertEqual(exit_code, 0) + self.assertIn("spam ", output) + self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + self.assertIn("spam\\040", pathlib.Path(hfile.name).read_text()) + + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl("exit\n", env=env) + self.assertEqual(exit_code, 0) + self.assertNotIn("\\040", pathlib.Path(hfile.name).read_text()) diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 91fd7dd13f9063..7d07906df20cc1 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -7,11 +7,12 @@ import tempfile import textwrap import unittest -from test.support import verbose +from test.support import requires_gil_enabled, verbose from test.support.import_helper import import_module from test.support.os_helper import unlink, temp_dir, TESTFN from test.support.pty_helper import run_pty from test.support.script_helper import assert_python_ok +from test.support.threading_helper import requires_working_threading # Skip tests if there is no readline module readline = import_module('readline') @@ -349,6 +350,31 @@ def test_history_size(self): self.assertEqual(len(lines), history_size) self.assertEqual(lines[-1].strip(), b"last input") + @requires_working_threading() + @requires_gil_enabled() + def test_gh123321_threadsafe(self): + """gh-123321: readline should be thread-safe and not crash""" + script = textwrap.dedent(r""" + import threading + from test.support.threading_helper import join_thread + + def func(): + input() + + thread1 = threading.Thread(target=func) + thread2 = threading.Thread(target=func) + thread1.start() + thread2.start() + join_thread(thread1) + join_thread(thread2) + print("done") + """) + + output = run_pty(script, input=b"input1\rinput2\r") + + self.assertIn(b"done", output) + + def test_write_read_limited_history(self): previous_length = readline.get_history_length() self.addCleanup(readline.set_history_length, previous_length) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 363808cb444322..cd8ef0f10579f3 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -41,7 +41,7 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): # path may be used by Py_GetPath() to build the default module search # path. stdin_fname = os.path.join(os.path.dirname(sys.executable), "") - cmd_line = [stdin_fname, '-E', '-i'] + cmd_line = [stdin_fname, '-I', '-i'] cmd_line.extend(args) # Set TERM=vt100, for the rationale see the comments in spawn_python() of @@ -228,6 +228,7 @@ def test_asyncio_repl_reaches_python_startup_script(self): f.write("exit(0)" + os.linesep) env = os.environ.copy() + env["PYTHON_HISTORY"] = os.path.join(tmpdir, ".asyncio_history") env["PYTHONSTARTUP"] = script subprocess.check_call( [sys.executable, "-m", "asyncio"], @@ -240,7 +241,7 @@ def test_asyncio_repl_reaches_python_startup_script(self): @unittest.skipUnless(pty, "requires pty") def test_asyncio_repl_is_ok(self): m, s = pty.openpty() - cmd = [sys.executable, "-m", "asyncio"] + cmd = [sys.executable, "-I", "-m", "asyncio"] proc = subprocess.Popen( cmd, stdin=s, diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 7c607a809aa428..628f806c78959d 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -6826,6 +6826,28 @@ class TestMacOSTCPFlags(unittest.TestCase): def test_tcp_keepalive(self): self.assertTrue(socket.TCP_KEEPALIVE) +@unittest.skipUnless(hasattr(socket, 'TCP_QUICKACK'), 'need socket.TCP_QUICKACK') +class TestQuickackFlag(unittest.TestCase): + def check_set_quickack(self, sock): + # quickack already true by default on some OS distributions + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + if opt: + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK, 0) + + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + self.assertFalse(opt) + + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK, 1) + + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + self.assertTrue(opt) + + def test_set_quickack(self): + sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP) + with sock: + self.check_set_quickack(sock) + @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows") class TestMSWindowsTCPFlags(unittest.TestCase): @@ -6839,7 +6861,9 @@ class TestMSWindowsTCPFlags(unittest.TestCase): 'TCP_KEEPCNT', # available starting with Windows 10 1709 'TCP_KEEPIDLE', - 'TCP_KEEPINTVL' + 'TCP_KEEPINTVL', + # available starting with Windows 7 / Server 2008 R2 + 'TCP_QUICKACK', } def test_new_tcp_flags(self): diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 8386e407f49aa3..b97738836d92cc 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.13", - url="https://www.openssl.org/source/openssl-3.0.13.tar.gz", - checksum='88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313', + name="OpenSSL 3.0.15", + url="https://github.com/openssl/openssl/releases/download/openssl-3.0.15/openssl-3.0.15.tar.gz", + checksum='23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533', buildrecipe=build_universal_openssl, configure=None, install=None, diff --git a/Misc/NEWS.d/next/Build/2024-09-04-12-01-43.gh-issue-123418.ynzspB.rst b/Misc/NEWS.d/next/Build/2024-09-04-12-01-43.gh-issue-123418.ynzspB.rst new file mode 100644 index 00000000000000..38d0e02f3ce404 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-09-04-12-01-43.gh-issue-123418.ynzspB.rst @@ -0,0 +1 @@ +Updated Android build to use OpenSSL 3.0.15. diff --git a/Misc/NEWS.d/next/C_API/2024-08-29-15-05-19.gh-issue-123465.eqwNWq.rst b/Misc/NEWS.d/next/C_API/2024-08-29-15-05-19.gh-issue-123465.eqwNWq.rst new file mode 100644 index 00000000000000..1935adfad8885b --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-08-29-15-05-19.gh-issue-123465.eqwNWq.rst @@ -0,0 +1,4 @@ +:c:macro:`Py_RELATIVE_OFFSET` is now allowed in :c:type:`PyMemberDef` for +the special offset member ``"__vectorcalloffset__"``, as well as the +discouraged special offset members ``"__dictoffset__"`` and +``"__weaklistoffset__"`` diff --git a/Misc/NEWS.d/next/C_API/2024-08-30-14-02-17.gh-issue-107954.TPvj4u.rst b/Misc/NEWS.d/next/C_API/2024-08-30-14-02-17.gh-issue-107954.TPvj4u.rst index 370e2690c458ba..e0ebd816bd6586 100644 --- a/Misc/NEWS.d/next/C_API/2024-08-30-14-02-17.gh-issue-107954.TPvj4u.rst +++ b/Misc/NEWS.d/next/C_API/2024-08-30-14-02-17.gh-issue-107954.TPvj4u.rst @@ -12,6 +12,7 @@ Add functions to configure the Python initialization (:pep:`741`): * :c:func:`PyInitConfig_SetInt` * :c:func:`PyInitConfig_SetStr` * :c:func:`PyInitConfig_SetStrList` +* :c:func:`PyInitConfig_AddModule` * :c:func:`Py_InitializeFromInitConfig` Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-08-26-00-58-26.gh-issue-123321.ApxcnE.rst b/Misc/NEWS.d/next/Core and Builtins/2024-08-26-00-58-26.gh-issue-123321.ApxcnE.rst new file mode 100644 index 00000000000000..b0547e0e588e3d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-08-26-00-58-26.gh-issue-123321.ApxcnE.rst @@ -0,0 +1,2 @@ +Prevent Parser/myreadline race condition from segfaulting on multi-threaded +use. Patch by Bar Harel and Amit Wienner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-23-18-31-10.gh-issue-123275.DprIrj.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-23-18-31-10.gh-issue-123275.DprIrj.rst new file mode 100644 index 00000000000000..ab344a8ca40e47 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-23-18-31-10.gh-issue-123275.DprIrj.rst @@ -0,0 +1 @@ +Support :option:`-X gil=1 <-X>` and :envvar:`PYTHON_GIL=1 ` on non-free-threaded builds. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-01-00-02-05.gh-issue-123545.8nQNbL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-01-00-02-05.gh-issue-123545.8nQNbL.rst new file mode 100644 index 00000000000000..4da4151416d13c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-01-00-02-05.gh-issue-123545.8nQNbL.rst @@ -0,0 +1 @@ +Fix a double decref in rare cases on experimental JIT builds. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-17-32-15.gh-issue-119034.HYh5Vj.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-17-32-15.gh-issue-119034.HYh5Vj.rst new file mode 100644 index 00000000000000..f528691e1b6f9f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-17-32-15.gh-issue-119034.HYh5Vj.rst @@ -0,0 +1,2 @@ +Change ```` and ```` keys of the Python REPL to history +search forward/backward. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-06-14-13-01.gh-issue-119310.WQxyDF.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-06-14-13-01.gh-issue-119310.WQxyDF.rst new file mode 100644 index 00000000000000..e7bc24b537d46a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-06-14-13-01.gh-issue-119310.WQxyDF.rst @@ -0,0 +1,3 @@ +Allow the new interactive shell to read history files written with the +editline library that use unicode-escaped entries. Patch by aorcajo and +Łukasz Langa. diff --git a/Misc/NEWS.d/next/Library/2024-06-18-14-45-38.gh-issue-118710.5GZZPX.rst b/Misc/NEWS.d/next/Library/2024-06-18-14-45-38.gh-issue-118710.5GZZPX.rst new file mode 100644 index 00000000000000..a02d286bcecd7f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-18-14-45-38.gh-issue-118710.5GZZPX.rst @@ -0,0 +1 @@ +:class:`ipaddress.IPv4Address` and :class:`ipaddress.IPv6Address` attributes ``version`` and ``max_prefixlen`` are now available on the class. diff --git a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst new file mode 100644 index 00000000000000..0fd89a99681292 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst @@ -0,0 +1,2 @@ +Improve import time of :mod:`socket` by lazy importing modules and +writing :data:`!socket.errorTab` as a constant. diff --git a/Misc/NEWS.d/next/Library/2024-08-24-00-03-01.gh-issue-123240.uFPG3l.rst b/Misc/NEWS.d/next/Library/2024-08-24-00-03-01.gh-issue-123240.uFPG3l.rst new file mode 100644 index 00000000000000..e6ea6c33f89762 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-24-00-03-01.gh-issue-123240.uFPG3l.rst @@ -0,0 +1 @@ +Raise audit events for the :func:`input` in the new REPL. diff --git a/Misc/NEWS.d/next/Library/2024-09-04-18-23-43.gh-issue-123657.Oks4So.rst b/Misc/NEWS.d/next/Library/2024-09-04-18-23-43.gh-issue-123657.Oks4So.rst new file mode 100644 index 00000000000000..efebd21e26962a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-04-18-23-43.gh-issue-123657.Oks4So.rst @@ -0,0 +1,2 @@ +Fix crash and memory leak in :func:`decimal.getcontext`. It crashed when using +a thread-local context by ``--with-decimal-contextvar=no``. diff --git a/Misc/NEWS.d/next/Security/2024-09-04-12-41-35.gh-issue-123678.N41y9n.rst b/Misc/NEWS.d/next/Security/2024-09-04-12-41-35.gh-issue-123678.N41y9n.rst new file mode 100644 index 00000000000000..b70f578415fdc2 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-09-04-12-41-35.gh-issue-123678.N41y9n.rst @@ -0,0 +1 @@ +Upgrade libexpat to 2.6.3 diff --git a/Misc/NEWS.d/next/Tools-Demos/2024-09-04-10-07-51.gh-issue-123418.1eIFZb.rst b/Misc/NEWS.d/next/Tools-Demos/2024-09-04-10-07-51.gh-issue-123418.1eIFZb.rst new file mode 100644 index 00000000000000..fb9ac9e4f96725 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2024-09-04-10-07-51.gh-issue-123418.1eIFZb.rst @@ -0,0 +1,2 @@ +Update GitHub CI workflows to use OpenSSL 3.0.15 and multissltests to use +3.0.15, 3.1.7, and 3.2.3. diff --git a/Misc/NEWS.d/next/Windows/2024-08-29-16-13-45.gh-issue-123476.m2DFS4.rst b/Misc/NEWS.d/next/Windows/2024-08-29-16-13-45.gh-issue-123476.m2DFS4.rst new file mode 100644 index 00000000000000..801214edc315ff --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-08-29-16-13-45.gh-issue-123476.m2DFS4.rst @@ -0,0 +1 @@ +Add support for ``socket.TCP_QUICKACK`` on Windows platforms. diff --git a/Misc/NEWS.d/next/Windows/2024-09-04-09-59-18.gh-issue-123418.QaMC12.rst b/Misc/NEWS.d/next/Windows/2024-09-04-09-59-18.gh-issue-123418.QaMC12.rst new file mode 100644 index 00000000000000..c2b47dc40652dc --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-09-04-09-59-18.gh-issue-123418.QaMC12.rst @@ -0,0 +1 @@ +Updated Windows build to use OpenSSL 3.0.15. diff --git a/Misc/NEWS.d/next/macOS/2024-09-04-11-55-29.gh-issue-123418.8P4bmN.rst b/Misc/NEWS.d/next/macOS/2024-09-04-11-55-29.gh-issue-123418.8P4bmN.rst new file mode 100644 index 00000000000000..d01afce8a12350 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-09-04-11-55-29.gh-issue-123418.8P4bmN.rst @@ -0,0 +1 @@ +Updated macOS installer build to use OpenSSL 3.0.15. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 758d41910054ce..f7aea9e8f990ba 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -70,21 +70,21 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "e6a77c273ebb284fedd8ea19b081fce74a9455936ffd47215f7c24713e2614b2" + "checksumValue": "1550c87996a0858474a9dd179deab2c55eb73726b9a140b32865b02fd3d8a86b" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.13.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.15.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.13:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.15:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "openssl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.0.13" + "versionInfo": "3.0.15" }, { "SPDXID": "SPDXRef-PACKAGE-sqlite", diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index dadc07c4f40177..05647c1fe417b8 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4076a884f0ca96873589b5c8159e2e5bfb8b829a" + "checksumValue": "6aaee1b194bea30f0a60d1cce71eada8b14d3526" }, { "algorithm": "SHA256", - "checksumValue": "1a434bf3d2f9fb8a0b5adb79201a942788d11824c3e5b46a0b9962c0c482016c" + "checksumValue": "7bd4e53a8015534b5bbb58afe1a131b3989d3d4fca29bca685c44d34bcaa2555" } ], "fileName": "Modules/expat/expat.h" @@ -146,11 +146,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4c49b5df2bc702f663ba3b5a52d1940ec363226b" + "checksumValue": "aca27f46d9fd387b63ce7ff2e4f172cad130b39b" }, { "algorithm": "SHA256", - "checksumValue": "b5ec29f6560acc183f1ee8ab92bb3aea17b87b4c2120cd2e3f78deba7a12491e" + "checksumValue": "f537add526ecda8389503b7ef45fb52b6217e4dc171dcc3a8dc6903ff6134726" } ], "fileName": "Modules/expat/siphash.h" @@ -188,11 +188,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "fed1311be8577491b7f63085a27014eabf2caec8" + "checksumValue": "b2ec0ad170ccc21e63fbcfc8d7404cdd756eedd3" }, { "algorithm": "SHA256", - "checksumValue": "3dc233eca5fa1bb7387c503f8a12d840707e4374b229e05d5657db9645725040" + "checksumValue": "92159d4e17393e56ee85f47d9fb31348695a58589899aa01e7536cdc88f60b85" } ], "fileName": "Modules/expat/xmlparse.c" @@ -538,11 +538,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e52071a35fc1893928804fe91b098ad5682c2508" + "checksumValue": "47ce34375d43a27312e1fffb96b8965610b05855" }, { "algorithm": "SHA256", - "checksumValue": "c4424a4851cd2d4f27633ca19faf5cb1135a680443727a8d1b134737f9a71e62" + "checksumValue": "8affd767d7644150064d8bccd05d7bf4c4ae41fd4bb5bf5b8e943eabf09f3d74" } ], "fileName": "Modules/_hacl/Lib_Memzero0.c" @@ -1590,14 +1590,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "d4cf38d26e21a56654ffe4acd9cd5481164619626802328506a2869afab29ab3" + "checksumValue": "17aa6cfc5c4c219c09287abfc10bc13f0c06f30bb654b28bfe6f567ca646eb79" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_6_2/expat-2.6.2.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_6_3/expat-2.6.3.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.6.2:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.6.3:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1605,7 +1605,7 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.6.2" + "versionInfo": "2.6.3" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index b55102639c6786..2b23be7b753e34 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -320,7 +320,7 @@ _ctypes_alloc_format_string_for_type(char code, int big_endian) indicator set. If called with a suffix of NULL the error indicator must already be set. */ -char * +static char * _ctypes_alloc_format_string(const char *prefix, const char *suffix) { size_t len; @@ -352,7 +352,7 @@ _ctypes_alloc_format_string(const char *prefix, const char *suffix) Returns NULL on failure, with the error indicator set. If called with a suffix of NULL the error indicator must already be set. */ -char * +static char * _ctypes_alloc_format_string_with_shape(int ndim, const Py_ssize_t *shape, const char *prefix, const char *suffix) { @@ -664,9 +664,6 @@ StructUnionType_init(PyObject *self, PyObject *args, PyObject *kwds, int isStruc Py_DECREF(attrdict); return -1; } - if (!isStruct) { - info->flags |= TYPEFLAG_HASUNION; - } info->format = _ctypes_alloc_format_string(NULL, "B"); if (info->format == NULL) { @@ -2534,6 +2531,10 @@ converters_from_argtypes(ctypes_state *st, PyObject *ob) return -1; } + // TYPEFLAG_HASUNION and TYPEFLAG_HASBITFIELD used to be set + // if there were any unions/bitfields; + // if the check is re-enabled we either need to loop here or + // restore the flag if (stginfo != NULL) { if (stginfo->flags & TYPEFLAG_HASUNION) { Py_DECREF(converters); @@ -5780,7 +5781,7 @@ _ctypes_add_types(PyObject *mod) * Simple classes */ - CREATE_TYPE(st->PyCField_Type, &cfield_spec, NULL, NULL); + MOD_ADD_TYPE(st->PyCField_Type, &cfield_spec, NULL, NULL); /************************************************* * diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 2c1fb9b862e12d..abcab6557de914 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -20,6 +20,13 @@ #define CTYPES_CFIELD_CAPSULE_NAME_PYMEM "_ctypes/cfield.c pymem" +/*[clinic input] +module _ctypes +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=476a19c49b31a75c]*/ + +#include "clinic/cfield.c.h" + static void pymem_destructor(PyObject *ptr) { void *p = PyCapsule_GetPointer(ptr, CTYPES_CFIELD_CAPSULE_NAME_PYMEM); @@ -33,6 +40,10 @@ static void pymem_destructor(PyObject *ptr) /* PyCField_Type */ +/*[clinic input] +class _ctypes.CField "PyObject *" "PyObject" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=602817ea3ffc709c]*/ static inline Py_ssize_t round_down(Py_ssize_t numToRound, Py_ssize_t multiple) @@ -61,238 +72,142 @@ Py_ssize_t LOW_BIT(Py_ssize_t offset); static inline Py_ssize_t BUILD_SIZE(Py_ssize_t bitsize, Py_ssize_t offset); -/* PyCField_FromDesc creates and returns a struct/union field descriptor. - -The function expects to be called repeatedly for all fields in a struct or -union. It uses helper functions PyCField_FromDesc_gcc and -PyCField_FromDesc_msvc to simulate the corresponding compilers. - -GCC mode places fields one after another, bit by bit. But "each bit field must -fit within a single object of its specified type" (GCC manual, section 15.8 -"Bit Field Packing"). When it doesn't, we insert a few bits of padding to -avoid that. -MSVC mode works similar except for bitfield packing. Adjacent bit-fields are -packed into the same 1-, 2-, or 4-byte allocation unit if the integral types -are the same size and if the next bit-field fits into the current allocation -unit without crossing the boundary imposed by the common alignment requirements -of the bit-fields. +/*[clinic input] +@classmethod +_ctypes.CField.__new__ as PyCField_new -See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields for details. + name: object(subclass_of='&PyUnicode_Type') + type as proto: object + size: Py_ssize_t + offset: Py_ssize_t + index: Py_ssize_t + bit_size as bit_size_obj: object = None -We do not support zero length bitfields. In fact we use bitsize != 0 elsewhere -to indicate a bitfield. Here, non-bitfields need bitsize set to size*8. +[clinic start generated code]*/ -PyCField_FromDesc manages: -- *psize: the size of the structure / union so far. -- *poffset, *pbitofs: 8* (*poffset) + *pbitofs points to where the next field - would start. -- *palign: the alignment requirements of the last field we placed. -*/ - -static int -PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - CFieldObject* self, StgInfo* info, - int is_bitfield - ) -{ - // We don't use poffset here, so clear it, if it has been set. - *pbitofs += *poffset * 8; - *poffset = 0; - - *palign = info->align; - - if (bitsize > 0) { - // Determine whether the bit field, if placed at the next free bit, - // fits within a single object of its specified type. - // That is: determine a "slot", sized & aligned for the specified type, - // which contains the bitfield's beginning: - Py_ssize_t slot_start_bit = round_down(*pbitofs, 8 * info->align); - Py_ssize_t slot_end_bit = slot_start_bit + 8 * info->size; - // And see if it also contains the bitfield's last bit: - Py_ssize_t field_end_bit = *pbitofs + bitsize; - if (field_end_bit > slot_end_bit) { - // It doesn't: add padding (bump up to the next alignment boundary) - *pbitofs = round_up(*pbitofs, 8*info->align); - } - } - assert(*poffset == 0); - - self->offset = round_down(*pbitofs, 8*info->align) / 8; - if(is_bitfield) { - Py_ssize_t effective_bitsof = *pbitofs - 8 * self->offset; - self->size = BUILD_SIZE(bitsize, effective_bitsof); - assert(effective_bitsof <= info->size * 8); - } else { - self->size = info->size; +static PyObject * +PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, + Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, + PyObject *bit_size_obj) +/*[clinic end generated code: output=43649ef9157c5f58 input=3d813f56373c4caa]*/ +{ + CFieldObject* self = NULL; + if (size < 0) { + PyErr_Format(PyExc_ValueError, + "size of field %R must not be negative, got %zd", + name, size); + goto error; } - - *pbitofs += bitsize; - *psize = round_up(*pbitofs, 8) / 8; - - return 0; -} - -static int -PyCField_FromDesc_msvc( - Py_ssize_t *pfield_size, Py_ssize_t bitsize, - Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, - Py_ssize_t *palign, int pack, - CFieldObject* self, StgInfo* info, - int is_bitfield - ) -{ - if (pack) { - *palign = Py_MIN(pack, info->align); - } else { - *palign = info->align; + // assert: no overflow; + if ((unsigned long long int) size + >= (1ULL << (8*sizeof(Py_ssize_t)-1)) / 8) { + PyErr_Format(PyExc_ValueError, + "size of field %R is too big: %zd", name, size); + goto error; } - // *poffset points to end of current bitfield. - // *pbitofs is generally non-positive, - // and 8 * (*poffset) + *pbitofs points just behind - // the end of the last field we placed. - if (0 < *pbitofs + bitsize || 8 * info->size != *pfield_size) { - // Close the previous bitfield (if any). - // and start a new bitfield: - *poffset = round_up(*poffset, *palign); - - *poffset += info->size; - - *pfield_size = info->size * 8; - // Reminder: 8 * (*poffset) + *pbitofs points to where we would start a - // new field. Ie just behind where we placed the last field plus an - // allowance for alignment. - *pbitofs = - *pfield_size; + PyTypeObject *tp = type; + ctypes_state *st = get_module_state_by_class(tp); + self = (CFieldObject *)tp->tp_alloc(tp, 0); + if (!self) { + return NULL; } - - assert(8 * info->size == *pfield_size); - - self->offset = *poffset - (*pfield_size) / 8; - if(is_bitfield) { - assert(0 <= (*pfield_size + *pbitofs)); - assert((*pfield_size + *pbitofs) < info->size * 8); - self->size = BUILD_SIZE(bitsize, *pfield_size + *pbitofs); + if (PyUnicode_CheckExact(name)) { + self->name = Py_NewRef(name); } else { - self->size = info->size; + self->name = PyObject_Str(name); + if (!self->name) { + goto error; + } } - assert(*pfield_size + *pbitofs <= info->size * 8); - - *pbitofs += bitsize; - *psize = *poffset; - return 0; -} - -PyObject * -PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, Py_ssize_t bitsize, - Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian, LayoutMode layout_mode) -{ - PyTypeObject *tp = st->PyCField_Type; - CFieldObject* self = (CFieldObject *)tp->tp_alloc(tp, 0); - if (self == NULL) { - return NULL; - } StgInfo *info; - if (PyStgInfo_FromType(st, desc, &info) < 0) { - Py_DECREF(self); - return NULL; + if (PyStgInfo_FromType(st, proto, &info) < 0) { + goto error; } - if (!info) { - PyErr_SetString(PyExc_TypeError, - "has no _stginfo_"); - Py_DECREF(self); - return NULL; + if (info == NULL) { + PyErr_Format(PyExc_TypeError, + "type of field %R must be a C type", self->name); + goto error; } - PyObject* proto = desc; + Py_ssize_t bit_size = NUM_BITS(size); + if (bit_size) { + assert(bit_size > 0); + assert(bit_size <= info->size * 8); + switch(info->ffi_type_pointer.type) { + case FFI_TYPE_UINT8: + case FFI_TYPE_UINT16: + case FFI_TYPE_UINT32: + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + break; + + case FFI_TYPE_SINT8: + case FFI_TYPE_SINT16: + case FFI_TYPE_SINT32: + if (info->getfunc != _ctypes_get_fielddesc("c")->getfunc + && info->getfunc != _ctypes_get_fielddesc("u")->getfunc) + { + break; + } + _Py_FALLTHROUGH; /* else fall through */ + default: + PyErr_Format(PyExc_TypeError, + "bit fields not allowed for type %s", + ((PyTypeObject*)proto)->tp_name); + goto error; + } + } + + self->proto = Py_NewRef(proto); + self->size = size; + self->offset = offset; + + self->index = index; - /* Field descriptors for 'c_char * n' are be scpecial cased to + /* Field descriptors for 'c_char * n' are be special cased to return a Python string instead of an Array object instance... */ - SETFUNC setfunc = NULL; - GETFUNC getfunc = NULL; + self->setfunc = NULL; + self->getfunc = NULL; if (PyCArrayTypeObject_Check(st, proto)) { StgInfo *ainfo; if (PyStgInfo_FromType(st, proto, &ainfo) < 0) { - Py_DECREF(self); - return NULL; + goto error; } if (ainfo && ainfo->proto) { StgInfo *iinfo; if (PyStgInfo_FromType(st, ainfo->proto, &iinfo) < 0) { - Py_DECREF(self); - return NULL; + goto error; } if (!iinfo) { PyErr_SetString(PyExc_TypeError, "has no _stginfo_"); - Py_DECREF(self); - return NULL; + goto error; } if (iinfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { struct fielddesc *fd = _ctypes_get_fielddesc("s"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; + self->getfunc = fd->getfunc; + self->setfunc = fd->setfunc; } if (iinfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { struct fielddesc *fd = _ctypes_get_fielddesc("U"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; + self->getfunc = fd->getfunc; + self->setfunc = fd->setfunc; } } } - self->setfunc = setfunc; - self->getfunc = getfunc; - self->index = index; - - self->proto = Py_NewRef(proto); - - int is_bitfield = !!bitsize; - if(!is_bitfield) { - assert(info->size >= 0); - // assert: no overflow; - assert((unsigned long long int) info->size - < (1ULL << (8*sizeof(Py_ssize_t)-1)) / 8); - bitsize = 8 * info->size; - // Caution: bitsize might still be 0 now. - } - assert(bitsize <= info->size * 8); - - int result; - if (layout_mode == LAYOUT_MODE_MS) { - result = PyCField_FromDesc_msvc( - pfield_size, bitsize, pbitofs, - psize, poffset, palign, - pack, - self, info, - is_bitfield - ); - } else { - assert(pack == 0); - result = PyCField_FromDesc_gcc( - bitsize, pbitofs, - psize, poffset, palign, - self, info, - is_bitfield - ); - } - if (result < 0) { - Py_DECREF(self); - return NULL; - } - assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); - if(big_endian && is_bitfield) { - self->size = BUILD_SIZE(NUM_BITS(self->size), 8*info->size - LOW_BIT(self->size) - bitsize); - } return (PyObject *)self; +error: + Py_XDECREF(self); + return NULL; } + static int PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) { @@ -371,8 +286,10 @@ PyCField_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - (void)PyCField_clear((CFieldObject *)self); - Py_TYPE(self)->tp_free((PyObject *)self); + CFieldObject *self_cf = (CFieldObject *)self; + (void)PyCField_clear(self_cf); + Py_CLEAR(self_cf->name); + Py_TYPE(self)->tp_free(self); Py_DECREF(tp); } @@ -398,6 +315,7 @@ PyCField_repr(CFieldObject *self) } static PyType_Slot cfield_slots[] = { + {Py_tp_new, PyCField_new}, {Py_tp_dealloc, PyCField_dealloc}, {Py_tp_repr, PyCField_repr}, {Py_tp_doc, (void *)PyDoc_STR("Structure/Union member")}, @@ -413,7 +331,7 @@ PyType_Spec cfield_spec = { .name = "_ctypes.CField", .basicsize = sizeof(CFieldObject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), + Py_TPFLAGS_IMMUTABLETYPE), .slots = cfield_slots, }; diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h new file mode 100644 index 00000000000000..df5da783e050ed --- /dev/null +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -0,0 +1,113 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +static PyObject * +PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, + Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, + PyObject *bit_size_obj); + +static PyObject * +PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "CField", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 5; + PyObject *name; + PyObject *proto; + Py_ssize_t size; + Py_ssize_t offset; + Py_ssize_t index; + PyObject *bit_size_obj = Py_None; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 6, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!PyUnicode_Check(fastargs[0])) { + _PyArg_BadArgument("CField", "argument 'name'", "str", fastargs[0]); + goto exit; + } + name = fastargs[0]; + proto = fastargs[1]; + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + size = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[3]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[4]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + index = ival; + } + if (!noptargs) { + goto skip_optional_pos; + } + bit_size_obj = fastargs[5]; +skip_optional_pos: + return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj); + +exit: + return return_value; +} +/*[clinic end generated code: output=27c010bae9be7213 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index a794cfe86b5f42..2eb1b6cae4d81b 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -216,18 +216,6 @@ extern int PyObject_stginfo(PyObject *self, Py_ssize_t *psize, Py_ssize_t *palig extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); -typedef enum { - LAYOUT_MODE_MS, - LAYOUT_MODE_GCC_SYSV, -} LayoutMode; - -extern PyObject * -PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, Py_ssize_t bitsize, - Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, - Py_ssize_t *palign, - int pack, int is_big_endian, LayoutMode layout_mode); - extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); extern PyObject *PyCData_FromBytes(ctypes_state *st, PyObject *type, char *data, Py_ssize_t length); @@ -259,16 +247,18 @@ struct fielddesc { GETFUNC getfunc_swapped; }; -typedef struct { +typedef struct CFieldObject { PyObject_HEAD Py_ssize_t offset; Py_ssize_t size; Py_ssize_t index; /* Index into CDataObject's object array */ - PyObject *proto; /* a type or NULL */ + PyObject *proto; /* underlying ctype; must have StgInfo */ GETFUNC getfunc; /* getter function if proto is NULL */ SETFUNC setfunc; /* setter function if proto is NULL */ int anonymous; + + PyObject *name; /* exact PyUnicode */ } CFieldObject; /**************************************************************** @@ -379,8 +369,6 @@ PyObject *_ctypes_callproc(ctypes_state *st, #define TYPEFLAG_ISPOINTER 0x100 #define TYPEFLAG_HASPOINTER 0x200 -#define TYPEFLAG_HASUNION 0x400 -#define TYPEFLAG_HASBITFIELD 0x800 #define DICTFLAG_FINAL 0x1000 @@ -436,10 +424,6 @@ extern void *_ctypes_alloc_closure(void); extern PyObject *PyCData_FromBaseObj(ctypes_state *st, PyObject *type, PyObject *base, Py_ssize_t index, char *adr); -extern char *_ctypes_alloc_format_string(const char *prefix, const char *suffix); -extern char *_ctypes_alloc_format_string_with_shape(int ndim, - const Py_ssize_t *shape, - const char *prefix, const char *suffix); extern int _ctypes_simple_instance(ctypes_state *st, PyObject *obj); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 970f0a033fbb0b..c9cd1c6e7381f6 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -210,29 +210,6 @@ MakeAnonFields(PyObject *type) return 0; } -/* - Allocate a memory block for a pep3118 format string, copy prefix (if - non-null) into it and append `{padding}x` to the end. - Returns NULL on failure, with the error indicator set. -*/ -char * -_ctypes_alloc_format_padding(const char *prefix, Py_ssize_t padding) -{ - /* int64 decimal characters + x + null */ - char buf[19 + 1 + 1]; - - assert(padding > 0); - - if (padding == 1) { - /* Use x instead of 1x, for brevity */ - return _ctypes_alloc_format_string(prefix, "x"); - } - - int ret = PyOS_snprintf(buf, sizeof(buf), "%zdx", padding); (void)ret; - assert(0 <= ret && ret < (Py_ssize_t)sizeof(buf)); - return _ctypes_alloc_format_string(prefix, buf); -} - /* Retrieve the (optional) _pack_ attribute from a type, the _fields_ attribute, and initialize StgInfo. Used for Structure and Union subclasses. @@ -240,125 +217,35 @@ _ctypes_alloc_format_padding(const char *prefix, Py_ssize_t padding) int PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct) { - Py_ssize_t len, offset, size, align, i; - Py_ssize_t union_size, total_align, aligned_size; - Py_ssize_t field_size = 0; - Py_ssize_t bitofs = 0; PyObject *tmp; - int pack; - int forced_alignment = 1; Py_ssize_t ffi_ofs; - int big_endian; int arrays_seen = 0; - if (fields == NULL) - return 0; - - int rc = PyObject_HasAttrWithError(type, &_Py_ID(_swappedbytes_)); - if (rc < 0) { - return -1; - } - if (rc) { - big_endian = !PY_BIG_ENDIAN; - } - else { - big_endian = PY_BIG_ENDIAN; - } - - if (PyObject_GetOptionalAttr(type, &_Py_ID(_pack_), &tmp) < 0) { - return -1; - } - if (tmp) { - pack = PyLong_AsInt(tmp); - Py_DECREF(tmp); - if (pack < 0) { - if (!PyErr_Occurred() || - PyErr_ExceptionMatches(PyExc_TypeError) || - PyErr_ExceptionMatches(PyExc_OverflowError)) - { - PyErr_SetString(PyExc_ValueError, - "_pack_ must be a non-negative integer"); - } - return -1; - } - } - else { - /* Setting `_pack_ = 0` amounts to using the default alignment */ - pack = 0; - } - - #ifdef MS_WIN32 - LayoutMode layout_mode = LAYOUT_MODE_MS; - #else - LayoutMode layout_mode = (pack > 0) ? LAYOUT_MODE_MS : LAYOUT_MODE_GCC_SYSV; - #endif - - if (PyObject_GetOptionalAttr(type, &_Py_ID(_layout_), &tmp) < 0) { - return -1; - } - if (tmp) { - if (!PyUnicode_Check(tmp)) { - PyErr_SetString(PyExc_TypeError, - "_layout_ must be a string"); - return -1; - } - if (PyUnicode_CompareWithASCIIString(tmp, "ms") == 0) { - layout_mode = LAYOUT_MODE_MS; - } - else if (PyUnicode_CompareWithASCIIString(tmp, "gcc-sysv") == 0) { - layout_mode = LAYOUT_MODE_GCC_SYSV; - if (pack > 0) { - PyErr_SetString(PyExc_ValueError, - "_pack_ is not compatible with _layout_=\"gcc-sysv\""); - return -1; - } - } - else { - PyErr_Format(PyExc_ValueError, - "unknown _layout_ %R", tmp); - return -1; - } - } - if (PyObject_GetOptionalAttr(type, &_Py_ID(_align_), &tmp) < 0) { - return -1; - } - if (tmp) { - forced_alignment = PyLong_AsInt(tmp); - Py_DECREF(tmp); - if (forced_alignment < 0) { - if (!PyErr_Occurred() || - PyErr_ExceptionMatches(PyExc_TypeError) || - PyErr_ExceptionMatches(PyExc_OverflowError)) - { - PyErr_SetString(PyExc_ValueError, - "_align_ must be a non-negative integer"); - } - return -1; - } - } - else { - /* Setting `_align_ = 0` amounts to using the default alignment */ - forced_alignment = 1; - } + int retval = -1; + // The following are NULL or hold strong references. + // They're cleared on error. + PyObject *layout_fields = NULL; + PyObject *layout = NULL; + PyObject *format_spec_obj = NULL; - len = PySequence_Size(fields); - if (len == -1) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of pairs"); - } - return -1; + if (fields == NULL) { + return 0; } ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); StgInfo *stginfo; if (PyStgInfo_FromType(st, type, &stginfo) < 0) { - return -1; + goto error; } if (!stginfo) { PyErr_SetString(PyExc_TypeError, "ctypes state is not initialized"); - return -1; + goto error; + } + PyObject *base = (PyObject *)((PyTypeObject *)type)->tp_base; + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, base, &baseinfo) < 0) { + goto error; } /* If this structure/union is already marked final we cannot assign @@ -367,40 +254,114 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (stginfo->flags & DICTFLAG_FINAL) {/* is final ? */ PyErr_SetString(PyExc_AttributeError, "_fields_ is final"); - return -1; + goto error; + } + + PyObject *layout_func = _PyImport_GetModuleAttrString("ctypes._layout", + "get_layout"); + if (!layout_func) { + goto error; + } + PyObject *kwnames = PyTuple_Pack( + 2, + &_Py_ID(is_struct), + &_Py_ID(base)); + if (!kwnames) { + goto error; + } + layout = PyObject_Vectorcall( + layout_func, + 1 + (PyObject*[]){ + NULL, + /* positional args */ + type, + fields, + /* keyword args */ + isStruct ? Py_True : Py_False, + baseinfo ? base : Py_None}, + 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + Py_DECREF(kwnames); + Py_DECREF(layout_func); + fields = NULL; // a borrowed reference we won't be using again + if (!layout) { + goto error; + } + + tmp = PyObject_GetAttr(layout, &_Py_ID(align)); + if (!tmp) { + goto error; + } + Py_ssize_t total_align = PyLong_AsInt(tmp); + Py_DECREF(tmp); + if (total_align < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "align must be a non-negative integer"); + } + goto error; + } + + tmp = PyObject_GetAttr(layout, &_Py_ID(size)); + if (!tmp) { + goto error; + } + Py_ssize_t total_size = PyLong_AsInt(tmp); + Py_DECREF(tmp); + if (total_size < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "size must be a non-negative integer"); + } + goto error; + } + + format_spec_obj = PyObject_GetAttr(layout, &_Py_ID(format_spec)); + if (!format_spec_obj) { + goto error; + } + Py_ssize_t format_spec_size; + const char *format_spec = PyUnicode_AsUTF8AndSize(format_spec_obj, + &format_spec_size); + if (!format_spec) { + goto error; } if (stginfo->format) { PyMem_Free(stginfo->format); stginfo->format = NULL; } + stginfo->format = PyMem_Malloc(format_spec_size + 1); + if (!stginfo->format) { + PyErr_NoMemory(); + goto error; + } + memcpy(stginfo->format, format_spec, format_spec_size + 1); - if (stginfo->ffi_type_pointer.elements) - PyMem_Free(stginfo->ffi_type_pointer.elements); - - StgInfo *baseinfo; - if (PyStgInfo_FromType(st, (PyObject *)((PyTypeObject *)type)->tp_base, - &baseinfo) < 0) { - return -1; + PyObject *layout_fields_obj = PyObject_GetAttr(layout, &_Py_ID(fields)); + if (!layout_fields_obj) { + goto error; } - if (baseinfo) { - stginfo->flags |= (baseinfo->flags & - (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD)); + layout_fields = PySequence_Tuple(layout_fields_obj); + Py_DECREF(layout_fields_obj); + if (!layout_fields) { + goto error; } - if (!isStruct) { - stginfo->flags |= TYPEFLAG_HASUNION; + Py_CLEAR(layout); + + Py_ssize_t len = PyTuple_GET_SIZE(layout_fields); + + if (stginfo->ffi_type_pointer.elements) { + PyMem_Free(stginfo->ffi_type_pointer.elements); + stginfo->ffi_type_pointer.elements = NULL; } + if (baseinfo) { - size = offset = baseinfo->size; - align = baseinfo->align; - union_size = 0; - total_align = align ? align : 1; - total_align = max(total_align, forced_alignment); stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, baseinfo->length + len + 1); if (stginfo->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); - return -1; + goto error; } memset(stginfo->ffi_type_pointer.elements, 0, sizeof(ffi_type *) * (baseinfo->length + len + 1)); @@ -411,231 +372,61 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } ffi_ofs = baseinfo->length; } else { - offset = 0; - size = 0; - align = 0; - union_size = 0; - total_align = forced_alignment; stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1); if (stginfo->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); - return -1; + goto error; } memset(stginfo->ffi_type_pointer.elements, 0, sizeof(ffi_type *) * (len + 1)); ffi_ofs = 0; } - assert(stginfo->format == NULL); - if (isStruct) { - stginfo->format = _ctypes_alloc_format_string(NULL, "T{"); - } else { - /* PEP3118 doesn't support union. Use 'B' for bytes. */ - stginfo->format = _ctypes_alloc_format_string(NULL, "B"); - } - if (stginfo->format == NULL) - return -1; + for (Py_ssize_t i = 0; i < len; ++i) { + PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); + assert(prop_obj); + if (!PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)) { + PyErr_Format(PyExc_TypeError, + "fields must be of type CField, got %T", prop_obj); + goto error; - for (i = 0; i < len; ++i) { - PyObject *name = NULL, *desc = NULL; - PyObject *pair = PySequence_GetItem(fields, i); - PyObject *prop; - Py_ssize_t bitsize = 0; + } + CFieldObject *prop = (CFieldObject *)prop_obj; // borrow from prop_obj - if (!pair || !PyArg_ParseTuple(pair, "UO|n", &name, &desc, &bitsize)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of (name, C type) pairs"); - Py_XDECREF(pair); - return -1; + if (prop->index != i) { + PyErr_Format(PyExc_ValueError, + "field %R index mismatch (expected %zd, got %zd)", + prop->name, i, prop->index); + goto error; } - if (PyCArrayTypeObject_Check(st, desc)) { + + if (PyCArrayTypeObject_Check(st, prop->proto)) { arrays_seen = 1; } StgInfo *info; - if (PyStgInfo_FromType(st, desc, &info) < 0) { - Py_DECREF(pair); - return -1; - } - if (info == NULL) { - Py_DECREF(pair); - PyErr_Format(PyExc_TypeError, - "second item in _fields_ tuple (index %zd) must be a C type", - i); - return -1; + if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { + goto error; } + assert(info); stginfo->ffi_type_pointer.elements[ffi_ofs + i] = &info->ffi_type_pointer; if (info->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) stginfo->flags |= TYPEFLAG_HASPOINTER; - stginfo->flags |= info->flags & (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD); info->flags |= DICTFLAG_FINAL; /* mark field type final */ - if (PyTuple_Size(pair) == 3) { /* bits specified */ - stginfo->flags |= TYPEFLAG_HASBITFIELD; - switch(info->ffi_type_pointer.type) { - case FFI_TYPE_UINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT64: - case FFI_TYPE_UINT64: - break; - - case FFI_TYPE_SINT8: - case FFI_TYPE_SINT16: - case FFI_TYPE_SINT32: - if (info->getfunc != _ctypes_get_fielddesc("c")->getfunc - && info->getfunc != _ctypes_get_fielddesc("u")->getfunc) - { - break; - } - _Py_FALLTHROUGH; /* else fall through */ - default: - PyErr_Format(PyExc_TypeError, - "bit fields not allowed for type %s", - ((PyTypeObject *)desc)->tp_name); - Py_DECREF(pair); - return -1; - } - if (bitsize <= 0 || bitsize > info->size * 8) { - PyErr_Format(PyExc_ValueError, - "number of bits invalid for bit field %R", - name); - Py_DECREF(pair); - return -1; - } - } else - bitsize = 0; - - if (isStruct) { - const char *fieldfmt = info->format ? info->format : "B"; - const char *fieldname = PyUnicode_AsUTF8(name); - char *ptr; - Py_ssize_t len; - char *buf; - Py_ssize_t last_size = size; - Py_ssize_t padding; - - if (fieldname == NULL) - { - Py_DECREF(pair); - return -1; - } - - /* construct the field now, as `prop->offset` is `offset` with - corrected alignment */ - prop = PyCField_FromDesc(st, desc, i, - &field_size, bitsize, &bitofs, - &size, &offset, &align, - pack, big_endian, layout_mode); - if (prop == NULL) { - Py_DECREF(pair); - return -1; - } - - /* number of bytes between the end of the last field and the start - of this one */ - padding = ((CFieldObject *)prop)->offset - last_size; - - if (padding > 0) { - ptr = stginfo->format; - stginfo->format = _ctypes_alloc_format_padding(ptr, padding); - PyMem_Free(ptr); - if (stginfo->format == NULL) { - Py_DECREF(pair); - Py_DECREF(prop); - return -1; - } - } - - len = strlen(fieldname) + strlen(fieldfmt); - - buf = PyMem_Malloc(len + 2 + 1); - if (buf == NULL) { - Py_DECREF(pair); - Py_DECREF(prop); - PyErr_NoMemory(); - return -1; - } - sprintf(buf, "%s:%s:", fieldfmt, fieldname); - - ptr = stginfo->format; - if (info->shape != NULL) { - stginfo->format = _ctypes_alloc_format_string_with_shape( - info->ndim, info->shape, stginfo->format, buf); - } else { - stginfo->format = _ctypes_alloc_format_string(stginfo->format, buf); - } - PyMem_Free(ptr); - PyMem_Free(buf); - - if (stginfo->format == NULL) { - Py_DECREF(pair); - Py_DECREF(prop); - return -1; - } - } else /* union */ { - field_size = 0; - size = 0; - bitofs = 0; - offset = 0; - align = 0; - prop = PyCField_FromDesc(st, desc, i, - &field_size, bitsize, &bitofs, - &size, &offset, &align, - pack, big_endian, layout_mode); - if (prop == NULL) { - Py_DECREF(pair); - return -1; - } - union_size = max(size, union_size); - } - total_align = max(align, total_align); - if (-1 == PyObject_SetAttr(type, name, prop)) { - Py_DECREF(prop); - Py_DECREF(pair); - return -1; + if (-1 == PyObject_SetAttr(type, prop->name, prop_obj)) { + goto error; } - Py_DECREF(pair); - Py_DECREF(prop); - } - - if (!isStruct) { - size = union_size; - } - - /* Adjust the size according to the alignment requirements */ - aligned_size = ((size + total_align - 1) / total_align) * total_align; - - if (isStruct) { - char *ptr; - Py_ssize_t padding; - - /* Pad up to the full size of the struct */ - padding = aligned_size - size; - if (padding > 0) { - ptr = stginfo->format; - stginfo->format = _ctypes_alloc_format_padding(ptr, padding); - PyMem_Free(ptr); - if (stginfo->format == NULL) { - return -1; - } - } - - ptr = stginfo->format; - stginfo->format = _ctypes_alloc_format_string(stginfo->format, "}"); - PyMem_Free(ptr); - if (stginfo->format == NULL) - return -1; } stginfo->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, Py_ssize_t, unsigned short); - stginfo->ffi_type_pointer.size = aligned_size; + stginfo->ffi_type_pointer.size = total_size; - stginfo->size = aligned_size; + stginfo->size = total_size; stginfo->align = total_align; stginfo->length = ffi_ofs + len; @@ -650,7 +441,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct # define MAX_STRUCT_SIZE 16 #endif - if (arrays_seen && (size <= MAX_STRUCT_SIZE)) { + if (arrays_seen && (total_size <= MAX_STRUCT_SIZE)) { /* * See bpo-22273 and gh-110190. Arrays are normally treated as * pointers, which is fine when an array name is being passed as @@ -725,35 +516,19 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_ssize_t struct_index = 0; /* index into dummy structs */ /* first pass to see how much memory to allocate */ - for (i = 0; i < len; ++i) { - PyObject *name, *desc; - PyObject *pair = PySequence_GetItem(fields, i); - int bitsize = 0; - - if (pair == NULL) { - return -1; - } - if (!PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of (name, C type) pairs"); - Py_DECREF(pair); - return -1; - } + for (Py_ssize_t i = 0; i < len; ++i) { + PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed + assert(prop_obj); + assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)); + CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed StgInfo *info; - if (PyStgInfo_FromType(st, desc, &info) < 0) { - Py_DECREF(pair); - return -1; - } - if (info == NULL) { - Py_DECREF(pair); - PyErr_Format(PyExc_TypeError, - "second item in _fields_ tuple (index %zd) must be a C type", - i); - return -1; + if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { + goto error; } + assert(info); - if (!PyCArrayTypeObject_Check(st, desc)) { + if (!PyCArrayTypeObject_Check(st, prop->proto)) { /* Not an array. Just need an ffi_type pointer. */ num_ffi_type_pointers++; } @@ -763,15 +538,13 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct StgInfo *einfo; if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { - Py_DECREF(pair); - return -1; + goto error; } if (einfo == NULL) { - Py_DECREF(pair); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); - return -1; + goto error; } /* * We need one extra ffi_type to hold the struct, and one @@ -781,7 +554,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct num_ffi_types++; num_ffi_type_pointers += length + 1; } - Py_DECREF(pair); } /* @@ -798,7 +570,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (type_block == NULL) { PyErr_NoMemory(); - return -1; + goto error; } /* * the first block takes up ffi_ofs + len + 1 which is the pointers * @@ -822,48 +594,21 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct element_index = ffi_ofs; /* second pass to actually set the type pointers */ - for (i = 0; i < len; ++i) { - PyObject *name, *desc; - PyObject *pair = PySequence_GetItem(fields, i); - int bitsize = 0; - - if (pair == NULL) { - PyMem_Free(type_block); - return -1; - } - /* In theory, we made this call in the first pass, so it *shouldn't* - * fail. However, you never know, and the code above might change - * later - keeping the check in here is a tad defensive but it - * will affect program size only slightly and performance hardly at - * all. - */ - if (!PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of (name, C type) pairs"); - Py_DECREF(pair); - PyMem_Free(type_block); - return -1; - } + for (Py_ssize_t i = 0; i < len; ++i) { + PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed + assert(prop_obj); + assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)); + CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed StgInfo *info; - if (PyStgInfo_FromType(st, desc, &info) < 0) { - Py_DECREF(pair); - PyMem_Free(type_block); - return -1; - } - - /* Possibly this check could be avoided, but see above comment. */ - if (info == NULL) { - Py_DECREF(pair); + if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { PyMem_Free(type_block); - PyErr_Format(PyExc_TypeError, - "second item in _fields_ tuple (index %zd) must be a C type", - i); - return -1; + goto error; } + assert(info); assert(element_index < (ffi_ofs + len)); /* will be used below */ - if (!PyCArrayTypeObject_Check(st, desc)) { + if (!PyCArrayTypeObject_Check(st, prop->proto)) { /* Not an array. Just copy over the element ffi_type. */ element_types[element_index++] = &info->ffi_type_pointer; } @@ -871,17 +616,15 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_ssize_t length = info->length; StgInfo *einfo; if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { - Py_DECREF(pair); PyMem_Free(type_block); - return -1; + goto error; } if (einfo == NULL) { - Py_DECREF(pair); PyMem_Free(type_block); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); - return -1; + goto error; } element_types[element_index++] = &structs[struct_index]; structs[struct_index].size = length * einfo->ffi_type_pointer.size; @@ -898,7 +641,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct assert(dummy_index < (num_ffi_type_pointers)); dummy_types[dummy_index++] = NULL; } - Py_DECREF(pair); } element_types[element_index] = NULL; @@ -916,9 +658,14 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (stginfo->flags & DICTFLAG_FINAL) { PyErr_SetString(PyExc_AttributeError, "Structure or union cannot contain itself"); - return -1; + goto error; } stginfo->flags |= DICTFLAG_FINAL; - return MakeAnonFields(type); + retval = MakeAnonFields(type); +error: + Py_XDECREF(layout_fields); + Py_XDECREF(layout); + Py_XDECREF(format_spec_obj); + return retval; } diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 0e743a609d33d6..96212875154cb8 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -76,8 +76,9 @@ typedef struct { #ifndef WITH_DECIMAL_CONTEXTVAR /* Key for thread state dictionary */ PyObject *tls_context_key; - /* Invariant: NULL or the most recently accessed thread local context */ - struct PyDecContextObject *cached_context; + /* Invariant: NULL or a strong reference to the most recently accessed + thread local context. */ + struct PyDecContextObject *cached_context; /* Not borrowed */ #else PyObject *current_context_var; #endif @@ -1419,12 +1420,6 @@ context_dealloc(PyDecContextObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); -#ifndef WITH_DECIMAL_CONTEXTVAR - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); - if (self == state->cached_context) { - state->cached_context = NULL; - } -#endif (void)context_clear(self); tp->tp_free(self); Py_DECREF(tp); @@ -1701,7 +1696,8 @@ current_context_from_dict(decimal_state *modstate) /* Cache the context of the current thread, assuming that it * will be accessed several times before a thread switch. */ - modstate->cached_context = (PyDecContextObject *)tl_context; + Py_XSETREF(modstate->cached_context, + (PyDecContextObject *)Py_NewRef(tl_context)); modstate->cached_context->tstate = tstate; /* Borrowed reference with refcount==1 */ @@ -1769,7 +1765,7 @@ PyDec_SetCurrentContext(PyObject *self, PyObject *v) Py_INCREF(v); } - state->cached_context = NULL; + Py_CLEAR(state->cached_context); if (PyDict_SetItem(dict, state->tls_context_key, v) < 0) { Py_DECREF(v); return NULL; @@ -6122,6 +6118,16 @@ decimal_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->Rational); Py_VISIT(state->SignalTuple); + if (state->signal_map != NULL) { + for (DecCondMap *cm = state->signal_map; cm->name != NULL; cm++) { + Py_VISIT(cm->ex); + } + } + if (state->cond_map != NULL) { + for (DecCondMap *cm = state->cond_map + 1; cm->name != NULL; cm++) { + Py_VISIT(cm->ex); + } + } return 0; } diff --git a/Modules/_hacl/Lib_Memzero0.c b/Modules/_hacl/Lib_Memzero0.c index 3d8a1e5f265605..5b1a2f7797db76 100644 --- a/Modules/_hacl/Lib_Memzero0.c +++ b/Modules/_hacl/Lib_Memzero0.c @@ -13,7 +13,7 @@ #include #endif -#ifdef __FreeBSD__ +#if defined(__FreeBSD__) || defined(__NetBSD__) #include #endif diff --git a/Modules/_testlimitedcapi/clinic/heaptype_relative.c.h b/Modules/_testlimitedcapi/clinic/heaptype_relative.c.h new file mode 100644 index 00000000000000..994f83102adfb0 --- /dev/null +++ b/Modules/_testlimitedcapi/clinic/heaptype_relative.c.h @@ -0,0 +1,44 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(make_heaptype_with_member__doc__, +"make_heaptype_with_member($module, /, extra_base_size=0, basicsize=0,\n" +" member_offset=0, add_relative_flag=False, *,\n" +" member_name=\'memb\', member_flags=0,\n" +" member_type=-1)\n" +"--\n" +"\n"); + +#define MAKE_HEAPTYPE_WITH_MEMBER_METHODDEF \ + {"make_heaptype_with_member", (PyCFunction)(void(*)(void))make_heaptype_with_member, METH_VARARGS|METH_KEYWORDS, make_heaptype_with_member__doc__}, + +static PyObject * +make_heaptype_with_member_impl(PyObject *module, int extra_base_size, + int basicsize, int member_offset, + int add_relative_flag, + const char *member_name, int member_flags, + int member_type); + +static PyObject * +make_heaptype_with_member(PyObject *module, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + static char *_keywords[] = {"extra_base_size", "basicsize", "member_offset", "add_relative_flag", "member_name", "member_flags", "member_type", NULL}; + int extra_base_size = 0; + int basicsize = 0; + int member_offset = 0; + int add_relative_flag = 0; + const char *member_name = "memb"; + int member_flags = 0; + int member_type = Py_T_BYTE; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iiip$sii:make_heaptype_with_member", _keywords, + &extra_base_size, &basicsize, &member_offset, &add_relative_flag, &member_name, &member_flags, &member_type)) + goto exit; + return_value = make_heaptype_with_member_impl(module, extra_base_size, basicsize, member_offset, add_relative_flag, member_name, member_flags, member_type); + +exit: + return return_value; +} +/*[clinic end generated code: output=01933185947faecc input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/heaptype_relative.c b/Modules/_testlimitedcapi/heaptype_relative.c index c2531518d86a51..45d65ee47349f9 100644 --- a/Modules/_testlimitedcapi/heaptype_relative.c +++ b/Modules/_testlimitedcapi/heaptype_relative.c @@ -8,6 +8,8 @@ #include // max_align_t #include // memset +#include "clinic/heaptype_relative.c.h" + static PyType_Slot empty_slots[] = { {0, NULL}, }; @@ -247,6 +249,81 @@ heaptype_with_member_set_memb_relative(PyObject *self, PyObject *value) Py_RETURN_NONE; } +typedef struct { + int padding; // just so the offset isn't 0 + PyObject *dict; +} HeapCTypeWithDictStruct; + +static void +heapctypewithrelativedict_dealloc(PyObject* self) +{ + PyTypeObject *tp = Py_TYPE(self); + HeapCTypeWithDictStruct *data = PyObject_GetTypeData(self, tp); + Py_XDECREF(data->dict); + PyObject_Free(self); + Py_DECREF(tp); +} + +static PyType_Spec HeapCTypeWithRelativeDict_spec = { + .name = "_testcapi.HeapCTypeWithRelativeDict", + .basicsize = -(int)sizeof(HeapCTypeWithDictStruct), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = (PyType_Slot[]) { + {Py_tp_dealloc, heapctypewithrelativedict_dealloc}, + {Py_tp_getset, (PyGetSetDef[]) { + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, + {NULL} /* Sentinel */ + }}, + {Py_tp_members, (PyMemberDef[]) { + {"dictobj", _Py_T_OBJECT, + offsetof(HeapCTypeWithDictStruct, dict), + Py_RELATIVE_OFFSET}, + {"__dictoffset__", Py_T_PYSSIZET, + offsetof(HeapCTypeWithDictStruct, dict), + Py_READONLY | Py_RELATIVE_OFFSET}, + {NULL} /* Sentinel */ + }}, + {0, 0}, + } +}; + +typedef struct { + char padding; // just so the offset isn't 0 + PyObject *weakreflist; +} HeapCTypeWithWeakrefStruct; + +static void +heapctypewithrelativeweakref_dealloc(PyObject* self) +{ + PyTypeObject *tp = Py_TYPE(self); + HeapCTypeWithWeakrefStruct *data = PyObject_GetTypeData(self, tp); + if (data->weakreflist != NULL) { + PyObject_ClearWeakRefs(self); + } + Py_XDECREF(data->weakreflist); + PyObject_Free(self); + Py_DECREF(tp); +} + +static PyType_Spec HeapCTypeWithRelativeWeakref_spec = { + .name = "_testcapi.HeapCTypeWithRelativeWeakref", + .basicsize = -(int)sizeof(HeapCTypeWithWeakrefStruct), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = (PyType_Slot[]) { + {Py_tp_dealloc, heapctypewithrelativeweakref_dealloc}, + {Py_tp_members, (PyMemberDef[]) { + {"weakreflist", _Py_T_OBJECT, + offsetof(HeapCTypeWithWeakrefStruct, weakreflist), + Py_RELATIVE_OFFSET}, + {"__weaklistoffset__", Py_T_PYSSIZET, + offsetof(HeapCTypeWithWeakrefStruct, weakreflist), + Py_READONLY | Py_RELATIVE_OFFSET}, + {NULL} /* Sentinel */ + }}, + {0, 0}, + } +}; + static PyMethodDef heaptype_with_member_methods[] = { {"get_memb", heaptype_with_member_get_memb, METH_NOARGS}, {"set_memb", heaptype_with_member_set_memb, METH_O}, @@ -256,19 +333,31 @@ static PyMethodDef heaptype_with_member_methods[] = { {NULL}, }; +/*[clinic input] +make_heaptype_with_member + + extra_base_size: int = 0 + basicsize: int = 0 + member_offset: int = 0 + add_relative_flag: bool = False + * + member_name: str = "memb" + member_flags: int = 0 + member_type: int(c_default="Py_T_BYTE") = -1 + +[clinic start generated code]*/ + static PyObject * -make_heaptype_with_member(PyObject *module, PyObject *args) +make_heaptype_with_member_impl(PyObject *module, int extra_base_size, + int basicsize, int member_offset, + int add_relative_flag, + const char *member_name, int member_flags, + int member_type) +/*[clinic end generated code: output=7005db9a07396997 input=007e29cdbe1d3390]*/ { PyObject *base = NULL; PyObject *result = NULL; - int extra_base_size, basicsize, offset, add_flag; - - int r = PyArg_ParseTuple(args, "iiip", &extra_base_size, &basicsize, &offset, &add_flag); - if (!r) { - goto finally; - } - PyType_Spec base_spec = { .name = "_testcapi.Base", .basicsize = sizeof(PyObject) + extra_base_size, @@ -281,7 +370,8 @@ make_heaptype_with_member(PyObject *module, PyObject *args) } PyMemberDef members[] = { - {"memb", Py_T_BYTE, offset, add_flag ? Py_RELATIVE_OFFSET : 0}, + {member_name, member_type, member_offset, + member_flags | (add_relative_flag ? Py_RELATIVE_OFFSET : 0)}, {0}, }; PyType_Slot slots[] = { @@ -325,7 +415,7 @@ static PyMethodDef TestMethods[] = { {"make_sized_heaptypes", make_sized_heaptypes, METH_VARARGS}, {"subclass_var_heaptype", subclass_var_heaptype, METH_VARARGS}, {"subclass_heaptype", subclass_heaptype, METH_VARARGS}, - {"make_heaptype_with_member", make_heaptype_with_member, METH_VARARGS}, + MAKE_HEAPTYPE_WITH_MEMBER_METHODDEF {"test_alignof_max_align_t", test_alignof_max_align_t, METH_NOARGS}, {NULL}, }; @@ -341,5 +431,42 @@ _PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *m) return -1; } +#define ADD_FROM_SPEC(SPEC) do { \ + PyObject *tp = PyType_FromSpec(SPEC); \ + if (!tp) { \ + return -1; \ + } \ + if (PyModule_AddType(m, (PyTypeObject *)tp) < 0) { \ + return -1; \ + } \ + } while (0) + + PyObject *tp; + + tp = PyType_FromSpec(&HeapCTypeWithRelativeDict_spec); + if (!tp) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject *)tp) < 0) { + return -1; + } + Py_DECREF(tp); + + tp = PyType_FromSpec(&HeapCTypeWithRelativeWeakref_spec); + if (!tp) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject *)tp) < 0) { + return -1; + } + Py_DECREF(tp); + + if (PyModule_AddIntMacro(m, Py_T_PYSSIZET) < 0) { + return -1; + } + if (PyModule_AddIntMacro(m, Py_READONLY) < 0) { + return -1; + } + return 0; } diff --git a/Modules/_testlimitedcapi/vectorcall_limited.c b/Modules/_testlimitedcapi/vectorcall_limited.c index 5ef97ca8a063e1..4a7af965776470 100644 --- a/Modules/_testlimitedcapi/vectorcall_limited.c +++ b/Modules/_testlimitedcapi/vectorcall_limited.c @@ -6,6 +6,8 @@ # define Py_LIMITED_API 0x030c0000 #endif +#include // offsetof + #include "parts.h" #include "clinic/vectorcall_limited.c.h" @@ -175,6 +177,41 @@ static PyType_Spec LimitedVectorCallClass_spec = { .slots = LimitedVectorallClass_slots, }; +typedef struct { + vectorcallfunc vfunc; +} LimitedRelativeVectorCallStruct; + +static PyObject * +LimitedRelativeVectorCallClass_new(PyTypeObject *tp, PyTypeObject *a, PyTypeObject *kw) +{ + PyObject *self = ((allocfunc)PyType_GetSlot(tp, Py_tp_alloc))(tp, 0); + if (!self) { + return NULL; + } + LimitedRelativeVectorCallStruct *data = PyObject_GetTypeData(self, tp); + data->vfunc = LimitedVectorCallClass_vectorcall; + return self; +} + + +static PyType_Spec LimitedRelativeVectorCallClass_spec = { + .name = "_testlimitedcapi.LimitedRelativeVectorCallClass", + .basicsize = -(int)sizeof(LimitedRelativeVectorCallStruct), + .flags = Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_HAVE_VECTORCALL, + .slots = (PyType_Slot[]) { + {Py_tp_new, LimitedRelativeVectorCallClass_new}, + {Py_tp_call, LimitedVectorCallClass_tpcall}, + {Py_tp_members, (PyMemberDef[]){ + {"__vectorcalloffset__", Py_T_PYSSIZET, + offsetof(LimitedRelativeVectorCallStruct, vfunc), + Py_READONLY | Py_RELATIVE_OFFSET}, + {NULL} + }}, + {0} + }, +}; + static PyMethodDef TestMethods[] = { _TESTLIMITEDCAPI_CALL_VECTORCALL_METHODDEF _TESTLIMITEDCAPI_CALL_VECTORCALL_METHOD_METHODDEF @@ -197,5 +234,16 @@ _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *m) return -1; } Py_DECREF(LimitedVectorCallClass); + + PyObject *LimitedRelativeVectorCallClass = PyType_FromModuleAndSpec( + m, &LimitedRelativeVectorCallClass_spec, NULL); + if (!LimitedRelativeVectorCallClass) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject *)LimitedRelativeVectorCallClass) < 0) { + return -1; + } + Py_DECREF(LimitedRelativeVectorCallClass); + return 0; } diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index e1e81082d9ec47..4f05cab375ed6b 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -3390,7 +3390,7 @@ DisableEventHook(void) } static int -module_clear(PyObject *mod) +module_clear(PyObject *Py_UNUSED(mod)) { Py_CLEAR(Tkinter_TclError); Py_CLEAR(Tkapp_Type); @@ -3399,16 +3399,28 @@ module_clear(PyObject *mod) return 0; } +static int +module_traverse(PyObject *Py_UNUSED(module), visitproc visit, void *arg) +{ + Py_VISIT(Tkinter_TclError); + Py_VISIT(Tkapp_Type); + Py_VISIT(Tktt_Type); + Py_VISIT(PyTclObject_Type); + return 0; +} + static void module_free(void *mod) { - module_clear((PyObject *)mod); + (void)module_clear((PyObject *)mod); } static struct PyModuleDef _tkintermodule = { PyModuleDef_HEAD_INIT, .m_name = "_tkinter", + .m_size = -1, .m_methods = moduleMethods, + .m_traverse = module_traverse, .m_clear = module_clear, .m_free = module_free }; diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index c2770be3897e58..d0d6015a66283f 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -1066,7 +1066,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ #define XML_MAJOR_VERSION 2 #define XML_MINOR_VERSION 6 -#define XML_MICRO_VERSION 2 +#define XML_MICRO_VERSION 3 #ifdef __cplusplus } diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index a1ed99e687bd6e..04f6f74585b5a2 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -126,8 +126,7 @@ | ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) \ | ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56)) -#define SIPHASH_INITIALIZER \ - { 0, 0, 0, 0, {0}, 0, 0 } +#define SIPHASH_INITIALIZER {0, 0, 0, 0, {0}, 0, 0} struct siphash { uint64_t v0, v1, v2, v3; diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 2951fec70c56cb..d9285b213b38bd 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* 2a14271ad4d35e82bde8ba210b4edb7998794bcbae54deab114046a300f9639a (2.6.2+) +/* ba4cdf9bdb534f355a9def4c9e25d20ee8e72f95b0a4d930be52e563f5080196 (2.6.3+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -39,6 +39,7 @@ Copyright (c) 2022 Sean McBride Copyright (c) 2023 Owain Davies Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow + Copyright (c) 2024 Berkay Eren Ürün Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -294,7 +295,7 @@ typedef struct { The name of the element is stored in both the document and API encodings. The memory buffer 'buf' is a separately-allocated memory area which stores the name. During the XML_Parse()/ - XMLParseBuffer() when the element is open, the memory for the 'raw' + XML_ParseBuffer() when the element is open, the memory for the 'raw' version of the name (in the document encoding) is shared with the document buffer. If the element is open across calls to XML_Parse()/XML_ParseBuffer(), the buffer is re-allocated to @@ -2038,6 +2039,12 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { if (parser == NULL) return XML_STATUS_ERROR; + + if (len < 0) { + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; + return XML_STATUS_ERROR; + } + switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -5846,18 +5853,17 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { /* Set a safe default value in case 'next' does not get set */ next = textStart; -#ifdef XML_DTD if (entity->is_param) { int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok, next, &next, XML_FALSE, XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); - } else -#endif /* XML_DTD */ + } else { result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding, textStart, textEnd, &next, XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); + } if (result == XML_ERROR_NONE) { if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { @@ -5894,18 +5900,17 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, /* Set a safe default value in case 'next' does not get set */ next = textStart; -#ifdef XML_DTD if (entity->is_param) { int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok, next, &next, XML_FALSE, XML_TRUE, XML_ACCOUNT_ENTITY_EXPANSION); - } else -#endif /* XML_DTD */ + } else { result = doContent(parser, openEntity->startTagLevel, parser->m_internalEncoding, textStart, textEnd, &next, XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); + } if (result != XML_ERROR_NONE) return result; @@ -5932,7 +5937,6 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return XML_ERROR_NONE; } -#ifdef XML_DTD if (entity->is_param) { int tok; parser->m_processor = prologProcessor; @@ -5940,9 +5944,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, XML_ACCOUNT_DIRECT); - } else -#endif /* XML_DTD */ - { + } else { parser->m_processor = contentProcessor; /* see externalEntityContentProcessor vs contentProcessor */ result = doContent(parser, parser->m_parentParser ? 1 : 0, @@ -7016,6 +7018,16 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (! newE) return 0; if (oldE->nDefaultAtts) { + /* Detect and prevent integer overflow. + * The preprocessor guard addresses the "always false" warning + * from -Wtype-limits on platforms where + * sizeof(int) < sizeof(size_t), e.g. on x86_64. */ +#if UINT_MAX >= SIZE_MAX + if ((size_t)oldE->nDefaultAtts + > ((size_t)(-1) / sizeof(DEFAULT_ATTRIBUTE))) { + return 0; + } +#endif newE->defaultAtts = ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (! newE->defaultAtts) { @@ -7558,6 +7570,15 @@ nextScaffoldPart(XML_Parser parser) { int next; if (! dtd->scaffIndex) { + /* Detect and prevent integer overflow. + * The preprocessor guard addresses the "always false" warning + * from -Wtype-limits on platforms where + * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ +#if UINT_MAX >= SIZE_MAX + if (parser->m_groupSize > ((size_t)(-1) / sizeof(int))) { + return -1; + } +#endif dtd->scaffIndex = (int *)MALLOC(parser, parser->m_groupSize * sizeof(int)); if (! dtd->scaffIndex) return -1; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 3ffdaa45f16ac7..77e09659514160 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -3166,6 +3166,17 @@ sock_setsockopt(PySocketSockObject *s, PyObject *args) /* setsockopt(level, opt, flag) */ if (PyArg_ParseTuple(args, "iii:setsockopt", &level, &optname, &flag)) { +#ifdef MS_WINDOWS + if (optname == SIO_TCP_SET_ACK_FREQUENCY) { + int dummy; + res = WSAIoctl(s->sock_fd, SIO_TCP_SET_ACK_FREQUENCY, &flag, + sizeof(flag), NULL, 0, &dummy, NULL, NULL); + if (res >= 0) { + s->quickack = flag; + } + goto done; + } +#endif res = setsockopt(s->sock_fd, level, optname, (char*)&flag, sizeof flag); goto done; @@ -3251,6 +3262,11 @@ sock_getsockopt(PySocketSockObject *s, PyObject *args) return s->errorhandler(); return PyLong_FromUnsignedLong(vflag); } +#endif +#ifdef MS_WINDOWS + if (optname == SIO_TCP_SET_ACK_FREQUENCY) { + return PyLong_FromLong(s->quickack); + } #endif flagsize = sizeof flag; res = getsockopt(s->sock_fd, level, optname, @@ -5316,6 +5332,9 @@ sock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) ((PySocketSockObject *)new)->sock_fd = INVALID_SOCKET; ((PySocketSockObject *)new)->sock_timeout = _PyTime_FromSeconds(-1); ((PySocketSockObject *)new)->errorhandler = &set_error; +#ifdef MS_WINDOWS + ((PySocketSockObject *)new)->quickack = 0; +#endif } return new; } @@ -8616,6 +8635,9 @@ socket_exec(PyObject *m) #ifdef TCP_CONNECTION_INFO ADD_INT_MACRO(m, TCP_CONNECTION_INFO); #endif +#ifdef SIO_TCP_SET_ACK_FREQUENCY +#define TCP_QUICKACK SIO_TCP_SET_ACK_FREQUENCY +#endif #ifdef TCP_QUICKACK ADD_INT_MACRO(m, TCP_QUICKACK); #endif diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 09fd70f351f1d8..a77c620c2ef630 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -325,6 +325,9 @@ typedef struct { PyTime_t sock_timeout; /* Operation timeout in seconds; 0.0 means non-blocking */ struct _socket_state *state; +#ifdef MS_WINDOWS + int quickack; +#endif } PySocketSockObject; /* --- C API ----------------------------------------------------*/ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 78f6931dc1d289..6740da108ace91 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4642,6 +4642,41 @@ check_basicsize_includes_size_and_offsets(PyTypeObject* type) return 1; } +/* Set *dest to the offset specified by a special "__*offset__" member. + * Return 0 on success, -1 on failure. + */ +static inline int +special_offset_from_member( + const PyMemberDef *memb /* may be NULL */, + Py_ssize_t type_data_offset, + Py_ssize_t *dest /* not NULL */) +{ + if (memb == NULL) { + *dest = 0; + return 0; + } + if (memb->type != Py_T_PYSSIZET) { + PyErr_Format( + PyExc_SystemError, + "type of %s must be Py_T_PYSSIZET", + memb->name); + return -1; + } + if (memb->flags == Py_READONLY) { + *dest = memb->offset; + return 0; + } + else if (memb->flags == (Py_READONLY | Py_RELATIVE_OFFSET)) { + *dest = memb->offset + type_data_offset; + return 0; + } + PyErr_Format( + PyExc_SystemError, + "flags for %s must be Py_READONLY or (Py_READONLY | Py_RELATIVE_OFFSET)", + memb->name); + return -1; +} + static PyObject * _PyType_FromMetaclass_impl( PyTypeObject *metaclass, PyObject *module, @@ -4667,10 +4702,11 @@ _PyType_FromMetaclass_impl( const PyType_Slot *slot; Py_ssize_t nmembers = 0; - Py_ssize_t weaklistoffset, dictoffset, vectorcalloffset; + const PyMemberDef *weaklistoffset_member = NULL; + const PyMemberDef *dictoffset_member = NULL; + const PyMemberDef *vectorcalloffset_member = NULL; char *res_start; - nmembers = weaklistoffset = dictoffset = vectorcalloffset = 0; for (slot = spec->slots; slot->slot; slot++) { if (slot->slot < 0 || (size_t)slot->slot >= Py_ARRAY_LENGTH(pyslot_offsets)) { @@ -4687,24 +4723,6 @@ _PyType_FromMetaclass_impl( } for (const PyMemberDef *memb = slot->pfunc; memb->name != NULL; memb++) { nmembers++; - if (strcmp(memb->name, "__weaklistoffset__") == 0) { - // The PyMemberDef must be a Py_ssize_t and readonly - assert(memb->type == Py_T_PYSSIZET); - assert(memb->flags == Py_READONLY); - weaklistoffset = memb->offset; - } - if (strcmp(memb->name, "__dictoffset__") == 0) { - // The PyMemberDef must be a Py_ssize_t and readonly - assert(memb->type == Py_T_PYSSIZET); - assert(memb->flags == Py_READONLY); - dictoffset = memb->offset; - } - if (strcmp(memb->name, "__vectorcalloffset__") == 0) { - // The PyMemberDef must be a Py_ssize_t and readonly - assert(memb->type == Py_T_PYSSIZET); - assert(memb->flags == Py_READONLY); - vectorcalloffset = memb->offset; - } if (memb->flags & Py_RELATIVE_OFFSET) { if (spec->basicsize > 0) { PyErr_SetString( @@ -4719,6 +4737,15 @@ _PyType_FromMetaclass_impl( goto finally; } } + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + weaklistoffset_member = memb; + } + if (strcmp(memb->name, "__dictoffset__") == 0) { + dictoffset_member = memb; + } + if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + vectorcalloffset_member = memb; + } } break; case Py_tp_doc: @@ -4882,6 +4909,24 @@ _PyType_FromMetaclass_impl( Py_ssize_t itemsize = spec->itemsize; + /* Compute special offsets */ + + Py_ssize_t weaklistoffset = 0; + if (special_offset_from_member(weaklistoffset_member, type_data_offset, + &weaklistoffset) < 0) { + goto finally; + } + Py_ssize_t dictoffset = 0; + if (special_offset_from_member(dictoffset_member, type_data_offset, + &dictoffset) < 0) { + goto finally; + } + Py_ssize_t vectorcalloffset = 0; + if (special_offset_from_member(vectorcalloffset_member, type_data_offset, + &vectorcalloffset) < 0) { + goto finally; + } + /* Allocate the new type * * Between here and PyType_Ready, we should limit: @@ -6895,7 +6940,7 @@ object_getstate_default(PyObject *obj, int required) iterate over it */ if (slotnames_size != PyList_GET_SIZE(slotnames)) { PyErr_Format(PyExc_RuntimeError, - "__slotsname__ changed size during iteration"); + "__slotnames__ changed size during iteration"); goto error; } diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 1927938ef0821c..a1a67966182863 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -53,7 +53,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.13 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.15 set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.45.3.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.14.0 @@ -77,7 +77,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.13 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.15 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.14.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/python.props b/PCbuild/python.props index 86fe8531d7df55..c8ecdb4515ae9a 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -75,8 +75,8 @@ $(libffiDir)$(ArchName)\ $(libffiOutDir)include $(ExternalsDir)\mpdecimal-4.0.0\ - $(ExternalsDir)openssl-3.0.13\ - $(ExternalsDir)openssl-bin-3.0.13\$(ArchName)\ + $(ExternalsDir)openssl-3.0.15\ + $(ExternalsDir)openssl-bin-3.0.15\$(ArchName)\ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ $(ExternalsDir)\zlib-1.3.1\ diff --git a/Parser/myreadline.c b/Parser/myreadline.c index 1825665354844b..6eab56ac52dc08 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -392,9 +392,14 @@ PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) } } - _PyOS_ReadlineTState = tstate; Py_BEGIN_ALLOW_THREADS + + // GH-123321: We need to acquire the lock before setting + // _PyOS_ReadlineTState and after the release of the GIL, otherwise + // the variable may be nullified by a different thread or a deadlock + // may occur if the GIL is taken in any sub-function. PyThread_acquire_lock(_PyOS_ReadlineLock, 1); + _PyOS_ReadlineTState = tstate; /* This is needed to handle the unlikely case that the * interpreter is in interactive mode *and* stdin/out are not @@ -418,11 +423,13 @@ PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) else { rv = (*PyOS_ReadlineFunctionPointer)(sys_stdin, sys_stdout, prompt); } - Py_END_ALLOW_THREADS + // gh-123321: Must set the variable and then release the lock before + // taking the GIL. Otherwise a deadlock or segfault may occur. + _PyOS_ReadlineTState = NULL; PyThread_release_lock(_PyOS_ReadlineLock); - _PyOS_ReadlineTState = NULL; + Py_END_ALLOW_THREADS if (rv == NULL) return NULL; diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 13f1db1cc708b5..342cc91cc58ced 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1806,6 +1806,16 @@ static int test_init_set_config(void) } +static int initconfig_getint(PyInitConfig *config, const char *name) +{ + int64_t value; + int res = PyInitConfig_GetInt(config, name, &value); + assert(res == 0); + assert(INT_MIN <= value && value <= INT_MAX); + return (int)value; +} + + static int test_initconfig_api(void) { PyInitConfig *config = PyInitConfig_Create(); @@ -1844,7 +1854,6 @@ static int test_initconfig_api(void) goto error; } - if (Py_InitializeFromInitConfig(config) < 0) { goto error; } @@ -1876,38 +1885,51 @@ static int test_initconfig_get_api(void) assert(PyInitConfig_HasOption(config, "non-existent") == 0); // test PyInitConfig_GetInt() - int64_t value; - assert(PyInitConfig_GetInt(config, "dev_mode", &value) == 0); - assert(value == 0); + assert(initconfig_getint(config, "dev_mode") == 0); assert(PyInitConfig_SetInt(config, "dev_mode", 1) == 0); - assert(PyInitConfig_GetInt(config, "dev_mode", &value) == 0); - assert(value == 1); + assert(initconfig_getint(config, "dev_mode") == 1); // test PyInitConfig_GetInt() on a PyPreConfig option - assert(PyInitConfig_GetInt(config, "utf8_mode", &value) == 0); - assert(value == 0); + assert(initconfig_getint(config, "utf8_mode") == 0); assert(PyInitConfig_SetInt(config, "utf8_mode", 1) == 0); - assert(PyInitConfig_GetInt(config, "utf8_mode", &value) == 0); - assert(value == 1); + assert(initconfig_getint(config, "utf8_mode") == 1); // test PyInitConfig_GetStr() char *str; + assert(PyInitConfig_GetStr(config, "program_name", &str) == 0); + assert(str == NULL); assert(PyInitConfig_SetStr(config, "program_name", PROGRAM_NAME_UTF8) == 0); assert(PyInitConfig_GetStr(config, "program_name", &str) == 0); assert(strcmp(str, PROGRAM_NAME_UTF8) == 0); free(str); // test PyInitConfig_GetStrList() and PyInitConfig_FreeStrList() + size_t length; + char **items; + assert(PyInitConfig_GetStrList(config, "xoptions", &length, &items) == 0); + assert(length == 0); + char* xoptions[] = {"faulthandler"}; assert(PyInitConfig_SetStrList(config, "xoptions", Py_ARRAY_LENGTH(xoptions), xoptions) == 0); - size_t length; - char **items; + assert(PyInitConfig_GetStrList(config, "xoptions", &length, &items) == 0); assert(length == 1); assert(strcmp(items[0], "faulthandler") == 0); PyInitConfig_FreeStrList(length, items); + // Setting hash_seed sets use_hash_seed + assert(initconfig_getint(config, "use_hash_seed") == 0); + assert(PyInitConfig_SetInt(config, "hash_seed", 123) == 0); + assert(initconfig_getint(config, "use_hash_seed") == 1); + + // Setting module_search_paths sets module_search_paths_set + assert(initconfig_getint(config, "module_search_paths_set") == 0); + char* paths[] = {"search", "path"}; + assert(PyInitConfig_SetStrList(config, "module_search_paths", + Py_ARRAY_LENGTH(paths), paths) == 0); + assert(initconfig_getint(config, "module_search_paths_set") == 1); + return 0; } @@ -1941,6 +1963,62 @@ static int test_initconfig_exit(void) } +static PyModuleDef_Slot extension_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL} +}; + +static struct PyModuleDef extension_module = { + PyModuleDef_HEAD_INIT, + .m_name = "my_test_extension", + .m_size = 0, + .m_slots = extension_slots, +}; + +static PyObject* init_my_test_extension(void) +{ + return PyModuleDef_Init(&extension_module); +} + + +static int test_initconfig_module(void) +{ + PyInitConfig *config = PyInitConfig_Create(); + if (config == NULL) { + printf("Init allocation error\n"); + return 1; + } + + if (PyInitConfig_SetStr(config, "program_name", PROGRAM_NAME_UTF8) < 0) { + goto error; + } + + if (PyInitConfig_AddModule(config, "my_test_extension", + init_my_test_extension) < 0) { + goto error; + } + + if (Py_InitializeFromInitConfig(config) < 0) { + goto error; + } + PyInitConfig_Free(config); + + if (PyRun_SimpleString("import my_test_extension") < 0) { + fprintf(stderr, "unable to import my_test_extension\n"); + exit(1); + } + + Py_Finalize(); + return 0; + + const char *err_msg; +error: + (void)PyInitConfig_GetError(config, &err_msg); + printf("Python init failed: %s\n", err_msg); + exit(1); +} + + static void configure_init_main(PyConfig *config) { wchar_t* argv[] = { @@ -2362,6 +2440,7 @@ static struct TestCase TestCases[] = { {"test_initconfig_api", test_initconfig_api}, {"test_initconfig_get_api", test_initconfig_get_api}, {"test_initconfig_exit", test_initconfig_exit}, + {"test_initconfig_module", test_initconfig_module}, {"test_run_main", test_run_main}, {"test_run_main_loop", test_run_main_loop}, {"test_get_argc_argv", test_get_argc_argv}, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c4cc8127bafbb4..b5a642dccd2aec 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4739,8 +4739,6 @@ dummy_func( if (optimized <= 0) { exit->temperature = restart_backoff_counter(temperature); if (optimized < 0) { - Py_DECREF(current_executor); - tstate->previous_executor = Py_None; GOTO_UNWIND(); } tstate->previous_executor = (PyObject *)current_executor; @@ -4822,8 +4820,6 @@ dummy_func( if (optimized <= 0) { exit->temperature = restart_backoff_counter(exit->temperature); if (optimized < 0) { - Py_DECREF(current_executor); - tstate->previous_executor = Py_None; GOTO_UNWIND(); } GOTO_TIER_ONE(target); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d4be7904a50c10..6d687bbb48b0ba 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -5305,8 +5305,6 @@ if (optimized <= 0) { exit->temperature = restart_backoff_counter(temperature); if (optimized < 0) { - Py_DECREF(current_executor); - tstate->previous_executor = Py_None; GOTO_UNWIND(); } tstate->previous_executor = (PyObject *)current_executor; @@ -5437,8 +5435,6 @@ if (optimized <= 0) { exit->temperature = restart_backoff_counter(exit->temperature); if (optimized < 0) { - Py_DECREF(current_executor); - tstate->previous_executor = Py_None; GOTO_UNWIND(); } GOTO_TIER_ONE(target); diff --git a/Python/initconfig.c b/Python/initconfig.c index d2bb46062209da..d93244f7f41084 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1031,7 +1031,6 @@ PyConfig_InitIsolatedConfig(PyConfig *config) config->dev_mode = 0; config->install_signal_handlers = 0; config->use_hash_seed = 0; - config->faulthandler = 0; config->tracemalloc = 0; config->perf_profiling = 0; config->int_max_str_digits = _PY_LONG_DEFAULT_MAX_STR_DIGITS; @@ -1715,20 +1714,24 @@ config_wstr_to_int(const wchar_t *wstr, int *result) static PyStatus config_read_gil(PyConfig *config, size_t len, wchar_t first_char) { -#ifdef Py_GIL_DISABLED if (len == 1 && first_char == L'0') { +#ifdef Py_GIL_DISABLED config->enable_gil = _PyConfig_GIL_DISABLE; +#else + return _PyStatus_ERR("Disabling the GIL is not supported by this build"); +#endif } else if (len == 1 && first_char == L'1') { +#ifdef Py_GIL_DISABLED config->enable_gil = _PyConfig_GIL_ENABLE; +#else + return _PyStatus_OK(); +#endif } else { return _PyStatus_ERR("PYTHON_GIL / -X gil must be \"0\" or \"1\""); } return _PyStatus_OK(); -#else - return _PyStatus_ERR("PYTHON_GIL / -X gil are not supported by this build"); -#endif } static PyStatus @@ -3424,6 +3427,8 @@ _Py_DumpPathConfig(PyThreadState *tstate) struct PyInitConfig { PyPreConfig preconfig; PyConfig config; + struct _inittab *inittab; + Py_ssize_t inittab_size; PyStatus status; char *err_msg; }; @@ -3753,7 +3758,7 @@ PyInitConfig_SetInt(PyInitConfig *config, const char *name, int64_t value) return -1; } - if (strcmp(name, "hash_seed")) { + if (strcmp(name, "hash_seed") == 0) { config->config.use_hash_seed = 1; } @@ -3863,13 +3868,53 @@ PyInitConfig_SetStrList(PyInitConfig *config, const char *name, return -1; } PyWideStringList *list = raw_member; - return _PyWideStringList_FromUTF8(config, list, length, items); + if (_PyWideStringList_FromUTF8(config, list, length, items) < 0) { + return -1; + } + + if (strcmp(name, "module_search_paths") == 0) { + config->config.module_search_paths_set = 1; + } + return 0; +} + + +int +PyInitConfig_AddModule(PyInitConfig *config, const char *name, + PyObject* (*initfunc)(void)) +{ + size_t size = sizeof(struct _inittab) * (config->inittab_size + 2); + struct _inittab *new_inittab = PyMem_RawRealloc(config->inittab, size); + if (new_inittab == NULL) { + config->status = _PyStatus_NO_MEMORY(); + return -1; + } + config->inittab = new_inittab; + + struct _inittab *entry = &config->inittab[config->inittab_size]; + entry->name = name; + entry->initfunc = initfunc; + + // Terminator entry + entry = &config->inittab[config->inittab_size + 1]; + entry->name = NULL; + entry->initfunc = NULL; + + config->inittab_size++; + return 0; } int Py_InitializeFromInitConfig(PyInitConfig *config) { + if (config->inittab_size >= 1) { + if (PyImport_ExtendInittab(config->inittab) < 0) { + config->status = _PyStatus_NO_MEMORY(); + return -1; + } + } + _PyPreConfig_GetConfig(&config->preconfig, &config->config); config->status = Py_PreInitializeFromArgs( diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index baa16102068aa0..eae0e0c5e8761f 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -43,13 +43,14 @@ log = logging.getLogger("multissl") OPENSSL_OLD_VERSIONS = [ + "1.1.1w", ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1w", - "3.0.13", - "3.1.5", - "3.2.1", + "3.0.15", + "3.1.7", + "3.2.3", + "3.3.2", ] LIBRESSL_OLD_VERSIONS = [ @@ -397,6 +398,7 @@ def run_python_tests(self, tests, network=True): class BuildOpenSSL(AbstractBuilder): library = "OpenSSL" url_templates = ( + "https://github.com/openssl/openssl/releases/download/openssl-{v}/openssl-{v}.tar.gz", "https://www.openssl.org/source/openssl-{v}.tar.gz", "https://www.openssl.org/source/old/{s}/openssl-{v}.tar.gz" ) @@ -439,6 +441,7 @@ def short_version(self): parsed = parsed[:2] return ".".join(str(i) for i in parsed) + class BuildLibreSSL(AbstractBuilder): library = "LibreSSL" url_templates = ( diff --git a/configure b/configure index 2c58af3eace84a..2e8044819083f3 100755 --- a/configure +++ b/configure @@ -4146,9 +4146,9 @@ if test -z "$CPP"; then fi if test -z "$CXX"; then case "$host" in - aarch64-apple-ios*-simulator) CXX=arm64-apple-ios-simulator-clang ;; - aarch64-apple-ios*) CXX=arm64-apple-ios-clang ;; - x86_64-apple-ios*-simulator) CXX=x86_64-apple-ios-simulator-clang ;; + aarch64-apple-ios*-simulator) CXX=arm64-apple-ios-simulator-clang++ ;; + aarch64-apple-ios*) CXX=arm64-apple-ios-clang++ ;; + x86_64-apple-ios*-simulator) CXX=x86_64-apple-ios-simulator-clang++ ;; *) esac fi @@ -4509,7 +4509,7 @@ if test "$cross_compiling" = yes; then # IPHONEOS_DEPLOYMENT_TARGET is the minimum supported iOS version { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking iOS deployment target" >&5 printf %s "checking iOS deployment target... " >&6; } - IPHONEOS_DEPLOYMENT_TARGET=${_host_os:3} + IPHONEOS_DEPLOYMENT_TARGET=$(echo ${_host_os} | cut -c4-) IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=13.0} { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $IPHONEOS_DEPLOYMENT_TARGET" >&5 printf "%s\n" "$IPHONEOS_DEPLOYMENT_TARGET" >&6; } diff --git a/configure.ac b/configure.ac index 3c1dc1cc5018d4..38582d9987dc28 100644 --- a/configure.ac +++ b/configure.ac @@ -418,9 +418,9 @@ if test -z "$CPP"; then fi if test -z "$CXX"; then case "$host" in - aarch64-apple-ios*-simulator) CXX=arm64-apple-ios-simulator-clang ;; - aarch64-apple-ios*) CXX=arm64-apple-ios-clang ;; - x86_64-apple-ios*-simulator) CXX=x86_64-apple-ios-simulator-clang ;; + aarch64-apple-ios*-simulator) CXX=arm64-apple-ios-simulator-clang++ ;; + aarch64-apple-ios*) CXX=arm64-apple-ios-clang++ ;; + x86_64-apple-ios*-simulator) CXX=x86_64-apple-ios-simulator-clang++ ;; *) esac fi @@ -758,7 +758,7 @@ if test "$cross_compiling" = yes; then # IPHONEOS_DEPLOYMENT_TARGET is the minimum supported iOS version AC_MSG_CHECKING([iOS deployment target]) - IPHONEOS_DEPLOYMENT_TARGET=${_host_os:3} + IPHONEOS_DEPLOYMENT_TARGET=$(echo ${_host_os} | cut -c4-) IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=13.0} AC_MSG_RESULT([$IPHONEOS_DEPLOYMENT_TARGET]) diff --git a/iOS/Resources/bin/arm64-apple-ios-clang++ b/iOS/Resources/bin/arm64-apple-ios-clang++ new file mode 100755 index 00000000000000..f24bec11268f7e --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ new file mode 100755 index 00000000000000..ef37d05b512959 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios-simulator $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ new file mode 100755 index 00000000000000..86f03ea32bc2fd --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios-simulator $@