From d8c0fe1944ac41787e16fa60e608f56c8235e100 Mon Sep 17 00:00:00 2001 From: Bradley Reynolds Date: Tue, 17 Sep 2024 19:51:15 -0500 Subject: [PATCH 001/105] gh-124194: Fix wrong issue number in What's New in Python 3.8 (#124195) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/whatsnew/3.8.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 35ae0e338d9c03..d0e60bc280a217 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -1755,7 +1755,7 @@ The following features and APIs have been removed from Python 3.8: * Starting with Python 3.3, importing ABCs from :mod:`collections` was deprecated, and importing should be done from :mod:`collections.abc`. Being able to import from collections was marked for removal in 3.8, but has been - delayed to 3.9. (See :issue:`36952`.) + delayed to 3.9. (See :gh:`81134`.) * The :mod:`macpath` module, deprecated in Python 3.7, has been removed. (Contributed by Victor Stinner in :issue:`35471`.) From 29a1a6e3ed6f606939b4aaf8d6955f368c3be3fc Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 17 Sep 2024 22:55:09 -0500 Subject: [PATCH 002/105] gh-124191: Disable -Wconversion in enable safety (gh-124192) --- Tools/build/.warningignore_macos | 227 +-------------------------- Tools/build/.warningignore_ubuntu | 252 +----------------------------- configure | 39 ----- configure.ac | 1 - 4 files changed, 5 insertions(+), 514 deletions(-) diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos index 2ed02ba6b634b0..e9307b44c7ee5f 100644 --- a/Tools/build/.warningignore_macos +++ b/Tools/build/.warningignore_macos @@ -3,226 +3,7 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) -Include/internal/mimalloc/mimalloc/internal.h 4 -Include/internal/pycore_backoff.h 1 -Include/internal/pycore_dict.h 2 -Modules/_asynciomodule.c 3 -Modules/_bisectmodule.c 2 -Modules/_bz2module.c 5 -Modules/_collectionsmodule.c 2 -Modules/_csv.c 3 -Modules/_ctypes/_ctypes.c 37 -Modules/_ctypes/_ctypes_test_generated.c.h 141 -Modules/_ctypes/callbacks.c 6 -Modules/_ctypes/callproc.c 15 -Modules/_ctypes/cfield.c 56 -Modules/_ctypes/malloc_closure.c 3 -Modules/_ctypes/stgdict.c 17 -Modules/_cursesmodule.c 24 -Modules/_datetimemodule.c 28 -Modules/_dbmmodule.c 8 -Modules/_decimal/_decimal.c 15 -Modules/_elementtree.c 42 -Modules/_functoolsmodule.c 6 -Modules/_gdbmmodule.c 5 -Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84 -Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84 -Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 24 -Modules/_hashopenssl.c 16 -Modules/_interpchannelsmodule.c 1 -Modules/_interpqueuesmodule.c 1 -Modules/_io/_iomodule.c 1 -Modules/_io/bufferedio.c 4 -Modules/_io/bytesio.c 11 -Modules/_io/fileio.c 9 -Modules/_io/stringio.c 8 -Modules/_io/textio.c 11 -Modules/_json.c 19 -Modules/_localemodule.c 3 -Modules/_lzmamodule.c 10 -Modules/_multiprocessing/semaphore.c 2 -Modules/_operator.c 5 -Modules/_pickle.c 71 -Modules/_posixsubprocess.c 8 -Modules/_queuemodule.c 4 -Modules/_randommodule.c 3 -Modules/_scproxy.c 3 -Modules/_sqlite/connection.c 4 -Modules/_sqlite/cursor.c 3 -Modules/_sqlite/module.c 2 -Modules/_sre/sre.c 18 -Modules/_sre/sre_lib.h 62 -Modules/_ssl.c 29 -Modules/_struct.c 1 -Modules/_testbuffer.c 22 -Modules/_testcapi/heaptype.c 1 -Modules/_testcapi/long.c 2 -Modules/_testcapi/mem.c 2 -Modules/_testcapi/monitoring.c 3 -Modules/_testcapi/pyatomic.c 1 -Modules/_testcapi/unicode.c 2 -Modules/_testcapi/vectorcall.c 3 -Modules/_testcapi/watchers.c 3 -Modules/_testcapimodule.c 3 -Modules/_testclinic.c 14 -Modules/_testexternalinspection.c 7 -Modules/_testinternalcapi.c 8 -Modules/_testinternalcapi/pytime.c 8 -Modules/_testinternalcapi/test_critical_sections.c 1 -Modules/_testinternalcapi/test_lock.c 2 -Modules/_testlimitedcapi/heaptype_relative.c 4 -Modules/_testlimitedcapi/object.c 2 -Modules/_testlimitedcapi/unicode.c 2 -Modules/_threadmodule.c 2 -Modules/_tkinter.c 6 -Modules/_xxtestfuzz/_xxtestfuzz.c 1 -Modules/_xxtestfuzz/fuzzer.c 11 -Modules/_zoneinfo.c 14 -Modules/arraymodule.c 32 -Modules/atexitmodule.c 1 -Modules/binascii.c 206 -Modules/blake2module.c 6 -Modules/cjkcodecs/_codecs_cn.c 1 -Modules/cjkcodecs/_codecs_iso2022.c 2 -Modules/cjkcodecs/_codecs_jp.c 14 -Modules/cjkcodecs/_codecs_kr.c 3 -Modules/cjkcodecs/cjkcodecs.h 1 -Modules/cjkcodecs/multibytecodec.c 2 -Modules/clinic/_testclinic.c.h 1 -Modules/clinic/arraymodule.c.h 1 -Modules/clinic/unicodedata.c.h 10 -Modules/cmathmodule.c 1 -Modules/expat/siphash.h 8 -Modules/expat/xmlparse.c 45 -Modules/expat/xmltok.c 17 -Modules/expat/xmltok_impl.c 34 -Modules/faulthandler.c 3 -Modules/fcntlmodule.c 1 -Modules/getpath.c 7 -Modules/grpmodule.c 4 -Modules/itertoolsmodule.c 7 -Modules/main.c 2 -Modules/mathmodule.c 15 -Modules/mmapmodule.c 20 -Modules/posixmodule.c 67 -Modules/pwdmodule.c 4 -Modules/pyexpat.c 20 -Modules/readline.c 1 -Modules/resource.c 3 -Modules/rotatingtree.c 1 -Modules/selectmodule.c 6 -Modules/sha3module.c 4 -Modules/signalmodule.c 1 -Modules/socketmodule.c 44 -Modules/syslogmodule.c 3 -Modules/timemodule.c 4 -Modules/unicodedata.c 28 -Modules/unicodedata_db.h 1 -Modules/xxsubtype.c 2 -Modules/zlibmodule.c 16 -Objects/abstract.c 2 -Objects/bytearrayobject.c 34 -Objects/bytes_methods.c 9 -Objects/bytesobject.c 35 -Objects/call.c 13 -Objects/classobject.c 4 -Objects/codeobject.c 15 -Objects/descrobject.c 2 -Objects/dictobject.c 28 -Objects/fileobject.c 3 -Objects/floatobject.c 30 -Objects/frameobject.c 19 -Objects/funcobject.c 1 -Objects/genobject.c 5 -Objects/listobject.c 43 -Objects/longobject.c 46 -Objects/memoryobject.c 6 -Objects/methodobject.c 1 -Objects/mimalloc/alloc.c 6 -Objects/mimalloc/arena.c 6 -Objects/mimalloc/heap.c 1 -Objects/mimalloc/init.c 2 -Objects/mimalloc/options.c 1 -Objects/mimalloc/os.c 4 -Objects/mimalloc/page-queue.c 2 -Objects/mimalloc/page.c 1 -Objects/mimalloc/prim/osx/../unix/prim.c 2 -Objects/mimalloc/random.c 1 -Objects/mimalloc/segment.c 11 -Objects/mimalloc/stats.c 1 -Objects/moduleobject.c 2 -Objects/object.c 1 -Objects/obmalloc.c 6 -Objects/odictobject.c 3 -Objects/rangeobject.c 10 -Objects/setobject.c 13 -Objects/sliceobject.c 4 -Objects/stringlib/codecs.h 26 -Objects/stringlib/eq.h 1 -Objects/stringlib/fastsearch.h 14 -Objects/stringlib/join.h 1 -Objects/stringlib/replace.h 4 -Objects/stringlib/repr.h 21 -Objects/stringlib/transmogrify.h 5 -Objects/structseq.c 14 -Objects/tupleobject.c 10 -Objects/typeobject.c 17 -Objects/unicodectype.c 7 -Objects/unicodeobject.c 113 -Parser/action_helpers.c 4 -Parser/lexer/buffer.c 1 -Parser/lexer/lexer.c 12 -Parser/parser.c 116 -Parser/string_parser.c 7 -Parser/tokenizer/file_tokenizer.c 8 -Parser/tokenizer/helpers.c 7 -Parser/tokenizer/readline_tokenizer.c 3 -Programs/_freeze_module.c 1 -Python/Python-ast.c 15 -Python/asdl.c 3 -Python/assemble.c 7 -Python/ast_opt.c 7 -Python/bltinmodule.c 9 -Python/bootstrap_hash.c 4 -Python/ceval.c 8 -Python/ceval_gil.c 2 -Python/codecs.c 32 -Python/codegen.c 6 -Python/compile.c 2 -Python/context.c 1 -Python/crossinterp.c 2 -Python/crossinterp_data_lookup.h 1 -Python/dtoa.c 34 -Python/errors.c 1 -Python/fileutils.c 7 -Python/flowgraph.c 8 -Python/formatter_unicode.c 7 -Python/frame.c 4 -Python/gc.c 7 -Python/generated_cases.c.h 35 -Python/getargs.c 11 -Python/import.c 5 -Python/initconfig.c 11 -Python/instrumentation.c 31 -Python/intrinsics.c 1 -Python/legacy_tracing.c 3 -Python/lock.c 4 -Python/marshal.c 11 -Python/modsupport.c 3 -Python/mystrtoul.c 4 -Python/pathconfig.c 1 -Python/preconfig.c 2 -Python/pyarena.c 1 -Python/pyhash.c 2 -Python/pylifecycle.c 7 -Python/pystate.c 6 -Python/pystrhex.c 19 -Python/pystrtod.c 3 -Python/qsbr.c 2 -Python/specialize.c 10 -Python/suggestions.c 12 -Python/symtable.c 18 -Python/sysmodule.c 2 -Python/thread_pthread.h 1 -Python/traceback.c 6 -Python/tracemalloc.c 6 +Modules/expat/siphash.h 7 +Modules/expat/xmlparse.c 8 +Modules/expat/xmltok.c 3 +Modules/expat/xmltok_impl.c 26 \ No newline at end of file diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index d010152a229fae..980785d59ae33b 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -3,254 +3,4 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) -/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/evp.h 2 -/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/ssl.h 4 -/usr/include/tcl8.6/tclTomMathDecls.h 1 -Include/cpython/bytearrayobject.h 1 -Include/cpython/bytesobject.h 3 -Include/cpython/dictobject.h 2 -Include/cpython/listobject.h 1 -Include/cpython/pyctype.h 2 -Include/cpython/tupleobject.h 1 -Include/cpython/unicodeobject.h 7 -Include/internal/mimalloc/mimalloc/internal.h 4 -Include/internal/mimalloc/mimalloc/types.h 2 -Include/internal/pycore_asdl.h 1 -Include/internal/pycore_backoff.h 3 -Include/internal/pycore_blocks_output_buffer.h 1 -Include/internal/pycore_dict.h 2 -Include/internal/pycore_interp.h 1 -Include/internal/pycore_obmalloc.h 1 -Include/internal/pycore_pymath.h 1 -Include/internal/pycore_runtime_init.h 1 -Include/longobject.h 1 -Include/object.h 4 -Include/opcode_ids.h 1 -Include/pymacro.h 4 -Include/pymath.h 1 -Include/pymem.h 2 -Include/pyport.h 2 -Modules/_asynciomodule.c 3 -Modules/_bisectmodule.c 4 -Modules/_bz2module.c 5 -Modules/_collectionsmodule.c 2 -Modules/_csv.c 2 -Modules/_ctypes/_ctypes.c 53 -Modules/_ctypes/_ctypes_test.c 7 -Modules/_ctypes/_ctypes_test_generated.c.h 2 -Modules/_ctypes/callbacks.c 3 -Modules/_ctypes/callproc.c 13 -Modules/_ctypes/cfield.c 33 -Modules/_ctypes/stgdict.c 17 -Modules/_cursesmodule.c 27 -Modules/_datetimemodule.c 38 -Modules/_datetimemodule.c 38 -Modules/_dbmmodule.c 7 -Modules/_decimal/_decimal.c 19 -Modules/_elementtree.c 37 -Modules/_functoolsmodule.c 6 -Modules/_gdbmmodule.c 4 -Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84 -Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84 -Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84 -Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84 -Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4 -Modules/_hashopenssl.c 13 -Modules/_io/_iomodule.c 1 -Modules/_io/bufferedio.c 15 -Modules/_io/bytesio.c 14 -Modules/_io/fileio.c 9 -Modules/_io/stringio.c 8 -Modules/_io/textio.c 17 -Modules/_json.c 19 -Modules/_localemodule.c 2 -Modules/_lsprof.c 5 -Modules/_lzmamodule.c 6 -Modules/_multiprocessing/posixshmem.c 1 -Modules/_multiprocessing/semaphore.c 1 -Modules/_operator.c 5 -Modules/_pickle.c 73 -Modules/_posixsubprocess.c 11 -Modules/_queuemodule.c 4 -Modules/_randommodule.c 3 -Modules/_sqlite/connection.c 5 -Modules/_sqlite/cursor.c 3 -Modules/_sqlite/module.c 2 -Modules/_sre/sre.c 14 -Modules/_sre/sre_lib.h 25 -Modules/_ssl.c 26 -Modules/_struct.c 3 -Modules/_testbuffer.c 27 -Modules/_testcapi/bytes.c 1 -Modules/_testcapi/heaptype.c 1 -Modules/_testcapi/long.c 2 -Modules/_testcapi/mem.c 2 -Modules/_testcapi/monitoring.c 3 -Modules/_testcapi/pyatomic.c 4 -Modules/_testcapi/pyatomic.c 4 -Modules/_testcapi/unicode.c 1 -Modules/_testcapi/vectorcall.c 3 -Modules/_testcapi/watchers.c 3 -Modules/_testcapimodule.c 1 -Modules/_testclinic.c 14 -Modules/_testclinic.c 14 -Modules/_testexternalinspection.c 6 -Modules/_testinternalcapi.c 10 -Modules/_testinternalcapi/test_critical_sections.c 1 -Modules/_testinternalcapi/test_lock.c 4 -Modules/_testlimitedcapi/heaptype_relative.c 3 -Modules/_testlimitedcapi/object.c 2 -Modules/_testlimitedcapi/unicode.c 1 -Modules/_testmultiphase.c 1 -Modules/_tkinter.c 8 -Modules/_xxtestfuzz/_xxtestfuzz.c 1 -Modules/_xxtestfuzz/fuzzer.c 13 -Modules/_zoneinfo.c 17 -Modules/arraymodule.c 48 -Modules/binascii.c 208 -Modules/blake2module.c 8 -Modules/cjkcodecs/_codecs_iso2022.c 1 -Modules/cjkcodecs/_codecs_jp.c 17 -Modules/cjkcodecs/_codecs_kr.c 7 -Modules/cjkcodecs/alg_jisx0201.h 2 -Modules/cjkcodecs/cjkcodecs.h 1 -Modules/cjkcodecs/multibytecodec.c 12 -Modules/expat/pyexpatns.h 3 -Modules/expat/siphash.h 1 -Modules/expat/xmlparse.c 43 -Modules/expat/xmltok.c 15 -Modules/expat/xmltok.c 15 -Modules/expat/xmltok_impl.c 8 -Modules/faulthandler.c 5 -Modules/fcntlmodule.c 6 -Modules/getpath.c 7 -Modules/grpmodule.c 4 -Modules/itertoolsmodule.c 4 -Modules/main.c 2 -Modules/mathmodule.c 14 -Modules/mmapmodule.c 22 -Modules/mmapmodule.c 22 -Modules/posixmodule.c 79 -Modules/pwdmodule.c 4 -Modules/pyexpat.c 10 -Modules/readline.c 1 -Modules/resource.c 4 -Modules/rotatingtree.c 2 -Modules/selectmodule.c 1 -Modules/sha3module.c 4 -Modules/signalmodule.c 3 -Modules/socketmodule.c 75 -Modules/syslogmodule.c 3 -Modules/termios.c 1 -Modules/timemodule.c 10 -Modules/unicodedata.c 24 -Modules/unicodedata_db.h 1 -Modules/zlibmodule.c 24 -Objects/abstract.c 6 -Objects/bytearrayobject.c 42 -Objects/bytes_methods.c 4 -Objects/bytesobject.c 45 -Objects/call.c 12 -Objects/classobject.c 4 -Objects/codeobject.c 19 -Objects/descrobject.c 2 -Objects/dictobject.c 31 -Objects/fileobject.c 3 -Objects/floatobject.c 10 -Objects/frameobject.c 16 -Objects/funcobject.c 1 -Objects/genobject.c 3 -Objects/listobject.c 38 -Objects/longobject.c 47 -Objects/memoryobject.c 12 -Objects/methodobject.c 1 -Objects/mimalloc/alloc.c 6 -Objects/mimalloc/arena.c 6 -Objects/mimalloc/heap.c 2 -Objects/mimalloc/init.c 2 -Objects/mimalloc/options.c 4 -Objects/mimalloc/os.c 4 -Objects/mimalloc/page-queue.c 2 -Objects/mimalloc/page.c 2 -Objects/mimalloc/prim/unix/prim.c 6 -Objects/mimalloc/random.c 1 -Objects/mimalloc/segment.c 11 -Objects/mimalloc/stats.c 5 -Objects/moduleobject.c 4 -Objects/object.c 1 -Objects/obmalloc.c 6 -Objects/odictobject.c 6 -Objects/rangeobject.c 10 -Objects/setobject.c 13 -Objects/sliceobject.c 2 -Objects/stringlib/codecs.h 12 -Objects/stringlib/eq.h 1 -Objects/stringlib/fastsearch.h 8 -Objects/stringlib/join.h 3 -Objects/stringlib/replace.h 4 -Objects/stringlib/repr.h 21 -Objects/stringlib/transmogrify.h 26 -Objects/structseq.c 10 -Objects/tupleobject.c 8 -Objects/typeobject.c 38 -Objects/unicodectype.c 7 -Objects/unicodeobject.c 135 -Parser/action_helpers.c 3 -Parser/lexer/buffer.c 1 -Parser/lexer/lexer.c 14 -Parser/parser.c 116 -Parser/string_parser.c 7 -Parser/tokenizer/file_tokenizer.c 9 -Parser/tokenizer/helpers.c 7 -Parser/tokenizer/readline_tokenizer.c 4 -Python/assemble.c 11 -Python/ast_opt.c 5 -Python/bltinmodule.c 8 -Python/bootstrap_hash.c 7 -Python/ceval.c 8 -Python/ceval_gil.c 2 -Python/codecs.c 28 -Python/codegen.c 6 -Python/compile.c 2 -Python/context.c 1 -Python/crossinterp.c 2 -Python/crossinterp_data_lookup.h 1 -Python/dtoa.c 30 -Python/errors.c 1 -Python/fileutils.c 11 -Python/flowgraph.c 7 -Python/formatter_unicode.c 6 -Python/frame.c 3 -Python/gc.c 8 -Python/generated_cases.c.h 27 -Python/getargs.c 7 -Python/hashtable.c 1 -Python/import.c 7 -Python/initconfig.c 11 -Python/instrumentation.c 43 -Python/intrinsics.c 1 -Python/legacy_tracing.c 3 -Python/lock.c 4 -Python/marshal.c 16 -Python/modsupport.c 3 -Python/mystrtoul.c 4 -Python/pathconfig.c 1 -Python/perf_jit_trampoline.c 32 -Python/perf_trampoline.c 12 -Python/preconfig.c 2 -Python/pyarena.c 1 -Python/pyhash.c 4 -Python/pylifecycle.c 3 -Python/pystate.c 4 -Python/pystrhex.c 15 -Python/pystrtod.c 12 -Python/pytime.c 2 -Python/qsbr.c 2 -Python/specialize.c 9 -Python/suggestions.c 12 -Python/symtable.c 15 -Python/sysmodule.c 2 -Python/thread.c 1 -Python/thread_pthread.h 6 -Python/traceback.c 6 -Python/tracemalloc.c 6 + diff --git a/configure b/configure index b7055a8f74f5c4..d88acc580b790b 100755 --- a/configure +++ b/configure @@ -9771,45 +9771,6 @@ then : else $as_nop { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wtrampolines not supported" >&5 printf "%s\n" "$as_me: WARNING: -Wtrampolines not supported" >&2;} -fi - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wconversion" >&5 -printf %s "checking whether C compiler accepts -Wconversion... " >&6; } -if test ${ax_cv_check_cflags__Werror__Wconversion+y} -then : - printf %s "(cached) " >&6 -else $as_nop - - ax_check_save_flags=$CFLAGS - CFLAGS="$CFLAGS -Werror -Wconversion" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main (void) -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO" -then : - ax_cv_check_cflags__Werror__Wconversion=yes -else $as_nop - ax_cv_check_cflags__Werror__Wconversion=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wconversion" >&5 -printf "%s\n" "$ax_cv_check_cflags__Werror__Wconversion" >&6; } -if test "x$ax_cv_check_cflags__Werror__Wconversion" = xyes -then : - CFLAGS_NODIST="$CFLAGS_NODIST -Wconversion" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wconversion not supported" >&5 -printf "%s\n" "$as_me: WARNING: -Wconversion not supported" >&2;} fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wimplicit-fallthrough" >&5 diff --git a/configure.ac b/configure.ac index ab6233a1341422..e83cba49d03433 100644 --- a/configure.ac +++ b/configure.ac @@ -2514,7 +2514,6 @@ if test "$enable_safety" = "yes" then AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [CFLAGS_NODIST="$CFLAGS_NODIST -fstack-protector-strong"], [AC_MSG_WARN([-fstack-protector-strong not supported])], [-Werror]) AX_CHECK_COMPILE_FLAG([-Wtrampolines], [CFLAGS_NODIST="$CFLAGS_NODIST -Wtrampolines"], [AC_MSG_WARN([-Wtrampolines not supported])], [-Werror]) - AX_CHECK_COMPILE_FLAG([-Wconversion], [CFLAGS_NODIST="$CFLAGS_NODIST -Wconversion"], [AC_MSG_WARN([-Wconversion not supported])], [-Werror]) AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough"], [AC_MSG_WARN([-Wimplicit-fallthrough not supported])], [-Werror]) AX_CHECK_COMPILE_FLAG([-Werror=format-security], [CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security"], [AC_MSG_WARN([-Werror=format-security not supported])], [-Werror]) AX_CHECK_COMPILE_FLAG([-Wbidi-chars=any], [CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any"], [AC_MSG_WARN([-Wbidi-chars=any not supported])], [-Werror]) From 79a74102362996bbd4ff5d410a0d57d43c236da4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 18 Sep 2024 09:29:24 +0300 Subject: [PATCH 003/105] Revert "gh-123974: Fix time.get_clock_info() on NetBSD (#123975)" (GH-124115) This reverts commit b1d6f8a2ee04215c64aa8752cc515b7e98a08d28. --- Modules/timemodule.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Modules/timemodule.c b/Modules/timemodule.c index cfbb26b0259504..46f85bc9c30f9c 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1509,19 +1509,15 @@ _PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) return -1; } if (info) { + struct timespec res; info->implementation = function; info->monotonic = 1; info->adjustable = 0; - #if defined(__NetBSD__) - info->resolution = 1e-9; - #else - struct timespec res; if (clock_getres(clk_id, &res)) { PyErr_SetFromErrno(PyExc_OSError); return -1; } info->resolution = res.tv_sec + res.tv_nsec * 1e-9; - #endif } if (_PyTime_FromTimespec(tp, &ts) < 0) { From 646f16bdeed6ebe1069e1d64886fbaa26edac75c Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 18 Sep 2024 16:18:19 +0900 Subject: [PATCH 004/105] gh-124153: Implement `PyType_GetBaseByToken()` and `Py_tp_token` slot (GH-124163) --- Doc/c-api/type.rst | 68 ++++++++- Doc/data/stable_abi.dat | 1 + Doc/whatsnew/3.14.rst | 5 + Include/cpython/object.h | 1 + Include/object.h | 4 + Include/typeslots.h | 4 + Lib/test/test_capi/test_misc.py | 71 +++++++++ Lib/test/test_stable_abi_ctypes.py | 1 + Lib/test/test_sys.py | 2 +- ...-09-17-05-23-35.gh-issue-124153.L8TWmx.rst | 2 + Misc/stable_abi.toml | 8 +- Modules/_ctypes/_ctypes.c | 5 +- Modules/_ctypes/ctypes.h | 20 ++- Modules/_testcapi/heaptype.c | 117 +++++++++++++++ Objects/typeobject.c | 139 +++++++++++++++++- Objects/typeslots.inc | 1 + Objects/typeslots.py | 6 +- PC/python3dll.c | 1 + 18 files changed, 443 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2024-09-17-05-23-35.gh-issue-124153.L8TWmx.rst diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index b25a407964be8d..b56da6954f41d4 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -264,6 +264,24 @@ Type Objects .. versionadded:: 3.11 +.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) + + Find the first superclass in *type*'s :term:`method resolution order` whose + :c:macro:`Py_tp_token` token is equal to the given one. + + * If found, set *\*result* to a new :term:`strong reference` + to it and return ``1``. + * If not found, set *\*result* to ``NULL`` and return ``0``. + * On error, set *\*result* to ``NULL`` and return ``-1`` with an + exception set. + + The *result* argument may be ``NULL``, in which case *\*result* is not set. + Use this if you need only the return value. + + The *token* argument may not be ``NULL``. + + .. versionadded:: 3.14 + .. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) Attempt to assign a version tag to the given type. @@ -488,6 +506,11 @@ The following functions and structs are used to create * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` + An additional slot is supported that does not correspond to a + :c:type:`!PyTypeObject` struct field: + + * :c:data:`Py_tp_token` + The following “offset” fields cannot be set using :c:type:`PyType_Slot`: * :c:member:`~PyTypeObject.tp_weaklistoffset` @@ -538,4 +561,47 @@ The following functions and structs are used to create The desired value of the slot. In most cases, this is a pointer to a function. - Slots other than ``Py_tp_doc`` may not be ``NULL``. + *pfunc* values may not be ``NULL``, except for the following slots: + + * ``Py_tp_doc`` + * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` + rather than ``NULL``) + +.. c:macro:: Py_tp_token + + A :c:member:`~PyType_Slot.slot` that records a static memory layout ID + for a class. + + If the :c:type:`PyType_Spec` of the class is statically + allocated, the token can be set to the spec using the special value + :c:data:`Py_TP_USE_SPEC`: + + .. code-block:: c + + static PyType_Slot foo_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, + + It can also be set to an arbitrary pointer, but you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + + Use :c:func:`PyType_GetBaseByToken` to check if a class's superclass has + a given token -- that is, check whether the memory layout is compatible. + + To get the token for a given class (without considering superclasses), + use :c:func:`PyType_GetSlot` with ``Py_tp_token``. + + .. versionadded:: 3.14 + + .. c:namespace:: NULL + + .. c:macro:: Py_TP_USE_SPEC + + Used as a value with :c:data:`Py_tp_token` to set the token to the + class's :c:type:`PyType_Spec`. + Expands to ``NULL``. + + .. versionadded:: 3.14 diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 7eeee270bb7f32..7c4b56d1dd652f 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -690,6 +690,7 @@ func,PyType_FromSpec,3.2,, func,PyType_FromSpecWithBases,3.3,, func,PyType_GenericAlloc,3.2,, func,PyType_GenericNew,3.2,, +func,PyType_GetBaseByToken,3.14,, func,PyType_GetFlags,3.2,, func,PyType_GetFullyQualifiedName,3.13,, func,PyType_GetModule,3.10,, diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 5e762336e547f6..53399aa4e50fa6 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -554,6 +554,11 @@ New Features (Contributed by Victor Stinner in :gh:`107954`.) +* Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot for easier + superclass identification, which attempts to resolve the `type checking issue + `__ mentioned in :pep:`630` + (:gh:`124153`). + Porting to Python 3.14 ---------------------- diff --git a/Include/cpython/object.h b/Include/cpython/object.h index e1024ddbdf6062..9d092749b90096 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -269,6 +269,7 @@ typedef struct _heaptypeobject { struct _dictkeysobject *ht_cached_keys; PyObject *ht_module; char *_ht_tpname; // Storage for "tp_name"; see PyType_FromModuleAndSpec + void *ht_token; // Storage for the "Py_tp_token" slot struct _specialization_cache _spec_cache; // For use by the specializer. #ifdef Py_GIL_DISABLED Py_ssize_t unique_id; // ID used for thread-local refcounting diff --git a/Include/object.h b/Include/object.h index abfdb6ce24df21..7124f58f6bdb37 100644 --- a/Include/object.h +++ b/Include/object.h @@ -391,6 +391,10 @@ PyAPI_FUNC(PyObject *) PyType_FromMetaclass(PyTypeObject*, PyObject*, PyType_Spe PyAPI_FUNC(void *) PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls); PyAPI_FUNC(Py_ssize_t) PyType_GetTypeDataSize(PyTypeObject *cls); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 +PyAPI_FUNC(int) PyType_GetBaseByToken(PyTypeObject *, void *, PyTypeObject **); +#define Py_TP_USE_SPEC NULL +#endif /* Generic type check */ PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *); diff --git a/Include/typeslots.h b/Include/typeslots.h index e91caa1509c34b..a7f3017ec02e92 100644 --- a/Include/typeslots.h +++ b/Include/typeslots.h @@ -90,3 +90,7 @@ /* New in 3.14 */ #define Py_tp_vectorcall 82 #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 +/* New in 3.14 */ +#define Py_tp_token 83 +#endif diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index ebc0a8a3d4efb5..5c6faa1626d380 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1144,6 +1144,77 @@ class MyType: MyType.__module__ = 123 self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + def test_get_base_by_token(self): + def get_base_by_token(src, key, comparable=True): + def run(use_mro): + find_first = _testcapi.pytype_getbasebytoken + ret1, result = find_first(src, key, use_mro, True) + ret2, no_result = find_first(src, key, use_mro, False) + self.assertIn(ret1, (0, 1)) + self.assertEqual(ret1, result is not None) + self.assertEqual(ret1, ret2) + self.assertIsNone(no_result) + return result + + found_in_mro = run(True) + found_in_bases = run(False) + if comparable: + self.assertIs(found_in_mro, found_in_bases) + return found_in_mro + return found_in_mro, found_in_bases + + create_type = _testcapi.create_type_with_token + get_token = _testcapi.get_tp_token + + Py_TP_USE_SPEC = _testcapi.Py_TP_USE_SPEC + self.assertEqual(Py_TP_USE_SPEC, 0) + + A1 = create_type('_testcapi.A1', Py_TP_USE_SPEC) + self.assertTrue(get_token(A1) != Py_TP_USE_SPEC) + + B1 = create_type('_testcapi.B1', id(self)) + self.assertTrue(get_token(B1) == id(self)) + + tokenA1 = get_token(A1) + # find A1 from A1 + found = get_base_by_token(A1, tokenA1) + self.assertIs(found, A1) + + # no token in static types + STATIC = type(1) + self.assertEqual(get_token(STATIC), 0) + found = get_base_by_token(STATIC, tokenA1) + self.assertIs(found, None) + + # no token in pure subtypes + class A2(A1): pass + self.assertEqual(get_token(A2), 0) + # find A1 + class Z(STATIC, B1, A2): pass + found = get_base_by_token(Z, tokenA1) + self.assertIs(found, A1) + + # searching for NULL token is an error + with self.assertRaises(SystemError): + get_base_by_token(Z, 0) + with self.assertRaises(SystemError): + get_base_by_token(STATIC, 0) + + # share the token with A1 + C1 = create_type('_testcapi.C1', tokenA1) + self.assertTrue(get_token(C1) == tokenA1) + + # find C1 first by shared token + class Z(C1, A2): pass + found = get_base_by_token(Z, tokenA1) + self.assertIs(found, C1) + # B1 not found + found = get_base_by_token(Z, get_token(B1)) + self.assertIs(found, None) + + with self.assertRaises(TypeError): + _testcapi.pytype_getbasebytoken( + 'not a type', id(self), True, False) def test_gen_get_code(self): def genf(): yield diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 4bca33b7451f80..4e509e30bee122 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -719,6 +719,7 @@ def test_windows_feature_macros(self): "PyType_FromSpecWithBases", "PyType_GenericAlloc", "PyType_GenericNew", + "PyType_GetBaseByToken", "PyType_GetFlags", "PyType_GetFullyQualifiedName", "PyType_GetModule", diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 42b5a7c94e7700..77009648bd26ed 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1718,7 +1718,7 @@ def delx(self): del self.__x '3P' # PyMappingMethods '10P' # PySequenceMethods '2P' # PyBufferProcs - '6P' + '7P' '1PIP' # Specializer cache + typeid # heap type id (free-threaded only) ) diff --git a/Misc/NEWS.d/next/C_API/2024-09-17-05-23-35.gh-issue-124153.L8TWmx.rst b/Misc/NEWS.d/next/C_API/2024-09-17-05-23-35.gh-issue-124153.L8TWmx.rst new file mode 100644 index 00000000000000..b8c0b4667cb730 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-09-17-05-23-35.gh-issue-124153.L8TWmx.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot for easier +type checking, related to :pep:`489` and :pep:`630`. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 6036fc96fdd995..d8a9d1f3335583 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2527,4 +2527,10 @@ [function.PyLong_AsUInt64] added = '3.14' [const.Py_tp_vectorcall] - added = '3.14' \ No newline at end of file + added = '3.14' +[function.PyType_GetBaseByToken] + added = '3.14' +[const.Py_tp_token] + added = '3.14' +[const.Py_TP_USE_SPEC] + added = '3.14' diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 2b23be7b753e34..1d9e1699022d3e 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -500,7 +500,7 @@ CType_Type_dealloc(PyObject *self) { StgInfo *info = _PyStgInfo_FromType_NoState(self); if (!info) { - PyErr_WriteUnraisable(self); + PyErr_WriteUnraisable(NULL); // NULL avoids segfault here } if (info) { PyMem_Free(info->ffi_type_pointer.elements); @@ -560,6 +560,7 @@ static PyMethodDef ctype_methods[] = { }; static PyType_Slot ctype_type_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_traverse, CType_Type_traverse}, {Py_tp_clear, CType_Type_clear}, {Py_tp_dealloc, CType_Type_dealloc}, @@ -569,7 +570,7 @@ static PyType_Slot ctype_type_slots[] = { {0, NULL}, }; -static PyType_Spec pyctype_type_spec = { +PyType_Spec pyctype_type_spec = { .name = "_ctypes.CType_Type", .basicsize = -(Py_ssize_t)sizeof(StgInfo), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 2eb1b6cae4d81b..738dcd1aaf8a01 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -108,6 +108,7 @@ get_module_state_by_def(PyTypeObject *cls) } +extern PyType_Spec pyctype_type_spec; extern PyType_Spec carg_spec; extern PyType_Spec cfield_spec; extern PyType_Spec cthunk_spec; @@ -490,16 +491,23 @@ PyStgInfo_FromAny(ctypes_state *state, PyObject *obj, StgInfo **result) /* A variant of PyStgInfo_FromType that doesn't need the state, * so it can be called from finalization functions when the module - * state is torn down. Does no checks; cannot fail. - * This inlines the current implementation PyObject_GetTypeData, - * so it might break in the future. + * state is torn down. */ static inline StgInfo * _PyStgInfo_FromType_NoState(PyObject *type) { - size_t type_basicsize =_Py_SIZE_ROUND_UP(PyType_Type.tp_basicsize, - ALIGNOF_MAX_ALIGN_T); - return (StgInfo *)((char *)type + type_basicsize); + PyTypeObject *PyCType_Type; + if (PyType_GetBaseByToken(Py_TYPE(type), &pyctype_type_spec, &PyCType_Type) < 0) { + return NULL; + } + if (PyCType_Type == NULL) { + PyErr_Format(PyExc_TypeError, "expected a ctypes type, got '%N'", type); + return NULL; + } + + StgInfo *info = PyObject_GetTypeData(type, PyCType_Type); + Py_DECREF(PyCType_Type); + return info; } // Initialize StgInfo on a newly created type diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index b3fb9ec056c6c2..cc88147dfcd7fb 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -410,6 +410,118 @@ pyobject_getitemdata(PyObject *self, PyObject *o) } +static PyObject * +create_type_with_token(PyObject *module, PyObject *args) +{ + const char *name; + PyObject *py_token; + if (!PyArg_ParseTuple(args, "sO", &name, &py_token)) { + return NULL; + } + void *token = PyLong_AsVoidPtr(py_token); + if (token == Py_TP_USE_SPEC) { + // Py_TP_USE_SPEC requires the spec that at least outlives the class + static PyType_Slot slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, + {0}, + }; + static PyType_Spec spec = { + .name = "_testcapi.DefaultTokenTest", + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = slots, + }; + PyObject *type = PyType_FromMetaclass(NULL, NULL, &spec, NULL); + if (!type) { + return NULL; + } + token = PyType_GetSlot((PyTypeObject *)type, Py_tp_token); + assert(!PyErr_Occurred()); + Py_DECREF(type); + if (token != &spec) { + PyErr_SetString(PyExc_AssertionError, + "failed to convert token from Py_TP_USE_SPEC"); + return NULL; + } + } + // Test non-NULL token that must also outlive the class + PyType_Slot slots[] = { + {Py_tp_token, token}, + {0}, + }; + PyType_Spec spec = { + .name = name, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = slots, + }; + return PyType_FromMetaclass(NULL, module, &spec, NULL); +} + +static PyObject * +get_tp_token(PyObject *self, PyObject *type) +{ + void *token = PyType_GetSlot((PyTypeObject *)type, Py_tp_token); + if (PyErr_Occurred()) { + return NULL; + } + return PyLong_FromVoidPtr(token); +} + +static PyObject * +pytype_getbasebytoken(PyObject *self, PyObject *args) +{ + PyTypeObject *type; + PyObject *py_token, *use_mro, *need_result; + if (!PyArg_ParseTuple(args, "OOOO", + &type, &py_token, &use_mro, &need_result)) { + return NULL; + } + + PyObject *mro_save = NULL; + if (use_mro != Py_True) { + // Test internal detail: PyType_GetBaseByToken works even with + // types that are only partially initialized (or torn down): + // if tp_mro=NULL we fall back to tp_bases. + assert(PyType_Check(type)); + mro_save = type->tp_mro; + type->tp_mro = NULL; + } + + void *token = PyLong_AsVoidPtr(py_token); + PyObject *result; + int ret; + if (need_result == Py_True) { + ret = PyType_GetBaseByToken(type, token, (PyTypeObject **)&result); + } + else { + result = NULL; + ret = PyType_GetBaseByToken(type, token, NULL); + } + + if (use_mro != Py_True) { + type->tp_mro = mro_save; + } + if (ret < 0) { + assert(result == NULL); + return NULL; + } + PyObject *py_ret = PyLong_FromLong(ret); + if (py_ret == NULL) { + goto error; + } + PyObject *tuple = PyTuple_New(2); + if (tuple == NULL) { + goto error; + } + PyTuple_SET_ITEM(tuple, 0, py_ret); + PyTuple_SET_ITEM(tuple, 1, result ? result : Py_None); + return tuple; +error: + Py_XDECREF(py_ret); + Py_XDECREF(result); + return NULL; +} + + static PyMethodDef TestMethods[] = { {"pytype_fromspec_meta", pytype_fromspec_meta, METH_O}, {"test_type_from_ephemeral_spec", test_type_from_ephemeral_spec, METH_NOARGS}, @@ -423,6 +535,9 @@ static PyMethodDef TestMethods[] = { {"make_immutable_type_with_base", make_immutable_type_with_base, METH_O}, {"make_type_with_base", make_type_with_base, METH_O}, {"pyobject_getitemdata", pyobject_getitemdata, METH_O}, + {"create_type_with_token", create_type_with_token, METH_VARARGS}, + {"get_tp_token", get_tp_token, METH_O}, + {"pytype_getbasebytoken", pytype_getbasebytoken, METH_VARARGS}, {NULL}, }; @@ -1287,6 +1402,8 @@ _PyTestCapi_Init_Heaptype(PyObject *m) { &PyType_Type, m, &HeapCTypeMetaclassNullNew_spec, (PyObject *) &PyType_Type); ADD("HeapCTypeMetaclassNullNew", HeapCTypeMetaclassNullNew); + ADD("Py_TP_USE_SPEC", PyLong_FromVoidPtr(Py_TP_USE_SPEC)); + PyObject *HeapCCollection = PyType_FromMetaclass( NULL, m, &HeapCCollection_spec, NULL); if (HeapCCollection == NULL) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 28edd801284b81..68e481f8e5163b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3926,6 +3926,7 @@ type_new_alloc(type_new_ctx *ctx) et->ht_name = Py_NewRef(ctx->name); et->ht_module = NULL; et->_ht_tpname = NULL; + et->ht_token = NULL; #ifdef Py_GIL_DISABLED _PyType_AssignId(et); @@ -4984,6 +4985,11 @@ PyType_FromMetaclass( } } break; + case Py_tp_token: + { + res->ht_token = slot->pfunc == Py_TP_USE_SPEC ? spec : slot->pfunc; + } + break; default: { /* Copy other slots directly */ @@ -5144,8 +5150,15 @@ PyType_GetSlot(PyTypeObject *type, int slot) PyErr_BadInternalCall(); return NULL; } + int slot_offset = pyslot_offsets[slot].slot_offset; - parent_slot = *(void**)((char*)type + pyslot_offsets[slot].slot_offset); + if (slot_offset >= (int)sizeof(PyTypeObject)) { + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + return NULL; + } + } + + parent_slot = *(void**)((char*)type + slot_offset); if (parent_slot == NULL) { return NULL; } @@ -5274,6 +5287,129 @@ _PyType_GetModuleByDef2(PyTypeObject *left, PyTypeObject *right, return module; } + +static PyTypeObject * +get_base_by_token_recursive(PyTypeObject *type, void *token) +{ + assert(PyType_GetSlot(type, Py_tp_token) != token); + PyObject *bases = lookup_tp_bases(type); + assert(bases != NULL); + Py_ssize_t n = PyTuple_GET_SIZE(bases); + for (Py_ssize_t i = 0; i < n; i++) { + PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, i)); + if (!_PyType_HasFeature(base, Py_TPFLAGS_HEAPTYPE)) { + continue; + } + if (((PyHeapTypeObject*)base)->ht_token == token) { + return base; + } + base = get_base_by_token_recursive(base, token); + if (base != NULL) { + return base; + } + } + return NULL; +} + +static inline PyTypeObject * +get_base_by_token_from_mro(PyTypeObject *type, void *token) +{ + // Bypass lookup_tp_mro() as PyType_IsSubtype() does + PyObject *mro = type->tp_mro; + assert(mro != NULL); + assert(PyTuple_Check(mro)); + // mro_invoke() ensures that the type MRO cannot be empty. + assert(PyTuple_GET_SIZE(mro) >= 1); + // Also, the first item in the MRO is the type itself, which is supposed + // to be already checked by the caller. We skip it in the loop. + assert(PyTuple_GET_ITEM(mro, 0) == (PyObject *)type); + assert(PyType_GetSlot(type, Py_tp_token) != token); + + Py_ssize_t n = PyTuple_GET_SIZE(mro); + for (Py_ssize_t i = 1; i < n; i++) { + PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(mro, i)); + if (!_PyType_HasFeature(base, Py_TPFLAGS_HEAPTYPE)) { + continue; + } + if (((PyHeapTypeObject*)base)->ht_token == token) { + return base; + } + } + return NULL; +} + +static int +check_base_by_token(PyTypeObject *type, void *token) { + // Chain the branches, which will be optimized exclusive here + if (token == NULL) { + PyErr_Format(PyExc_SystemError, + "PyType_GetBaseByToken called with token=NULL"); + return -1; + } + else if (!PyType_Check(type)) { + PyErr_Format(PyExc_TypeError, + "expected a type, got a '%T' object", type); + return -1; + } + else if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + return 0; + } + else if (((PyHeapTypeObject*)type)->ht_token == token) { + return 1; + } + else if (type->tp_mro != NULL) { + // This will not be inlined + return get_base_by_token_from_mro(type, token) ? 1 : 0; + } + else { + return get_base_by_token_recursive(type, token) ? 1 : 0; + } +} + +int +PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) +{ + if (result == NULL) { + // If the `result` is checked only once here, the subsequent + // branches will become trivial to optimize. + return check_base_by_token(type, token); + } + if (token == NULL || !PyType_Check(type)) { + *result = NULL; + return check_base_by_token(type, token); + } + + // Chain the branches, which will be optimized exclusive here + PyTypeObject *base; + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + // No static type has a heaptype superclass, + // which is ensured by type_ready_mro(). + *result = NULL; + return 0; + } + else if (((PyHeapTypeObject*)type)->ht_token == token) { + *result = (PyTypeObject *)Py_NewRef(type); + return 1; + } + else if (type->tp_mro != NULL) { + // Expect this to be inlined + base = get_base_by_token_from_mro(type, token); + } + else { + base = get_base_by_token_recursive(type, token); + } + + if (base != NULL) { + *result = (PyTypeObject *)Py_NewRef(base); + return 1; + } + else { + *result = NULL; + return 0; + } +} + + void * PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls) { @@ -5966,6 +6102,7 @@ type_dealloc(PyObject *self) #ifdef Py_GIL_DISABLED _PyType_ReleaseId(et); #endif + et->ht_token = NULL; Py_TYPE(type)->tp_free((PyObject *)type); } diff --git a/Objects/typeslots.inc b/Objects/typeslots.inc index ffb85ff56adff1..642160fe0bd8bc 100644 --- a/Objects/typeslots.inc +++ b/Objects/typeslots.inc @@ -81,3 +81,4 @@ {-1, offsetof(PyTypeObject, tp_finalize)}, {offsetof(PyAsyncMethods, am_send), offsetof(PyTypeObject, tp_as_async)}, {-1, offsetof(PyTypeObject, tp_vectorcall)}, +{-1, offsetof(PyHeapTypeObject, ht_token)}, diff --git a/Objects/typeslots.py b/Objects/typeslots.py index 8ab05f91be12b0..c7f8a33bb1e74e 100755 --- a/Objects/typeslots.py +++ b/Objects/typeslots.py @@ -13,7 +13,11 @@ def generate_typeslots(out=sys.stdout): continue member = m.group(1) - if member.startswith("tp_"): + if member == "tp_token": + # The heap type structure (ht_*) is an implementation detail; + # the public slot for it has a familiar `tp_` prefix + member = '{-1, offsetof(PyHeapTypeObject, ht_token)}' + elif member.startswith("tp_"): member = f'{{-1, offsetof(PyTypeObject, {member})}}' elif member.startswith("am_"): member = (f'{{offsetof(PyAsyncMethods, {member}),'+ diff --git a/PC/python3dll.c b/PC/python3dll.c index 1845334b244d8c..7bd04cb483b547 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -651,6 +651,7 @@ EXPORT_FUNC(PyType_FromSpec) EXPORT_FUNC(PyType_FromSpecWithBases) EXPORT_FUNC(PyType_GenericAlloc) EXPORT_FUNC(PyType_GenericNew) +EXPORT_FUNC(PyType_GetBaseByToken) EXPORT_FUNC(PyType_GetFlags) EXPORT_FUNC(PyType_GetFullyQualifiedName) EXPORT_FUNC(PyType_GetModule) From 81480e6edb34774d783d018d1f0e61ab5c3f0a9a Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Wed, 18 Sep 2024 02:49:43 -0500 Subject: [PATCH 005/105] gh-124190: Ignore files directories check warning tooling (#124193) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- ...-09-17-22-21-58.gh-issue-124190.3fWhiX.rst | 1 + Tools/build/.warningignore_macos | 2 +- Tools/build/.warningignore_ubuntu | 1 - Tools/build/check_warnings.py | 148 ++++++++++++------ 4 files changed, 105 insertions(+), 47 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-09-17-22-21-58.gh-issue-124190.3fWhiX.rst diff --git a/Misc/NEWS.d/next/Tests/2024-09-17-22-21-58.gh-issue-124190.3fWhiX.rst b/Misc/NEWS.d/next/Tests/2024-09-17-22-21-58.gh-issue-124190.3fWhiX.rst new file mode 100644 index 00000000000000..819b1ca49235fc --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-09-17-22-21-58.gh-issue-124190.3fWhiX.rst @@ -0,0 +1 @@ +Add capability to ignore entire files or directories in check warning CI tool diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos index e9307b44c7ee5f..e72309229cc60f 100644 --- a/Tools/build/.warningignore_macos +++ b/Tools/build/.warningignore_macos @@ -6,4 +6,4 @@ Modules/expat/siphash.h 7 Modules/expat/xmlparse.c 8 Modules/expat/xmltok.c 3 -Modules/expat/xmltok_impl.c 26 \ No newline at end of file +Modules/expat/xmltok_impl.c 26 diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 980785d59ae33b..469c727abfb11c 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -3,4 +3,3 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) - diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index e58ee2a7cc8571..7210cc8365ead6 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -11,9 +11,49 @@ from typing import NamedTuple -class FileWarnings(NamedTuple): - name: str +class IgnoreRule(NamedTuple): + file_path: str count: int + ignore_all: bool = False + is_directory: bool = False + + +def parse_warning_ignore_file(file_path: str) -> set[IgnoreRule]: + """ + Parses the warning ignore file and returns a set of IgnoreRules + """ + files_with_expected_warnings = set() + with Path(file_path).open(encoding="UTF-8") as ignore_rules_file: + files_with_expected_warnings = set() + for i, line in enumerate(ignore_rules_file): + line = line.strip() + if line and not line.startswith("#"): + line_parts = line.split() + if len(line_parts) >= 2: + file_name = line_parts[0] + count = line_parts[1] + ignore_all = count == "*" + is_directory = file_name.endswith("/") + + # Directories must have a wildcard count + if is_directory and count != "*": + print( + f"Error parsing ignore file: {file_path} at line: {i}" + ) + print( + f"Directory {file_name} must have count set to *" + ) + sys.exit(1) + if ignore_all: + count = 0 + + files_with_expected_warnings.add( + IgnoreRule( + file_name, int(count), ignore_all, is_directory + ) + ) + + return files_with_expected_warnings def extract_warnings_from_compiler_output( @@ -48,11 +88,15 @@ def extract_warnings_from_compiler_output( "line": match.group("line"), "column": match.group("column"), "message": match.group("message"), - "option": match.group("option").lstrip("[").rstrip("]"), + "option": match.group("option") + .lstrip("[") + .rstrip("]"), } ) except: - print(f"Error parsing compiler output. Unable to extract warning on line {i}:\n{line}") + print( + f"Error parsing compiler output. Unable to extract warning on line {i}:\n{line}" + ) sys.exit(1) return compiler_warnings @@ -78,9 +122,24 @@ def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: return warnings_by_file +def is_file_ignored( + file_path: str, ignore_rules: set[IgnoreRule] +) -> IgnoreRule | None: + """ + Returns the IgnoreRule object for the file path if there is a related rule for it + """ + for rule in ignore_rules: + if rule.is_directory: + if file_path.startswith(rule.file_path): + return rule + elif file_path == rule.file_path: + return rule + return None + + def get_unexpected_warnings( - files_with_expected_warnings: set[FileWarnings], - files_with_warnings: set[FileWarnings], + ignore_rules: set[IgnoreRule], + files_with_warnings: set[IgnoreRule], ) -> int: """ Returns failure status if warnings discovered in list of warnings @@ -89,14 +148,21 @@ def get_unexpected_warnings( """ unexpected_warnings = {} for file in files_with_warnings.keys(): - found_file_in_ignore_list = False - for ignore_file in files_with_expected_warnings: - if file == ignore_file.name: - if len(files_with_warnings[file]) > ignore_file.count: - unexpected_warnings[file] = (files_with_warnings[file], ignore_file.count) - found_file_in_ignore_list = True - break - if not found_file_in_ignore_list: + + rule = is_file_ignored(file, ignore_rules) + + if rule: + if rule.ignore_all: + continue + + if len(files_with_warnings[file]) > rule.count: + unexpected_warnings[file] = ( + files_with_warnings[file], + rule.count, + ) + continue + elif rule is None: + # If the file is not in the ignore list, then it is unexpected unexpected_warnings[file] = (files_with_warnings[file], 0) if unexpected_warnings: @@ -115,19 +181,27 @@ def get_unexpected_warnings( def get_unexpected_improvements( - files_with_expected_warnings: set[FileWarnings], - files_with_warnings: set[FileWarnings], + ignore_rules: set[IgnoreRule], + files_with_warnings: set[IgnoreRule], ) -> int: """ - Returns failure status if there are no warnings in the list of warnings - for a file that is in the list of files with expected warnings + Returns failure status if the number of warnings for a file is greater + than the expected number of warnings for that file based on the ignore + rules """ unexpected_improvements = [] - for file in files_with_expected_warnings: - if file.name not in files_with_warnings.keys(): - unexpected_improvements.append((file.name, file.count, 0)) - elif len(files_with_warnings[file.name]) < file.count: - unexpected_improvements.append((file.name, file.count, len(files_with_warnings[file.name]))) + for rule in ignore_rules: + if not rule.ignore_all and rule.file_path not in files_with_warnings.keys(): + if rule.file_path not in files_with_warnings.keys(): + unexpected_improvements.append((rule.file_path, rule.count, 0)) + elif len(files_with_warnings[rule.file_path]) < rule.count: + unexpected_improvements.append( + ( + rule.file_path, + rule.count, + len(files_with_warnings[rule.file_path]), + ) + ) if unexpected_improvements: print("Unexpected improvements:") @@ -202,7 +276,7 @@ def main(argv: list[str] | None = None) -> int: "Warning ignore file not specified." " Continuing without it (no warnings ignored)." ) - files_with_expected_warnings = set() + ignore_rules = set() else: if not Path(args.warning_ignore_file_path).is_file(): print( @@ -210,19 +284,7 @@ def main(argv: list[str] | None = None) -> int: f" {args.warning_ignore_file_path}" ) return 1 - with Path(args.warning_ignore_file_path).open( - encoding="UTF-8" - ) as clean_files: - # Files with expected warnings are stored as a set of tuples - # where the first element is the file name and the second element - # is the number of warnings expected in that file - files_with_expected_warnings = { - FileWarnings( - file.strip().split()[0], int(file.strip().split()[1]) - ) - for file in clean_files - if file.strip() and not file.startswith("#") - } + ignore_rules = parse_warning_ignore_file(args.warning_ignore_file_path) with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f: compiler_output_file_contents = f.read() @@ -230,27 +292,23 @@ def main(argv: list[str] | None = None) -> int: warnings = extract_warnings_from_compiler_output( compiler_output_file_contents, args.compiler_output_type, - args.path_prefix + args.path_prefix, ) files_with_warnings = get_warnings_by_file(warnings) - status = get_unexpected_warnings( - files_with_expected_warnings, files_with_warnings - ) + status = get_unexpected_warnings(ignore_rules, files_with_warnings) if args.fail_on_regression: exit_code |= status - status = get_unexpected_improvements( - files_with_expected_warnings, files_with_warnings - ) + status = get_unexpected_improvements(ignore_rules, files_with_warnings) if args.fail_on_improvement: exit_code |= status print( "For information about this tool and its configuration" " visit https://devguide.python.org/development-tools/warnings/" - ) + ) return exit_code From 8a284e189673582e262744618f293f9901a32e49 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 18 Sep 2024 11:39:11 +0300 Subject: [PATCH 006/105] gh-119771: Set errno on overflows in _Py_c_pow() (#120256) Before we did this in complex_pow() and behavior of the public C API function _Py_c_pow() was different from the pure-python pow(). --- Doc/c-api/complex.rst | 2 ++ Lib/test/test_capi/test_complex.py | 6 +++++- .../C API/2024-06-08-08-33-40.gh-issue-119771.Oip2dL.rst | 2 ++ Objects/complexobject.c | 4 +++- 4 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-06-08-08-33-40.gh-issue-119771.Oip2dL.rst diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 67d0c5f144e075..16bd79475dc1e6 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -79,6 +79,8 @@ pointers. This is consistent throughout the API. If *num* is null and *exp* is not a positive real number, this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. + Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + Complex Numbers as Python Objects ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Lib/test/test_capi/test_complex.py b/Lib/test/test_capi/test_complex.py index 328ea12f97462c..368edfbf2ce97e 100644 --- a/Lib/test/test_capi/test_complex.py +++ b/Lib/test/test_capi/test_complex.py @@ -226,7 +226,11 @@ def test_py_c_pow(self): self.assertEqual(_py_c_pow(0j, -1)[1], errno.EDOM) self.assertEqual(_py_c_pow(0j, 1j)[1], errno.EDOM) - self.assertEqual(_py_c_pow(*[DBL_MAX+1j]*2)[0], complex(*[INF]*2)) + max_num = DBL_MAX+1j + self.assertEqual(_py_c_pow(max_num, max_num), + (complex(INF, INF), errno.ERANGE)) + self.assertEqual(_py_c_pow(max_num, 2), + (complex(INF, INF), errno.ERANGE)) def test_py_c_abs(self): diff --git a/Misc/NEWS.d/next/C API/2024-06-08-08-33-40.gh-issue-119771.Oip2dL.rst b/Misc/NEWS.d/next/C API/2024-06-08-08-33-40.gh-issue-119771.Oip2dL.rst new file mode 100644 index 00000000000000..61619082487c3b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-06-08-08-33-40.gh-issue-119771.Oip2dL.rst @@ -0,0 +1,2 @@ +Set :data:`errno` in :c:func:`_Py_c_pow` on overflows. Patch by Sergey B +Kirpichev. diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 4a8dac6c53f529..787235c63a6be1 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -173,6 +173,8 @@ _Py_c_pow(Py_complex a, Py_complex b) } r.real = len*cos(phase); r.imag = len*sin(phase); + + _Py_ADJUST_ERANGE2(r.real, r.imag); } return r; } @@ -567,12 +569,12 @@ complex_pow(PyObject *v, PyObject *w, PyObject *z) // a faster and more accurate algorithm. if (b.imag == 0.0 && b.real == floor(b.real) && fabs(b.real) <= 100.0) { p = c_powi(a, (long)b.real); + _Py_ADJUST_ERANGE2(p.real, p.imag); } else { p = _Py_c_pow(a, b); } - _Py_ADJUST_ERANGE2(p.real, p.imag); if (errno == EDOM) { PyErr_SetString(PyExc_ZeroDivisionError, "zero to a negative or complex power"); From 42c8b0556c02d29e32f4c7c95e7128a343716250 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 18 Sep 2024 14:14:34 +0200 Subject: [PATCH 007/105] gh-123085: _compile_importlib: Avoid copying sources before compilation (GH-124131) Co-authored-by: Jason R. Coombs --- Lib/test/test_importlib/resources/test_files.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 3cdbee302c5e75..933894dce2c045 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -134,18 +134,17 @@ def test_implicit_files_submodule(self): def _compile_importlib(self): """ Make a compiled-only copy of the importlib resources package. + + Currently only code is copied, as importlib resources doesn't itself + have any resources. """ bin_site = self.fixtures.enter_context(os_helper.temp_dir()) c_resources = pathlib.Path(bin_site, 'c_resources') sources = pathlib.Path(resources.__file__).parent - shutil.copytree(sources, c_resources, ignore=lambda *_: ['__pycache__']) - - for dirpath, _, filenames in os.walk(c_resources): - for filename in filenames: - source_path = pathlib.Path(dirpath) / filename - cfile = source_path.with_suffix('.pyc') - py_compile.compile(source_path, cfile) - pathlib.Path.unlink(source_path) + + for source_path in sources.glob('**/*.py'): + c_path = c_resources.joinpath(source_path.relative_to(sources)).with_suffix('.pyc') + py_compile.compile(source_path, c_path) self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site)) def test_implicit_files_with_compiled_importlib(self): From 32119fc377a4d9df524a7bac02b6922a990361dd Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 18 Sep 2024 14:15:43 +0200 Subject: [PATCH 008/105] gh-118915: Add/fix docs entries for some new 3.13 C API (GH-124134) --- Doc/c-api/exceptions.rst | 3 +++ Doc/c-api/long.rst | 2 ++ Doc/c-api/module.rst | 2 ++ Doc/c-api/monitoring.rst | 34 +++++++++++++++++++++++++++++++--- Doc/conf.py | 5 +++++ 5 files changed, 43 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 372fc1dde8fc0d..05349590975160 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -1004,6 +1004,7 @@ the variables: single: PyExc_OverflowError (C var) single: PyExc_PermissionError (C var) single: PyExc_ProcessLookupError (C var) + single: PyExc_PythonFinalizationError (C var) single: PyExc_RecursionError (C var) single: PyExc_ReferenceError (C var) single: PyExc_RuntimeError (C var) @@ -1096,6 +1097,8 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PythonFinalizationError` | :exc:`PythonFinalizationError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | | diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 30133a9c5cfa2f..098a55c50e219a 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -509,6 +509,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Currently, ``-1`` corresponds to ``Py_ASNATIVEBYTES_NATIVE_ENDIAN | Py_ASNATIVEBYTES_UNSIGNED_BUFFER``. + .. c:namespace:: NULL + ============================================= ====== Flag Value ============================================= ====== diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 3c779488813383..ec61be284caad9 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -421,6 +421,8 @@ The available slot types are: Specifies one of the following values: + .. c:namespace:: NULL + .. c:macro:: Py_MOD_GIL_USED The module depends on the presence of the global interpreter lock (GIL), diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index b34035b5548f02..4db26d82b9200d 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -133,7 +133,7 @@ Managing the Monitoring State Monitoring states can be managed with the help of monitoring scopes. A scope would typically correspond to a python function. -.. :c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) +.. c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) Enter a monitored scope. ``event_types`` is an array of the event IDs for events that may be fired from the scope. For example, the ID of a ``PY_START`` @@ -158,7 +158,35 @@ would typically correspond to a python function. execution is paused, such as when emulating a generator, the scope needs to be exited and re-entered. - -.. :c:function:: int PyMonitoring_ExitScope(void) + The macros for *event_types* are: + + .. c:namespace:: NULL + + .. The table is here to make the docs searchable, and to allow automatic + links to the identifiers. + + ================================================== ===================================== + Macro Event + ================================================== ===================================== + .. c:macro:: PY_MONITORING_EVENT_BRANCH :monitoring-event:`BRANCH` + .. c:macro:: PY_MONITORING_EVENT_CALL :monitoring-event:`CALL` + .. c:macro:: PY_MONITORING_EVENT_C_RAISE :monitoring-event:`C_RAISE` + .. c:macro:: PY_MONITORING_EVENT_C_RETURN :monitoring-event:`C_RETURN` + .. c:macro:: PY_MONITORING_EVENT_EXCEPTION_HANDLED :monitoring-event:`EXCEPTION_HANDLED` + .. c:macro:: PY_MONITORING_EVENT_INSTRUCTION :monitoring-event:`INSTRUCTION` + .. c:macro:: PY_MONITORING_EVENT_JUMP :monitoring-event:`JUMP` + .. c:macro:: PY_MONITORING_EVENT_LINE :monitoring-event:`LINE` + .. c:macro:: PY_MONITORING_EVENT_PY_RESUME :monitoring-event:`PY_RESUME` + .. c:macro:: PY_MONITORING_EVENT_PY_RETURN :monitoring-event:`PY_RETURN` + .. c:macro:: PY_MONITORING_EVENT_PY_START :monitoring-event:`PY_START` + .. c:macro:: PY_MONITORING_EVENT_PY_THROW :monitoring-event:`PY_THROW` + .. c:macro:: PY_MONITORING_EVENT_PY_UNWIND :monitoring-event:`PY_UNWIND` + .. c:macro:: PY_MONITORING_EVENT_PY_YIELD :monitoring-event:`PY_YIELD` + .. c:macro:: PY_MONITORING_EVENT_RAISE :monitoring-event:`RAISE` + .. c:macro:: PY_MONITORING_EVENT_RERAISE :monitoring-event:`RERAISE` + .. c:macro:: PY_MONITORING_EVENT_STOP_ITERATION :monitoring-event:`STOP_ITERATION` + ================================================== ===================================== + +.. c:function:: int PyMonitoring_ExitScope(void) Exit the last scope that was entered with ``PyMonitoring_EnterScope``. diff --git a/Doc/conf.py b/Doc/conf.py index 9f860363eabd09..27cf03d6bea05a 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -132,6 +132,8 @@ ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), + ('c:type', 'int8_t'), + ('c:type', 'int16_t'), ('c:type', 'int32_t'), ('c:type', 'int64_t'), ('c:type', 'intmax_t'), @@ -141,6 +143,8 @@ ('c:type', 'size_t'), ('c:type', 'ssize_t'), ('c:type', 'time_t'), + ('c:type', 'uint8_t'), + ('c:type', 'uint16_t'), ('c:type', 'uint32_t'), ('c:type', 'uint64_t'), ('c:type', 'uintmax_t'), @@ -244,6 +248,7 @@ ('c:data', 'PyExc_OverflowError'), ('c:data', 'PyExc_PermissionError'), ('c:data', 'PyExc_ProcessLookupError'), + ('c:data', 'PyExc_PythonFinalizationError'), ('c:data', 'PyExc_RecursionError'), ('c:data', 'PyExc_ReferenceError'), ('c:data', 'PyExc_RuntimeError'), From 0c4884d8aa51df2dd013c3e78fcc3f2077d743c3 Mon Sep 17 00:00:00 2001 From: Sam James Date: Wed, 18 Sep 2024 13:44:32 +0100 Subject: [PATCH 009/105] test: fix _is_perf_version_at_least typo (#124199) --- Lib/test/test_perf_profiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index ac1911ca24eafe..b68a55259c62e1 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -479,7 +479,7 @@ def compile_trampolines_for_all_functions(): self.assertIn(line, child_perf_file_contents) -def _is_perf_vesion_at_least(major, minor): +def _is_perf_version_at_least(major, minor): # The output of perf --version looks like "perf version 6.7-3" but # it can also be perf version "perf version 5.15.143" try: @@ -494,7 +494,7 @@ def _is_perf_vesion_at_least(major, minor): @unittest.skipUnless(perf_command_works(), "perf command doesn't work") -@unittest.skipUnless(_is_perf_vesion_at_least(6, 6), "perf command may not work due to a perf bug") +@unittest.skipUnless(_is_perf_version_at_least(6, 6), "perf command may not work due to a perf bug") class TestPerfProfilerWithDwarf(unittest.TestCase, TestPerfProfilerMixin): def run_perf(self, script_dir, script, activate_trampoline=True): if activate_trampoline: From 3b6bfa77aa4da2ce1f3a15e39831f8b85882698c Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 18 Sep 2024 17:17:32 +0300 Subject: [PATCH 010/105] gh-102511: Change the `os.path.splitroot` param name from `path` back to `p` (GH-124097) --- Modules/clinic/posixmodule.c.h | 8 ++++---- Modules/posixmodule.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 4b9dbac9af031f..9722f06a5935b9 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2367,7 +2367,7 @@ os__path_isjunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P #endif /* defined(MS_WINDOWS) */ PyDoc_STRVAR(os__path_splitroot_ex__doc__, -"_path_splitroot_ex($module, /, path)\n" +"_path_splitroot_ex($module, /, p)\n" "--\n" "\n" "Split a pathname into drive, root and tail.\n" @@ -2393,7 +2393,7 @@ os__path_splitroot_ex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), }, + .ob_item = { _Py_LATIN1_CHR('p'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2402,7 +2402,7 @@ os__path_splitroot_ex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"path", NULL}; + static const char * const _keywords[] = {"p", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "_path_splitroot_ex", @@ -12837,4 +12837,4 @@ os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=2fafa0d2814948f8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a736ad3f7205176e input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f24ab81cbcb77b..8b2c4bd35a8da7 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5497,7 +5497,7 @@ os__path_isjunction_impl(PyObject *module, path_t *path) /*[clinic input] os._path_splitroot_ex - path: path_t(make_wide=True, nonstrict=True) + p as path: path_t(make_wide=True, nonstrict=True) Split a pathname into drive, root and tail. @@ -5506,7 +5506,7 @@ The tail contains anything after the root. static PyObject * os__path_splitroot_ex_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=4b0072b6cdf4b611 input=6eb76e9173412c92]*/ +/*[clinic end generated code: output=4b0072b6cdf4b611 input=4556b615c7cc13f2]*/ { Py_ssize_t drvsize, rootsize; PyObject *drv = NULL, *root = NULL, *tail = NULL, *result = NULL; From 96f619faa74a8a32c2c297833cdeb0393c0b6b13 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 18 Sep 2024 08:39:22 -0700 Subject: [PATCH 011/105] gh-124206: Fix calling get_annotate_function() on static types (#124208) Fixes #124206. No news entry because the bug this fixes was never released. --- Lib/annotationlib.py | 6 +++++- Lib/test/test_annotationlib.py | 21 +++++++++++++++++++++ Lib/test/test_typing.py | 19 +++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 9d1943b27e8e9c..09a844ddb56f06 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -575,7 +575,11 @@ def get_annotate_function(obj): Returns the __annotate__ function or None. """ if isinstance(obj, type): - return _BASE_GET_ANNOTATE(obj) + try: + return _BASE_GET_ANNOTATE(obj) + except AttributeError: + # AttributeError is raised for static types. + return None return getattr(obj, "__annotate__", None) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index ce4f92624d9036..309f6d2120109a 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -928,6 +928,27 @@ class D(metaclass=Meta): self.assertIs(annotate_func, None) +class TestGetAnnotateFunction(unittest.TestCase): + def test_static_class(self): + self.assertIsNone(get_annotate_function(object)) + self.assertIsNone(get_annotate_function(int)) + + def test_unannotated_class(self): + class C: + pass + + self.assertIsNone(get_annotate_function(C)) + + D = type("D", (), {}) + self.assertIsNone(get_annotate_function(D)) + + def test_annotated_class(self): + class C: + a: int + + self.assertEqual(get_annotate_function(C)(Format.VALUE), {"a": int}) + + class TestAnnotationLib(unittest.TestCase): def test__all__(self): support.check__all__(self, annotationlib) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 6e036b600330c1..3ac6b97383fcef 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7043,6 +7043,25 @@ def h(x: collections.abc.Callable[P, int]): ... self.assertEqual(get_type_hints(g), {'x': collections.abc.Callable[..., int]}) self.assertEqual(get_type_hints(h), {'x': collections.abc.Callable[P, int]}) + def test_get_type_hints_format(self): + class C: + x: undefined + + with self.assertRaises(NameError): + get_type_hints(C) + + with self.assertRaises(NameError): + get_type_hints(C, format=annotationlib.Format.VALUE) + + annos = get_type_hints(C, format=annotationlib.Format.FORWARDREF) + self.assertIsInstance(annos, dict) + self.assertEqual(list(annos), ['x']) + self.assertIsInstance(annos['x'], annotationlib.ForwardRef) + self.assertEqual(annos['x'].__arg__, 'undefined') + + self.assertEqual(get_type_hints(C, format=annotationlib.Format.SOURCE), + {'x': 'undefined'}) + class GetUtilitiesTestCase(TestCase): def test_get_origin(self): From 8b6c7c7877c26f0201f37f69d4db2f35d7abd760 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 18 Sep 2024 08:47:57 -0700 Subject: [PATCH 012/105] gh-120754: Refactor I/O modules to stash whole stat result rather than individual members (#123412) Multiple places in the I/O stack optimize common cases by using the information from stat. Currently individual members are extracted from the stat and stored into the fileio struct. Refactor the code to store the whole stat struct instead. Parallels the changes to _io. The `stat` Python object doesn't allow changing members, so rather than modifying estimated_size, just clear the value. --- Lib/_pyio.py | 44 ++++++++++++----------- Modules/_io/fileio.c | 83 ++++++++++++++++++++++++++++++-------------- 2 files changed, 81 insertions(+), 46 deletions(-) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 75b5ad1b1a47d2..18849b309b8605 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -242,14 +242,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, buffering = -1 line_buffering = True if buffering < 0: - buffering = DEFAULT_BUFFER_SIZE - try: - bs = os.fstat(raw.fileno()).st_blksize - except (OSError, AttributeError): - pass - else: - if bs > 1: - buffering = bs + buffering = raw._blksize if buffering < 0: raise ValueError("invalid buffering size") if buffering == 0: @@ -1565,19 +1558,15 @@ def __init__(self, file, mode='r', closefd=True, opener=None): os.set_inheritable(fd, False) self._closefd = closefd - fdfstat = os.fstat(fd) + self._stat_atopen = os.fstat(fd) try: - if stat.S_ISDIR(fdfstat.st_mode): + if stat.S_ISDIR(self._stat_atopen.st_mode): raise IsADirectoryError(errno.EISDIR, os.strerror(errno.EISDIR), file) except AttributeError: # Ignore the AttributeError if stat.S_ISDIR or errno.EISDIR # don't exist. pass - self._blksize = getattr(fdfstat, 'st_blksize', 0) - if self._blksize <= 1: - self._blksize = DEFAULT_BUFFER_SIZE - self._estimated_size = fdfstat.st_size if _setmode: # don't translate newlines (\r\n <=> \n) @@ -1623,6 +1612,17 @@ def __repr__(self): return ('<%s name=%r mode=%r closefd=%r>' % (class_name, name, self.mode, self._closefd)) + @property + def _blksize(self): + if self._stat_atopen is None: + return DEFAULT_BUFFER_SIZE + + blksize = getattr(self._stat_atopen, "st_blksize", 0) + # WASI sets blsize to 0 + if not blksize: + return DEFAULT_BUFFER_SIZE + return blksize + def _checkReadable(self): if not self._readable: raise UnsupportedOperation('File not open for reading') @@ -1655,16 +1655,20 @@ def readall(self): """ self._checkClosed() self._checkReadable() - if self._estimated_size <= 0: + if self._stat_atopen is None or self._stat_atopen.st_size <= 0: bufsize = DEFAULT_BUFFER_SIZE else: - bufsize = self._estimated_size + 1 + # In order to detect end of file, need a read() of at least 1 + # byte which returns size 0. Oversize the buffer by 1 byte so the + # I/O can be completed with two read() calls (one for all data, one + # for EOF) without needing to resize the buffer. + bufsize = self._stat_atopen.st_size + 1 - if self._estimated_size > 65536: + if self._stat_atopen.st_size > 65536: try: pos = os.lseek(self._fd, 0, SEEK_CUR) - if self._estimated_size >= pos: - bufsize = self._estimated_size - pos + 1 + if self._stat_atopen.st_size >= pos: + bufsize = self._stat_atopen.st_size - pos + 1 except OSError: pass @@ -1742,7 +1746,7 @@ def truncate(self, size=None): if size is None: size = self.tell() os.ftruncate(self._fd, size) - self._estimated_size = size + self._stat_atopen = None return size def close(self): diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 5d9d87d6118a75..865b0e3634f3b4 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -74,8 +74,13 @@ typedef struct { signed int seekable : 2; /* -1 means unknown */ unsigned int closefd : 1; char finalizing; - unsigned int blksize; - Py_off_t estimated_size; + /* Stat result which was grabbed at file open, useful for optimizing common + File I/O patterns to be more efficient. This is only guidance / an + estimate, as it is subject to Time-Of-Check to Time-Of-Use (TOCTOU) + issues / bugs. Both the underlying file descriptor and file may be + modified outside of the fileio object / Python (ex. gh-90102, GH-121941, + gh-109523). */ + struct _Py_stat_struct *stat_atopen; PyObject *weakreflist; PyObject *dict; } fileio; @@ -199,8 +204,7 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->writable = 0; self->appending = 0; self->seekable = -1; - self->blksize = 0; - self->estimated_size = -1; + self->stat_atopen = NULL; self->closefd = 1; self->weakreflist = NULL; } @@ -256,7 +260,6 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, #elif !defined(MS_WINDOWS) int *atomic_flag_works = NULL; #endif - struct _Py_stat_struct fdfstat; int fstat_result; int async_err = 0; @@ -454,9 +457,13 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, #endif } - self->blksize = DEFAULT_BUFFER_SIZE; + self->stat_atopen = PyMem_New(struct _Py_stat_struct, 1); + if (self->stat_atopen == NULL) { + PyErr_NoMemory(); + goto error; + } Py_BEGIN_ALLOW_THREADS - fstat_result = _Py_fstat_noraise(self->fd, &fdfstat); + fstat_result = _Py_fstat_noraise(self->fd, self->stat_atopen); Py_END_ALLOW_THREADS if (fstat_result < 0) { /* Tolerate fstat() errors other than EBADF. See Issue #25717, where @@ -471,25 +478,21 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, #endif goto error; } + + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; } else { #if defined(S_ISDIR) && defined(EISDIR) /* On Unix, open will succeed for directories. In Python, there should be no file objects referring to directories, so we need a check. */ - if (S_ISDIR(fdfstat.st_mode)) { + if (S_ISDIR(self->stat_atopen->st_mode)) { errno = EISDIR; PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj); goto error; } #endif /* defined(S_ISDIR) */ -#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE - if (fdfstat.st_blksize > 1) - self->blksize = fdfstat.st_blksize; -#endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */ - if (fdfstat.st_size < PY_SSIZE_T_MAX) { - self->estimated_size = (Py_off_t)fdfstat.st_size; - } } #if defined(MS_WINDOWS) || defined(__CYGWIN__) @@ -521,6 +524,10 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, internal_close(self); _PyErr_ChainExceptions1(exc); } + if (self->stat_atopen != NULL) { + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; + } done: #ifdef MS_WINDOWS @@ -553,6 +560,10 @@ fileio_dealloc(fileio *self) if (_PyIOBase_finalize((PyObject *) self) < 0) return; _PyObject_GC_UNTRACK(self); + if (self->stat_atopen != NULL) { + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; + } if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); (void)fileio_clear(self); @@ -725,20 +736,27 @@ _io_FileIO_readall_impl(fileio *self) return err_closed(); } - end = self->estimated_size; + if (self->stat_atopen != NULL && self->stat_atopen->st_size < _PY_READ_MAX) { + end = (Py_off_t)self->stat_atopen->st_size; + } + else { + end = -1; + } if (end <= 0) { /* Use a default size and resize as needed. */ bufsize = SMALLCHUNK; } else { - /* This is probably a real file, so we try to allocate a - buffer one byte larger than the rest of the file. If the - calculation is right then we should get EOF without having - to enlarge the buffer. */ + /* This is probably a real file. */ if (end > _PY_READ_MAX - 1) { bufsize = _PY_READ_MAX; } else { + /* In order to detect end of file, need a read() of at + least 1 byte which returns size 0. Oversize the buffer + by 1 byte so the I/O can be completed with two read() + calls (one for all data, one for EOF) without needing + to resize the buffer. */ bufsize = (size_t)end + 1; } @@ -1094,11 +1112,13 @@ _io_FileIO_truncate_impl(fileio *self, PyTypeObject *cls, PyObject *posobj) return NULL; } - /* Sometimes a large file is truncated. While estimated_size is used as a - estimate, that it is much larger than the actual size can result in a - significant over allocation and sometimes a MemoryError / running out of - memory. */ - self->estimated_size = pos; + /* Since the file was truncated, its size at open is no longer accurate + as an estimate. Clear out the stat result, and rely on dynamic resize + code if a readall is requested. */ + if (self->stat_atopen != NULL) { + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; + } return posobj; } @@ -1229,16 +1249,27 @@ get_mode(fileio *self, void *closure) return PyUnicode_FromString(mode_string(self)); } +static PyObject * +get_blksize(fileio *self, void *closure) +{ +#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE + if (self->stat_atopen != NULL && self->stat_atopen->st_blksize > 1) { + return PyLong_FromLong(self->stat_atopen->st_blksize); + } +#endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */ + return PyLong_FromLong(DEFAULT_BUFFER_SIZE); +} + static PyGetSetDef fileio_getsetlist[] = { {"closed", (getter)get_closed, NULL, "True if the file is closed"}, {"closefd", (getter)get_closefd, NULL, "True if the file descriptor will be closed by close()."}, {"mode", (getter)get_mode, NULL, "String giving the file mode"}, + {"_blksize", (getter)get_blksize, NULL, "Stat st_blksize if available"}, {NULL}, }; static PyMemberDef fileio_members[] = { - {"_blksize", Py_T_UINT, offsetof(fileio, blksize), 0}, {"_finalizing", Py_T_BOOL, offsetof(fileio, finalizing), 0}, {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(fileio, weakreflist), Py_READONLY}, {"__dictoffset__", Py_T_PYSSIZET, offsetof(fileio, dict), Py_READONLY}, From 5cd50cb6eb28e525f0c838e049e900ea982a5a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 18 Sep 2024 18:42:33 +0200 Subject: [PATCH 013/105] gh-122145: Handle an empty AST body when reporting tracebacks (#122161) --- Lib/test/test_traceback.py | 35 +++++++++++++++++++ Lib/traceback.py | 2 ++ ...-07-23-12-38-14.gh-issue-122145.sTO8nX.rst | 3 ++ 3 files changed, 40 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-07-23-12-38-14.gh-issue-122145.sTO8nX.rst diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index b568221212d71f..455fea034198a6 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -3307,6 +3307,41 @@ def format_frame_summary(self, frame_summary, colorize=False): f' File "{__file__}", line {lno}, in f\n 1/0\n' ) + def test_summary_should_show_carets(self): + # See: https://github.com/python/cpython/issues/122353 + + # statement to execute and to get a ZeroDivisionError for a traceback + statement = "abcdef = 1 / 0 and 2.0" + colno = statement.index('1 / 0') + end_colno = colno + len('1 / 0') + + # Actual line to use when rendering the traceback + # and whose AST will be extracted (it will be empty). + cached_line = '# this line will be used during rendering' + self.addCleanup(unlink, TESTFN) + with open(TESTFN, "w") as file: + file.write(cached_line) + linecache.updatecache(TESTFN, {}) + + try: + exec(compile(statement, TESTFN, "exec")) + except ZeroDivisionError as exc: + # This is the simplest way to create a StackSummary + # whose FrameSummary items have their column offsets. + s = traceback.TracebackException.from_exception(exc).stack + self.assertIsInstance(s, traceback.StackSummary) + with unittest.mock.patch.object(s, '_should_show_carets', + wraps=s._should_show_carets) as ff: + self.assertEqual(len(s), 2) + self.assertListEqual( + s.format_frame_summary(s[1]).splitlines(), + [ + f' File "{TESTFN}", line 1, in ', + f' {cached_line}' + ] + ) + ff.assert_called_with(colno, end_colno, [cached_line], None) + class Unrepresentable: def __repr__(self) -> str: raise Exception("Unrepresentable") diff --git a/Lib/traceback.py b/Lib/traceback.py index 3e708c6f86a4c5..0fe7187a0c6193 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -698,6 +698,8 @@ def _should_show_carets(self, start_offset, end_offset, all_lines, anchors): with suppress(SyntaxError, ImportError): import ast tree = ast.parse('\n'.join(all_lines)) + if not tree.body: + return False statement = tree.body[0] value = None def _spawns_full_line(value): diff --git a/Misc/NEWS.d/next/Library/2024-07-23-12-38-14.gh-issue-122145.sTO8nX.rst b/Misc/NEWS.d/next/Library/2024-07-23-12-38-14.gh-issue-122145.sTO8nX.rst new file mode 100644 index 00000000000000..a4282f12d9742a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-23-12-38-14.gh-issue-122145.sTO8nX.rst @@ -0,0 +1,3 @@ +Fix an issue when reporting tracebacks corresponding to Python code +emitting an empty AST body. +Patch by Nikita Sobolev and Bénédikt Tran. From f9fa6ba4f8d90ae12bc1f6a792d66903bb169ba8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 Sep 2024 19:10:56 +0200 Subject: [PATCH 014/105] gh-124064: Fix -Wconversion warnings in Parser/string_parser.c (#124204) Fix integer overflow check in decode_unicode_with_escapes(): use PY_SSIZE_T_MAX instead of SIZE_MAX. --- Parser/string_parser.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Parser/string_parser.c b/Parser/string_parser.c index 93ad92b823581e..9537c543b0eb93 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -18,7 +18,7 @@ warn_invalid_escape_sequence(Parser *p, const char *first_invalid_escape, Token // to avoid showing the warning twice. return 0; } - unsigned char c = *first_invalid_escape; + unsigned char c = (unsigned char)*first_invalid_escape; if ((t->type == FSTRING_MIDDLE || t->type == FSTRING_END) && (c == '{' || c == '}')) { // in this case the tokenizer has already emitted a warning, // see Parser/tokenizer/helpers.c:warn_invalid_escape_sequence @@ -90,12 +90,12 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t) const char *end; /* check for integer overflow */ - if (len > SIZE_MAX / 6) { + if (len > (size_t)PY_SSIZE_T_MAX / 6) { return NULL; } /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5 "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */ - u = PyBytes_FromStringAndSize((char *)NULL, len * 6); + u = PyBytes_FromStringAndSize((char *)NULL, (Py_ssize_t)len * 6); if (u == NULL) { return NULL; } @@ -142,11 +142,11 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t) *p++ = *s++; } } - len = p - buf; + len = (size_t)(p - buf); s = buf; const char *first_invalid_escape; - v = _PyUnicode_DecodeUnicodeEscapeInternal(s, len, NULL, NULL, &first_invalid_escape); + v = _PyUnicode_DecodeUnicodeEscapeInternal(s, (Py_ssize_t)len, NULL, NULL, &first_invalid_escape); // HACK: later we can simply pass the line no, since we don't preserve the tokens // when we are decoding the string but we preserve the line numbers. @@ -185,7 +185,7 @@ PyObject * _PyPegen_decode_string(Parser *p, int raw, const char *s, size_t len, Token *t) { if (raw) { - return PyUnicode_DecodeUTF8Stateful(s, len, NULL, NULL); + return PyUnicode_DecodeUTF8Stateful(s, (Py_ssize_t)len, NULL, NULL); } return decode_unicode_with_escapes(p, s, len, t); } @@ -274,9 +274,9 @@ _PyPegen_parse_string(Parser *p, Token *t) } } if (rawmode) { - return PyBytes_FromStringAndSize(s, len); + return PyBytes_FromStringAndSize(s, (Py_ssize_t)len); } - return decode_bytes_with_escapes(p, s, len, t); + return decode_bytes_with_escapes(p, s, (Py_ssize_t)len, t); } return _PyPegen_decode_string(p, rawmode, s, len, t); } From 21d2a9ab2f4dcbf1be462d3b7f7a231a46bc1cb7 Mon Sep 17 00:00:00 2001 From: Tomas R Date: Wed, 18 Sep 2024 19:28:22 +0200 Subject: [PATCH 015/105] gh-116022: Improve `repr()` of AST nodes (#117046) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: AN Long Co-authored-by: Jelle Zijlstra Co-authored-by: Alex Waygood Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/ast.rst | 5 + Lib/test/test_ast/data/ast_repr.txt | 209 +++++++++++++++++ Lib/test/test_ast/test_ast.py | 24 +- Makefile.pre.in | 1 + ...-03-19-22-21-22.gh-issue-116022.iyHENN.rst | 1 + Parser/asdl_c.py | 222 ++++++++++++++++++ Python/Python-ast.c | 222 ++++++++++++++++++ 7 files changed, 682 insertions(+), 2 deletions(-) create mode 100644 Lib/test/test_ast/data/ast_repr.txt create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-19-22-21-22.gh-issue-116022.iyHENN.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index f2994739b48932..55007624c876fa 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -134,6 +134,11 @@ Node classes Simple indices are represented by their value, extended slices are represented as tuples. +.. versionchanged:: 3.14 + + The :meth:`~object.__repr__` output of :class:`~ast.AST` nodes includes + the values of the node fields. + .. deprecated:: 3.8 Old classes :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, diff --git a/Lib/test/test_ast/data/ast_repr.txt b/Lib/test/test_ast/data/ast_repr.txt new file mode 100644 index 00000000000000..3778b9e70a4605 --- /dev/null +++ b/Lib/test/test_ast/data/ast_repr.txt @@ -0,0 +1,209 @@ +Module(body=[Expr(value=Constant(value='module docstring', kind=None))], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=arg(...), defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...), ..., arg(...)], vararg=arg(...), kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=arg(...), defaults=[Constant(...), ..., Dict(...)]), body=[Expr(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Expr(value=Constant(...))], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[Name(id='object', ctx=Load(...))], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[Name(id='A', ctx=Load(...)), Name(id='B', ctx=Load(...))], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Return(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Return(value=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Delete(targets=[Name(id='v', ctx=Del(...))])], type_ignores=[]) +Module(body=[Assign(targets=[Name(id='v', ctx=Store(...))], value=Constant(value=1, kind=None), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Tuple(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Tuple(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[List(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Subscript(value=Name(...), slice=Name(...), ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Add(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Sub(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Mult(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=MatMult(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Div(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Mod(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Pow(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=LShift(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=RShift(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitOr(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitXor(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitAnd(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=FloorDiv(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[While(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[While(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[Pass()])], type_ignores=[]) +Module(body=[If(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[])])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[Pass()])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[Pass(...)])])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[If(...)])])], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None), withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...)), withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None), withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[Raise(exc=None, cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Call(func=Name(...), args=[Constant(...)], keywords=[]), cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Name(id='Exception', ctx=Load(...)), cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Call(func=Name(...), args=[Constant(...)], keywords=[]), cause=Constant(value=None, kind=None))], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[], orelse=[], finalbody=[Pass()])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=None)], type_ignores=[]) +Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=Constant(value='message', kind=None))], type_ignores=[]) +Module(body=[Import(names=[alias(name='sys', asname=None)])], type_ignores=[]) +Module(body=[Import(names=[alias(name='foo', asname='bar')])], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0)], type_ignores=[]) +Module(body=[Global(names=['v'])], type_ignores=[]) +Module(body=[Expr(value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[Pass()], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Break()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Continue()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Tuple(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Tuple(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=List(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...), comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Constant(...)), Expr(value=Await(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[AsyncFor(target=Name(...), iter=Name(...), body=[Expr(...)], orelse=[Expr(...)], type_comment=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[AsyncWith(items=[withitem(...)], body=[Expr(...)], type_comment=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[None, Constant(...)], values=[Dict(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Starred(...), Constant(...)]))], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Yield(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=YieldFrom(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=ListComp(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Call(func=Name(...), args=[GeneratorExp(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Attribute(value=Attribute(...), attr='c', ctx=Load(...))], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Expr(value=NamedExpr(target=Name(...), value=Constant(...)))], type_ignores=[]) +Module(body=[If(test=NamedExpr(target=Name(...), value=Call(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[While(test=NamedExpr(target=Name(...), value=Call(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...), ..., arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...), arg(...)], kw_defaults=[None, None], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...), arg(...)], kw_defaults=[None, None], kwarg=arg(...), defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...), arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...), ..., Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=None, defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[None], kwarg=None, defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=arg(...), defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[None], kwarg=arg(...), defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[], value=Name(id='int', ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=None, default_value=None)], value=Name(id='int', ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=None, default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=None, default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))])], type_ignores=[]) +Module(body=[Match(subject=Name(id='x', ctx=Load(...)), cases=[match_case(pattern=MatchValue(...), guard=None, body=[Pass(...)])])], type_ignores=[]) +Module(body=[Match(subject=Name(id='x', ctx=Load(...)), cases=[match_case(pattern=MatchValue(...), guard=None, body=[Pass(...)]), match_case(pattern=MatchAs(...), guard=None, body=[Pass(...)])])], type_ignores=[]) +Module(body=[Expr(value=Constant(value=None, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=True, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=False, kind=None))], type_ignores=[]) +Module(body=[Expr(value=BoolOp(op=And(...), values=[Name(...), Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=BoolOp(op=Or(...), values=[Name(...), Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Add(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Sub(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Mult(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Div(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=MatMult(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=FloorDiv(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Pow(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Mod(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=RShift(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=LShift(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitXor(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitOr(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitAnd(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=Not(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=UAdd(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=USub(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=Invert(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=Lambda(args=arguments(...), body=Constant(...)))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[Constant(...)], values=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[], values=[]))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[Constant(...)], values=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[Constant(...), Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Constant(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Constant(...), ops=[Lt(...), Lt(...)], comparators=[Constant(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[Eq(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[LtE(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[GtE(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[NotEq(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[Is(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[IsNot(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[In(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[NotIn(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[Constant(...), ..., Starred(...)], keywords=[keyword(...), keyword(...)]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[Starred(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[GeneratorExp(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=10, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=1j, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value='string', kind=None))], type_ignores=[]) +Module(body=[Expr(value=Attribute(value=Name(...), attr='b', ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=Name(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Name(id='v', ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Attribute(...), args=[Subscript(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=IfExp(test=Name(...), body=Call(...), orelse=Call(...)))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[Constant(...), ..., Constant(...)]))], type_ignores=[]) \ No newline at end of file diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 77596eca5d8b74..f052822cb45273 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -10,6 +10,7 @@ import types import unittest import weakref +from pathlib import Path from textwrap import dedent try: import _testinternalcapi @@ -29,6 +30,16 @@ STDLIB_FILES = [fn for fn in os.listdir(STDLIB) if fn.endswith(".py")] STDLIB_FILES.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) +AST_REPR_DATA_FILE = Path(__file__).parent / "data" / "ast_repr.txt" + +def ast_repr_get_test_cases() -> list[str]: + return exec_tests + eval_tests + + +def ast_repr_update_snapshots() -> None: + data = [repr(ast.parse(test)) for test in ast_repr_get_test_cases()] + AST_REPR_DATA_FILE.write_text("\n".join(data)) + class AST_Tests(unittest.TestCase): maxDiff = None @@ -408,7 +419,7 @@ def test_invalid_sum(self): m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) with self.assertRaises(TypeError) as cm: compile(m, "", "exec") - self.assertIn("but got None: for node, attr, source in tests: self.assert_none_check(node, attr, source) + def test_repr(self) -> None: + snapshots = AST_REPR_DATA_FILE.read_text().split("\n") + for test, snapshot in zip(ast_repr_get_test_cases(), snapshots, strict=True): + with self.subTest(test_input=test): + self.assertEqual(repr(ast.parse(test)), snapshot) + class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" @@ -3332,5 +3349,8 @@ def test_folding_type_param_in_type_alias(self): self.assert_ast(result_code, non_optimized_target, optimized_target) -if __name__ == "__main__": +if __name__ == '__main__': + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + ast_repr_update_snapshots() + sys.exit(0) unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 4947680985e831..a4d99262702a17 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2416,6 +2416,7 @@ LIBSUBDIRS= asyncio \ TESTSUBDIRS= idlelib/idle_test \ test \ test/test_ast \ + test/test_ast/data \ test/archivetestdata \ test/audiodata \ test/certdata \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-19-22-21-22.gh-issue-116022.iyHENN.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-19-22-21-22.gh-issue-116022.iyHENN.rst new file mode 100644 index 00000000000000..659ffb289129e2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-19-22-21-22.gh-issue-116022.iyHENN.rst @@ -0,0 +1 @@ +Improve the :meth:`~object.__repr__` output of :class:`~ast.AST` nodes. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 9fed69b12479d6..fac9a7740a1fe6 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1435,8 +1435,230 @@ def visitModule(self, mod): {NULL} }; +static PyObject * +ast_repr_max_depth(AST_object *self, int depth); + +/* Format list and tuple properties of AST nodes. + Note that, only the first and last elements are shown. + Anything in between is represented with an ellipsis ('...'). + For example, the list [1, 2, 3] is formatted as + 'List(elts=[Constant(1), ..., Constant(3)])'. */ +static PyObject * +ast_repr_list(PyObject *list, int depth) +{ + assert(PyList_Check(list) || PyTuple_Check(list)); + + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + + Py_ssize_t length = PySequence_Size(list); + if (length < 0) { + return NULL; + } + else if (length == 0) { + return PyObject_Repr(list); + } + + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + writer.overallocate = 1; + PyObject *items[2] = {NULL, NULL}; + + items[0] = PySequence_GetItem(list, 0); + if (!items[0]) { + goto error; + } + if (length > 1) { + items[1] = PySequence_GetItem(list, length - 1); + if (!items[1]) { + goto error; + } + } + + bool is_list = PyList_Check(list); + if (_PyUnicodeWriter_WriteChar(&writer, is_list ? '[' : '(') < 0) { + goto error; + } + + for (Py_ssize_t i = 0; i < Py_MIN(length, 2); i++) { + PyObject *item = items[i]; + PyObject *item_repr; + + if (PyType_IsSubtype(Py_TYPE(item), (PyTypeObject *)state->AST_type)) { + item_repr = ast_repr_max_depth((AST_object*)item, depth - 1); + } else { + item_repr = PyObject_Repr(item); + } + if (!item_repr) { + goto error; + } + if (i > 0) { + if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { + goto error; + } + } + if (_PyUnicodeWriter_WriteStr(&writer, item_repr) < 0) { + Py_DECREF(item_repr); + goto error; + } + if (i == 0 && length > 2) { + if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ...", 5) < 0) { + Py_DECREF(item_repr); + goto error; + } + } + Py_DECREF(item_repr); + } + + if (_PyUnicodeWriter_WriteChar(&writer, is_list ? ']' : ')') < 0) { + goto error; + } + + Py_XDECREF(items[0]); + Py_XDECREF(items[1]); + return _PyUnicodeWriter_Finish(&writer); + +error: + Py_XDECREF(items[0]); + Py_XDECREF(items[1]); + _PyUnicodeWriter_Dealloc(&writer); + return NULL; +} + +static PyObject * +ast_repr_max_depth(AST_object *self, int depth) +{ + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + + if (depth <= 0) { + return PyUnicode_FromFormat("%s(...)", Py_TYPE(self)->tp_name); + } + + int status = Py_ReprEnter((PyObject *)self); + if (status != 0) { + if (status < 0) { + return NULL; + } + return PyUnicode_FromFormat("%s(...)", Py_TYPE(self)->tp_name); + } + + PyObject *fields; + if (PyObject_GetOptionalAttr((PyObject *)Py_TYPE(self), state->_fields, &fields) < 0) { + Py_ReprLeave((PyObject *)self); + return NULL; + } + + Py_ssize_t numfields = PySequence_Size(fields); + if (numfields < 0) { + Py_ReprLeave((PyObject *)self); + Py_DECREF(fields); + return NULL; + } + + if (numfields == 0) { + Py_ReprLeave((PyObject *)self); + Py_DECREF(fields); + return PyUnicode_FromFormat("%s()", Py_TYPE(self)->tp_name); + } + + const char* tp_name = Py_TYPE(self)->tp_name; + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + writer.overallocate = 1; + + if (_PyUnicodeWriter_WriteASCIIString(&writer, tp_name, strlen(tp_name)) < 0) { + goto error; + } + if (_PyUnicodeWriter_WriteChar(&writer, '(') < 0) { + goto error; + } + + for (Py_ssize_t i = 0; i < numfields; i++) { + PyObject *name = PySequence_GetItem(fields, i); + if (!name) { + goto error; + } + + PyObject *value = PyObject_GetAttr((PyObject *)self, name); + if (!value) { + Py_DECREF(name); + goto error; + } + + PyObject *value_repr; + if (PyList_Check(value) || PyTuple_Check(value)) { + value_repr = ast_repr_list(value, depth); + } + else if (PyType_IsSubtype(Py_TYPE(value), (PyTypeObject *)state->AST_type)) { + value_repr = ast_repr_max_depth((AST_object*)value, depth - 1); + } + else { + value_repr = PyObject_Repr(value); + } + + Py_DECREF(value); + + if (!value_repr) { + Py_DECREF(name); + Py_DECREF(value); + goto error; + } + + if (i > 0) { + if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { + Py_DECREF(name); + Py_DECREF(value_repr); + goto error; + } + } + if (_PyUnicodeWriter_WriteStr(&writer, name) < 0) { + Py_DECREF(name); + Py_DECREF(value_repr); + goto error; + } + + Py_DECREF(name); + + if (_PyUnicodeWriter_WriteChar(&writer, '=') < 0) { + Py_DECREF(value_repr); + goto error; + } + if (_PyUnicodeWriter_WriteStr(&writer, value_repr) < 0) { + Py_DECREF(value_repr); + goto error; + } + + Py_DECREF(value_repr); + } + + if (_PyUnicodeWriter_WriteChar(&writer, ')') < 0) { + goto error; + } + Py_ReprLeave((PyObject *)self); + Py_DECREF(fields); + return _PyUnicodeWriter_Finish(&writer); + +error: + Py_ReprLeave((PyObject *)self); + Py_DECREF(fields); + _PyUnicodeWriter_Dealloc(&writer); + return NULL; +} + +static PyObject * +ast_repr(AST_object *self) +{ + return ast_repr_max_depth(self, 3); +} + static PyType_Slot AST_type_slots[] = { {Py_tp_dealloc, ast_dealloc}, + {Py_tp_repr, ast_repr}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_setattro, PyObject_GenericSetAttr}, {Py_tp_traverse, ast_traverse}, diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 4d0db457a8b172..860447ef9ed702 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -5636,8 +5636,230 @@ static PyGetSetDef ast_type_getsets[] = { {NULL} }; +static PyObject * +ast_repr_max_depth(AST_object *self, int depth); + +/* Format list and tuple properties of AST nodes. + Note that, only the first and last elements are shown. + Anything in between is represented with an ellipsis ('...'). + For example, the list [1, 2, 3] is formatted as + 'List(elts=[Constant(1), ..., Constant(3)])'. */ +static PyObject * +ast_repr_list(PyObject *list, int depth) +{ + assert(PyList_Check(list) || PyTuple_Check(list)); + + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + + Py_ssize_t length = PySequence_Size(list); + if (length < 0) { + return NULL; + } + else if (length == 0) { + return PyObject_Repr(list); + } + + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + writer.overallocate = 1; + PyObject *items[2] = {NULL, NULL}; + + items[0] = PySequence_GetItem(list, 0); + if (!items[0]) { + goto error; + } + if (length > 1) { + items[1] = PySequence_GetItem(list, length - 1); + if (!items[1]) { + goto error; + } + } + + bool is_list = PyList_Check(list); + if (_PyUnicodeWriter_WriteChar(&writer, is_list ? '[' : '(') < 0) { + goto error; + } + + for (Py_ssize_t i = 0; i < Py_MIN(length, 2); i++) { + PyObject *item = items[i]; + PyObject *item_repr; + + if (PyType_IsSubtype(Py_TYPE(item), (PyTypeObject *)state->AST_type)) { + item_repr = ast_repr_max_depth((AST_object*)item, depth - 1); + } else { + item_repr = PyObject_Repr(item); + } + if (!item_repr) { + goto error; + } + if (i > 0) { + if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { + goto error; + } + } + if (_PyUnicodeWriter_WriteStr(&writer, item_repr) < 0) { + Py_DECREF(item_repr); + goto error; + } + if (i == 0 && length > 2) { + if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ...", 5) < 0) { + Py_DECREF(item_repr); + goto error; + } + } + Py_DECREF(item_repr); + } + + if (_PyUnicodeWriter_WriteChar(&writer, is_list ? ']' : ')') < 0) { + goto error; + } + + Py_XDECREF(items[0]); + Py_XDECREF(items[1]); + return _PyUnicodeWriter_Finish(&writer); + +error: + Py_XDECREF(items[0]); + Py_XDECREF(items[1]); + _PyUnicodeWriter_Dealloc(&writer); + return NULL; +} + +static PyObject * +ast_repr_max_depth(AST_object *self, int depth) +{ + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + + if (depth <= 0) { + return PyUnicode_FromFormat("%s(...)", Py_TYPE(self)->tp_name); + } + + int status = Py_ReprEnter((PyObject *)self); + if (status != 0) { + if (status < 0) { + return NULL; + } + return PyUnicode_FromFormat("%s(...)", Py_TYPE(self)->tp_name); + } + + PyObject *fields; + if (PyObject_GetOptionalAttr((PyObject *)Py_TYPE(self), state->_fields, &fields) < 0) { + Py_ReprLeave((PyObject *)self); + return NULL; + } + + Py_ssize_t numfields = PySequence_Size(fields); + if (numfields < 0) { + Py_ReprLeave((PyObject *)self); + Py_DECREF(fields); + return NULL; + } + + if (numfields == 0) { + Py_ReprLeave((PyObject *)self); + Py_DECREF(fields); + return PyUnicode_FromFormat("%s()", Py_TYPE(self)->tp_name); + } + + const char* tp_name = Py_TYPE(self)->tp_name; + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + writer.overallocate = 1; + + if (_PyUnicodeWriter_WriteASCIIString(&writer, tp_name, strlen(tp_name)) < 0) { + goto error; + } + if (_PyUnicodeWriter_WriteChar(&writer, '(') < 0) { + goto error; + } + + for (Py_ssize_t i = 0; i < numfields; i++) { + PyObject *name = PySequence_GetItem(fields, i); + if (!name) { + goto error; + } + + PyObject *value = PyObject_GetAttr((PyObject *)self, name); + if (!value) { + Py_DECREF(name); + goto error; + } + + PyObject *value_repr; + if (PyList_Check(value) || PyTuple_Check(value)) { + value_repr = ast_repr_list(value, depth); + } + else if (PyType_IsSubtype(Py_TYPE(value), (PyTypeObject *)state->AST_type)) { + value_repr = ast_repr_max_depth((AST_object*)value, depth - 1); + } + else { + value_repr = PyObject_Repr(value); + } + + Py_DECREF(value); + + if (!value_repr) { + Py_DECREF(name); + Py_DECREF(value); + goto error; + } + + if (i > 0) { + if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { + Py_DECREF(name); + Py_DECREF(value_repr); + goto error; + } + } + if (_PyUnicodeWriter_WriteStr(&writer, name) < 0) { + Py_DECREF(name); + Py_DECREF(value_repr); + goto error; + } + + Py_DECREF(name); + + if (_PyUnicodeWriter_WriteChar(&writer, '=') < 0) { + Py_DECREF(value_repr); + goto error; + } + if (_PyUnicodeWriter_WriteStr(&writer, value_repr) < 0) { + Py_DECREF(value_repr); + goto error; + } + + Py_DECREF(value_repr); + } + + if (_PyUnicodeWriter_WriteChar(&writer, ')') < 0) { + goto error; + } + Py_ReprLeave((PyObject *)self); + Py_DECREF(fields); + return _PyUnicodeWriter_Finish(&writer); + +error: + Py_ReprLeave((PyObject *)self); + Py_DECREF(fields); + _PyUnicodeWriter_Dealloc(&writer); + return NULL; +} + +static PyObject * +ast_repr(AST_object *self) +{ + return ast_repr_max_depth(self, 3); +} + static PyType_Slot AST_type_slots[] = { {Py_tp_dealloc, ast_dealloc}, + {Py_tp_repr, ast_repr}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_setattro, PyObject_GenericSetAttr}, {Py_tp_traverse, ast_traverse}, From 9a6e2336e4b54fc13064b77826a67b03b3b45133 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 18 Sep 2024 16:02:20 -0500 Subject: [PATCH 016/105] Fix `make htmllive` target (GH-124219) Allow `make -C Doc htmllive` to work without manual venv activation Set PATH to ensure that `sphinx-autobuild` can find `sphinx-build`. --- Doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/Makefile b/Doc/Makefile index 505b8953b50b32..fc0d1e4d9a2a55 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -152,7 +152,7 @@ htmlview: html $(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))" .PHONY: htmllive -htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild +htmllive: SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-autobuild htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0 htmllive: _ensure-sphinx-autobuild html From 36682c091407dc9c7e750c22fb71e62466952662 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Thu, 19 Sep 2024 00:22:00 +0300 Subject: [PATCH 017/105] gh-124083: Skip test_signal.test_strsignal() on NetBSD (#124084) Skip test_strsignal() on NetBSD due to TypeError. --- Lib/test/test_signal.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index da36b8576be1ad..704a0090bdbc0f 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -123,6 +123,8 @@ def __repr__(self): self.assertEqual(signal.getsignal(signal.SIGHUP), hup) self.assertEqual(0, argument.repr_count) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124083: strsignal is not supported on NetBSD") def test_strsignal(self): self.assertIn("Interrupt", signal.strsignal(signal.SIGINT)) self.assertIn("Terminated", signal.strsignal(signal.SIGTERM)) From ea7fe1fe2e162f2375562467ad834c6224a62daf Mon Sep 17 00:00:00 2001 From: Jacek Date: Thu, 19 Sep 2024 00:05:18 +0200 Subject: [PATCH 018/105] gh-124212: Fix undefined variable in error message in venv (GH-124211) --- Lib/test/test_venv.py | 15 +++++++++++++++ Lib/venv/__init__.py | 2 +- ...2024-09-18-17-45-52.gh-issue-124212.n6kIby.rst | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-18-17-45-52.gh-issue-124212.n6kIby.rst diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index b9fcc59d49668c..1ef08da326c18c 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -504,6 +504,21 @@ def test_unicode_in_batch_file(self): ) self.assertEqual(out.strip(), '0') + @unittest.skipUnless(os.name == 'nt' and can_symlink(), + 'symlinks on Windows') + def test_failed_symlink(self): + """ + Test handling of failed symlinks on Windows. + """ + rmtree(self.env_dir) + env_dir = os.path.join(os.path.realpath(self.env_dir), 'venv') + with patch('os.symlink') as mock_symlink: + mock_symlink.side_effect = OSError() + builder = venv.EnvBuilder(clear=True, symlinks=True) + _, err = self.run_with_capture(builder.create, env_dir) + filepath_regex = r"'[A-Z]:\\\\(?:[^\\\\]+\\\\)*[^\\\\]+'" + self.assertRegex(err, rf"Unable to symlink {filepath_regex} to {filepath_regex}") + @requireVenvCreate def test_multiprocessing(self): """ diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index fa69d5846f2fa7..028e9483196694 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -393,7 +393,7 @@ def setup_python(self, context): os.symlink(src, dest) to_unlink.append(dest) except OSError: - logger.warning('Unable to symlink %r to %r', src, dst) + logger.warning('Unable to symlink %r to %r', src, dest) do_copies = True for f in to_unlink: try: diff --git a/Misc/NEWS.d/next/Library/2024-09-18-17-45-52.gh-issue-124212.n6kIby.rst b/Misc/NEWS.d/next/Library/2024-09-18-17-45-52.gh-issue-124212.n6kIby.rst new file mode 100644 index 00000000000000..7848f26511e282 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-18-17-45-52.gh-issue-124212.n6kIby.rst @@ -0,0 +1 @@ +Fix invalid variable in :mod:`venv` handling of failed symlink on Windows From 43cd7aa8cd88624f7211e47b98bc1e8e63e7660f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 19 Sep 2024 00:11:50 +0200 Subject: [PATCH 019/105] gh-120754: Fix memory leak in FileIO.__init__() (#124225) Free 'self->stat_atopen' before assigning it, since io.FileIO.__init__() can be called multiple times manually (especially by test_io). --- Modules/_io/fileio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 865b0e3634f3b4..8dae465fd20f8b 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -457,6 +457,7 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, #endif } + PyMem_Free(self->stat_atopen); self->stat_atopen = PyMem_New(struct _Py_stat_struct, 1); if (self->stat_atopen == NULL) { PyErr_NoMemory(); From 7628f67d55cb65bad9c9266e0457e468cd7e3775 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 19 Sep 2024 10:55:47 +0300 Subject: [PATCH 020/105] gh-123934: Fix `MagicMock` not to reset magic method return values (#124038) --- .../testmock/testmagicmethods.py | 39 +++++++++++++++++++ Lib/unittest/mock.py | 13 ++++++- ...-09-13-10-34-19.gh-issue-123934.yMe7mL.rst | 2 + 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-13-10-34-19.gh-issue-123934.yMe7mL.rst diff --git a/Lib/test/test_unittest/testmock/testmagicmethods.py b/Lib/test/test_unittest/testmock/testmagicmethods.py index 5ca753b8f20811..2a8aa11b3284f6 100644 --- a/Lib/test/test_unittest/testmock/testmagicmethods.py +++ b/Lib/test/test_unittest/testmock/testmagicmethods.py @@ -331,6 +331,45 @@ def test_magic_methods_fspath(self): self.assertEqual(os.fspath(mock), expected_path) mock.__fspath__.assert_called_once() + def test_magic_mock_does_not_reset_magic_returns(self): + # https://github.com/python/cpython/issues/123934 + for reset in (True, False): + with self.subTest(reset=reset): + mm = MagicMock() + self.assertIs(type(mm.__str__()), str) + mm.__str__.assert_called_once() + + self.assertIs(type(mm.__hash__()), int) + mm.__hash__.assert_called_once() + + for _ in range(3): + # Repeat reset several times to be sure: + mm.reset_mock(return_value=reset) + + self.assertIs(type(mm.__str__()), str) + mm.__str__.assert_called_once() + + self.assertIs(type(mm.__hash__()), int) + mm.__hash__.assert_called_once() + + def test_magic_mock_resets_manual_mocks(self): + mm = MagicMock() + mm.__iter__ = MagicMock(return_value=iter([1])) + mm.custom = MagicMock(return_value=2) + self.assertEqual(list(iter(mm)), [1]) + self.assertEqual(mm.custom(), 2) + + mm.reset_mock(return_value=True) + self.assertEqual(list(iter(mm)), []) + self.assertIsInstance(mm.custom(), MagicMock) + + def test_magic_mock_resets_manual_mocks_empty_iter(self): + mm = MagicMock() + mm.__iter__.return_value = [] + self.assertEqual(list(iter(mm)), []) + + mm.reset_mock(return_value=True) + self.assertEqual(list(iter(mm)), []) def test_magic_methods_and_spec(self): class Iterable(object): diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 1fa90277e08ec3..bb34c7436047ad 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -628,7 +628,7 @@ def __set_side_effect(self, value): side_effect = property(__get_side_effect, __set_side_effect) - def reset_mock(self, visited=None,*, return_value=False, side_effect=False): + def reset_mock(self, visited=None, *, return_value=False, side_effect=False): "Restore the mock object to its initial state." if visited is None: visited = [] @@ -2218,6 +2218,17 @@ def mock_add_spec(self, spec, spec_set=False): self._mock_add_spec(spec, spec_set) self._mock_set_magics() + def reset_mock(self, /, *args, return_value=False, **kwargs): + if ( + return_value + and self._mock_name + and _is_magic(self._mock_name) + ): + # Don't reset return values for magic methods, + # otherwise `m.__str__` will start + # to return `MagicMock` instances, instead of `str` instances. + return_value = False + super().reset_mock(*args, return_value=return_value, **kwargs) class MagicProxy(Base): diff --git a/Misc/NEWS.d/next/Library/2024-09-13-10-34-19.gh-issue-123934.yMe7mL.rst b/Misc/NEWS.d/next/Library/2024-09-13-10-34-19.gh-issue-123934.yMe7mL.rst new file mode 100644 index 00000000000000..cec7741bcabbda --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-13-10-34-19.gh-issue-123934.yMe7mL.rst @@ -0,0 +1,2 @@ +Fix :class:`unittest.mock.MagicMock` reseting magic methods return values +after ``.reset_mock(return_value=True)`` was called. From 4420cf4dc9ef7bd3c1c9b5465fa9397304bf0110 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Thu, 19 Sep 2024 11:11:03 +0300 Subject: [PATCH 021/105] gh-124040: Adjust few tests in testHypot/testDist to get exactly computed results (GH-124042) --- Lib/test/test_math.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index cf452960729aba..a3eebc97ada23b 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -812,11 +812,13 @@ def testHypot(self): # Test allowable types (those with __float__) self.assertEqual(hypot(12.0, 5.0), 13.0) self.assertEqual(hypot(12, 5), 13) - self.assertEqual(hypot(1, -1), math.sqrt(2)) - self.assertEqual(hypot(1, FloatLike(-1.)), math.sqrt(2)) + self.assertEqual(hypot(0.75, -1), 1.25) + self.assertEqual(hypot(-1, 0.75), 1.25) + self.assertEqual(hypot(0.75, FloatLike(-1.)), 1.25) + self.assertEqual(hypot(FloatLike(-1.), 0.75), 1.25) self.assertEqual(hypot(Decimal(12), Decimal(5)), 13) self.assertEqual(hypot(Fraction(12, 32), Fraction(5, 32)), Fraction(13, 32)) - self.assertEqual(hypot(bool(1), bool(0), bool(1), bool(1)), math.sqrt(3)) + self.assertEqual(hypot(True, False, True, True, True), 2.0) # Test corner cases self.assertEqual(hypot(0.0, 0.0), 0.0) # Max input is zero @@ -972,9 +974,9 @@ def testDist(self): self.assertEqual(dist((D(14), D(1)), (D(2), D(-4))), D(13)) self.assertEqual(dist((F(14, 32), F(1, 32)), (F(2, 32), F(-4, 32))), F(13, 32)) - self.assertEqual(dist((True, True, False, True, False), - (True, False, True, True, False)), - sqrt(2.0)) + self.assertEqual(dist((True, True, False, False, True, True), + (True, False, True, False, False, False)), + 2.0) # Test corner cases self.assertEqual(dist((13.25, 12.5, -3.25), From 426569eb8ca1edaa68026aa2bab6b8d1c9105f93 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 19 Sep 2024 06:18:24 -0700 Subject: [PATCH 022/105] Support the "pager" binary in _pyrepl (#122878) Debian (and derivatives) provide a /usr/bin/pager binary, managed by the alternatives system, that always points to an available pager utility. Allow _pyrepl to use it, to follow system policy. This is a very trivial change, from a patch that Debian has been carrying since 2.7 era. Seems appropriate to upstream. https://bugs.debian.org/799555 --- Lib/_pyrepl/pager.py | 2 ++ .../2024-09-19-13-17-31.gh-issue-122878.4iFpsB.rst | 1 + 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-19-13-17-31.gh-issue-122878.4iFpsB.rst diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py index 66dcd99111adfc..1fddc63e3ee3ad 100644 --- a/Lib/_pyrepl/pager.py +++ b/Lib/_pyrepl/pager.py @@ -36,6 +36,8 @@ def get_pager() -> Pager: return plain_pager if sys.platform == 'win32': return lambda text, title='': tempfile_pager(plain(text), 'more <') + if hasattr(os, 'system') and os.system('(pager) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'pager', title) if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: return lambda text, title='': pipe_pager(text, 'less', title) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-19-13-17-31.gh-issue-122878.4iFpsB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-19-13-17-31.gh-issue-122878.4iFpsB.rst new file mode 100644 index 00000000000000..85dd0fd769be68 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-19-13-17-31.gh-issue-122878.4iFpsB.rst @@ -0,0 +1 @@ +Use the ``pager`` binary, if available (e.g. on Debian and derivatives), to display REPL ``help()``. From 8f82d9aa2191db7826bb7a453fe06ce65f966cf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Thu, 19 Sep 2024 16:47:05 +0200 Subject: [PATCH 023/105] bpo-41843: Reenable use of sendfile in shutil module on Solaris (GH-23893) --- Doc/library/shutil.rst | 5 ++++- Lib/shutil.py | 6 +++--- .../next/Library/2020-12-22-18-08-12.bpo-41843.q9Nh2r.rst | 2 ++ 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-12-22-18-08-12.bpo-41843.q9Nh2r.rst diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 220207e5f3cbbf..e623c3df7beba6 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -517,7 +517,7 @@ the use of userspace buffers in Python as in "``outfd.write(infd.read())``". On macOS `fcopyfile`_ is used to copy the file content (not metadata). -On Linux :func:`os.sendfile` is used. +On Linux and Solaris :func:`os.sendfile` is used. On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB instead of 64 KiB) and a :func:`memoryview`-based variant of @@ -529,6 +529,9 @@ file then shutil will silently fallback on using less efficient .. versionchanged:: 3.8 +.. versionchanged:: 3.14 + Solaris now uses :func:`os.sendfile`. + .. _shutil-copytree-example: copytree example diff --git a/Lib/shutil.py b/Lib/shutil.py index 6037092a5e09f2..89c12b76b61dfc 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -48,7 +48,7 @@ # This should never be removed, see rationale in: # https://bugs.python.org/issue43743#msg393429 _USE_CP_SENDFILE = (hasattr(os, "sendfile") - and sys.platform.startswith(("linux", "android"))) + and sys.platform.startswith(("linux", "android", "solaris"))) _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS # CMD defaults in Windows 10 @@ -110,7 +110,7 @@ def _fastcopy_fcopyfile(fsrc, fdst, flags): def _fastcopy_sendfile(fsrc, fdst): """Copy data from one regular mmap-like fd to another by using high-performance sendfile(2) syscall. - This should work on Linux >= 2.6.33 only. + This should work on Linux >= 2.6.33, Android and Solaris. """ # Note: copyfileobj() is left alone in order to not introduce any # unexpected breakage. Possible risks by using zero-copy calls @@ -265,7 +265,7 @@ def copyfile(src, dst, *, follow_symlinks=True): return dst except _GiveupOnFastCopy: pass - # Linux + # Linux / Android / Solaris elif _USE_CP_SENDFILE: try: _fastcopy_sendfile(fsrc, fdst) diff --git a/Misc/NEWS.d/next/Library/2020-12-22-18-08-12.bpo-41843.q9Nh2r.rst b/Misc/NEWS.d/next/Library/2020-12-22-18-08-12.bpo-41843.q9Nh2r.rst new file mode 100644 index 00000000000000..4e525f7ed6a757 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-12-22-18-08-12.bpo-41843.q9Nh2r.rst @@ -0,0 +1,2 @@ +Solaris now uses :func:`os.sendfile` fast-copy syscall for more efficient +:mod:`shutil` file copy related functions. From 7331d0f70bc9fbac177b76b6ec03486430383425 Mon Sep 17 00:00:00 2001 From: luk1337 Date: Thu, 19 Sep 2024 18:05:20 +0200 Subject: [PATCH 024/105] gh-124160: Pass main_tstate to update_global_state_for_extension() (#124164) --- .../C_API/2024-09-18-18-40-30.gh-issue-124160.Zy-VKi.rst | 2 ++ Programs/_testembed.c | 9 +++++++++ Python/import.c | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/C_API/2024-09-18-18-40-30.gh-issue-124160.Zy-VKi.rst diff --git a/Misc/NEWS.d/next/C_API/2024-09-18-18-40-30.gh-issue-124160.Zy-VKi.rst b/Misc/NEWS.d/next/C_API/2024-09-18-18-40-30.gh-issue-124160.Zy-VKi.rst new file mode 100644 index 00000000000000..26e7aef08ea4f3 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-09-18-18-40-30.gh-issue-124160.Zy-VKi.rst @@ -0,0 +1,2 @@ +Fix crash when importing modules containing state and single-phase +initialization in a subinterpreter. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 342cc91cc58ced..10ee6b7be23e21 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -95,6 +95,14 @@ static void _testembed_Py_Initialize(void) } +static int test_import_in_subinterpreters(void) +{ + _testembed_Py_InitializeFromConfig(); + PyThreadState_Swap(Py_NewInterpreter()); + return PyRun_SimpleString("import readline"); // gh-124160 +} + + /***************************************************** * Test repeated initialisation and subinterpreters *****************************************************/ @@ -2398,6 +2406,7 @@ static struct TestCase TestCases[] = { {"test_repeated_init_exec", test_repeated_init_exec}, {"test_repeated_simple_init", test_repeated_simple_init}, {"test_forced_io_encoding", test_forced_io_encoding}, + {"test_import_in_subinterpreters", test_import_in_subinterpreters}, {"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters}, {"test_repeated_init_and_inittab", test_repeated_init_and_inittab}, {"test_pre_initialization_api", test_pre_initialization_api}, diff --git a/Python/import.c b/Python/import.c index a5ea0e2f81df57..26ad84372cea68 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2045,7 +2045,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, singlephase.m_init = p0; } cached = update_global_state_for_extension( - tstate, info->path, info->name, def, &singlephase); + main_tstate, info->path, info->name, def, &singlephase); if (cached == NULL) { assert(PyErr_Occurred()); goto main_finally; From 992e8f6102e317b4967a762fbefea82f9fcf9dfb Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 19 Sep 2024 21:35:12 +0300 Subject: [PATCH 025/105] gh-124245: Fix UserWarning in test_argparse (GH-124246) --- Lib/test/test_argparse.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 584462b741ee99..435db65d52fddb 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -5811,9 +5811,8 @@ def test_invalid_args(self): parser = ErrorRaisingArgumentParser(prog='PROG') parser.add_argument('--foo', nargs="*") parser.add_argument('foo') - with captured_stderr() as stderr: + with self.assertWarns(UserWarning): parser.parse_intermixed_args(['hello', '--foo']) - self.assertIn("UserWarning", stderr.getvalue()) class TestIntermixedMessageContentError(TestCase): # case where Intermixed gives different error message From 5f011115943933ff36adf997c886d73ea88003fb Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 19 Sep 2024 21:50:05 +0300 Subject: [PATCH 026/105] Docs: Update two FAQs for Python 3 (#124247) Co-authored-by: Alex Waygood --- Doc/faq/extending.rst | 7 +++---- Doc/faq/general.rst | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 1cff2c4091df06..3147fda7c37124 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -246,13 +246,12 @@ Then, when you run GDB: I want to compile a Python module on my Linux system, but some files are missing. Why? -------------------------------------------------------------------------------------- -Most packaged versions of Python don't include the -:file:`/usr/lib/python2.{x}/config/` directory, which contains various files +Most packaged versions of Python omit some files required for compiling Python extensions. -For Red Hat, install the python-devel RPM to get the necessary files. +For Red Hat, install the python3-devel RPM to get the necessary files. -For Debian, run ``apt-get install python-dev``. +For Debian, run ``apt-get install python3-dev``. How do I tell "incomplete input" from "invalid input"? ------------------------------------------------------ diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 31df5ebbfb83dd..578777d7f23621 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -309,10 +309,9 @@ guaranteed that interfaces will remain the same throughout a series of bugfix releases. The latest stable releases can always be found on the `Python download page -`_. There are two production-ready versions -of Python: 2.x and 3.x. The recommended version is 3.x, which is supported by -most widely used libraries. Although 2.x is still widely used, :pep:`it is not -maintained anymore <0373>`. +`_. +Python 3.x is the recommended version and supported by most widely used libraries. +Python 2.x :pep:`is not maintained anymore <373>`. How many people are using Python? --------------------------------- From 7a2d77c903f29d7ea08b870b8e3fa2130f667a59 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 19 Sep 2024 22:32:01 +0200 Subject: [PATCH 027/105] test_cext, test_cppext: enable /W4 warnings on Windows (#124253) Add an explicit cast to (void*) and add Py_UNUSED() to fix some warnings in extension.c. --- Lib/test/test_cext/extension.c | 10 ++++++++-- Lib/test/test_cext/setup.py | 2 ++ Lib/test/test_cppext/setup.py | 2 ++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index eb23dbe20353ba..b76abe1d74c628 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -37,7 +37,13 @@ static PyMethodDef _testcext_methods[] = { static int -_testcext_exec(PyObject *module) +_testcext_exec( +#ifdef __STDC_VERSION__ + PyObject *module +#else + PyObject *Py_UNUSED(module) +#endif + ) { #ifdef __STDC_VERSION__ if (PyModule_AddIntMacro(module, __STDC_VERSION__) < 0) { @@ -53,7 +59,7 @@ _testcext_exec(PyObject *module) } static PyModuleDef_Slot _testcext_slots[] = { - {Py_mod_exec, _testcext_exec}, + {Py_mod_exec, (void*)_testcext_exec}, {0, NULL} }; diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 19edc5e663c55d..e97749b45ea6f3 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -31,6 +31,8 @@ else: # MSVC compiler flags CFLAGS = [ + # Display warnings level 1 to 4 + '/W4', # Treat all compiler warnings as compiler errors '/WX', ] diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index f1848f2fd42a46..d97b238b8d1477 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -22,6 +22,8 @@ else: # MSVC compiler flags CPPFLAGS = [ + # Display warnings level 1 to 4 + '/W4', # Treat all compiler warnings as compiler errors '/WX', ] From aae126748ff3d442fdbcd07933855ffd7ae6f59c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 20 Sep 2024 09:20:47 +0300 Subject: [PATCH 028/105] gh-81691: Fix handling of multiple "--" (double dashes) in argparse (GH-124233) Only the first one has now been removed, all subsequent ones are now taken literally. --- Lib/argparse.py | 12 ++-- Lib/test/test_argparse.py | 60 ++++++++++++++++++- ...4-09-19-10-36-18.gh-issue-81691.Hyhp_U.rst | 3 + 3 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-19-10-36-18.gh-issue-81691.Hyhp_U.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index a88a8c65c40a1d..7988c447d03584 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2070,6 +2070,11 @@ def consume_positionals(start_index): # and add the Positional and its args to the list for action, arg_count in zip(positionals, arg_counts): args = arg_strings[start_index: start_index + arg_count] + # Strip out the first '--' if it is not in PARSER or REMAINDER arg. + if (action.nargs not in [PARSER, REMAINDER] + and arg_strings_pattern.find('-', start_index, + start_index + arg_count) >= 0): + args.remove('--') start_index += arg_count if args and action.deprecated and action.dest not in warned: self._warning(_("argument '%(argument_name)s' is deprecated") % @@ -2470,13 +2475,6 @@ def parse_known_intermixed_args(self, args=None, namespace=None): # Value conversion methods # ======================== def _get_values(self, action, arg_strings): - # for everything but PARSER, REMAINDER args, strip out first '--' - if not action.option_strings and action.nargs not in [PARSER, REMAINDER]: - try: - arg_strings.remove('--') - except ValueError: - pass - # optional argument produces a default when not present if not arg_strings and action.nargs == OPTIONAL: if action.option_strings: diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 435db65d52fddb..609881879551b6 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -5721,7 +5721,30 @@ def test_zero_or_more_optional(self): self.assertEqual(NS(x=[]), args) def test_double_dash(self): - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('-f', '--foo') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--foo=--']) + self.assertEqual(NS(foo='--', bar=[]), args) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument -f/--foo: expected one argument', + parser.parse_args, ['--foo', '--']) + args = parser.parse_args(['-f--']) + self.assertEqual(NS(foo='--', bar=[]), args) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument -f/--foo: expected one argument', + parser.parse_args, ['-f', '--']) + args = parser.parse_args(['--foo', 'a', '--', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', 'b', '--foo', 'c']) + self.assertEqual(NS(foo='c', bar=['a', 'b']), args) + args = parser.parse_args(['a', '--', 'b', '--foo', 'c']) + self.assertEqual(NS(foo=None, bar=['a', 'b', '--foo', 'c']), args) + args = parser.parse_args(['a', '--', 'b', '--', 'c', '--foo', 'd']) + self.assertEqual(NS(foo=None, bar=['a', 'b', '--', 'c', '--foo', 'd']), args) + + parser = argparse.ArgumentParser(exit_on_error=False) parser.add_argument('-f', '--foo', nargs='*') parser.add_argument('bar', nargs='*') @@ -5735,6 +5758,41 @@ def test_double_dash(self): self.assertEqual(NS(foo=[], bar=[]), args) args = parser.parse_args(['--foo', 'a', 'b', '--', 'c', 'd']) self.assertEqual(NS(foo=['a', 'b'], bar=['c', 'd']), args) + args = parser.parse_args(['a', 'b', '--foo', 'c', 'd']) + self.assertEqual(NS(foo=['c', 'd'], bar=['a', 'b']), args) + args = parser.parse_args(['a', '--', 'b', '--foo', 'c', 'd']) + self.assertEqual(NS(foo=None, bar=['a', 'b', '--foo', 'c', 'd']), args) + args, argv = parser.parse_known_args(['a', 'b', '--foo', 'c', '--', 'd']) + self.assertEqual(NS(foo=['c'], bar=['a', 'b']), args) + self.assertEqual(argv, ['--', 'd']) + + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('foo') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--', 'a', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', '--', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', '--', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', '--', 'c']), args) + args = parser.parse_args(['--', '--', 'a', '--', 'b', 'c']) + self.assertEqual(NS(foo='--', bar=['a', '--', 'b', 'c']), args) + + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('foo') + parser.add_argument('bar', nargs=argparse.REMAINDER) + + args = parser.parse_args(['--', 'a', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', '--', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', '--', 'c']), args) + args = parser.parse_args(['a', '--', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', '--', 'c']), args) # =========================== diff --git a/Misc/NEWS.d/next/Library/2024-09-19-10-36-18.gh-issue-81691.Hyhp_U.rst b/Misc/NEWS.d/next/Library/2024-09-19-10-36-18.gh-issue-81691.Hyhp_U.rst new file mode 100644 index 00000000000000..8f0108502efde6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-19-10-36-18.gh-issue-81691.Hyhp_U.rst @@ -0,0 +1,3 @@ +Fix handling of multiple ``"--"`` (double dashes) in :mod:`argparse`. Only +the first one has now been removed, all subsequent ones are now taken +literally. From 3e36e5aef18e326f5d1081d73ee8d8fefa1d82f8 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 20 Sep 2024 09:30:27 +0300 Subject: [PATCH 029/105] gh-123797: Check for runtime availability of `ptsname_r` on macos (#123806) --- Lib/test/test_posix.py | 7 +++ ...-09-07-12-14-54.gh-issue-123797.yFDeug.rst | 1 + Modules/posixmodule.c | 46 +++++++++++++++---- 3 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2024-09-07-12-14-54.gh-issue-123797.yFDeug.rst diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 659b9a5a73305d..35016b83a477fc 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -2142,6 +2142,13 @@ def test_stat(self): with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): os.stat("file", dir_fd=0) + def test_ptsname_r(self): + self._verify_available("HAVE_PTSNAME_R") + if self.mac_ver >= (10, 13, 4): + self.assertIn("HAVE_PTSNAME_R", posix._have_functions) + else: + self.assertNotIn("HAVE_PTSNAME_R", posix._have_functions) + def test_access(self): self._verify_available("HAVE_FACCESSAT") if self.mac_ver >= (10, 10): diff --git a/Misc/NEWS.d/next/macOS/2024-09-07-12-14-54.gh-issue-123797.yFDeug.rst b/Misc/NEWS.d/next/macOS/2024-09-07-12-14-54.gh-issue-123797.yFDeug.rst new file mode 100644 index 00000000000000..f126bd0d39bf59 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-09-07-12-14-54.gh-issue-123797.yFDeug.rst @@ -0,0 +1 @@ +Check for runtime availability of ``ptsname_r`` function on macos. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8b2c4bd35a8da7..86366c66c46552 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -125,6 +125,7 @@ # define HAVE_PWRITEV_RUNTIME __builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) # define HAVE_MKFIFOAT_RUNTIME __builtin_available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) # define HAVE_MKNODAT_RUNTIME __builtin_available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) +# define HAVE_PTSNAME_R_RUNTIME __builtin_available(macOS 10.13.4, iOS 11.3, tvOS 11.3, watchOS 4.3, *) # define HAVE_POSIX_SPAWN_SETSID_RUNTIME __builtin_available(macOS 10.15, *) @@ -206,6 +207,10 @@ # define HAVE_MKNODAT_RUNTIME (mknodat != NULL) # endif +# ifdef HAVE_PTSNAME_R +# define HAVE_PTSNAME_R_RUNTIME (ptsname_r != NULL) +# endif + #endif #ifdef HAVE_FUTIMESAT @@ -231,6 +236,7 @@ # define HAVE_PWRITEV_RUNTIME 1 # define HAVE_MKFIFOAT_RUNTIME 1 # define HAVE_MKNODAT_RUNTIME 1 +# define HAVE_PTSNAME_R_RUNTIME 1 #endif @@ -8635,6 +8641,19 @@ os_unlockpt_impl(PyObject *module, int fd) #endif /* HAVE_UNLOCKPT */ #if defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) +static PyObject * +py_ptsname(int fd) +{ + // POSIX manpage: Upon failure, ptsname() shall return a null pointer + // and may set errno. Always initialize errno to avoid undefined behavior. + errno = 0; + char *name = ptsname(fd); + if (name == NULL) { + return posix_error(); + } + return PyUnicode_DecodeFSDefault(name); +} + /*[clinic input] os.ptsname @@ -8656,22 +8675,22 @@ os_ptsname_impl(PyObject *module, int fd) int ret; char name[MAXPATHLEN+1]; - ret = ptsname_r(fd, name, sizeof(name)); + if (HAVE_PTSNAME_R_RUNTIME) { + ret = ptsname_r(fd, name, sizeof(name)); + } + else { + // fallback to ptsname() if ptsname_r() is not available in runtime. + return py_ptsname(fd); + } if (ret != 0) { errno = ret; return posix_error(); } -#else - char *name; - - name = ptsname(fd); - /* POSIX manpage: Upon failure, ptsname() shall return a null pointer and may set errno. - *MAY* set errno? Hmm... */ - if (name == NULL) - return posix_error(); -#endif /* HAVE_PTSNAME_R */ return PyUnicode_DecodeFSDefault(name); +#else + return py_ptsname(fd); +#endif /* HAVE_PTSNAME_R */ } #endif /* defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) */ @@ -17751,6 +17770,9 @@ PROBE(probe_futimens, HAVE_FUTIMENS_RUNTIME) PROBE(probe_utimensat, HAVE_UTIMENSAT_RUNTIME) #endif +#ifdef HAVE_PTSNAME_R +PROBE(probe_ptsname_r, HAVE_PTSNAME_R_RUNTIME) +#endif @@ -17891,6 +17913,10 @@ static const struct have_function { { "HAVE_UTIMENSAT", probe_utimensat }, #endif +#ifdef HAVE_PTSNAME_R + { "HAVE_PTSNAME_R", probe_ptsname_r }, +#endif + #ifdef MS_WINDOWS { "MS_WINDOWS", NULL }, #endif From aee219f4558dda619bd86e4b0e028ce47a5e4b77 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 20 Sep 2024 10:27:34 +0200 Subject: [PATCH 030/105] gh-123880: Allow recursive import of single-phase-init modules (GH-123950) Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: Brett Cannon --- Lib/test/test_import/__init__.py | 69 ++++++++++++++----- .../data/circular_imports/singlephase.py | 13 ++++ ...-09-12-16-16-24.gh-issue-123880.2-8vcj.rst | 2 + Modules/_testsinglephase.c | 63 ++++++++++++++++- Python/import.c | 18 +++-- Tools/c-analyzer/cpython/ignored.tsv | 1 + 6 files changed, 141 insertions(+), 25 deletions(-) create mode 100644 Lib/test/test_import/data/circular_imports/singlephase.py create mode 100644 Misc/NEWS.d/next/C_API/2024-09-12-16-16-24.gh-issue-123880.2-8vcj.rst diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 3d89d69955bb07..5d0d02480b3929 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -111,6 +111,24 @@ def require_frozen(module, *, skip=True): def require_pure_python(module, *, skip=False): _require_loader(module, SourceFileLoader, skip) +def create_extension_loader(modname, filename): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + return AppleFrameworkLoader(modname, filename) + else: + return ExtensionFileLoader(modname, filename) + +def import_extension_from_file(modname, filename, *, put_in_sys_modules=True): + loader = create_extension_loader(modname, filename) + spec = importlib.util.spec_from_loader(modname, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + if put_in_sys_modules: + sys.modules[modname] = module + return module + + def remove_files(name): for f in (name + ".py", name + ".pyc", @@ -1894,6 +1912,37 @@ def test_absolute_circular_submodule(self): str(cm.exception), ) + @requires_singlephase_init + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + def test_singlephase_circular(self): + """Regression test for gh-123950 + + Import a single-phase-init module that imports itself + from the PyInit_* function (before it's added to sys.modules). + Manages its own cache (which is `static`, and so incompatible + with multiple interpreters or interpreter reset). + """ + name = '_testsinglephase_circular' + helper_name = 'test.test_import.data.circular_imports.singlephase' + with uncache(name, helper_name): + filename = _testsinglephase.__file__ + # We don't put the module in sys.modules: that the *inner* + # import should do that. + mod = import_extension_from_file(name, filename, + put_in_sys_modules=False) + + self.assertEqual(mod.helper_mod_name, helper_name) + self.assertIn(name, sys.modules) + self.assertIn(helper_name, sys.modules) + + self.assertIn(name, sys.modules) + self.assertIn(helper_name, sys.modules) + self.assertNotIn(name, sys.modules) + self.assertNotIn(helper_name, sys.modules) + self.assertIs(mod.clear_static_var(), mod) + _testinternalcapi.clear_extension('_testsinglephase_circular', + mod.__spec__.origin) + def test_unwritable_module(self): self.addCleanup(unload, "test.test_import.data.unwritable") self.addCleanup(unload, "test.test_import.data.unwritable.x") @@ -1933,14 +1982,6 @@ def pipe(self): os.set_blocking(r, False) return (r, w) - def create_extension_loader(self, modname, filename): - # Apple extensions must be distributed as frameworks. This requires - # a specialist loader. - if is_apple_mobile: - return AppleFrameworkLoader(modname, filename) - else: - return ExtensionFileLoader(modname, filename) - def import_script(self, name, fd, filename=None, check_override=None): override_text = '' if check_override is not None: @@ -2157,11 +2198,7 @@ def test_multi_init_extension_compat(self): def test_multi_init_extension_non_isolated_compat(self): modname = '_test_non_isolated' filename = _testmultiphase.__file__ - loader = self.create_extension_loader(modname, filename) - spec = importlib.util.spec_from_loader(modname, loader) - module = importlib.util.module_from_spec(spec) - loader.exec_module(module) - sys.modules[modname] = module + module = import_extension_from_file(modname, filename) require_extension(module) with self.subTest(f'{modname}: isolated'): @@ -2176,11 +2213,7 @@ def test_multi_init_extension_non_isolated_compat(self): def test_multi_init_extension_per_interpreter_gil_compat(self): modname = '_test_shared_gil_only' filename = _testmultiphase.__file__ - loader = self.create_extension_loader(modname, filename) - spec = importlib.util.spec_from_loader(modname, loader) - module = importlib.util.module_from_spec(spec) - loader.exec_module(module) - sys.modules[modname] = module + module = import_extension_from_file(modname, filename) require_extension(module) with self.subTest(f'{modname}: isolated, strict'): diff --git a/Lib/test/test_import/data/circular_imports/singlephase.py b/Lib/test/test_import/data/circular_imports/singlephase.py new file mode 100644 index 00000000000000..05618bc72f9327 --- /dev/null +++ b/Lib/test/test_import/data/circular_imports/singlephase.py @@ -0,0 +1,13 @@ +"""Circular import involving a single-phase-init extension. + +This module is imported from the _testsinglephase_circular module from +_testsinglephase, and imports that module again. +""" + +import importlib +import _testsinglephase +from test.test_import import import_extension_from_file + +name = '_testsinglephase_circular' +filename = _testsinglephase.__file__ +mod = import_extension_from_file(name, filename) diff --git a/Misc/NEWS.d/next/C_API/2024-09-12-16-16-24.gh-issue-123880.2-8vcj.rst b/Misc/NEWS.d/next/C_API/2024-09-12-16-16-24.gh-issue-123880.2-8vcj.rst new file mode 100644 index 00000000000000..8a31c962ec7d93 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-09-12-16-16-24.gh-issue-123880.2-8vcj.rst @@ -0,0 +1,2 @@ +Fixed a bug that prevented circular imports of extension modules that use +single-phase initialization. diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 066e0dbfb63fbf..2c59085d15b5be 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -1,7 +1,7 @@ /* Testing module for single-phase initialization of extension modules -This file contains 8 distinct modules, meaning each as its own name +This file contains several distinct modules, meaning each as its own name and its own init function (PyInit_...). The default import system will only find the one matching the filename: _testsinglephase. To load the others you must do so manually. For example: @@ -12,9 +12,13 @@ filename = _testsinglephase.__file__ loader = importlib.machinery.ExtensionFileLoader(name, filename) spec = importlib.util.spec_from_file_location(name, filename, loader=loader) mod = importlib._bootstrap._load(spec) +loader.exec_module(module) +sys.modules[modname] = module ``` -Here are the 8 modules: +(The last two lines are just for completeness.) + +Here are the modules: * _testsinglephase * def: _testsinglephase_basic, @@ -163,6 +167,11 @@ Here are the 8 modules: * functions: none * import system: same as _testsinglephase_with_state +* _testsinglephase_circular + Regression test for gh-123880. + Does not have the common attributes & methods. + See test_singlephase_circular test.test_import.SinglephaseInitTests. + Module state: * fields @@ -740,3 +749,53 @@ PyInit__testsinglephase_with_state_check_cache_first(void) } return PyModule_Create(&_testsinglephase_with_state_check_cache_first); } + + +/****************************************/ +/* the _testsinglephase_circular module */ +/****************************************/ + +static PyObject *static_module_circular; + +static PyObject * +circularmod_clear_static_var(PyObject *self, PyObject *arg) +{ + PyObject *result = static_module_circular; + static_module_circular = NULL; + return result; +} + +static struct PyModuleDef _testsinglephase_circular = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_circular", + .m_doc = PyDoc_STR("Test module _testsinglephase_circular"), + .m_methods = (PyMethodDef[]) { + {"clear_static_var", circularmod_clear_static_var, METH_NOARGS, + "Clear the static variable and return its previous value."}, + {NULL, NULL} /* sentinel */ + } +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_circular(void) +{ + if (!static_module_circular) { + static_module_circular = PyModule_Create(&_testsinglephase_circular); + if (!static_module_circular) { + return NULL; + } + } + static const char helper_mod_name[] = ( + "test.test_import.data.circular_imports.singlephase"); + PyObject *helper_mod = PyImport_ImportModule(helper_mod_name); + Py_XDECREF(helper_mod); + if (!helper_mod) { + return NULL; + } + if(PyModule_AddStringConstant(static_module_circular, + "helper_mod_name", + helper_mod_name) < 0) { + return NULL; + } + return Py_NewRef(static_module_circular); +} diff --git a/Python/import.c b/Python/import.c index 26ad84372cea68..460b1fe225c72e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -815,6 +815,8 @@ static int clear_singlephase_extension(PyInterpreterState *interp, // Currently, this is only used for testing. // (See _testinternalcapi.clear_extension().) +// If adding another use, be careful about modules that import themselves +// recursively (see gh-123880). int _PyImport_ClearExtension(PyObject *name, PyObject *filename) { @@ -1322,12 +1324,16 @@ _extensions_cache_set(PyObject *path, PyObject *name, value = entry == NULL ? NULL : (struct extensions_cache_value *)entry->value; - /* We should never be updating an existing cache value. */ - assert(value == NULL); if (value != NULL) { - PyErr_Format(PyExc_SystemError, - "extension module %R is already cached", name); - goto finally; + /* gh-123880: If there's an existing cache value, it means a module is + * being imported recursively from its PyInit_* or Py_mod_* function. + * (That function presumably handles returning a partially + * constructed module in such a case.) + * We can reuse the existing cache value; it is owned by the cache. + * (Entries get removed from it in exceptional circumstances, + * after interpreter shutdown, and in runtime shutdown.) + */ + goto finally_oldvalue; } newvalue = alloc_extensions_cache_value(); if (newvalue == NULL) { @@ -1392,6 +1398,7 @@ _extensions_cache_set(PyObject *path, PyObject *name, cleanup_old_cached_def(&olddefbase); } +finally_oldvalue: extensions_lock_release(); if (key != NULL) { hashtable_destroy_str(key); @@ -2128,6 +2135,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, } +// Used in _PyImport_ClearExtension; see notes there. static int clear_singlephase_extension(PyInterpreterState *interp, PyObject *name, PyObject *path) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index fabb5de921acf7..b002ada1a8d588 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -590,6 +590,7 @@ Modules/_testmultiphase.c - slots_nonmodule_with_exec_slots - Modules/_testmultiphase.c - testexport_methods - Modules/_testmultiphase.c - uninitialized_def - Modules/_testsinglephase.c - global_state - +Modules/_testsinglephase.c - static_module_circular - Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule - Modules/_xxtestfuzz/_xxtestfuzz.c - module_methods - Modules/_xxtestfuzz/fuzzer.c - RE_FLAG_DEBUG - From baa3550bc3a119f41cc4eaed5373f9d695208e8e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 20 Sep 2024 12:54:20 +0300 Subject: [PATCH 031/105] gh-95468: Add more tests for "--" (double dash) in test_argparse (GH-124274) --- Lib/test/test_argparse.py | 43 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 609881879551b6..138ff19e86acf4 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -5720,7 +5720,9 @@ def test_zero_or_more_optional(self): args = parser.parse_args([]) self.assertEqual(NS(x=[]), args) - def test_double_dash(self): + +class TestDoubleDash(TestCase): + def test_single_argument_option(self): parser = argparse.ArgumentParser(exit_on_error=False) parser.add_argument('-f', '--foo') parser.add_argument('bar', nargs='*') @@ -5744,6 +5746,7 @@ def test_double_dash(self): args = parser.parse_args(['a', '--', 'b', '--', 'c', '--foo', 'd']) self.assertEqual(NS(foo=None, bar=['a', 'b', '--', 'c', '--foo', 'd']), args) + def test_multiple_argument_option(self): parser = argparse.ArgumentParser(exit_on_error=False) parser.add_argument('-f', '--foo', nargs='*') parser.add_argument('bar', nargs='*') @@ -5766,6 +5769,7 @@ def test_double_dash(self): self.assertEqual(NS(foo=['c'], bar=['a', 'b']), args) self.assertEqual(argv, ['--', 'd']) + def test_multiple_double_dashes(self): parser = argparse.ArgumentParser(exit_on_error=False) parser.add_argument('foo') parser.add_argument('bar', nargs='*') @@ -5781,9 +5785,10 @@ def test_double_dash(self): args = parser.parse_args(['--', '--', 'a', '--', 'b', 'c']) self.assertEqual(NS(foo='--', bar=['a', '--', 'b', 'c']), args) + def test_remainder(self): parser = argparse.ArgumentParser(exit_on_error=False) parser.add_argument('foo') - parser.add_argument('bar', nargs=argparse.REMAINDER) + parser.add_argument('bar', nargs='...') args = parser.parse_args(['--', 'a', 'b', 'c']) self.assertEqual(NS(foo='a', bar=['b', 'c']), args) @@ -5794,6 +5799,40 @@ def test_double_dash(self): args = parser.parse_args(['a', '--', 'b', '--', 'c']) self.assertEqual(NS(foo='a', bar=['b', '--', 'c']), args) + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo') + parser.add_argument('bar', nargs='...') + args = parser.parse_args(['--foo', 'a', '--', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['--', 'b', '--', 'c']), args) + + def test_subparser(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('foo') + subparsers = parser.add_subparsers() + parser1 = subparsers.add_parser('run') + parser1.add_argument('-f') + parser1.add_argument('bar', nargs='*') + + args = parser.parse_args(['x', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f='c', bar=['a', 'b']), args) + args = parser.parse_args(['x', 'run', 'a', 'b', '--', '-f', 'c']) + self.assertEqual(NS(foo='x', f=None, bar=['a', 'b', '-f', 'c']), args) + args = parser.parse_args(['x', 'run', 'a', '--', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f=None, bar=['a', 'b', '-f', 'c']), args) + args = parser.parse_args(['x', 'run', '--', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f=None, bar=['a', 'b', '-f', 'c']), args) + args = parser.parse_args(['x', '--', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f='c', bar=['a', 'b']), args) + args = parser.parse_args(['--', 'x', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f='c', bar=['a', 'b']), args) + args = parser.parse_args(['x', 'run', '--', 'a', '--', 'b']) + self.assertEqual(NS(foo='x', f=None, bar=['a', '--', 'b']), args) + args = parser.parse_args(['x', '--', 'run', '--', 'a', '--', 'b']) + self.assertEqual(NS(foo='x', f=None, bar=['a', '--', 'b']), args) + self.assertRaisesRegex(argparse.ArgumentError, + "invalid choice: '--'", + parser.parse_args, ['--', 'x', '--', 'run', 'a', 'b']) + # =========================== # parse_intermixed_args tests From 63f196090f90cbfe5f698824655f74dea5cb2b29 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Fri, 20 Sep 2024 06:08:59 -0400 Subject: [PATCH 032/105] gh-124248: Fix crash in struct when processing 0p fields (#124251) --- Lib/test/test_struct.py | 9 +++++++++ Misc/ACKS | 1 + ...-09-19-11-47-39.gh-issue-124248.g7rufd.rst | 2 ++ Modules/_struct.c | 19 +++++++++++++++---- 4 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-19-11-47-39.gh-issue-124248.g7rufd.rst diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 5508cc3eec85c8..bdbf8800cfd8f6 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -96,6 +96,13 @@ def test_new_features(self): ('10s', b'helloworld', b'helloworld', b'helloworld', 0), ('11s', b'helloworld', b'helloworld\0', b'helloworld\0', 1), ('20s', b'helloworld', b'helloworld'+10*b'\0', b'helloworld'+10*b'\0', 1), + ('0p', b'helloworld', b'', b'', 1), + ('1p', b'helloworld', b'\x00', b'\x00', 1), + ('2p', b'helloworld', b'\x01h', b'\x01h', 1), + ('10p', b'helloworld', b'\x09helloworl', b'\x09helloworl', 1), + ('11p', b'helloworld', b'\x0Ahelloworld', b'\x0Ahelloworld', 0), + ('12p', b'helloworld', b'\x0Ahelloworld\0', b'\x0Ahelloworld\0', 1), + ('20p', b'helloworld', b'\x0Ahelloworld'+9*b'\0', b'\x0Ahelloworld'+9*b'\0', 1), ('b', 7, b'\7', b'\7', 0), ('b', -7, b'\371', b'\371', 0), ('B', 7, b'\7', b'\7', 0), @@ -339,6 +346,7 @@ def assertStructError(func, *args, **kwargs): def test_p_code(self): # Test p ("Pascal string") code. for code, input, expected, expectedback in [ + ('0p', b'abc', b'', b''), ('p', b'abc', b'\x00', b''), ('1p', b'abc', b'\x00', b''), ('2p', b'abc', b'\x01a', b'a'), @@ -580,6 +588,7 @@ def test__sizeof__(self): self.check_sizeof('187s', 1) self.check_sizeof('20p', 1) self.check_sizeof('0s', 1) + self.check_sizeof('0p', 1) self.check_sizeof('0c', 0) def test_boundary_error_message(self): diff --git a/Misc/ACKS b/Misc/ACKS index b031eb7c11f73f..2db17adcc2ee66 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1653,6 +1653,7 @@ Scott Schram Robin Schreiber Chad J. Schroeder Simon-Martin Schroeder +Brian Schubert Christian Schubert Sam Schulenburg Andreas Schwab diff --git a/Misc/NEWS.d/next/Library/2024-09-19-11-47-39.gh-issue-124248.g7rufd.rst b/Misc/NEWS.d/next/Library/2024-09-19-11-47-39.gh-issue-124248.g7rufd.rst new file mode 100644 index 00000000000000..1bd333f485a2ab --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-19-11-47-39.gh-issue-124248.g7rufd.rst @@ -0,0 +1,2 @@ +Fixed potential crash when using :mod:`struct` to process zero-width +'Pascal string' fields (``0p``). diff --git a/Modules/_struct.c b/Modules/_struct.c index f744193469e2dc..2ae5060ba34163 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1669,9 +1669,16 @@ s_unpack_internal(PyStructObject *soself, const char *startfrom, if (e->format == 's') { v = PyBytes_FromStringAndSize(res, code->size); } else if (e->format == 'p') { - Py_ssize_t n = *(unsigned char*)res; - if (n >= code->size) - n = code->size - 1; + Py_ssize_t n; + if (code->size == 0) { + n = 0; + } + else { + n = *(unsigned char*)res; + if (n >= code->size) { + n = code->size - 1; + } + } v = PyBytes_FromStringAndSize(res + 1, n); } else { v = e->unpack(state, res, e); @@ -1982,8 +1989,12 @@ s_pack_internal(PyStructObject *soself, PyObject *const *args, int offset, n = PyByteArray_GET_SIZE(v); p = PyByteArray_AS_STRING(v); } - if (n > (code->size - 1)) + if (code->size == 0) { + n = 0; + } + else if (n > (code->size - 1)) { n = code->size - 1; + } if (n > 0) memcpy(res + 1, p, n); if (n > 255) From 622368d99c986ca1a9bdba951ac53f42d7ee6fca Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Fri, 20 Sep 2024 07:42:44 -0300 Subject: [PATCH 033/105] Fix typo in XMLParser doc (#124129) --- Doc/library/xml.etree.elementtree.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 9fad463d936660..1daf6628013bf0 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -1375,7 +1375,7 @@ XMLParser Objects .. versionchanged:: 3.8 Parameters are now :ref:`keyword-only `. - The *html* argument no longer supported. + The *html* argument is no longer supported. .. method:: close() From db6eb3640a7d98db6fea17cf9da4cb14504e5571 Mon Sep 17 00:00:00 2001 From: Y5 <124019959+y5c4l3@users.noreply.github.com> Date: Fri, 20 Sep 2024 11:01:28 +0000 Subject: [PATCH 034/105] gh-124217, ipaddress: Add RFC 9637 reserved IPv6 block `3fff::/20` (#124240) Signed-off-by: y5c4l3 --- Lib/ipaddress.py | 2 ++ Lib/test/test_ipaddress.py | 2 ++ .../next/Library/2024-09-19-20-15-00.gh-issue-124217.j0KlQB.rst | 1 + 3 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-09-19-20-15-00.gh-issue-124217.j0KlQB.rst diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index cb41c522883d74..703fa289dda1fb 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -2375,6 +2375,8 @@ class _IPv6Constants: IPv6Network('2001:db8::/32'), # IANA says N/A, let's consider it not globally reachable to be safe IPv6Network('2002::/16'), + # RFC 9637: https://www.rfc-editor.org/rfc/rfc9637.html#section-6-2.2 + IPv6Network('3fff::/20'), IPv6Network('fc00::/7'), IPv6Network('fe80::/10'), ] diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index aad779dd9b25e7..b1ac2b94f41b38 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -2433,6 +2433,8 @@ def testReservedIpv6(self): self.assertTrue(ipaddress.ip_address('2001:30::').is_global) self.assertFalse(ipaddress.ip_address('2001:40::').is_global) self.assertFalse(ipaddress.ip_address('2002::').is_global) + # gh-124217: conform with RFC 9637 + self.assertFalse(ipaddress.ip_address('3fff::').is_global) # some generic IETF reserved addresses self.assertEqual(True, ipaddress.ip_address('100::').is_reserved) diff --git a/Misc/NEWS.d/next/Library/2024-09-19-20-15-00.gh-issue-124217.j0KlQB.rst b/Misc/NEWS.d/next/Library/2024-09-19-20-15-00.gh-issue-124217.j0KlQB.rst new file mode 100644 index 00000000000000..46f9866f8d427c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-19-20-15-00.gh-issue-124217.j0KlQB.rst @@ -0,0 +1 @@ +Add RFC 9637 reserved IPv6 block ``3fff::/20`` in :mod:`ipaddress` module. From 1a577729e347714eb819fa3a3a00149406c24e5e Mon Sep 17 00:00:00 2001 From: AN Long Date: Fri, 20 Sep 2024 20:42:43 +0800 Subject: [PATCH 035/105] gh-111513: Improve datetime.fromtimestamp's error message (#124249) --- .../Library/2024-09-19-16-00-22.gh-issue-111513.6jHm02.rst | 1 + Python/pytime.c | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-09-19-16-00-22.gh-issue-111513.6jHm02.rst diff --git a/Misc/NEWS.d/next/Library/2024-09-19-16-00-22.gh-issue-111513.6jHm02.rst b/Misc/NEWS.d/next/Library/2024-09-19-16-00-22.gh-issue-111513.6jHm02.rst new file mode 100644 index 00000000000000..c6b85f9cd72255 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-19-16-00-22.gh-issue-111513.6jHm02.rst @@ -0,0 +1 @@ +Improve the error message that may be raised by :meth:`datetime.date.fromtimestamp`. diff --git a/Python/pytime.c b/Python/pytime.c index cd76970718622f..2b37cd991ef4e4 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -387,6 +387,10 @@ pytime_object_to_denominator(PyObject *obj, time_t *sec, long *numerator, *sec = _PyLong_AsTime_t(obj); *numerator = 0; if (*sec == (time_t)-1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Format(PyExc_TypeError, + "argument must be int or float, not %T", obj); + } return -1; } return 0; From 342e654b8eda24c68da64cc21bc9583e480d9e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Fri, 20 Sep 2024 15:37:49 +0200 Subject: [PATCH 036/105] gh-124213: Skip tests failing inside systemd-nspawn --suppress-sync=true (#124215) Add a helper function that checks whether the test suite is running inside a systemd-nspawn container, and skip the few tests failing with `--suppress-sync=true` in that case. The tests are failing because `--suppress-sync=true` stubs out `fsync()`, `fdatasync()` and `msync()` calls, and therefore they always return success without checking for invalid arguments. Call `os.open(__file__, os.O_RDONLY | os.O_SYNC)` and check the errno to detect whether `--suppress-sync=true` is actually used, and skip the tests only in that scenario. --- Lib/test/support/__init__.py | 33 +++++++++++++++++++ Lib/test/test_mmap.py | 4 ++- Lib/test/test_os.py | 10 ++++-- ...-09-18-18-39-21.gh-issue-124213.AQq_xg.rst | 3 ++ 4 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-09-18-18-39-21.gh-issue-124213.AQq_xg.rst diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index dbf479dddff7b3..628529b8664c77 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -61,6 +61,7 @@ "without_optimizer", "force_not_colorized", "BrokenIter", + "in_systemd_nspawn_sync_suppressed", ] @@ -2873,3 +2874,35 @@ def __iter__(self): if self.iter_raises: 1/0 return self + + +def in_systemd_nspawn_sync_suppressed() -> bool: + """ + Test whether the test suite is runing in systemd-nspawn + with ``--suppress-sync=true``. + + This can be used to skip tests that rely on ``fsync()`` calls + and similar not being intercepted. + """ + + if not hasattr(os, "O_SYNC"): + return False + + try: + with open("/run/systemd/container", "rb") as fp: + if fp.read().rstrip() != b"systemd-nspawn": + return False + except FileNotFoundError: + return False + + # If systemd-nspawn is used, O_SYNC flag will immediately + # trigger EINVAL. Otherwise, ENOENT will be given instead. + import errno + try: + with os.open(__file__, os.O_RDONLY | os.O_SYNC): + pass + except OSError as err: + if err.errno == errno.EINVAL: + return True + + return False diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index a1cf5384ada5b5..b2a299ed172967 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1,5 +1,6 @@ from test.support import ( requires, _2G, _4G, gc_collect, cpython_only, is_emscripten, is_apple, + in_systemd_nspawn_sync_suppressed, ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -839,7 +840,8 @@ def test_flush_return_value(self): mm.write(b'python') result = mm.flush() self.assertIsNone(result) - if sys.platform.startswith(('linux', 'android')): + if (sys.platform.startswith(('linux', 'android')) + and not in_systemd_nspawn_sync_suppressed()): # 'offset' must be a multiple of mmap.PAGESIZE on Linux. # See bpo-34754 for details. self.assertRaises(OSError, mm.flush, 1, len(b'python')) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 1e570c757fccbc..9fa4e406bd2468 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2351,9 +2351,13 @@ def test_chmod(self): @unittest.skipIf(support.is_wasi, "Cannot create invalid FD on WASI.") class TestInvalidFD(unittest.TestCase): - singles = ["fchdir", "dup", "fdatasync", "fstat", - "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] - singles_fildes = {"fchdir", "fdatasync", "fsync"} + singles = ["fchdir", "dup", "fstat", "fstatvfs", "tcgetpgrp", "ttyname"] + singles_fildes = {"fchdir"} + # systemd-nspawn --suppress-sync=true does not verify fd passed + # fdatasync() and fsync(), and always returns success + if not support.in_systemd_nspawn_sync_suppressed(): + singles += ["fdatasync", "fsync"] + singles_fildes |= {"fdatasync", "fsync"} #singles.append("close") #We omit close because it doesn't raise an exception on some platforms def get_single(f): diff --git a/Misc/NEWS.d/next/Tests/2024-09-18-18-39-21.gh-issue-124213.AQq_xg.rst b/Misc/NEWS.d/next/Tests/2024-09-18-18-39-21.gh-issue-124213.AQq_xg.rst new file mode 100644 index 00000000000000..021fbefb635af1 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-09-18-18-39-21.gh-issue-124213.AQq_xg.rst @@ -0,0 +1,3 @@ +Detect whether the test suite is running inside a systemd-nspawn container +with ``--suppress-sync=true`` option, and skip the ``test_os`` +and ``test_mmap`` tests that are failing in this scenario. From d5f95ec07bb47a4d6554e04d13a979dbeac05f74 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 22 Sep 2024 15:51:09 -0400 Subject: [PATCH 037/105] gh-112938: IDLE - Fix uninteruptable hang when Shell gets rapid continuous output. (#124310) https://github.com/python/cpython/issues/88496 replaced text.update with text.update_idletasks in colorizer.py and outwin.py to fix test failures on macOS. While theoretically correct, the result was Shell freezing when receiving continuous short strings to print. Test: `while 1: 1`. The guess is that there is no idle time in which to do the screen update. Reverting the change in one of the files, outwin, fixes the issue. Colorizer runs ever 1/20 second and seems to work fine. When running test-outwin on macOS, alias 'update' to 'update_idletasks on the text used for testing. --- Lib/idlelib/idle_test/test_outwin.py | 5 +++++ Lib/idlelib/outwin.py | 2 +- .../next/IDLE/2024-09-21-23-12-18.gh-issue-112938.OeiDru.rst | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/IDLE/2024-09-21-23-12-18.gh-issue-112938.OeiDru.rst diff --git a/Lib/idlelib/idle_test/test_outwin.py b/Lib/idlelib/idle_test/test_outwin.py index d6e85ad674417c..81f4aad7e95e95 100644 --- a/Lib/idlelib/idle_test/test_outwin.py +++ b/Lib/idlelib/idle_test/test_outwin.py @@ -1,6 +1,7 @@ "Test outwin, coverage 76%." from idlelib import outwin +import sys import unittest from test.support import requires from tkinter import Tk, Text @@ -18,6 +19,10 @@ def setUpClass(cls): root.withdraw() w = cls.window = outwin.OutputWindow(None, None, None, root) cls.text = w.text = Text(root) + if sys.platform == 'darwin': # Issue 112938 + cls.text.update = cls.text.update_idletasks + # Without this, test write, writelines, and goto... fail. + # The reasons and why macOS-specific are unclear. @classmethod def tearDownClass(cls): diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py index 5ed3f35a7af655..8baa657550de94 100644 --- a/Lib/idlelib/outwin.py +++ b/Lib/idlelib/outwin.py @@ -112,7 +112,7 @@ def write(self, s, tags=(), mark="insert"): assert isinstance(s, str) self.text.insert(mark, s, tags) self.text.see(mark) - self.text.update_idletasks() + self.text.update() return len(s) def writelines(self, lines): diff --git a/Misc/NEWS.d/next/IDLE/2024-09-21-23-12-18.gh-issue-112938.OeiDru.rst b/Misc/NEWS.d/next/IDLE/2024-09-21-23-12-18.gh-issue-112938.OeiDru.rst new file mode 100644 index 00000000000000..0cd058eeffb1d5 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2024-09-21-23-12-18.gh-issue-112938.OeiDru.rst @@ -0,0 +1 @@ +Fix uninteruptable hang when Shell gets rapid continuous output. From 2e8c769481d5729d86be8c6cff5881c4c5fbb8d2 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 23 Sep 2024 09:54:36 +0300 Subject: [PATCH 038/105] gh-124228: Fix UUID test in configure files for NetBSD (#124229) Fix UUID configuration in configure files for NetBSD compatibility. --- configure | 8 ++++++++ configure.ac | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/configure b/configure index d88acc580b790b..8c69b44ed7318e 100755 --- a/configure +++ b/configure @@ -14068,6 +14068,14 @@ done fi +# gh-124228: While the libuuid library is available on NetBSD, it supports only UUID version 4. +# This restriction inhibits the proper generation of time-based UUIDs. +if test "$ac_sys_system" = "NetBSD"; then + have_uuid=missing + printf "%s\n" "#define HAVE_UUID_H 0" >>confdefs.h + +fi + if test "x$have_uuid" = xmissing then : have_uuid=no diff --git a/configure.ac b/configure.ac index e83cba49d03433..d3cdeb8a252a24 100644 --- a/configure.ac +++ b/configure.ac @@ -3802,6 +3802,13 @@ AS_VAR_IF([have_uuid], [missing], [ ]) ]) +# gh-124228: While the libuuid library is available on NetBSD, it supports only UUID version 4. +# This restriction inhibits the proper generation of time-based UUIDs. +if test "$ac_sys_system" = "NetBSD"; then + have_uuid=missing + AC_DEFINE([HAVE_UUID_H], [0]) +fi + AS_VAR_IF([have_uuid], [missing], [have_uuid=no]) # 'Real Time' functions on Solaris From d3e79d75d164c338a64fd66edb26e69c501cee58 Mon Sep 17 00:00:00 2001 From: Y5 <124019959+y5c4l3@users.noreply.github.com> Date: Mon, 23 Sep 2024 06:58:14 +0000 Subject: [PATCH 039/105] gh-124130: Notes on empty string corner case of category `\B` (#124133) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: y5c4l3 Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/re.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index efb2722f18038f..9db6f1da3be4db 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -572,6 +572,12 @@ character ``'$'``. Word boundaries are determined by the current locale if the :py:const:`~re.LOCALE` flag is used. + .. note:: + + Note that ``\B`` does not match an empty string, which differs from + RE implementations in other programming languages such as Perl. + This behavior is kept for compatibility reasons. + .. index:: single: \d; in regular expressions ``\d`` From 0e89f7abd494bd6c658083c1c1d50a640a1f2309 Mon Sep 17 00:00:00 2001 From: Sam James Date: Mon, 23 Sep 2024 08:03:30 +0100 Subject: [PATCH 040/105] GH-113655: Lower the C recursion limit for HPPA, PPC64 and SPARC (#124264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lower the C recursion limit for HPPA, PPC64 and SPARC, as they use relatively large stack frames that cause e.g. `test_descr` to hit a stack overflow. According to quick testing, it seems that values around 8000 are max for HPPA and PPC64 (ELFv1 ABI) and 7000 for SPARC64. To keep things safe, let's use 5000 for PPC64 and 4000 for SPARC. Co-authored-by: Michał Górny --- Include/cpython/pystate.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index f005729fff11b6..32f68378ea5d72 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -218,9 +218,15 @@ struct _ts { # define Py_C_RECURSION_LIMIT 3000 #elif defined(_Py_ADDRESS_SANITIZER) # define Py_C_RECURSION_LIMIT 4000 +#elif defined(__sparc__) + // test_descr crashed on sparc64 with >7000 but let's keep a margin of error. +# define Py_C_RECURSION_LIMIT 4000 #elif defined(__wasi__) // Based on wasmtime 16. # define Py_C_RECURSION_LIMIT 5000 +#elif defined(__hppa__) || defined(__powerpc64__) + // test_descr crashed with >8000 but let's keep a margin of error. +# define Py_C_RECURSION_LIMIT 5000 #else // This value is duplicated in Lib/test/support/__init__.py # define Py_C_RECURSION_LIMIT 10000 From 5f5c0b9c23238dc0a1fdb764f625ae0cc5604519 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 23 Sep 2024 02:08:03 -0700 Subject: [PATCH 041/105] GH-79714: Add mention of stderr for clarity to ArgumentParser.exit() (GH-123932) --- Doc/library/argparse.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index aa1341c8d4d4a8..c4e2dbcac2156c 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1455,7 +1455,7 @@ The ``deprecated`` keyword argument of specifies if the argument is deprecated and will be removed in the future. For arguments, if ``deprecated`` is ``True``, then a warning will be -printed to standard error when the argument is used:: +printed to :data:`sys.stderr` when the argument is used:: >>> import argparse >>> parser = argparse.ArgumentParser(prog='snake.py') @@ -2235,8 +2235,8 @@ Exiting methods .. method:: ArgumentParser.exit(status=0, message=None) This method terminates the program, exiting with the specified *status* - and, if given, it prints a *message* before that. The user can override - this method to handle these steps differently:: + and, if given, it prints a *message* to :data:`sys.stderr` before that. + The user can override this method to handle these steps differently:: class ErrorCatchingArgumentParser(argparse.ArgumentParser): def exit(self, status=0, message=None): @@ -2246,8 +2246,8 @@ Exiting methods .. method:: ArgumentParser.error(message) - This method prints a usage message including the *message* to the - standard error and terminates the program with a status code of 2. + This method prints a usage message, including the *message*, to + :data:`sys.stderr` and terminates the program with a status code of 2. Intermixed parsing From 9d0a75269c6ae361b1ed5910c3b3424ed93b6f6d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 14:28:17 +0100 Subject: [PATCH 042/105] GH-109975: Copyedit 3.13 What's New: C API (#124313) --- Doc/c-api/monitoring.rst | 10 +- Doc/c-api/time.rst | 2 + Doc/library/sys.monitoring.rst | 2 +- Doc/whatsnew/3.13.rst | 797 +++++++++++++++++++-------------- 4 files changed, 457 insertions(+), 354 deletions(-) diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index 4db26d82b9200d..285ddb2889a67f 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -1,6 +1,6 @@ .. highlight:: c -.. _monitoring: +.. _c-api-monitoring: Monitoring C API ================ @@ -141,18 +141,18 @@ would typically correspond to a python function. to the base-2 logarithm of ``sys.monitoring.events.PY_START``. ``state_array`` is an array with a monitoring state entry for each event in ``event_types``, it is allocated by the user but populated by - ``PyMonitoring_EnterScope`` with information about the activation state of + :c:func:`!PyMonitoring_EnterScope` with information about the activation state of the event. The size of ``event_types`` (and hence also of ``state_array``) is given in ``length``. The ``version`` argument is a pointer to a value which should be allocated by the user together with ``state_array`` and initialized to 0, - and then set only by ``PyMonitoring_EnterScope`` itelf. It allows this + and then set only by :c:func:`!PyMonitoring_EnterScope` itelf. It allows this function to determine whether event states have changed since the previous call, and to return quickly if they have not. The scopes referred to here are lexical scopes: a function, class or method. - ``PyMonitoring_EnterScope`` should be called whenever the lexical scope is + :c:func:`!PyMonitoring_EnterScope` should be called whenever the lexical scope is entered. Scopes can be reentered, reusing the same *state_array* and *version*, in situations like when emulating a recursive Python function. When a code-like's execution is paused, such as when emulating a generator, the scope needs to @@ -189,4 +189,4 @@ would typically correspond to a python function. .. c:function:: int PyMonitoring_ExitScope(void) - Exit the last scope that was entered with ``PyMonitoring_EnterScope``. + Exit the last scope that was entered with :c:func:`!PyMonitoring_EnterScope`. diff --git a/Doc/c-api/time.rst b/Doc/c-api/time.rst index 5cfdef71b3e191..7032cc48aa6913 100644 --- a/Doc/c-api/time.rst +++ b/Doc/c-api/time.rst @@ -1,5 +1,7 @@ .. highlight:: c +.. _c-api-time: + PyTime C API ============ diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 3ead20815fa30e..ac8bcceaca5aeb 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -262,7 +262,7 @@ Per code object events Events can also be controlled on a per code object basis. The functions defined below which accept a :class:`types.CodeType` should be prepared to accept a look-alike object from functions which are not defined -in Python (see :ref:`monitoring`). +in Python (see :ref:`c-api-monitoring`). .. function:: get_local_events(tool_id: int, code: CodeType, /) -> int diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 4d1a10155c4617..96fee0f604ccc7 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -159,7 +159,8 @@ C API improvements: * The :doc:`PyTime C API ` has been added, providing access to system clocks. * :c:type:`PyMutex` is a new lightweight mutex that occupies a single byte. -* :doc:`C-API support ` was added for generating :pep:`669` monitoring events. +* There is a new :ref:`suite of functions ` + for generating :pep:`669` monitoring events in the C API. New typing features: @@ -2008,228 +2009,497 @@ C API Changes New Features ------------ -* You no longer have to define the ``PY_SSIZE_T_CLEAN`` macro before including - :file:`Python.h` when using ``#`` formats in - :ref:`format codes `. - APIs accepting the format codes always use ``Py_ssize_t`` for ``#`` formats. - (Contributed by Inada Naoki in :gh:`104922`.) +* Add the :ref:`PyMonitoring C API ` + for generating :pep:`669` monitoring events: + + * :c:type:`PyMonitoringState` + * :c:func:`PyMonitoring_FirePyStartEvent` + * :c:func:`PyMonitoring_FirePyResumeEvent` + * :c:func:`PyMonitoring_FirePyReturnEvent` + * :c:func:`PyMonitoring_FirePyYieldEvent` + * :c:func:`PyMonitoring_FireCallEvent` + * :c:func:`PyMonitoring_FireLineEvent` + * :c:func:`PyMonitoring_FireJumpEvent` + * :c:func:`PyMonitoring_FireBranchEvent` + * :c:func:`PyMonitoring_FireCReturnEvent` + * :c:func:`PyMonitoring_FirePyThrowEvent` + * :c:func:`PyMonitoring_FireRaiseEvent` + * :c:func:`PyMonitoring_FireCRaiseEvent` + * :c:func:`PyMonitoring_FireReraiseEvent` + * :c:func:`PyMonitoring_FireExceptionHandledEvent` + * :c:func:`PyMonitoring_FirePyUnwindEvent` + * :c:func:`PyMonitoring_FireStopIterationEvent` + * :c:func:`PyMonitoring_EnterScope` + * :c:func:`PyMonitoring_ExitScope` -* The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` and - :c:func:`PyArg_VaParseTupleAndKeywords` now has type :c:expr:`char * const *` - in C and :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. - It makes these functions compatible with arguments of type - :c:expr:`const char * const *`, :c:expr:`const char **` or - :c:expr:`char * const *` in C++ and :c:expr:`char * const *` in C - without an explicit type cast. - This can be overridden with the :c:macro:`PY_CXX_CONST` macro. - (Contributed by Serhiy Storchaka in :gh:`65210`.) + (Contributed by Irit Katriel in :gh:`111997`). -* Add :c:func:`PyImport_AddModuleRef`: similar to - :c:func:`PyImport_AddModule`, but return a :term:`strong reference` instead - of a :term:`borrowed reference`. - (Contributed by Victor Stinner in :gh:`105922`.) +* Add :c:type:`PyMutex`, a lightweight mutex that occupies a single byte, + and the new :c:func:`PyMutex_Lock` and :c:func:`PyMutex_Unlock` functions. + :c:func:`!PyMutex_Lock` will release the :term:`GIL` (if currently held) + if the operation needs to block. + (Contributed by Sam Gross in :gh:`108724`.) -* Add :c:func:`PyWeakref_GetRef` function: similar to - :c:func:`PyWeakref_GetObject` but returns a :term:`strong reference`, or - ``NULL`` if the referent is no longer live. - (Contributed by Victor Stinner in :gh:`105927`.) +* Add the :ref:`PyTime C API ` to provide access to system clocks: -* Add :c:func:`PyObject_GetOptionalAttr` and - :c:func:`PyObject_GetOptionalAttrString`, variants of - :c:func:`PyObject_GetAttr` and :c:func:`PyObject_GetAttrString` which - don't raise :exc:`AttributeError` if the attribute is not found. - These variants are more convenient and faster if the missing attribute - should not be treated as a failure. - (Contributed by Serhiy Storchaka in :gh:`106521`.) + * :c:type:`PyTime_t`. + * :c:var:`PyTime_MIN` and :c:var:`PyTime_MAX`. + * :c:func:`PyTime_AsSecondsDouble`. + * :c:func:`PyTime_Monotonic`. + * :c:func:`PyTime_MonotonicRaw`. + * :c:func:`PyTime_PerfCounter`. + * :c:func:`PyTime_PerfCounterRaw`. + * :c:func:`PyTime_Time`. + * :c:func:`PyTime_TimeRaw`. -* Add :c:func:`PyMapping_GetOptionalItem` and - :c:func:`PyMapping_GetOptionalItemString`: variants of - :c:func:`PyObject_GetItem` and :c:func:`PyMapping_GetItemString` which don't - raise :exc:`KeyError` if the key is not found. - These variants are more convenient and faster if the missing key should not - be treated as a failure. - (Contributed by Serhiy Storchaka in :gh:`106307`.) + (Contributed by Victor Stinner and Petr Viktorin in :gh:`110850`.) -* Add fixed variants of functions which silently ignore errors: +* Add the :c:func:`PyDict_ContainsString` function + with the same behaviour as :c:func:`PyDict_Contains`, + but *key* is specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + (Contributed by Victor Stinner in :gh:`108314`.) + +* Add the :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` + functions, + which behave similarly to :c:func:`PyDict_GetItemWithError`, + but return a :term:`strong reference` instead of a :term:`borrowed reference`. + Moreover, these functions return ``-1`` on error, + removing the need to check :c:func:`!PyErr_Occurred`. + (Contributed by Victor Stinner in :gh:`106004`.) - - :c:func:`PyObject_HasAttrWithError` replaces :c:func:`PyObject_HasAttr`. - - :c:func:`PyObject_HasAttrStringWithError` replaces :c:func:`PyObject_HasAttrString`. - - :c:func:`PyMapping_HasKeyWithError` replaces :c:func:`PyMapping_HasKey`. - - :c:func:`PyMapping_HasKeyStringWithError` replaces :c:func:`PyMapping_HasKeyString`. +* Add the :c:func:`PyDict_SetDefaultRef` function, + which behaves similarly to :c:func:`PyDict_SetDefault`, + but returns a :term:`strong reference` instead of a :term:`borrowed reference`. + This function returns ``-1`` on error, + ``0`` on insertion, + and ``1`` if the key was already present in the dictionary. + (Contributed by Sam Gross in :gh:`112066`.) - New functions return not only ``1`` for true and ``0`` for false, but also - ``-1`` for error. +* Add the :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions + to remove a key from a dictionary and optionally return the removed value. + This is similar to :meth:`dict.pop`, + though there is no default value, + and :exc:`KeyError` is not raised for missing keys. + (Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.) - (Contributed by Serhiy Storchaka in :gh:`108511`.) +* Add the :c:func:`PyMapping_GetOptionalItem` + and :c:func:`PyMapping_GetOptionalItemString` functions + as alternatives to :c:func:`PyObject_GetItem` + and :c:func:`PyMapping_GetItemString` respectively. + The new functions do not raise :exc:`KeyError` + if the requested key is missing from the mapping. + These variants are more convenient and faster + if a missing key should not be treated as a failure. + (Contributed by Serhiy Storchaka in :gh:`106307`.) -* If Python is built in :ref:`debug mode ` or :option:`with - assertions <--with-assertions>`, :c:func:`PyTuple_SET_ITEM` and - :c:func:`PyList_SET_ITEM` now check the index argument with an assertion. - (Contributed by Victor Stinner in :gh:`106168`.) +* Add the :c:func:`PyObject_GetOptionalAttr` + and :c:func:`PyObject_GetOptionalAttrString` functions + as alternatives to :c:func:`PyObject_GetAttr` + and :c:func:`PyObject_GetAttrString` respectively. + The new functions do not raise :exc:`AttributeError` + if the requested attribute is not found on the object. + These variants are more convenient and faster + if the missing attribute should not be treated as a failure. + (Contributed by Serhiy Storchaka in :gh:`106521`.) -* Add :c:func:`PyModule_Add` function: similar to - :c:func:`PyModule_AddObjectRef` and :c:func:`PyModule_AddObject` but - always steals a reference to the value. - (Contributed by Serhiy Storchaka in :gh:`86493`.) +* Add the :c:func:`PyErr_FormatUnraisable` function + as an extension to :c:func:`PyErr_WriteUnraisable` + that allows customizing the warning message. + (Contributed by Serhiy Storchaka in :gh:`108082`.) -* Add :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` - functions: similar to :c:func:`PyDict_GetItemWithError` but returning a - :term:`strong reference` instead of a :term:`borrowed reference`. Moreover, - these functions return -1 on error and so checking ``PyErr_Occurred()`` is - not needed. - (Contributed by Victor Stinner in :gh:`106004`.) +* Add new functions that return a :term:`strong reference` instead of + a :term:`borrowed reference` for frame locals, globals, and builtins, + as part of :ref:`PEP 667 `: -* Added :c:func:`PyDict_SetDefaultRef`, which is similar to - :c:func:`PyDict_SetDefault` but returns a :term:`strong reference` instead of - a :term:`borrowed reference`. This function returns ``-1`` on error, ``0`` on - insertion, and ``1`` if the key was already present in the dictionary. - (Contributed by Sam Gross in :gh:`112066`.) + * :c:func:`PyEval_GetFrameBuiltins` replaces :c:func:`PyEval_GetBuiltins` + * :c:func:`PyEval_GetFrameGlobals` replaces :c:func:`PyEval_GetGlobals` + * :c:func:`PyEval_GetFrameLocals` replaces :c:func:`PyEval_GetLocals` -* Add :c:func:`PyDict_ContainsString` function: same as - :c:func:`PyDict_Contains`, but *key* is specified as a :c:expr:`const char*` - UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. - (Contributed by Victor Stinner in :gh:`108314`.) + (Contributed by Mark Shannon and Tian Gao in :gh:`74929`.) -* Added :c:func:`PyList_GetItemRef` function: similar to - :c:func:`PyList_GetItem` but returns a :term:`strong reference` instead of - a :term:`borrowed reference`. +* Add the :c:func:`Py_GetConstant` and :c:func:`Py_GetConstantBorrowed` + functions to get :term:`strong ` + or :term:`borrowed ` references to constants. + For example, ``Py_GetConstant(Py_CONSTANT_ZERO)`` returns a strong reference + to the constant zero. + (Contributed by Victor Stinner in :gh:`115754`.) + +* Add the :c:func:`PyImport_AddModuleRef` function + as a replacement for :c:func:`PyImport_AddModule` + that returns a :term:`strong reference` instead of a :term:`borrowed reference`. + (Contributed by Victor Stinner in :gh:`105922`.) -* Add :c:func:`Py_IsFinalizing` function: check if the main Python interpreter is +* Add the :c:func:`Py_IsFinalizing` function to check + whether the main Python interpreter is :term:`shutting down `. (Contributed by Victor Stinner in :gh:`108014`.) -* Add :c:func:`PyLong_AsInt` function: similar to :c:func:`PyLong_AsLong`, but - store the result in a C :c:expr:`int` instead of a C :c:expr:`long`. - Previously, it was known as the private function :c:func:`!_PyLong_AsInt` - (with an underscore prefix). +* Add the :c:func:`PyList_GetItemRef` function + as a replacement for :c:func:`PyList_GetItem` + that returns a :term:`strong reference` instead of a :term:`borrowed reference`. + (Contributed by Sam Gross in :gh:`114329`.) + +* Add the :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions, + mirroring the Python :meth:`!list.extend` and :meth:`!list.clear` methods. + (Contributed by Victor Stinner in :gh:`111138`.) + +* Add the :c:func:`PyLong_AsInt` function. + It behaves similarly to :c:func:`PyLong_AsLong`, + but stores the result in a C :c:expr:`int` instead of a C :c:expr:`long`. (Contributed by Victor Stinner in :gh:`108014`.) -* Python built with :file:`configure` :option:`--with-trace-refs` (tracing - references) now supports the :ref:`Limited API `. - (Contributed by Victor Stinner in :gh:`108634`.) +* Add the :c:func:`PyLong_AsNativeBytes`, :c:func:`PyLong_FromNativeBytes`, + and :c:func:`PyLong_FromUnsignedNativeBytes` functions + to simplify converting between native integer types + and Python :class:`int` objects. + (Contributed by Steve Dower in :gh:`111140`.) -* Add :c:func:`PyObject_VisitManagedDict` and - :c:func:`PyObject_ClearManagedDict` functions which must be called by the - traverse and clear functions of a type using - :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag. The `pythoncapi-compat project - `__ can be used to get these - functions on Python 3.11 and 3.12. +* Add :c:func:`PyModule_Add` function, which is similar to + :c:func:`PyModule_AddObjectRef` and :c:func:`PyModule_AddObject`, + but always steals a reference to the value. + (Contributed by Serhiy Storchaka in :gh:`86493`.) + +* Add the :c:func:`PyObject_GenericHash` function + that implements the default hashing function of a Python object. + (Contributed by Serhiy Storchaka in :gh:`113024`.) + +* Add the :c:func:`Py_HashPointer` function to hash a raw pointer. + (Contributed by Victor Stinner in :gh:`111545`.) + +* Add the :c:func:`PyObject_VisitManagedDict` and + :c:func:`PyObject_ClearManagedDict` functions. + which must be called by the traverse and clear functions of a type using + the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag. + The `pythoncapi-compat project`_ can be used to + use these functions with Python 3.11 and 3.12. (Contributed by Victor Stinner in :gh:`107073`.) -* Add :c:func:`PyUnicode_EqualToUTF8AndSize` and :c:func:`PyUnicode_EqualToUTF8` - functions: compare Unicode object with a :c:expr:`const char*` UTF-8 encoded - string and return true (``1``) if they are equal, or false (``0``) otherwise. +* Add the :c:func:`PyRefTracer_SetTracer` + and :c:func:`PyRefTracer_GetTracer` functions, + which enable tracking object creation and destruction + in the same way that the :mod:`tracemalloc` module does. + (Contributed by Pablo Galindo in :gh:`93502`.) + +* Add the :c:func:`PySys_AuditTuple` function + as an alternative to :c:func:`PySys_Audit` + that takes event arguments as a Python :class:`tuple` object. + (Contributed by Victor Stinner in :gh:`85283`.) + +* Add the :c:func:`PyThreadState_GetUnchecked()` function + as an alternative to :c:func:`PyThreadState_Get()` + that doesn't kill the process with a fatal error if it is ``NULL``. + The caller is responsible for checking if the result is ``NULL``. + (Contributed by Victor Stinner in :gh:`108867`.) + +* Add the :c:func:`PyType_GetFullyQualifiedName` function + to get the type's fully qualified name. + The module name is prepended if ``type.__module__`` is a string + and is not equal to either ``'builtins'`` or ``'__main__'``. + (Contributed by Victor Stinner in :gh:`111696`.) + +* Add the :c:func:`PyType_GetModuleName` function + to get the type's module name. + This is equivalent to getting the ``type.__module__`` attribute. + (Contributed by Eric Snow and Victor Stinner in :gh:`111696`.) + +* Add the :c:func:`PyUnicode_EqualToUTF8AndSize` + and :c:func:`PyUnicode_EqualToUTF8` functions + to compare a Unicode object with a :c:expr:`const char*` UTF-8 encoded string + and ``1`` if they are equal or ``0`` otherwise. These functions do not raise exceptions. (Contributed by Serhiy Storchaka in :gh:`110289`.) -* Add :c:func:`PyThreadState_GetUnchecked()` function: similar to - :c:func:`PyThreadState_Get()`, but don't kill the process with a fatal error - if it is NULL. The caller is responsible to check if the result is NULL. - Previously, the function was private and known as - ``_PyThreadState_UncheckedGet()``. - (Contributed by Victor Stinner in :gh:`108867`.) +* Add the :c:func:`PyWeakref_GetRef` function + as an alternative to :c:func:`PyWeakref_GetObject` + that returns a :term:`strong reference` + or ``NULL`` if the referent is no longer live. + (Contributed by Victor Stinner in :gh:`105927`.) -* Add :c:func:`PySys_AuditTuple` function: similar to :c:func:`PySys_Audit`, - but pass event arguments as a Python :class:`tuple` object. - (Contributed by Victor Stinner in :gh:`85283`.) +* Add fixed variants of functions which silently ignore errors: + + * :c:func:`PyObject_HasAttrWithError` replaces :c:func:`PyObject_HasAttr`. + * :c:func:`PyObject_HasAttrStringWithError` + replaces :c:func:`PyObject_HasAttrString`. + * :c:func:`PyMapping_HasKeyWithError` replaces :c:func:`PyMapping_HasKey`. + * :c:func:`PyMapping_HasKeyStringWithError` + replaces :c:func:`PyMapping_HasKeyString`. + + The new functions return ``-1`` for errors + and the standard ``1`` for true and ``0`` for false. + + (Contributed by Serhiy Storchaka in :gh:`108511`.) -* :c:func:`PyArg_ParseTupleAndKeywords` now supports non-ASCII keyword - parameter names. + +Changed C APIs +-------------- + +* The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` + and :c:func:`PyArg_VaParseTupleAndKeywords` + now has type :c:expr:`char * const *` in C + and :c:expr:`const char * const *` in C++, + instead of :c:expr:`char **`. + In C++, this makes these functions compatible with arguments + of type :c:expr:`const char * const *`, :c:expr:`const char **`, + or :c:expr:`char * const *` without an explicit type cast. + In C, the functions only support arguments of type :c:expr:`char * const *`. + This can be overridden with the :c:macro:`PY_CXX_CONST` macro. + (Contributed by Serhiy Storchaka in :gh:`65210`.) + +* :c:func:`PyArg_ParseTupleAndKeywords` now supports + non-ASCII keyword parameter names. (Contributed by Serhiy Storchaka in :gh:`110815`.) -* Add :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawCalloc`, - :c:func:`PyMem_RawRealloc` and :c:func:`PyMem_RawFree` to the limited C API - (version 3.13). - (Contributed by Victor Stinner in :gh:`85283`.) +* Add support for the ``%T``, ``%#T``, ``%N`` and ``%#N`` formats + to :c:func:`PyUnicode_FromFormat`: -* Add :c:func:`PySys_Audit` and :c:func:`PySys_AuditTuple` functions to the - limited C API. - (Contributed by Victor Stinner in :gh:`85283`.) + * ``%T``: Get the fully qualified name of an object type + * ``%#T``: As above, but use a colon as the separator + * ``%N``: Get the fully qualified name of a type + * ``%#N``: As above, but use a colon as the separator -* Add :c:func:`PyErr_FormatUnraisable` function: similar to - :c:func:`PyErr_WriteUnraisable`, but allow customizing the warning message. - (Contributed by Serhiy Storchaka in :gh:`108082`.) + See :pep:`737` for more information. + (Contributed by Victor Stinner in :gh:`111696`.) -* Add :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions: similar to - Python ``list.extend()`` and ``list.clear()`` methods. - (Contributed by Victor Stinner in :gh:`111138`.) +* You no longer have to define the ``PY_SSIZE_T_CLEAN`` macro before + including :file:`Python.h` when using ``#`` formats in + :ref:`format codes `. + APIs accepting the format codes always use ``Py_ssize_t`` for ``#`` formats. + (Contributed by Inada Naoki in :gh:`104922`.) -* Add :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions: remove a - key from a dictionary and optionally return the removed value. This is - similar to :meth:`dict.pop`, but without the default value and not raising - :exc:`KeyError` if the key is missing. - (Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.) +* If Python is built in :ref:`debug mode ` + or :option:`with assertions <--with-assertions>`, + :c:func:`PyTuple_SET_ITEM` and :c:func:`PyList_SET_ITEM` + now check the index argument with an assertion. + (Contributed by Victor Stinner in :gh:`106168`.) -* Add :c:func:`Py_HashPointer` function to hash a pointer. - (Contributed by Victor Stinner in :gh:`111545`.) -* Add :c:func:`PyObject_GenericHash` function that implements the default - hashing function of a Python object. - (Contributed by Serhiy Storchaka in :gh:`113024`.) +Limited C API Changes +--------------------- -* Add PyTime C API: +* The following functions are now included in the Limited C API: - * :c:type:`PyTime_t` type. - * :c:var:`PyTime_MIN` and :c:var:`PyTime_MAX` constants. - * Add functions: + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawFree` + * :c:func:`PySys_Audit` + * :c:func:`PySys_AuditTuple` + * :c:func:`PyType_GetModuleByDef` - * :c:func:`PyTime_AsSecondsDouble`. - * :c:func:`PyTime_Monotonic`. - * :c:func:`PyTime_MonotonicRaw`. - * :c:func:`PyTime_PerfCounter`. - * :c:func:`PyTime_PerfCounterRaw`. - * :c:func:`PyTime_Time`. - * :c:func:`PyTime_TimeRaw`. + (Contributed by Victor Stinner in :gh:`85283`, :gh:`85283`, and :gh:`116936`.) - (Contributed by Victor Stinner and Petr Viktorin in :gh:`110850`.) +* Python built with :option:`--with-trace-refs` (tracing references) + now supports the :ref:`Limited API `. + (Contributed by Victor Stinner in :gh:`108634`.) -* Add :c:func:`PyLong_AsNativeBytes`, :c:func:`PyLong_FromNativeBytes` and - :c:func:`PyLong_FromUnsignedNativeBytes` functions to simplify converting - between native integer types and Python :class:`int` objects. - (Contributed by Steve Dower in :gh:`111140`.) -* Add :c:func:`PyType_GetFullyQualifiedName` function to get the type's fully - qualified name. Equivalent to ``f"{type.__module__}.{type.__qualname__}"``, - or ``type.__qualname__`` if ``type.__module__`` is not a string or is equal - to ``"builtins"``. - (Contributed by Victor Stinner in :gh:`111696`.) +Removed C APIs +-------------- -* Add :c:func:`PyType_GetModuleName` function to get the type's module name. - Equivalent to getting the ``type.__module__`` attribute. - (Contributed by Eric Snow and Victor Stinner in :gh:`111696`.) +* Remove several functions, macros, variables, etc + with names prefixed by ``_Py`` or ``_PY`` (which are considered private). + If your project is affected by one of these removals + and you believe that the removed API should remain available, + please :ref:`open a new issue ` to request a public C API + and add ``cc: @vstinner`` to the issue to notify Victor Stinner. + (Contributed by Victor Stinner in :gh:`106320`.) -* Add support for ``%T``, ``%#T``, ``%N`` and ``%#N`` formats to - :c:func:`PyUnicode_FromFormat`: format the fully qualified name of an object - type and of a type: call :c:func:`PyType_GetModuleName`. See :pep:`737` for - more information. - (Contributed by Victor Stinner in :gh:`111696`.) +* Remove old buffer protocols deprecated in Python 3.0. + Use :ref:`bufferobjects` instead. -* Add :c:func:`Py_GetConstant` and :c:func:`Py_GetConstantBorrowed` functions - to get constants. For example, ``Py_GetConstant(Py_CONSTANT_ZERO)`` returns a - :term:`strong reference` to the constant zero. - (Contributed by Victor Stinner in :gh:`115754`.) + * :c:func:`!PyObject_CheckReadBuffer`: + Use :c:func:`PyObject_CheckBuffer` to test + whether the object supports the buffer protocol. + Note that :c:func:`PyObject_CheckBuffer` doesn't guarantee + that :c:func:`PyObject_GetBuffer` will succeed. + To test if the object is actually readable, + see the next example of :c:func:`PyObject_GetBuffer`. -* Add :c:func:`PyType_GetModuleByDef` to the limited C API - (Contributed by Victor Stinner in :gh:`116936`.) + * :c:func:`!PyObject_AsCharBuffer`, :c:func:`!PyObject_AsReadBuffer`: + Use :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: -* Add two new functions to the C-API, :c:func:`PyRefTracer_SetTracer` and - :c:func:`PyRefTracer_GetTracer`, that allow to track object creation and - destruction the same way the :mod:`tracemalloc` module does. (Contributed - by Pablo Galindo in :gh:`93502`.) + .. code-block:: c -* Add :c:func:`PyEval_GetFrameBuiltins`, :c:func:`PyEval_GetFrameGlobals`, and - :c:func:`PyEval_GetFrameLocals` to the C API. These replacements for - :c:func:`PyEval_GetBuiltins`, :c:func:`PyEval_GetGlobals`, and - :c:func:`PyEval_GetLocals` return :term:`strong references ` - rather than borrowed references. (Added as part of :pep:`667`.) + Py_buffer view; + if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) < 0) { + return NULL; + } + // Use `view.buf` and `view.len` to read from the buffer. + // You may need to cast buf as `(const char*)view.buf`. + PyBuffer_Release(&view); -* Add :c:type:`PyMutex` API, a lightweight mutex that occupies a single byte. - The :c:func:`PyMutex_Lock` function will release the GIL (if currently held) - if the operation needs to block. - (Contributed by Sam Gross in :gh:`108724`.) + * :c:func:`!PyObject_AsWriteBuffer`: + Use :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: -* Add C API functions for generating :pep:`669` monitoring events. - (Contributed by Irit Katriel in :gh:`111997`). + .. code-block:: c + + Py_buffer view; + if (PyObject_GetBuffer(obj, &view, PyBUF_WRITABLE) < 0) { + return NULL; + } + // Use `view.buf` and `view.len` to write to the buffer. + PyBuffer_Release(&view); + + (Contributed by Inada Naoki in :gh:`85275`.) + +* Remove various functions deprecated in Python 3.9: + + * :c:func:`!PyEval_CallObject`, :c:func:`!PyEval_CallObjectWithKeywords`: + Use :c:func:`PyObject_CallNoArgs` or :c:func:`PyObject_Call` instead. + + .. warning:: + + In :c:func:`PyObject_Call`, positional arguments must be a :class:`tuple` + and must not be ``NULL``, + and keyword arguments must be a :class:`dict` or ``NULL``, + whereas the removed functions checked argument types + and accepted ``NULL`` positional and keyword arguments. + To replace ``PyEval_CallObjectWithKeywords(func, NULL, kwargs)`` with + :c:func:`PyObject_Call`, + pass an empty tuple as positional arguments using + :c:func:`PyTuple_New(0) `. + + * :c:func:`!PyEval_CallFunction`: + Use :c:func:`PyObject_CallFunction` instead. + * :c:func:`!PyEval_CallMethod`: + Use :c:func:`PyObject_CallMethod` instead. + * :c:func:`!PyCFunction_Call`: + Use :c:func:`PyObject_Call` instead. + + (Contributed by Victor Stinner in :gh:`105107`.) + +* Remove the following old functions to configure the Python initialization, + deprecated in Python 3.11: + + * :c:func:`!PySys_AddWarnOptionUnicode`: + Use :c:member:`PyConfig.warnoptions` instead. + * :c:func:`!PySys_AddWarnOption`: + Use :c:member:`PyConfig.warnoptions` instead. + * :c:func:`!PySys_AddXOption`: + Use :c:member:`PyConfig.xoptions` instead. + * :c:func:`!PySys_HasWarnOptions`: + Use :c:member:`PyConfig.xoptions` instead. + * :c:func:`!PySys_SetPath`: + Set :c:member:`PyConfig.module_search_paths` instead. + * :c:func:`!Py_SetPath`: + Set :c:member:`PyConfig.module_search_paths` instead. + * :c:func:`!Py_SetStandardStreamEncoding`: + Set :c:member:`PyConfig.stdio_encoding` instead, + and set also maybe :c:member:`PyConfig.legacy_windows_stdio` (on Windows). + * :c:func:`!_Py_SetProgramFullPath`: + Set :c:member:`PyConfig.executable` instead. + + Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization + Configuration ` instead (:pep:`587`), added to Python 3.8. + (Contributed by Victor Stinner in :gh:`105145`.) + +* Remove :c:func:`!PyEval_AcquireLock` and :c:func:`!PyEval_ReleaseLock` functions, + deprecated in Python 3.2. + They didn't update the current thread state. + They can be replaced with: + + * :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread`; + * low-level :c:func:`PyEval_AcquireThread` and :c:func:`PyEval_RestoreThread`; + * or :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`. + + (Contributed by Victor Stinner in :gh:`105182`.) + +* Remove the :c:func:`!PyEval_ThreadsInitialized` function, + deprecated in Python 3.9. + Since Python 3.7, :c:func:`!Py_Initialize` always creates the GIL: + calling :c:func:`!PyEval_InitThreads` does nothing and + :c:func:`!PyEval_ThreadsInitialized` always returns non-zero. + (Contributed by Victor Stinner in :gh:`105182`.) + +* Remove the :c:func:`!_PyInterpreterState_Get` alias to + :c:func:`PyInterpreterState_Get()` + which was kept for backward compatibility with Python 3.8. + The `pythoncapi-compat project`_ can be used to get + :c:func:`PyInterpreterState_Get()` on Python 3.8 and older. + (Contributed by Victor Stinner in :gh:`106320`.) + +* Remove the private :c:func:`!_PyObject_FastCall` function: + use :c:func:`!PyObject_Vectorcall` which is available since Python 3.8 + (:pep:`590`). + (Contributed by Victor Stinner in :gh:`106023`.) + +* Remove the ``cpython/pytime.h`` header file, + which only contained private functions. + (Contributed by Victor Stinner in :gh:`106316`.) + +* Remove the undocumented ``PY_TIMEOUT_MAX`` constant from the limited C API. + (Contributed by Victor Stinner in :gh:`110014`.) + +* Remove the old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` + and ``Py_TRASHCAN_SAFE_END``. + Replace both with the new macros ``Py_TRASHCAN_BEGIN`` + and ``Py_TRASHCAN_END``. + (Contributed by Irit Katriel in :gh:`105111`.) + +Deprecated C APIs +----------------- + +* Deprecate old Python initialization functions: + + * :c:func:`PySys_ResetWarnOptions`: + Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + * :c:func:`Py_GetExecPrefix`: + Get :data:`sys.exec_prefix` instead. + * :c:func:`Py_GetPath`: + Get :data:`sys.path` instead. + * :c:func:`Py_GetPrefix`: + Get :data:`sys.prefix` instead. + * :c:func:`Py_GetProgramFullPath`: + Get :data:`sys.executable` instead. + * :c:func:`Py_GetProgramName`: + Get :data:`sys.executable` instead. + * :c:func:`Py_GetPythonHome`: + Get :c:member:`PyConfig.home` + or the :envvar:`PYTHONHOME` environment variable instead. + + (Contributed by Victor Stinner in :gh:`105145`.) + +* :term:`Soft deprecate ` the + :c:func:`PyEval_GetBuiltins`, :c:func:`PyEval_GetGlobals`, + and :c:func:`PyEval_GetLocals` functions, + which return a :term:`borrowed reference`. + (Soft deprecated as part of :pep:`667`.) + +* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function, + which is just an alias to :c:func:`PyImport_ImportModule` since Python 3.3. + (Contributed by Victor Stinner in :gh:`105396`.) + +* :term:`Soft deprecate ` the + :c:func:`PyModule_AddObject` function. + It should be replaced with :c:func:`PyModule_Add` + or :c:func:`PyModule_AddObjectRef`. + (Contributed by Serhiy Storchaka in :gh:`86493`.) + +* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types. + Use the :c:type:`wchar_t` type directly instead. + Since Python 3.3, ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` + are just aliases to :c:type:`!wchar_t`. + (Contributed by Victor Stinner in :gh:`105156`.) + +* Deprecate the :c:func:`PyWeakref_GetObject` and + :c:func:`PyWeakref_GET_OBJECT` functions, + which return a :term:`borrowed reference`. + Replace them with the new :c:func:`PyWeakref_GetRef` function, + which returns a :term:`strong reference`. + The `pythoncapi-compat project`_ can be used to get + :c:func:`PyWeakref_GetRef` on Python 3.12 and older. + (Contributed by Victor Stinner in :gh:`105927`.) + +.. Add deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/c-api-pending-removal-in-3.14.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst + +.. include:: ../deprecations/c-api-pending-removal-in-future.rst + +.. _pythoncapi-compat project: https://github.com/python/pythoncapi-compat/ Build Changes ============= @@ -2295,7 +2565,7 @@ Changes in the Python API main interpreter. Any library or application that provides a custom ``_thread`` module - must provide ``_is_main_interpreter()``, just like the module's + must provide :func:`!_is_main_interpreter`, just like the module's other "private" attributes. (See :gh:`112826`.) @@ -2347,22 +2617,22 @@ Changes in the C API -------------------- * ``Python.h`` no longer includes the ```` standard header. It was - included for the ``finite()`` function which is now provided by the + included for the :c:func:`!finite` function which is now provided by the ```` header. It should now be included explicitly if needed. Remove also the ``HAVE_IEEEFP_H`` macro. (Contributed by Victor Stinner in :gh:`108765`.) * ``Python.h`` no longer includes these standard header files: ````, ```` and ````. If needed, they should now be - included explicitly. For example, ```` provides the ``clock()`` and - ``gmtime()`` functions, ```` provides the ``select()`` - function, and ```` provides the ``futimes()``, ``gettimeofday()`` - and ``setitimer()`` functions. + included explicitly. For example, ```` provides the :c:func:`!clock` and + :c:func:`!gmtime` functions, ```` provides the :c:func:`!select` + function, and ```` provides the :c:func:`!futimes`, :c:func:`!gettimeofday` + and :c:func:`!setitimer` functions. (Contributed by Victor Stinner in :gh:`108765`.) * On Windows, ``Python.h`` no longer includes the ```` standard header file. If needed, it should now be included explicitly. For example, it - provides ``offsetof()`` function, and ``size_t`` and ``ptrdiff_t`` types. + provides :c:func:`!offsetof` function, and ``size_t`` and ``ptrdiff_t`` types. Including ```` explicitly was already needed by all other platforms, the ``HAVE_STDDEF_H`` macro is only defined on Windows. (Contributed by Victor Stinner in :gh:`108765`.) @@ -2443,175 +2713,6 @@ Changes in the C API is redundant now that :c:func:`PyFrame_GetLocals` returns a write-through proxy for :term:`optimized scopes `. (Changed as part of :pep:`667`.) -Removed C APIs --------------- - -* Remove many APIs (functions, macros, variables) with names prefixed by - ``_Py`` or ``_PY`` (considered as private API). If your project is affected - by one of these removals and you consider that the removed API should remain - available, please open a new issue to request a public C API and - add ``cc @vstinner`` to the issue to notify Victor Stinner. - (Contributed by Victor Stinner in :gh:`106320`.) - -* Remove functions deprecated in Python 3.9: - - * ``PyEval_CallObject()``, ``PyEval_CallObjectWithKeywords()``: use - :c:func:`PyObject_CallNoArgs` or :c:func:`PyObject_Call` instead. - Warning: :c:func:`PyObject_Call` positional arguments must be a - :class:`tuple` and must not be ``NULL``, keyword arguments must be a - :class:`dict` or ``NULL``, whereas removed functions checked arguments type - and accepted ``NULL`` positional and keyword arguments. - To replace ``PyEval_CallObjectWithKeywords(func, NULL, kwargs)`` with - :c:func:`PyObject_Call`, pass an empty tuple as positional arguments using - :c:func:`PyTuple_New(0) `. - * ``PyEval_CallFunction()``: use :c:func:`PyObject_CallFunction` instead. - * ``PyEval_CallMethod()``: use :c:func:`PyObject_CallMethod` instead. - * ``PyCFunction_Call()``: use :c:func:`PyObject_Call` instead. - - (Contributed by Victor Stinner in :gh:`105107`.) - -* Remove old buffer protocols deprecated in Python 3.0. Use :ref:`bufferobjects` instead. - - * :c:func:`!PyObject_CheckReadBuffer`: Use :c:func:`PyObject_CheckBuffer` to - test if the object supports the buffer protocol. - Note that :c:func:`PyObject_CheckBuffer` doesn't guarantee that - :c:func:`PyObject_GetBuffer` will succeed. - To test if the object is actually readable, see the next example - of :c:func:`PyObject_GetBuffer`. - - * :c:func:`!PyObject_AsCharBuffer`, :c:func:`!PyObject_AsReadBuffer`: - :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: - - .. code-block:: c - - Py_buffer view; - if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) < 0) { - return NULL; - } - // Use `view.buf` and `view.len` to read from the buffer. - // You may need to cast buf as `(const char*)view.buf`. - PyBuffer_Release(&view); - - * :c:func:`!PyObject_AsWriteBuffer`: Use - :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: - - .. code-block:: c - - Py_buffer view; - if (PyObject_GetBuffer(obj, &view, PyBUF_WRITABLE) < 0) { - return NULL; - } - // Use `view.buf` and `view.len` to write to the buffer. - PyBuffer_Release(&view); - - (Contributed by Inada Naoki in :gh:`85275`.) - -* Remove the following old functions to configure the Python initialization, - deprecated in Python 3.11: - - * ``PySys_AddWarnOptionUnicode()``: use :c:member:`PyConfig.warnoptions` instead. - * ``PySys_AddWarnOption()``: use :c:member:`PyConfig.warnoptions` instead. - * ``PySys_AddXOption()``: use :c:member:`PyConfig.xoptions` instead. - * ``PySys_HasWarnOptions()``: use :c:member:`PyConfig.xoptions` instead. - * ``PySys_SetPath()``: set :c:member:`PyConfig.module_search_paths` instead. - * ``Py_SetPath()``: set :c:member:`PyConfig.module_search_paths` instead. - * ``Py_SetStandardStreamEncoding()``: set :c:member:`PyConfig.stdio_encoding` - instead, and set also maybe :c:member:`PyConfig.legacy_windows_stdio` (on - Windows). - * ``_Py_SetProgramFullPath()``: set :c:member:`PyConfig.executable` instead. - - Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization - Configuration ` instead (:pep:`587`), added to Python 3.8. - (Contributed by Victor Stinner in :gh:`105145`.) - -* Remove ``PyEval_ThreadsInitialized()`` - function, deprecated in Python 3.9. Since Python 3.7, ``Py_Initialize()`` - always creates the GIL: calling ``PyEval_InitThreads()`` does nothing and - ``PyEval_ThreadsInitialized()`` always returned non-zero. - (Contributed by Victor Stinner in :gh:`105182`.) - -* Remove ``PyEval_AcquireLock()`` and ``PyEval_ReleaseLock()`` functions, - deprecated in Python 3.2. They didn't update the current thread state. - They can be replaced with: - - * :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread`; - * low-level :c:func:`PyEval_AcquireThread` and :c:func:`PyEval_RestoreThread`; - * or :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`. - - (Contributed by Victor Stinner in :gh:`105182`.) - -* Remove private ``_PyObject_FastCall()`` function: - use ``PyObject_Vectorcall()`` which is available since Python 3.8 - (:pep:`590`). - (Contributed by Victor Stinner in :gh:`106023`.) - -* Remove ``cpython/pytime.h`` header file: it only contained private functions. - (Contributed by Victor Stinner in :gh:`106316`.) - -* Remove ``_PyInterpreterState_Get()`` alias to - :c:func:`PyInterpreterState_Get()` which was kept for backward compatibility - with Python 3.8. The `pythoncapi-compat project - `__ can be used to get - :c:func:`PyInterpreterState_Get()` on Python 3.8 and older. - (Contributed by Victor Stinner in :gh:`106320`.) - -* The :c:func:`PyModule_AddObject` function is now :term:`soft deprecated`: - :c:func:`PyModule_Add` or :c:func:`PyModule_AddObjectRef` functions should - be used instead. - (Contributed by Serhiy Storchaka in :gh:`86493`.) - -* Remove undocumented ``PY_TIMEOUT_MAX`` constant from the limited C API. - (Contributed by Victor Stinner in :gh:`110014`.) - -Deprecated C APIs ------------------ - -* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly - the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and - ``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. - (Contributed by Victor Stinner in :gh:`105156`.) - -* Deprecate old Python initialization functions: - - * :c:func:`PySys_ResetWarnOptions`: - clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. - * :c:func:`Py_GetPath`: get :data:`sys.path` instead. - * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. - * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. - * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. - * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or - :envvar:`PYTHONHOME` environment variable instead. - - Functions scheduled for removal in Python 3.15. - (Contributed by Victor Stinner in :gh:`105145`.) - -* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function which is just - an alias to :c:func:`PyImport_ImportModule` since Python 3.3. - Scheduled for removal in Python 3.15. - (Contributed by Victor Stinner in :gh:`105396`.) - -* Deprecate the :c:func:`PyWeakref_GetObject` and - :c:func:`PyWeakref_GET_OBJECT` functions, which return a :term:`borrowed - reference`: use the new :c:func:`PyWeakref_GetRef` function instead, it - returns a :term:`strong reference`. The `pythoncapi-compat project - `__ can be used to get - :c:func:`PyWeakref_GetRef` on Python 3.12 and older. - (Contributed by Victor Stinner in :gh:`105927`.) - -* Deprecate the :c:func:`PyEval_GetBuiltins`, :c:func:`PyEval_GetGlobals`, and - :c:func:`PyEval_GetLocals` functions, which return a :term:`borrowed reference`. - Refer to the deprecation notices on each function for their recommended replacements. - (Soft deprecated as part of :pep:`667`.) - -.. Add deprecations above alphabetically, not here at the end. - -.. include:: ../deprecations/c-api-pending-removal-in-3.14.rst - -.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst - -.. include:: ../deprecations/c-api-pending-removal-in-future.rst - Regression Test Changes ======================= From 6ab634840c662ae07d90655e5e50ca43421da4be Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Mon, 23 Sep 2024 17:16:19 +0300 Subject: [PATCH 043/105] Add Kirill Podoprigora for AST codeowners (#124302) --- .github/CODEOWNERS | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6e934232756c5d..680f2ed5be031a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -38,7 +38,6 @@ Python/compile.c @markshannon @iritkatriel Python/assemble.c @markshannon @iritkatriel Python/flowgraph.c @markshannon @iritkatriel Python/instruction_sequence.c @iritkatriel -Python/ast_opt.c @isidentical Python/bytecodes.c @markshannon Python/optimizer*.c @markshannon Python/optimizer_analysis.c @Fidget-Spinner @@ -158,10 +157,12 @@ Include/internal/pycore_time.h @pganssle @abalkin /Tools/cases_generator/ @markshannon # AST -Python/ast.c @isidentical @JelleZijlstra -Parser/asdl.py @isidentical @JelleZijlstra -Parser/asdl_c.py @isidentical @JelleZijlstra -Lib/ast.py @isidentical @JelleZijlstra +Python/ast.c @isidentical @JelleZijlstra @eclips4 +Python/ast_opt.c @isidentical @eclips4 +Parser/asdl.py @isidentical @JelleZijlstra @eclips4 +Parser/asdl_c.py @isidentical @JelleZijlstra @eclips4 +Lib/ast.py @isidentical @JelleZijlstra @eclips4 +Lib/test/test_ast/ @eclips4 # Mock /Lib/unittest/mock.py @cjw296 From df7228ce140ecb005d44a0c171ba4d098b3fa67c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 23 Sep 2024 16:27:46 +0100 Subject: [PATCH 044/105] gh-124254: Detect freethreaded MSI component when doing an upgrade on Windows (GH-124279) --- .../next/Windows/2024-09-20-11-18-50.gh-issue-124254.iPin-L.rst | 1 + Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp | 1 + 2 files changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Windows/2024-09-20-11-18-50.gh-issue-124254.iPin-L.rst diff --git a/Misc/NEWS.d/next/Windows/2024-09-20-11-18-50.gh-issue-124254.iPin-L.rst b/Misc/NEWS.d/next/Windows/2024-09-20-11-18-50.gh-issue-124254.iPin-L.rst new file mode 100644 index 00000000000000..b93e356edb501d --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-09-20-11-18-50.gh-issue-124254.iPin-L.rst @@ -0,0 +1 @@ +Ensures experimental free-threaded binaries remain installed when updating. diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp index 7cddda9b06555d..094ddba4f1ad8f 100644 --- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp +++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp @@ -213,6 +213,7 @@ static struct { LPCWSTR regName; LPCWSTR variableName; } OPTIONAL_FEATURES[] = { { L"Shortcuts", L"Shortcuts" }, // Include_launcher and AssociateFiles are handled separately and so do // not need to be included in this list. + { L"freethreaded", L"Include_freethreaded" }, { nullptr, nullptr } }; From 6203ef35dd4ee9dd59759ce83eace8eacac69685 Mon Sep 17 00:00:00 2001 From: decorator-factory <42166884+decorator-factory@users.noreply.github.com> Date: Mon, 23 Sep 2024 19:34:08 +0300 Subject: [PATCH 045/105] Replace the term `Immutable` with a `Hashable` in the `sequence` entry of the Glossary (#124350) The term `Immutable` in the `sequence` entry of the glossary is used incorrectly, in fact dicts accepts hashable keys, which is not the same as immutable. --- Doc/glossary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index ea2f4da093caa0..b3fd3c96b5c217 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -1131,7 +1131,7 @@ Glossary :class:`tuple`, and :class:`bytes`. Note that :class:`dict` also supports :meth:`~object.__getitem__` and :meth:`!__len__`, but is considered a mapping rather than a sequence because the lookups use arbitrary - :term:`immutable` keys rather than integers. + :term:`hashable` keys rather than integers. The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just From c87b0e4a462f98c418f750c6c95d4d8715c38332 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 23 Sep 2024 19:10:55 +0100 Subject: [PATCH 046/105] GH-124284: Add stats for refcount operations on immortal objects (GH-124288) --- Include/cpython/pystats.h | 8 ++++++++ Include/internal/pycore_object.h | 5 +++++ Include/pystats.h | 2 ++ Include/refcount.h | 7 +++++++ Python/ceval.c | 3 +++ Python/specialize.c | 12 ++++++++---- Tools/scripts/summarize_stats.py | 18 ++++++++++++------ 7 files changed, 45 insertions(+), 10 deletions(-) diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index c4480758f48514..f1ca54839fbc38 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -70,6 +70,10 @@ typedef struct _object_stats { uint64_t decrefs; uint64_t interpreter_increfs; uint64_t interpreter_decrefs; + uint64_t immortal_increfs; + uint64_t immortal_decrefs; + uint64_t interpreter_immortal_increfs; + uint64_t interpreter_immortal_decrefs; uint64_t allocations; uint64_t allocations512; uint64_t allocations4k; @@ -163,7 +167,11 @@ PyAPI_DATA(PyStats*) _Py_stats; #ifdef _PY_INTERPRETER # define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0) # define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0) +# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_increfs++; } while (0) +# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_decrefs++; } while (0) #else # define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0) # define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0) +# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_increfs++; } while (0) +# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_decrefs++; } while (0) #endif diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index ad92a74d2b6b56..0d885b4630d2f0 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -214,6 +214,7 @@ static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { if (_Py_IsImmortal(op)) { + _Py_DECREF_IMMORTAL_STAT_INC(); return; } _Py_DECREF_STAT_INC(); @@ -235,6 +236,7 @@ static inline void _Py_DECREF_NO_DEALLOC(PyObject *op) { if (_Py_IsImmortal(op)) { + _Py_DECREF_IMMORTAL_STAT_INC(); return; } _Py_DECREF_STAT_INC(); @@ -315,6 +317,7 @@ _Py_INCREF_TYPE(PyTypeObject *type) { if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { assert(_Py_IsImmortalLoose(type)); + _Py_INCREF_IMMORTAL_STAT_INC(); return; } @@ -355,6 +358,7 @@ _Py_DECREF_TYPE(PyTypeObject *type) { if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { assert(_Py_IsImmortalLoose(type)); + _Py_DECREF_IMMORTAL_STAT_INC(); return; } @@ -511,6 +515,7 @@ _Py_TryIncrefFast(PyObject *op) { local += 1; if (local == 0) { // immortal + _Py_INCREF_IMMORTAL_STAT_INC(); return 1; } if (_Py_IsOwnedByCurrentThread(op)) { diff --git a/Include/pystats.h b/Include/pystats.h index acfa32201711e0..a515570d1bb3bc 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -18,6 +18,8 @@ extern "C" { #else # define _Py_INCREF_STAT_INC() ((void)0) # define _Py_DECREF_STAT_INC() ((void)0) +# define _Py_INCREF_IMMORTAL_STAT_INC() ((void)0) +# define _Py_DECREF_IMMORTAL_STAT_INC() ((void)0) #endif // !Py_STATS #ifdef __cplusplus diff --git a/Include/refcount.h b/Include/refcount.h index a0bd2087fb1b57..1d736b194dcc77 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -227,6 +227,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); uint32_t new_local = local + 1; if (new_local == 0) { + _Py_INCREF_IMMORTAL_STAT_INC(); // local is equal to _Py_IMMORTAL_REFCNT: do nothing return; } @@ -241,6 +242,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; PY_UINT32_T new_refcnt = cur_refcnt + 1; if (new_refcnt == 0) { + _Py_INCREF_IMMORTAL_STAT_INC(); // cur_refcnt is equal to _Py_IMMORTAL_REFCNT: the object is immortal, // do nothing return; @@ -249,6 +251,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) #else // Explicitly check immortality against the immortal value if (_Py_IsImmortal(op)) { + _Py_INCREF_IMMORTAL_STAT_INC(); return; } op->ob_refcnt++; @@ -295,6 +298,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) { uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + _Py_DECREF_IMMORTAL_STAT_INC(); return; } _Py_DECREF_STAT_INC(); @@ -320,6 +324,7 @@ static inline void Py_DECREF(PyObject *op) { uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + _Py_DECREF_IMMORTAL_STAT_INC(); return; } _Py_DECREF_STAT_INC(); @@ -343,6 +348,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) _Py_NegativeRefcount(filename, lineno, op); } if (_Py_IsImmortal(op)) { + _Py_DECREF_IMMORTAL_STAT_INC(); return; } _Py_DECREF_STAT_INC(); @@ -359,6 +365,7 @@ static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) // Non-limited C API and limited C API for Python 3.9 and older access // directly PyObject.ob_refcnt. if (_Py_IsImmortal(op)) { + _Py_DECREF_IMMORTAL_STAT_INC(); return; } _Py_DECREF_STAT_INC(); diff --git a/Python/ceval.c b/Python/ceval.c index 6236c668ee65eb..44b39f5d36c93c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -79,6 +79,7 @@ do { \ PyObject *op = _PyObject_CAST(arg); \ if (_Py_IsImmortal(op)) { \ + _Py_DECREF_IMMORTAL_STAT_INC(); \ break; \ } \ _Py_DECREF_STAT_INC(); \ @@ -93,6 +94,7 @@ do { \ PyObject *op = _PyObject_CAST(arg); \ if (_Py_IsImmortal(op)) { \ + _Py_DECREF_IMMORTAL_STAT_INC(); \ break; \ } \ _Py_DECREF_STAT_INC(); \ @@ -110,6 +112,7 @@ PyObject *op = _PyObject_CAST(arg); \ uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \ if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \ + _Py_DECREF_IMMORTAL_STAT_INC(); \ break; \ } \ _Py_DECREF_STAT_INC(); \ diff --git a/Python/specialize.c b/Python/specialize.c index da618952e85978..d8bff39511cf12 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -205,10 +205,14 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big); fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values); - fprintf(out, "Object interpreter increfs: %" PRIu64 "\n", stats->interpreter_increfs); - fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); - fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs); - fprintf(out, "Object decrefs: %" PRIu64 "\n", stats->decrefs); + fprintf(out, "Object interpreter mortal increfs: %" PRIu64 "\n", stats->interpreter_increfs); + fprintf(out, "Object interpreter mortal decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); + fprintf(out, "Object mortal increfs: %" PRIu64 "\n", stats->increfs); + fprintf(out, "Object mortal decrefs: %" PRIu64 "\n", stats->decrefs); + fprintf(out, "Object interpreter immortal increfs: %" PRIu64 "\n", stats->interpreter_immortal_increfs); + fprintf(out, "Object interpreter immortal decrefs: %" PRIu64 "\n", stats->interpreter_immortal_decrefs); + fprintf(out, "Object immortal increfs: %" PRIu64 "\n", stats->immortal_increfs); + fprintf(out, "Object immortal decrefs: %" PRIu64 "\n", stats->immortal_decrefs); fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request); fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index ffbc40e6a37f3d..5793e5c649d6b3 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -398,12 +398,18 @@ def get_object_stats(self) -> dict[str, tuple[int, int]]: total_allocations = self._data.get("Object allocations", 0) + self._data.get( "Object allocations from freelist", 0 ) - total_increfs = self._data.get( - "Object interpreter increfs", 0 - ) + self._data.get("Object increfs", 0) - total_decrefs = self._data.get( - "Object interpreter decrefs", 0 - ) + self._data.get("Object decrefs", 0) + total_increfs = ( + self._data.get("Object interpreter mortal increfs", 0) + + self._data.get("Object mortal increfs", 0) + + self._data.get("Object interpreter immortal increfs", 0) + + self._data.get("Object immortal increfs", 0) + ) + total_decrefs = ( + self._data.get("Object interpreter mortal decrefs", 0) + + self._data.get("Object mortal decrefs", 0) + + self._data.get("Object interpreter immortal decrefs", 0) + + self._data.get("Object immortal decrefs", 0) + ) result = {} for key, value in self._data.items(): From 2f6d4109b84d40b76e8814233ecfcc02291f71be Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 19:14:37 +0100 Subject: [PATCH 047/105] GH-109975: Copyedit 3.13 What's New: Porting to Python 3.13 (#124341) Copyedit Porting to Python 3.13 --- Doc/whatsnew/3.13.rst | 189 +++++++++++++++++++++++------------------- 1 file changed, 104 insertions(+), 85 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 96fee0f604ccc7..91ddc1c3212d5a 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -520,7 +520,7 @@ old generation, instead of collecting one or more generations. The behavior of :func:`!gc.collect` changes slightly: -* ``gc.collect(1)``: Performs an increment of GC, +* ``gc.collect(1)``: Performs an increment of garbage collection, rather than collecting generation 1. * Other calls to :func:`!gc.collect` are unchanged. @@ -2250,6 +2250,19 @@ Changed C APIs non-ASCII keyword parameter names. (Contributed by Serhiy Storchaka in :gh:`110815`.) +* The :c:func:`!PyCode_GetFirstFree` function is now unstable API + and is now named :c:func:`PyUnstable_Code_GetFirstFree`. + (Contributed by Bogdan Romanyuk in :gh:`115781`.) + +* The :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, + :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, + :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, + and :c:func:`PySys_GetObject` functions, + each of which clears all errors which occurred when calling them + now reports these errors using :func:`sys.unraisablehook`. + You may replace them with other functions as recommended in the documentation. + (Contributed by Serhiy Storchaka in :gh:`106672`.) + * Add support for the ``%T``, ``%#T``, ``%N`` and ``%#N`` formats to :c:func:`PyUnicode_FromFormat`: @@ -2555,63 +2568,75 @@ that may require changes to your code. Changes in the Python API ------------------------- -* An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to - retrieve a username, instead of :exc:`ImportError` on non-Unix platforms or - :exc:`KeyError` on Unix platforms where the password database is empty. +.. _pep667-porting-notes-py: -* The :mod:`threading` module now expects the :mod:`!_thread` module to have - an ``_is_main_interpreter`` attribute. It is a function with no - arguments that returns ``True`` if the current interpreter is the - main interpreter. +* :ref:`PEP 667 ` introduces several changes + to the semantics of :func:`locals` and :attr:`f_locals `: + + * Calling :func:`locals` in an :term:`optimized scope` now produces an + independent snapshot on each call, and hence no longer implicitly updates + previously returned references. Obtaining the legacy CPython behaviour now + requires explicit calls to update the initially returned dictionary with the + results of subsequent calls to :func:`!locals`. Code execution functions that + implicitly target :func:`!locals` (such as ``exec`` and ``eval``) must be + passed an explicit namespace to access their results in an optimized scope. + (Changed as part of :pep:`667`.) + + * Calling :func:`locals` from a comprehension at module or class scope + (including via ``exec`` or ``eval``) once more behaves as if the comprehension + were running as an independent nested function (i.e. the local variables from + the containing scope are not included). In Python 3.12, this had changed + to include the local variables from the containing scope when implementing + :pep:`709`. (Changed as part of :pep:`667`.) + + * Accessing :attr:`FrameType.f_locals ` in an + :term:`optimized scope` now returns a write-through proxy rather than a + snapshot that gets updated at ill-specified times. If a snapshot is desired, + it must be created explicitly with ``dict`` or the proxy's ``.copy()`` method. + (Changed as part of :pep:`667`.) + +* :class:`functools.partial` now emits a :exc:`FutureWarning` + when used as a method. + The behavior will change in future Python versions. + Wrap it in :func:`staticmethod` if you want to preserve the old behavior. + (Contributed by Serhiy Storchaka in :gh:`121027`.) - Any library or application that provides a custom ``_thread`` module - must provide :func:`!_is_main_interpreter`, just like the module's - other "private" attributes. - (See :gh:`112826`.) +* The :ref:`garbage collector is now incremental `, + which means that the behavior of :func:`gc.collect` changes slightly: -* :class:`mailbox.Maildir` now ignores files with a leading dot. - (Contributed by Zackery Spytz in :gh:`65559`.) + * ``gc.collect(1)``: Performs an increment of garbage collection, + rather than collecting generation 1. + * Other calls to :func:`!gc.collect` are unchanged. -* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both - files and directories if a pattern that ends with "``**``" is given, rather - than directories only. Users may add a trailing slash to match only - directories. +* An :exc:`OSError` is now raised by :func:`getpass.getuser` + for any failure to retrieve a username, + instead of :exc:`ImportError` on non-Unix platforms + or :exc:`KeyError` on Unix platforms where the password database is empty. -* The value of the :attr:`!mode` attribute of :class:`gzip.GzipFile` was - changed from integer (``1`` or ``2``) to string (``'rb'`` or ``'wb'``). +* The value of the :attr:`!mode` attribute of :class:`gzip.GzipFile` + is now a string (``'rb'`` or ``'wb'``) instead of an integer (``1`` or ``2``). The value of the :attr:`!mode` attribute of the readable file-like object - returned by :meth:`zipfile.ZipFile.open` was changed from ``'r'`` to ``'rb'``. + returned by :meth:`zipfile.ZipFile.open` is now ``'rb'`` instead of ``'r'``. (Contributed by Serhiy Storchaka in :gh:`115961`.) -* :class:`functools.partial` now emits a :exc:`FutureWarning` when it is - used as a method. - Its behavior will be changed in future Python versions. - Wrap it in :func:`staticmethod` if you want to preserve the old behavior. - (Contributed by Serhiy Storchaka in :gh:`121027`.) +* :class:`mailbox.Maildir` now ignores files with a leading dot (``.``). + (Contributed by Zackery Spytz in :gh:`65559`.) -.. _pep667-porting-notes-py: +* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both + files and directories if a pattern that ends with "``**``" is given, + rather than directories only. + Add a trailing slash to keep the previous behavior and only match directories. + +* The :mod:`threading` module now expects the :mod:`!_thread` module + to have an :func:`!_is_main_interpreter` function. + This function takes no arguments and returns ``True`` + if the current interpreter is the main interpreter. + + Any library or application that provides a custom :mod:`!_thread` module + must provide :func:`!_is_main_interpreter`, + just like the module's other "private" attributes. + (:gh:`112826`.) -* Calling :func:`locals` in an :term:`optimized scope` now produces an - independent snapshot on each call, and hence no longer implicitly updates - previously returned references. Obtaining the legacy CPython behaviour now - requires explicit calls to update the initially returned dictionary with the - results of subsequent calls to :func:`!locals`. Code execution functions that - implicitly target :func:`!locals` (such as ``exec`` and ``eval``) must be - passed an explicit namespace to access their results in an optimized scope. - (Changed as part of :pep:`667`.) - -* Calling :func:`locals` from a comprehension at module or class scope - (including via ``exec`` or ``eval``) once more behaves as if the comprehension - were running as an independent nested function (i.e. the local variables from - the containing scope are not included). In Python 3.12, this had changed - to include the local variables from the containing scope when implementing - :pep:`709`. (Changed as part of :pep:`667`.) - -* Accessing :attr:`FrameType.f_locals ` in an - :term:`optimized scope` now returns a write-through proxy rather than a - snapshot that gets updated at ill-specified times. If a snapshot is desired, - it must be created explicitly with ``dict`` or the proxy's ``.copy()`` method. - (Changed as part of :pep:`667`.) Changes in the C API -------------------- @@ -2673,45 +2698,39 @@ Changes in the C API added in Python 3.8 and the old macros were deprecated in Python 3.11. (Contributed by Irit Katriel in :gh:`105111`.) -* Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, - :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, - :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and - :c:func:`PySys_GetObject`, which clear all errors which occurred when calling - them, now report them using :func:`sys.unraisablehook`. - You may replace them with other functions as - recommended in the documentation. - (Contributed by Serhiy Storchaka in :gh:`106672`.) - -* :c:func:`!PyCode_GetFirstFree` is an unstable API now and has been renamed - to :c:func:`PyUnstable_Code_GetFirstFree`. - (Contributed by Bogdan Romanyuk in :gh:`115781`.) - .. _pep667-porting-notes-c: -* The effects of mutating the dictionary returned from :c:func:`PyEval_GetLocals` in an - :term:`optimized scope` have changed. New dict entries added this way will now *only* be - visible to subsequent :c:func:`PyEval_GetLocals` calls in that frame, as - :c:func:`PyFrame_GetLocals`, :func:`locals`, and - :attr:`FrameType.f_locals ` no longer access the same underlying cached - dictionary. Changes made to entries for actual variable names and names added via the - write-through proxy interfaces will be overwritten on subsequent calls to - :c:func:`PyEval_GetLocals` in that frame. The recommended code update depends on how the - function was being used, so refer to the deprecation notice on the function for details. - (Changed as part of :pep:`667`.) - -* Calling :c:func:`PyFrame_GetLocals` in an :term:`optimized scope` now returns a - write-through proxy rather than a snapshot that gets updated at ill-specified times. - If a snapshot is desired, it must be created explicitly (e.g. with :c:func:`PyDict_Copy`) - or by calling the new :c:func:`PyEval_GetFrameLocals` API. (Changed as part of :pep:`667`.) - -* :c:func:`!PyFrame_FastToLocals` and :c:func:`!PyFrame_FastToLocalsWithError` - no longer have any effect. Calling these functions has been redundant since - Python 3.11, when :c:func:`PyFrame_GetLocals` was first introduced. - (Changed as part of :pep:`667`.) - -* :c:func:`!PyFrame_LocalsToFast` no longer has any effect. Calling this function - is redundant now that :c:func:`PyFrame_GetLocals` returns a write-through proxy - for :term:`optimized scopes `. (Changed as part of :pep:`667`.) +* :ref:`PEP 667 ` introduces several changes + to frame-related functions: + + * The effects of mutating the dictionary returned from + :c:func:`PyEval_GetLocals` in an :term:`optimized scope` have changed. + New dict entries added this way will now *only* be visible to + subsequent :c:func:`PyEval_GetLocals` calls in that frame, + as :c:func:`PyFrame_GetLocals`, :func:`locals`, + and :attr:`FrameType.f_locals ` no longer access + the same underlying cached dictionary. + Changes made to entries for actual variable names and names added via + the write-through proxy interfaces will be overwritten on subsequent calls + to :c:func:`PyEval_GetLocals` in that frame. + The recommended code update depends on how the function was being used, + so refer to the deprecation notice on the function for details. + + * Calling :c:func:`PyFrame_GetLocals` in an :term:`optimized scope` + now returns a write-through proxy rather than a snapshot + that gets updated at ill-specified times. + If a snapshot is desired, it must be created explicitly + (e.g. with :c:func:`PyDict_Copy`), + or by calling the new :c:func:`PyEval_GetFrameLocals` API. + + * :c:func:`!PyFrame_FastToLocals` and :c:func:`!PyFrame_FastToLocalsWithError` + no longer have any effect. + Calling these functions has been redundant since Python 3.11, + when :c:func:`PyFrame_GetLocals` was first introduced. + + * :c:func:`!PyFrame_LocalsToFast` no longer has any effect. + Calling this function is redundant now that :c:func:`PyFrame_GetLocals` + returns a write-through proxy for :term:`optimized scopes `. Regression Test Changes ======================= From dc48312717142ec902197da504fad333f13c9937 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 23 Sep 2024 11:16:55 -0700 Subject: [PATCH 048/105] GH-124321: Fix argparse negative number parsing to capture -.5(GH-124322) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Lib/argparse.py | 2 +- Lib/test/test_argparse.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/argparse.py b/Lib/argparse.py index 7988c447d03584..b77da29417b920 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1360,7 +1360,7 @@ def __init__(self, self._defaults = {} # determines whether an "option" looks like a negative number - self._negative_number_matcher = _re.compile(r'^-\d[\d_]*(\.\d[\d_]*)?$') + self._negative_number_matcher = _re.compile(r'^-(?:\d+(?:_\d+)*(?:\.\d+(?:_\d+)*)?|\.\d+(?:_\d+)*)$') # whether or not there are any optionals that look like negative # numbers -- uses a list so it can be shared and edited diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 138ff19e86acf4..37d07968b22543 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2111,6 +2111,8 @@ class TestNegativeNumber(ParserTestCase): ('--int -1_000_000 --float -1_000_000.0', NS(int=-1000000, float=-1000000.0)), ('--float -1_000.0', NS(int=None, float=-1000.0)), ('--float -1_000_000.0_0', NS(int=None, float=-1000000.0)), + ('--float -.5', NS(int=None, float=-0.5)), + ('--float -.5_000', NS(int=None, float=-0.5)), ] class TestInvalidAction(TestCase): From f3b2c36deb1436b7c5abac826f5639efa1fb9fd4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 19:30:45 +0100 Subject: [PATCH 049/105] GH-109975: Copyedit 3.13 What's New: Build Changes (#124343) Co-authored-by: Malcolm Smith --- Doc/whatsnew/3.13.rst | 57 +++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 91ddc1c3212d5a..430eaf004a9828 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -2517,24 +2517,48 @@ Deprecated C APIs Build Changes ============= -* The :file:`configure` option :option:`--with-system-libmpdec` now defaults - to ``yes``. The bundled copy of ``libmpdecimal`` will be removed in Python - 3.15. +* ``arm64-apple-ios`` and ``arm64-apple-ios-simulator`` are both + now :pep:`11` tier 3 platforms. + (:ref:`PEP 730 ` written + and implementation contributed by Russell Keith-Magee in :gh:`114099`.) + +* ``wasm32-wasi`` is now a :pep:`11` tier 2 platform. + (Contributed by Brett Cannon in :gh:`115192`.) + +* ``wasm32-emscripten`` is no longer a :pep:`11` supported platform. + (Contributed by Brett Cannon in :gh:`115192`.) + +* Building CPython now requires a compiler with support for the C11 atomic + library, GCC built-in atomic functions, or MSVC interlocked intrinsics. * Autoconf 2.71 and aclocal 1.16.4 are now required to regenerate the :file:`configure` script. (Contributed by Christian Heimes in :gh:`89886`.) -* SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module. +* SQLite 3.15.2 or newer is required to build + the :mod:`sqlite3` extension module. (Contributed by Erlend Aasland in :gh:`105875`.) -* Python built with :file:`configure` :option:`--with-trace-refs` (tracing - references) is now ABI compatible with the Python release build and - :ref:`debug build `. +* CPython now bundles the `mimalloc library`_ by default. + It is licensed under the MIT license; + see :ref:`mimalloc license `. + The bundled mimalloc has custom changes, see :gh:`113141` for details. + (Contributed by Dino Viehland in :gh:`109914`.) + + .. _mimalloc library: https://github.com/microsoft/mimalloc/ + +* The :file:`configure` option :option:`--with-system-libmpdec` + now defaults to ``yes``. + The bundled copy of ``libmpdecimal`` will be removed in Python 3.15. + +* Python built with :file:`configure` :option:`--with-trace-refs` + (tracing references) is now ABI compatible with the Python release build + and :ref:`debug build `. (Contributed by Victor Stinner in :gh:`108634`.) -* Building CPython now requires a compiler with support for the C11 atomic - library, GCC built-in atomic functions, or MSVC interlocked intrinsics. +* On POSIX systems, the pkg-config (``.pc``) filenames now include the ABI + flags. For example, the free-threaded build generates ``python-3.13t.pc`` + and the debug build generates ``python-3.13d.pc``. * The ``errno``, ``fcntl``, ``grp``, ``md5``, ``pwd``, ``resource``, ``termios``, ``winsound``, @@ -2543,21 +2567,6 @@ Build Changes C extensions are now built with the :ref:`limited C API `. (Contributed by Victor Stinner in :gh:`85283`.) -* ``wasm32-wasi`` is now a :pep:`11` tier 2 platform. - (Contributed by Brett Cannon in :gh:`115192`.) - -* ``wasm32-emscripten`` is no longer a :pep:`11` supported platform. - (Contributed by Brett Cannon in :gh:`115192`.) - -* Python now bundles the `mimalloc library `__. - It is licensed under the MIT license; see :ref:`mimalloc license `. - The bundled mimalloc has custom changes, see :gh:`113141` for details. - (Contributed by Dino Viehland in :gh:`109914`.) - -* On POSIX systems, the pkg-config (``.pc``) filenames now include the ABI - flags. For example, the free-threaded build generates ``python-3.13t.pc`` - and the debug build generates ``python-3.13d.pc``. - Porting to Python 3.13 ====================== From e7d465a607b77a552d1e07f3fafa81ef5fc799d2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 19:40:20 +0100 Subject: [PATCH 050/105] GH-109975: Copyedit 3.13 What's New: Copyedit C API deprecations pending removal (#124336) --- .../c-api-pending-removal-in-3.14.rst | 76 +++++++++++++------ .../c-api-pending-removal-in-3.15.rst | 35 +++++---- .../c-api-pending-removal-in-future.rst | 60 ++++++++++----- Doc/whatsnew/3.13.rst | 3 +- 4 files changed, 114 insertions(+), 60 deletions(-) diff --git a/Doc/deprecations/c-api-pending-removal-in-3.14.rst b/Doc/deprecations/c-api-pending-removal-in-3.14.rst index 369892a75b16eb..d16da66c29abe7 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.14.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.14.rst @@ -9,38 +9,64 @@ Pending Removal in Python 3.14 * Functions to configure Python's initialization, deprecated in Python 3.11: - * ``PySys_SetArgvEx()``: set :c:member:`PyConfig.argv` instead. - * ``PySys_SetArgv()``: set :c:member:`PyConfig.argv` instead. - * ``Py_SetProgramName()``: set :c:member:`PyConfig.program_name` instead. - * ``Py_SetPythonHome()``: set :c:member:`PyConfig.home` instead. + * :c:func:`!PySys_SetArgvEx()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!PySys_SetArgv()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!Py_SetProgramName()`: + Set :c:member:`PyConfig.program_name` instead. + * :c:func:`!Py_SetPythonHome()`: + Set :c:member:`PyConfig.home` instead. The :c:func:`Py_InitializeFromConfig` API should be used with :c:type:`PyConfig` instead. * Global configuration variables: - * :c:var:`Py_DebugFlag`: use :c:member:`PyConfig.parser_debug` instead. - * :c:var:`Py_VerboseFlag`: use :c:member:`PyConfig.verbose` instead. - * :c:var:`Py_QuietFlag`: use :c:member:`PyConfig.quiet` instead. - * :c:var:`Py_InteractiveFlag`: use :c:member:`PyConfig.interactive` instead. - * :c:var:`Py_InspectFlag`: use :c:member:`PyConfig.inspect` instead. - * :c:var:`Py_OptimizeFlag`: use :c:member:`PyConfig.optimization_level` instead. - * :c:var:`Py_NoSiteFlag`: use :c:member:`PyConfig.site_import` instead. - * :c:var:`Py_BytesWarningFlag`: use :c:member:`PyConfig.bytes_warning` instead. - * :c:var:`Py_FrozenFlag`: use :c:member:`PyConfig.pathconfig_warnings` instead. - * :c:var:`Py_IgnoreEnvironmentFlag`: use :c:member:`PyConfig.use_environment` instead. - * :c:var:`Py_DontWriteBytecodeFlag`: use :c:member:`PyConfig.write_bytecode` instead. - * :c:var:`Py_NoUserSiteDirectory`: use :c:member:`PyConfig.user_site_directory` instead. - * :c:var:`Py_UnbufferedStdioFlag`: use :c:member:`PyConfig.buffered_stdio` instead. - * :c:var:`Py_HashRandomizationFlag`: use :c:member:`PyConfig.use_hash_seed` + * :c:var:`Py_DebugFlag`: + Use :c:member:`PyConfig.parser_debug` instead. + * :c:var:`Py_VerboseFlag`: + Use :c:member:`PyConfig.verbose` instead. + * :c:var:`Py_QuietFlag`: + Use :c:member:`PyConfig.quiet` instead. + * :c:var:`Py_InteractiveFlag`: + Use :c:member:`PyConfig.interactive` instead. + * :c:var:`Py_InspectFlag`: + Use :c:member:`PyConfig.inspect` instead. + * :c:var:`Py_OptimizeFlag`: + Use :c:member:`PyConfig.optimization_level` instead. + * :c:var:`Py_NoSiteFlag`: + Use :c:member:`PyConfig.site_import` instead. + * :c:var:`Py_BytesWarningFlag`: + Use :c:member:`PyConfig.bytes_warning` instead. + * :c:var:`Py_FrozenFlag`: + Use :c:member:`PyConfig.pathconfig_warnings` instead. + * :c:var:`Py_IgnoreEnvironmentFlag`: + Use :c:member:`PyConfig.use_environment` instead. + * :c:var:`Py_DontWriteBytecodeFlag`: + Use :c:member:`PyConfig.write_bytecode` instead. + * :c:var:`Py_NoUserSiteDirectory`: + Use :c:member:`PyConfig.user_site_directory` instead. + * :c:var:`Py_UnbufferedStdioFlag`: + Use :c:member:`PyConfig.buffered_stdio` instead. + * :c:var:`Py_HashRandomizationFlag`: + Use :c:member:`PyConfig.use_hash_seed` and :c:member:`PyConfig.hash_seed` instead. - * :c:var:`Py_IsolatedFlag`: use :c:member:`PyConfig.isolated` instead. - * :c:var:`Py_LegacyWindowsFSEncodingFlag`: use :c:member:`PyPreConfig.legacy_windows_fs_encoding` instead. - * :c:var:`Py_LegacyWindowsStdioFlag`: use :c:member:`PyConfig.legacy_windows_stdio` instead. - * :c:var:`!Py_FileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` instead. - * :c:var:`!Py_HasFileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` instead. - * :c:var:`!Py_FileSystemDefaultEncodeErrors`: use :c:member:`PyConfig.filesystem_errors` instead. - * :c:var:`!Py_UTF8Mode`: use :c:member:`PyPreConfig.utf8_mode` instead. (see :c:func:`Py_PreInitialize`) + * :c:var:`Py_IsolatedFlag`: + Use :c:member:`PyConfig.isolated` instead. + * :c:var:`Py_LegacyWindowsFSEncodingFlag`: + Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` instead. + * :c:var:`Py_LegacyWindowsStdioFlag`: + Use :c:member:`PyConfig.legacy_windows_stdio` instead. + * :c:var:`!Py_FileSystemDefaultEncoding`: + Use :c:member:`PyConfig.filesystem_encoding` instead. + * :c:var:`!Py_HasFileSystemDefaultEncoding`: + Use :c:member:`PyConfig.filesystem_encoding` instead. + * :c:var:`!Py_FileSystemDefaultEncodeErrors`: + Use :c:member:`PyConfig.filesystem_errors` instead. + * :c:var:`!Py_UTF8Mode`: + Use :c:member:`PyPreConfig.utf8_mode` instead. + (see :c:func:`Py_PreInitialize`) The :c:func:`Py_InitializeFromConfig` API should be used with :c:type:`PyConfig` instead. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst index c676927ed69212..e3974415e0cc89 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.15.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -2,19 +2,26 @@ Pending Removal in Python 3.15 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * The bundled copy of ``libmpdecimal``. -* :c:func:`PyImport_ImportModuleNoBlock`: use :c:func:`PyImport_ImportModule` instead. -* :c:func:`PyWeakref_GET_OBJECT`: use :c:func:`PyWeakref_GetRef` instead. -* :c:func:`PyWeakref_GetObject`: use :c:func:`PyWeakref_GetRef` instead. -* :c:type:`!Py_UNICODE_WIDE` type: use :c:type:`wchar_t` instead. -* :c:type:`Py_UNICODE` type: use :c:type:`wchar_t` instead. +* The :c:func:`PyImport_ImportModuleNoBlock`: + Use :c:func:`PyImport_ImportModule` instead. +* :c:func:`PyWeakref_GetObject` and :c:func:`PyWeakref_GET_OBJECT`: + Use :c:func:`PyWeakref_GetRef` instead. +* :c:type:`Py_UNICODE` type and the :c:macro:`!Py_UNICODE_WIDE` macro: + Use :c:type:`wchar_t` instead. * Python initialization functions: - * :c:func:`PySys_ResetWarnOptions`: clear :data:`sys.warnoptions` and - :data:`!warnings.filters` instead. - * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. - * :c:func:`Py_GetPath`: get :data:`sys.path` instead. - * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. - * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. - * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. - * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or - the :envvar:`PYTHONHOME` environment variable instead. + * :c:func:`PySys_ResetWarnOptions`: + Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + * :c:func:`Py_GetExecPrefix`: + Get :data:`sys.exec_prefix` instead. + * :c:func:`Py_GetPath`: + Get :data:`sys.path` instead. + * :c:func:`Py_GetPrefix`: + Get :data:`sys.prefix` instead. + * :c:func:`Py_GetProgramFullPath`: + Get :data:`sys.executable` instead. + * :c:func:`Py_GetProgramName`: + Get :data:`sys.executable` instead. + * :c:func:`Py_GetPythonHome`: + Get :c:member:`PyConfig.home` + or the :envvar:`PYTHONHOME` environment variable instead. diff --git a/Doc/deprecations/c-api-pending-removal-in-future.rst b/Doc/deprecations/c-api-pending-removal-in-future.rst index f646be45c8a770..0c3ae52b87ff74 100644 --- a/Doc/deprecations/c-api-pending-removal-in-future.rst +++ b/Doc/deprecations/c-api-pending-removal-in-future.rst @@ -4,28 +4,48 @@ Pending Removal in Future Versions The following APIs are deprecated and will be removed, although there is currently no date scheduled for their removal. -* :c:macro:`Py_TPFLAGS_HAVE_FINALIZE`: unneeded since Python 3.8. -* :c:func:`PyErr_Fetch`: use :c:func:`PyErr_GetRaisedException` instead. -* :c:func:`PyErr_NormalizeException`: use :c:func:`PyErr_GetRaisedException` instead. -* :c:func:`PyErr_Restore`: use :c:func:`PyErr_SetRaisedException` instead. -* :c:func:`PyModule_GetFilename`: use :c:func:`PyModule_GetFilenameObject` instead. -* :c:func:`PyOS_AfterFork`: use :c:func:`PyOS_AfterFork_Child` instead. -* :c:func:`PySlice_GetIndicesEx`: use :c:func:`PySlice_Unpack` and :c:func:`PySlice_AdjustIndices` instead. -* :c:func:`!PyUnicode_AsDecodedObject`: use :c:func:`PyCodec_Decode` instead. -* :c:func:`!PyUnicode_AsDecodedUnicode`: use :c:func:`PyCodec_Decode` instead. -* :c:func:`!PyUnicode_AsEncodedObject`: use :c:func:`PyCodec_Encode` instead. -* :c:func:`!PyUnicode_AsEncodedUnicode`: use :c:func:`PyCodec_Encode` instead. -* :c:func:`PyUnicode_READY`: unneeded since Python 3.12 -* :c:func:`!PyErr_Display`: use :c:func:`PyErr_DisplayException` instead. -* :c:func:`!_PyErr_ChainExceptions`: use ``_PyErr_ChainExceptions1`` instead. +* :c:macro:`Py_TPFLAGS_HAVE_FINALIZE`: + Unneeded since Python 3.8. +* :c:func:`PyErr_Fetch`: + Use :c:func:`PyErr_GetRaisedException` instead. +* :c:func:`PyErr_NormalizeException`: + Use :c:func:`PyErr_GetRaisedException` instead. +* :c:func:`PyErr_Restore`: + Use :c:func:`PyErr_SetRaisedException` instead. +* :c:func:`PyModule_GetFilename`: + Use :c:func:`PyModule_GetFilenameObject` instead. +* :c:func:`PyOS_AfterFork`: + Use :c:func:`PyOS_AfterFork_Child` instead. +* :c:func:`PySlice_GetIndicesEx`: + Use :c:func:`PySlice_Unpack` and :c:func:`PySlice_AdjustIndices` instead. +* :c:func:`!PyUnicode_AsDecodedObject`: + Use :c:func:`PyCodec_Decode` instead. +* :c:func:`!PyUnicode_AsDecodedUnicode`: + Use :c:func:`PyCodec_Decode` instead. +* :c:func:`!PyUnicode_AsEncodedObject`: + Use :c:func:`PyCodec_Encode` instead. +* :c:func:`!PyUnicode_AsEncodedUnicode`: + Use :c:func:`PyCodec_Encode` instead. +* :c:func:`PyUnicode_READY`: + Unneeded since Python 3.12 +* :c:func:`!PyErr_Display`: + Use :c:func:`PyErr_DisplayException` instead. +* :c:func:`!_PyErr_ChainExceptions`: + Use :c:func:`!_PyErr_ChainExceptions1` instead. * :c:member:`!PyBytesObject.ob_shash` member: call :c:func:`PyObject_Hash` instead. * :c:member:`!PyDictObject.ma_version_tag` member. * Thread Local Storage (TLS) API: - * :c:func:`PyThread_create_key`: use :c:func:`PyThread_tss_alloc` instead. - * :c:func:`PyThread_delete_key`: use :c:func:`PyThread_tss_free` instead. - * :c:func:`PyThread_set_key_value`: use :c:func:`PyThread_tss_set` instead. - * :c:func:`PyThread_get_key_value`: use :c:func:`PyThread_tss_get` instead. - * :c:func:`PyThread_delete_key_value`: use :c:func:`PyThread_tss_delete` instead. - * :c:func:`PyThread_ReInitTLS`: unneeded since Python 3.7. + * :c:func:`PyThread_create_key`: + Use :c:func:`PyThread_tss_alloc` instead. + * :c:func:`PyThread_delete_key`: + Use :c:func:`PyThread_tss_free` instead. + * :c:func:`PyThread_set_key_value`: + Use :c:func:`PyThread_tss_set` instead. + * :c:func:`PyThread_get_key_value`: + Use :c:func:`PyThread_tss_get` instead. + * :c:func:`PyThread_delete_key_value`: + Use :c:func:`PyThread_tss_delete` instead. + * :c:func:`PyThread_ReInitTLS`: + Unneeded since Python 3.7. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 430eaf004a9828..e62051e0dc9028 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -2489,7 +2489,8 @@ Deprecated C APIs or :c:func:`PyModule_AddObjectRef`. (Contributed by Serhiy Storchaka in :gh:`86493`.) -* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types. +* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types + and the :c:macro:`!Py_UNICODE_WIDE` define. Use the :c:type:`wchar_t` type directly instead. Since Python 3.3, ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` are just aliases to :c:type:`!wchar_t`. From 2e0d445364f6d9832e81263c1c68440654924fc4 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 23 Sep 2024 12:06:19 -0700 Subject: [PATCH 051/105] gh-119180: Fix annotationlib.ForwardRef.evaluate with no globals (#124326) We were sometimes passing None as the globals argument to eval(), which makes it inherit the globals from the calling scope. Instead, ensure that globals is always non-None. The test was passing accidentally because I passed "annotationlib" as a module object; fix that. Also document the parameters to ForwardRef() and remove two unused private ones. Co-authored-by: Alex Waygood --- Lib/annotationlib.py | 32 +++++++++++++++++++------------- Lib/test/test_annotationlib.py | 18 ++++++++++++++++-- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 09a844ddb56f06..0a67742a2b3081 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -45,7 +45,17 @@ class Format(enum.IntEnum): class ForwardRef: - """Wrapper that holds a forward reference.""" + """Wrapper that holds a forward reference. + + Constructor arguments: + * arg: a string representing the code to be evaluated. + * module: the module where the forward reference was created. + Must be a string, not a module object. + * owner: The owning object (module, class, or function). + * is_argument: Does nothing, retained for compatibility. + * is_class: True if the forward reference was created in class scope. + + """ __slots__ = _SLOTS @@ -57,8 +67,6 @@ def __init__( owner=None, is_argument=True, is_class=False, - _globals=None, - _cell=None, ): if not isinstance(arg, str): raise TypeError(f"Forward reference must be a string -- got {arg!r}") @@ -71,8 +79,8 @@ def __init__( self.__forward_module__ = module self.__code__ = None self.__ast_node__ = None - self.__globals__ = _globals - self.__cell__ = _cell + self.__globals__ = None + self.__cell__ = None self.__owner__ = owner def __init_subclass__(cls, /, *args, **kwds): @@ -115,6 +123,10 @@ def evaluate(self, *, globals=None, locals=None, type_params=None, owner=None): elif callable(owner): globals = getattr(owner, "__globals__", None) + # If we pass None to eval() below, the globals of this module are used. + if globals is None: + globals = {} + if locals is None: locals = {} if isinstance(owner, type): @@ -134,14 +146,8 @@ def evaluate(self, *, globals=None, locals=None, type_params=None, owner=None): # but should in turn be overridden by names in the class scope # (which here are called `globalns`!) if type_params is not None: - if globals is None: - globals = {} - else: - globals = dict(globals) - if locals is None: - locals = {} - else: - locals = dict(locals) + globals = dict(globals) + locals = dict(locals) for param in type_params: param_name = param.__name__ if not self.__forward_is_class__ or param_name not in globals: diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 309f6d2120109a..dd8ceb55a411fb 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1,6 +1,7 @@ """Tests for the annotations module.""" import annotationlib +import collections import functools import itertools import pickle @@ -278,11 +279,24 @@ class Gen[T]: ) def test_fwdref_with_module(self): - self.assertIs(ForwardRef("Format", module=annotationlib).evaluate(), Format) + self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) + self.assertIs(ForwardRef("Counter", module="collections").evaluate(), collections.Counter) with self.assertRaises(NameError): # If globals are passed explicitly, we don't look at the module dict - ForwardRef("Format", module=annotationlib).evaluate(globals={}) + ForwardRef("Format", module="annotationlib").evaluate(globals={}) + + def test_fwdref_to_builtin(self): + self.assertIs(ForwardRef("int").evaluate(), int) + self.assertIs(ForwardRef("int", module="collections").evaluate(), int) + self.assertIs(ForwardRef("int", owner=str).evaluate(), int) + + # builtins are still searched with explicit globals + self.assertIs(ForwardRef("int").evaluate(globals={}), int) + + # explicit values in globals have precedence + obj = object() + self.assertIs(ForwardRef("int").evaluate(globals={"int": obj}), obj) def test_fwdref_value_is_cached(self): fr = ForwardRef("hello") From d9d5b3d2ef770224b39987dbce7bb5c616fe3477 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 24 Sep 2024 04:14:15 +0900 Subject: [PATCH 052/105] gh-124344: Make `_PyObject_IS_GC()` use underscored `PyType_IS_GC()` (#124349) move up _PyType_IS_GC and use it --- Include/internal/pycore_object.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 0d885b4630d2f0..80b588815bc9cf 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -732,12 +732,15 @@ _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(PyObject *op) return (PyWeakReference **)((char *)op + offset); } +// Fast inlined version of PyType_IS_GC() +#define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) + // Fast inlined version of PyObject_IS_GC() static inline int _PyObject_IS_GC(PyObject *obj) { PyTypeObject *type = Py_TYPE(obj); - return (PyType_IS_GC(type) + return (_PyType_IS_GC(type) && (type->tp_is_gc == NULL || type->tp_is_gc(obj))); } @@ -755,9 +758,6 @@ _PyObject_HashFast(PyObject *op) return PyObject_Hash(op); } -// Fast inlined version of PyType_IS_GC() -#define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) - static inline size_t _PyType_PreHeaderSize(PyTypeObject *tp) { From 9e55a02fab77794c421b91429292ed227de0e24a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 20:32:23 +0100 Subject: [PATCH 053/105] GH-109975: Copyedit 3.13 What's New: Trivia (#124348) --- Doc/whatsnew/3.13.rst | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index e62051e0dc9028..ad2cf021232ac9 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -3,7 +3,7 @@ What's New In Python 3.13 **************************** -:Editor: Thomas Wouters +:Editors: Adam Turner and Thomas Wouters .. Rules for maintenance: @@ -53,12 +53,6 @@ For full details, see the :ref:`changelog `. :pep:`719` -- Python 3.13 Release Schedule -.. note:: - - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.13 moves towards release, - so it's worth checking back even after reading earlier versions. - Summary -- Release Highlights ============================= @@ -142,7 +136,7 @@ Significant improvements in the standard library: * The :mod:`copy` module now has a :func:`copy.replace` function, with support for many builtin types and any class defining the :func:`~object.__replace__` method. -* The :mod:`dbm.sqlite3` module is now the default :mod:`dbm` backend. +* The new :mod:`dbm.sqlite3` module is now the default :mod:`dbm` backend. * The :mod:`os` module has a :ref:`suite of new functions ` for working with Linux's timer notification file descriptors. * The :mod:`random` module now has a :ref:`command-line interface `. @@ -167,7 +161,7 @@ New typing features: * :pep:`696`: Type parameters (:data:`typing.TypeVar`, :data:`typing.ParamSpec`, and :data:`typing.TypeVarTuple`) now support defaults. * :pep:`702`: The new :func:`warnings.deprecated` decorator adds support - for marking deprecations in the type system. + for marking deprecations in the type system and at runtime. * :pep:`705`: :data:`typing.ReadOnly` can be used to mark an item of a :class:`typing.TypedDict` as read-only for type checkers. * :pep:`742`: :data:`typing.TypeIs` provides more intuitive @@ -177,14 +171,14 @@ Platform support: * :pep:`730`: Apple's iOS is now an :ref:`officially supported platform `, at :pep:`tier 3 <11#tier-3>`. - Official Android support (:pep:`738`) is in the works as well. + Official Android support (:pep:`738`) is also in the works. * ``wasm32-wasi`` is now supported as a :pep:`tier 2 <11#tier-2>` platform. * ``wasm32-emscripten`` is no longer an officially supported platform. Important removals: * :ref:`PEP 594 `: The remaining 19 "dead batteries" - have been removed from the standard library: + (legacy stdlib modules) have been removed from the standard library: :mod:`!aifc`, :mod:`!audioop`, :mod:`!cgi`, :mod:`!cgitb`, :mod:`!chunk`, :mod:`!crypt`, :mod:`!imghdr`, :mod:`!mailcap`, :mod:`!msilib`, :mod:`!nis`, :mod:`!nntplib`, :mod:`!ossaudiodev`, :mod:`!pipes`, :mod:`!sndhdr`, @@ -192,8 +186,8 @@ Important removals: * Remove the :program:`2to3` tool and :mod:`!lib2to3` module (deprecated in Python 3.11). * Remove the :mod:`!tkinter.tix` module (deprecated in Python 3.6). -* Remove :func:`!locale.resetlocale`. -* Remove :mod:`!typing.io` and :mod:`!typing.re`. +* Remove the :func:`!locale.resetlocale` function. +* Remove the :mod:`!typing.io` and :mod:`!typing.re` namespaces. * Remove chained :class:`classmethod` descriptors. Release schedule changes: @@ -338,7 +332,7 @@ designed with threading in mind will run faster on multi-core hardware. expect some bugs and a substantial single-threaded performance hit. Free-threaded builds of CPython support optionally running with the GIL enabled at runtime using the environment variable :envvar:`PYTHON_GIL` or -the command-line option :option:`-X gil`. +the command-line option :option:`-X gil=1`. To check if the current interpreter supports free-threading, :option:`python -VV <-V>` and :attr:`sys.version` contain "experimental free-threading build". @@ -439,14 +433,14 @@ Defined mutation semantics for :py:func:`locals` Historically, the expected result of mutating the return value of :func:`locals` has been left to individual Python implementations to define. Starting from Python 3.13, :pep:`667` standardises -the historical behaviour of CPython for most code execution scopes, +the historical behavior of CPython for most code execution scopes, but changes :term:`optimized scopes ` (functions, generators, coroutines, comprehensions, and generator expressions) to explicitly return independent snapshots of the currently assigned local variables, including locally referenced nonlocal variables captured in closures. This change to the semantics of :func:`locals` in optimized scopes also -affects the default behaviour of code execution functions that implicitly +affects the default behavior of code execution functions that implicitly target :func:`!locals` if no explicit namespace is provided (such as :func:`exec` and :func:`eval`). In previous versions, whether or not changes could be accessed by calling @@ -610,6 +604,9 @@ Other Language Changes the value of the *optimize* argument. (Contributed by Irit Katriel in :gh:`108113`). +* Add a :attr:`~property.__name__` attribute on :class:`property` objects. + (Contributed by Eugene Toder in :gh:`101860`.) + * Add :exc:`PythonFinalizationError`, a new exception derived from :exc:`RuntimeError` and used to signal when operations are blocked during :term:`finalization `. @@ -636,9 +633,6 @@ Other Language Changes the :mod:`bz2`, :mod:`lzma`, :mod:`tarfile`, and :mod:`zipfile` modules. (Contributed by Serhiy Storchaka in :gh:`115961`.) -* Add a :attr:`~property.__name__` attribute on :class:`property` objects. - (Contributed by Eugene Toder in :gh:`101860`.) - New Modules =========== @@ -678,7 +672,7 @@ ast * The constructors of node types in the :mod:`ast` module are now stricter in the arguments they accept, - with more intuitive behaviour when arguments are omitted. + with more intuitive behavior when arguments are omitted. If an optional field on an AST node is not included as an argument when constructing an instance, the field will now be set to ``None``. Similarly, @@ -889,7 +883,7 @@ email return ``('', '')`` pairs in more situations where invalid email addresses are encountered instead of potentially inaccurate values. The two functions have a new optional *strict* parameter (default ``True``). - To get the old behaviour (accepting malformed input), use ``strict=False``. + To get the old behavior (accepting malformed input), use ``strict=False``. ``getattr(email.utils, 'supports_strict_parsing', False)`` can be used to check if the *strict* parameter is available. (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve @@ -2055,7 +2049,7 @@ New Features (Contributed by Victor Stinner and Petr Viktorin in :gh:`110850`.) * Add the :c:func:`PyDict_ContainsString` function - with the same behaviour as :c:func:`PyDict_Contains`, + with the same behavior as :c:func:`PyDict_Contains`, but *key* is specified as a :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. (Contributed by Victor Stinner in :gh:`108314`.) @@ -2585,7 +2579,7 @@ Changes in the Python API * Calling :func:`locals` in an :term:`optimized scope` now produces an independent snapshot on each call, and hence no longer implicitly updates - previously returned references. Obtaining the legacy CPython behaviour now + previously returned references. Obtaining the legacy CPython behavior now requires explicit calls to update the initially returned dictionary with the results of subsequent calls to :func:`!locals`. Code execution functions that implicitly target :func:`!locals` (such as ``exec`` and ``eval``) must be From 7ee99217345af3010bf05b1f5241c661a5e0ea9b Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 23 Sep 2024 12:45:36 -0700 Subject: [PATCH 054/105] GH-87041: Fix incorrect indentation in argparse help (GH-124230) In case of usage a long command along with max_help_position more than the length of the command, the command's help was incorrectly started on the new line. Co-authored-by: Pavel Ditenbir --- Lib/argparse.py | 7 ++-- Lib/test/test_argparse.py | 40 +++++++++++++++++++ ...4-09-19-03-46-59.gh-issue-87041.9Ox7Bv.rst | 1 + 3 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-19-03-46-59.gh-issue-87041.9Ox7Bv.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index b77da29417b920..98d65312417ab8 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -261,13 +261,12 @@ def add_argument(self, action): # find all invocations get_invocation = self._format_action_invocation - invocations = [get_invocation(action)] + invocation_lengths = [len(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): - invocations.append(get_invocation(subaction)) + invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent) # update the maximum item length - invocation_length = max(map(len, invocations)) - action_length = invocation_length + self._current_indent + action_length = max(invocation_lengths) self._action_max_length = max(self._action_max_length, action_length) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 37d07968b22543..f51a690f7ddf1a 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -4960,6 +4960,46 @@ def custom_type(string): version = '' +class TestHelpUsageLongSubparserCommand(TestCase): + """Test that subparser commands are formatted correctly in help""" + maxDiff = None + + def test_parent_help(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, max_help_position=50) + + parent_parser = argparse.ArgumentParser( + prog='PROG', + formatter_class=custom_formatter + ) + + cmd_subparsers = parent_parser.add_subparsers(title="commands", + metavar='CMD', + help='command to use') + cmd_subparsers.add_parser("add", + help="add something") + + cmd_subparsers.add_parser("remove", + help="remove something") + + cmd_subparsers.add_parser("a-very-long-command", + help="command that does something") + + parser_help = parent_parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] CMD ... + + options: + -h, --help show this help message and exit + + commands: + CMD command to use + add add something + remove remove something + a-very-long-command command that does something + ''')) + + # ===================================== # Optional/Positional constructor tests # ===================================== diff --git a/Misc/NEWS.d/next/Library/2024-09-19-03-46-59.gh-issue-87041.9Ox7Bv.rst b/Misc/NEWS.d/next/Library/2024-09-19-03-46-59.gh-issue-87041.9Ox7Bv.rst new file mode 100644 index 00000000000000..47a5f0c7ba520f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-19-03-46-59.gh-issue-87041.9Ox7Bv.rst @@ -0,0 +1 @@ +Fix a bug in :mod:`argparse` where lengthy subparser argument help is incorrectly indented. From 41e7992e319faba91b0c75a767b36ff8628a8368 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 23 Sep 2024 21:28:03 +0100 Subject: [PATCH 055/105] gh-99108: Disable HACL SIMD code on older versions of Android (#124304) Disable HACL SIMD code on older versions of Android --- configure | 68 ++++++++++++++++++++++++------------------ configure.ac | 84 +++++++++++++++++++++++++++++----------------------- 2 files changed, 86 insertions(+), 66 deletions(-) diff --git a/configure b/configure index 8c69b44ed7318e..56b923b764ec46 100755 --- a/configure +++ b/configure @@ -30654,7 +30654,10 @@ case "$ac_sys_system" in esac -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -msse -msse2 -msse3 -msse4.1 -msse4.2" >&5 +# The SIMD files use aligned_alloc, which is not available on older versions of +# Android. +if test "$ac_sys_system" != "Linux-android" || test "$ANDROID_API_LEVEL" -ge 28; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -msse -msse2 -msse3 -msse4.1 -msse4.2" >&5 printf %s "checking whether C compiler accepts -msse -msse2 -msse3 -msse4.1 -msse4.2... " >&6; } if test ${ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2+y} then : @@ -30688,37 +30691,44 @@ printf "%s\n" "$ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2 if test "x$ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2" = xyes then : - LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2" + LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2" printf "%s\n" "#define HACL_CAN_COMPILE_SIMD128 1" >>confdefs.h - # macOS universal2 builds *support* the -msse etc flags because they're - # available on x86_64. However, performance of the HACL SIMD128 implementation - # isn't great, so it's disabled on ARM64. - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD128 implementation" >&5 + # macOS universal2 builds *support* the -msse etc flags because they're + # available on x86_64. However, performance of the HACL SIMD128 implementation + # isn't great, so it's disabled on ARM64. + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD128 implementation" >&5 printf %s "checking for HACL* SIMD128 implementation... " >&6; } - if test "$UNIVERSAL_ARCHS" == "universal2"; then - LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 + if test "$UNIVERSAL_ARCHS" == "universal2"; then + LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 printf "%s\n" "universal2" >&6; } - else - LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128.o" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: standard" >&5 + else + LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128.o" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: standard" >&5 printf "%s\n" "standard" >&6; } - fi + fi else $as_nop : fi +fi - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -mavx2" >&5 +# The SIMD files use aligned_alloc, which is not available on older versions of +# Android. +# +# Although AVX support is not guaranteed on Android +# (https://developer.android.com/ndk/guides/abis#86-64), this is safe because we do a +# runtime CPUID check. +if test "$ac_sys_system" != "Linux-android" || test "$ANDROID_API_LEVEL" -ge 28; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -mavx2" >&5 printf %s "checking whether C compiler accepts -mavx2... " >&6; } if test ${ax_cv_check_cflags__Werror__mavx2+y} then : @@ -30752,32 +30762,32 @@ printf "%s\n" "$ax_cv_check_cflags__Werror__mavx2" >&6; } if test "x$ax_cv_check_cflags__Werror__mavx2" = xyes then : - LIBHACL_SIMD256_FLAGS="-mavx2" + LIBHACL_SIMD256_FLAGS="-mavx2" printf "%s\n" "#define HACL_CAN_COMPILE_SIMD256 1" >>confdefs.h - # macOS universal2 builds *support* the -mavx2 compiler flag because it's - # available on x86_64; but the HACL SIMD256 build then fails because the - # implementation requires symbols that aren't available on ARM64. Use a - # wrapped implementation if we're building for universal2. - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD256 implementation" >&5 + # macOS universal2 builds *support* the -mavx2 compiler flag because it's + # available on x86_64; but the HACL SIMD256 build then fails because the + # implementation requires symbols that aren't available on ARM64. Use a + # wrapped implementation if we're building for universal2. + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD256 implementation" >&5 printf %s "checking for HACL* SIMD256 implementation... " >&6; } - if test "$UNIVERSAL_ARCHS" == "universal2"; then - LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 + if test "$UNIVERSAL_ARCHS" == "universal2"; then + LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 printf "%s\n" "universal2" >&6; } - else - LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256.o" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: standard" >&5 + else + LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256.o" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: standard" >&5 printf "%s\n" "standard" >&6; } - fi + fi else $as_nop : fi - +fi diff --git a/configure.ac b/configure.ac index d3cdeb8a252a24..c4df726cd6a8ae 100644 --- a/configure.ac +++ b/configure.ac @@ -7817,47 +7817,57 @@ case "$ac_sys_system" in esac AC_SUBST([LIBHACL_CFLAGS]) -dnl This can be extended here to detect e.g. Power8, which HACL* should also support. -AX_CHECK_COMPILE_FLAG([-msse -msse2 -msse3 -msse4.1 -msse4.2],[ - [LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2"] - - AC_DEFINE([HACL_CAN_COMPILE_SIMD128], [1], [HACL* library can compile SIMD128 implementations]) - - # macOS universal2 builds *support* the -msse etc flags because they're - # available on x86_64. However, performance of the HACL SIMD128 implementation - # isn't great, so it's disabled on ARM64. - AC_MSG_CHECKING([for HACL* SIMD128 implementation]) - if test "$UNIVERSAL_ARCHS" == "universal2"; then - [LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o"] - AC_MSG_RESULT([universal2]) - else - [LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128.o"] - AC_MSG_RESULT([standard]) - fi - -], [], [-Werror]) +# The SIMD files use aligned_alloc, which is not available on older versions of +# Android. +if test "$ac_sys_system" != "Linux-android" || test "$ANDROID_API_LEVEL" -ge 28; then + dnl This can be extended here to detect e.g. Power8, which HACL* should also support. + AX_CHECK_COMPILE_FLAG([-msse -msse2 -msse3 -msse4.1 -msse4.2],[ + [LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2"] + + AC_DEFINE([HACL_CAN_COMPILE_SIMD128], [1], [HACL* library can compile SIMD128 implementations]) + + # macOS universal2 builds *support* the -msse etc flags because they're + # available on x86_64. However, performance of the HACL SIMD128 implementation + # isn't great, so it's disabled on ARM64. + AC_MSG_CHECKING([for HACL* SIMD128 implementation]) + if test "$UNIVERSAL_ARCHS" == "universal2"; then + [LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o"] + AC_MSG_RESULT([universal2]) + else + [LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128.o"] + AC_MSG_RESULT([standard]) + fi + ], [], [-Werror]) +fi AC_SUBST([LIBHACL_SIMD128_FLAGS]) AC_SUBST([LIBHACL_SIMD128_OBJS]) -AX_CHECK_COMPILE_FLAG([-mavx2],[ - [LIBHACL_SIMD256_FLAGS="-mavx2"] - AC_DEFINE([HACL_CAN_COMPILE_SIMD256], [1], [HACL* library can compile SIMD256 implementations]) - - # macOS universal2 builds *support* the -mavx2 compiler flag because it's - # available on x86_64; but the HACL SIMD256 build then fails because the - # implementation requires symbols that aren't available on ARM64. Use a - # wrapped implementation if we're building for universal2. - AC_MSG_CHECKING([for HACL* SIMD256 implementation]) - if test "$UNIVERSAL_ARCHS" == "universal2"; then - [LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o"] - AC_MSG_RESULT([universal2]) - else - [LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256.o"] - AC_MSG_RESULT([standard]) - fi -], [], [-Werror]) - +# The SIMD files use aligned_alloc, which is not available on older versions of +# Android. +# +# Although AVX support is not guaranteed on Android +# (https://developer.android.com/ndk/guides/abis#86-64), this is safe because we do a +# runtime CPUID check. +if test "$ac_sys_system" != "Linux-android" || test "$ANDROID_API_LEVEL" -ge 28; then + AX_CHECK_COMPILE_FLAG([-mavx2],[ + [LIBHACL_SIMD256_FLAGS="-mavx2"] + AC_DEFINE([HACL_CAN_COMPILE_SIMD256], [1], [HACL* library can compile SIMD256 implementations]) + + # macOS universal2 builds *support* the -mavx2 compiler flag because it's + # available on x86_64; but the HACL SIMD256 build then fails because the + # implementation requires symbols that aren't available on ARM64. Use a + # wrapped implementation if we're building for universal2. + AC_MSG_CHECKING([for HACL* SIMD256 implementation]) + if test "$UNIVERSAL_ARCHS" == "universal2"; then + [LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o"] + AC_MSG_RESULT([universal2]) + else + [LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256.o"] + AC_MSG_RESULT([standard]) + fi + ], [], [-Werror]) +fi AC_SUBST([LIBHACL_SIMD256_FLAGS]) AC_SUBST([LIBHACL_SIMD256_OBJS]) From 67201ad53ff11576c69a9b762540b77128285f8d Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 23 Sep 2024 14:14:33 -0700 Subject: [PATCH 056/105] GH-65961: Document the deprecation of `__package__` and `__cached__` (GH-124377) The code changes for warning related to `__package__` landed in Python 3.12. `__cached__` doesn't have any changes as it isn't used but only set by the import system. --- Doc/deprecations/pending-removal-in-3.14.rst | 3 --- Doc/deprecations/pending-removal-in-3.15.rst | 3 +++ Doc/library/importlib.rst | 4 ++-- Doc/reference/import.rst | 8 ++++++++ .../2024-09-23-13-25-27.gh-issue-65961.LDqXV2.rst | 1 + 5 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-23-13-25-27.gh-issue-65961.LDqXV2.rst diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst index 6c831ae366ced5..452d6643e1d146 100644 --- a/Doc/deprecations/pending-removal-in-3.14.rst +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -47,9 +47,6 @@ Pending Removal in Python 3.14 * :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. (Contributed by Alan Williams in :gh:`72346`.) -* :mod:`importlib`: ``__package__`` and ``__cached__`` will cease to be set or - taken into consideration by the import system (:gh:`97879`). - * :mod:`importlib.abc` deprecated classes: * :class:`!importlib.abc.ResourceReader` diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index f7145a85bd2994..b921b4f97d524e 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -17,6 +17,9 @@ Pending Removal in Python 3.15 * The :option:`!--cgi` flag to the :program:`python -m http.server` command-line interface has been deprecated since Python 3.13. +* :mod:`importlib`: ``__package__`` and ``__cached__`` will cease to be set or + taken into consideration by the import system (:gh:`97879`). + * :class:`locale`: * The :func:`~locale.getdefaultlocale` function diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index d0a3d9d578e0cd..e4cef1f3e3b7c0 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1237,8 +1237,8 @@ find and load modules. .. attribute:: has_location ``True`` if the spec's :attr:`origin` refers to a loadable location, - ``False`` otherwise. This value impacts how :attr:`origin` is interpreted - and how the module's :attr:`__file__` is populated. + ``False`` otherwise. This value impacts how :attr:`origin` is interpreted + and how the module's :attr:`__file__` is populated. .. class:: AppleFrameworkLoader(name, path) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 7de995b12702ec..19b8aa05072c73 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -596,6 +596,10 @@ listed below. Raise :exc:`DeprecationWarning` instead of :exc:`ImportWarning` when falling back to ``__package__``. + .. deprecated-removed:: 3.13 3.15 + ``__package__`` will cease to be set or taken into consideration + by the import system or standard library. + .. attribute:: __spec__ @@ -653,6 +657,10 @@ listed below. It is **strongly** recommended that you rely on :attr:`__spec__` instead of ``__cached__``. + .. deprecated-removed:: 3.13 3.15 + ``__cached__`` will cease to be set or taken into consideration + by the import system or standard library. + .. _package-path-rules: module.__path__ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-23-13-25-27.gh-issue-65961.LDqXV2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-23-13-25-27.gh-issue-65961.LDqXV2.rst new file mode 100644 index 00000000000000..d380027f3c5776 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-23-13-25-27.gh-issue-65961.LDqXV2.rst @@ -0,0 +1 @@ +Deprecate the setting and using ``__package__`` and ``__cached__``. From 0060486862bfa8e6583beb627be154daaaaa9e2a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 23:11:34 +0100 Subject: [PATCH 057/105] Doc: Add ``make dist-no-html`` (#124383) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/Makefile | 32 ++++++++++++++++++++++++++++++- Doc/tools/templates/download.html | 2 +- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/Doc/Makefile b/Doc/Makefile index fc0d1e4d9a2a55..a2d89343648dc1 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -182,13 +182,26 @@ venv: echo "The venv has been created in the $(VENVDIR) directory"; \ fi +.PHONY: dist-no-html +dist-no-html: dist-text dist-pdf dist-epub dist-texinfo + .PHONY: dist dist: rm -rf dist mkdir -p dist - + $(MAKE) dist-html + $(MAKE) dist-text + $(MAKE) dist-pdf + $(MAKE) dist-epub + $(MAKE) dist-texinfo + +.PHONY: dist-html +dist-html: # archive the HTML @echo "Building HTML..." + mkdir -p dist + rm -rf build/html + find dist -name 'python-$(DISTVERSION)-docs-html*' -exec rm -rf {} \; $(MAKE) html cp -pPR build/html dist/python-$(DISTVERSION)-docs-html tar -C dist -cf dist/python-$(DISTVERSION)-docs-html.tar python-$(DISTVERSION)-docs-html @@ -198,8 +211,13 @@ dist: rm dist/python-$(DISTVERSION)-docs-html.tar @echo "Build finished and archived!" +.PHONY: dist-text +dist-text: # archive the text build @echo "Building text..." + mkdir -p dist + rm -rf build/text + find dist -name 'python-$(DISTVERSION)-docs-text*' -exec rm -rf {} \; $(MAKE) text cp -pPR build/text dist/python-$(DISTVERSION)-docs-text tar -C dist -cf dist/python-$(DISTVERSION)-docs-text.tar python-$(DISTVERSION)-docs-text @@ -209,9 +227,13 @@ dist: rm dist/python-$(DISTVERSION)-docs-text.tar @echo "Build finished and archived!" +.PHONY: dist-pdf +dist-pdf: # archive the A4 latex @echo "Building LaTeX (A4 paper)..." + mkdir -p dist rm -rf build/latex + find dist -name 'python-$(DISTVERSION)-docs-pdf*' -exec rm -rf {} \; $(MAKE) latex PAPER=a4 # remove zip & bz2 dependency on all-pdf, # as otherwise the full latexmk process is run twice. @@ -222,16 +244,24 @@ dist: cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 @echo "Build finished and archived!" +.PHONY: dist-epub +dist-epub: # copy the epub build @echo "Building EPUB..." + mkdir -p dist rm -rf build/epub + rm -f dist/python-$(DISTVERSION)-docs.epub $(MAKE) epub cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub @echo "Build finished and archived!" +.PHONY: dist-texinfo +dist-texinfo: # archive the texinfo build @echo "Building Texinfo..." + mkdir -p dist rm -rf build/texinfo + find dist -name 'python-$(DISTVERSION)-docs-texinfo*' -exec rm -rf {} \; $(MAKE) texinfo $(MAKE) info --directory=build/texinfo cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index f69adc71f937c4..b4217908cc63c9 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -7,7 +7,7 @@ The link below returns HTTP 404 until the first related alpha release. This is expected; use daily documentation builds for CPython development. #} - {% set dlbase = 'https://docs.python.org/ftp/python/doc/' + release %} + {% set dlbase = 'https://www.python.org/ftp/python/doc/' + release %} {% endif %} {% block body %} From 8a2baedc4bcb606da937e4e066b4b3a18961cace Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 23 Sep 2024 15:19:35 -0700 Subject: [PATCH 058/105] Bump Ruff to 0.6.7 (#124384) --- .pre-commit-config.yaml | 2 +- Doc/tools/extensions/c_annotations.py | 5 +---- Lib/test/pickletester.py | 4 +++- Lib/test/test_bz2.py | 1 - Lib/test/test_contextlib.py | 2 -- Lib/test/test_io.py | 2 -- Lib/test/test_os.py | 3 ++- Lib/test/test_with.py | 6 ++++-- 8 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ce4cb81511d8e..4cdb7545454589 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.10 + rev: v0.6.7 hooks: - id: ruff name: Run Ruff (lint) on Doc/ diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index a65cf71e4affe3..50065d34a2c27a 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -124,10 +124,7 @@ def add_annotations(app: Sphinx, doctree: nodes.document) -> None: continue if not par[0].get("ids", None): continue - name = par[0]["ids"][0] - if name.startswith("c."): - name = name[2:] - + name = par[0]["ids"][0].removeprefix("c.") objtype = par["objtype"] # Stable ABI annotation. diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 334d4dfebdf893..1722cc8612ca6b 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -4345,7 +4345,9 @@ class MyIntWithNew2(MyIntWithNew): class SlotList(MyList): __slots__ = ["foo"] -class SimpleNewObj(int): +# Ruff "redefined while unused" false positive here due to `global` variables +# being assigned (and then restored) from within test methods earlier in the file +class SimpleNewObj(int): # noqa: F811 def __init__(self, *args, **kwargs): # raise an error, to make sure this isn't called raise TypeError("SimpleNewObj.__init__() didn't expect to get called") diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index e4d1381be5f340..7d786be1d25b1c 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -476,7 +476,6 @@ def testReadlinesNoNewline(self): self.assertEqual(xlines, [b'Test']) def testContextProtocol(self): - f = None with BZ2File(self.filename, "wb") as f: f.write(b"xxx") f = BZ2File(self.filename, "rb") diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 36c3abca80f894..cf6519598037e9 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -444,12 +444,10 @@ class FileContextTestCase(unittest.TestCase): def testWithOpen(self): tfn = tempfile.mktemp() try: - f = None with open(tfn, "w", encoding="utf-8") as f: self.assertFalse(f.closed) f.write("Booh\n") self.assertTrue(f.closed) - f = None with self.assertRaises(ZeroDivisionError): with open(tfn, "r", encoding="utf-8") as f: self.assertFalse(f.closed) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 1ca3edac8c8dc9..aa1b8268592ff7 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -639,11 +639,9 @@ def test_large_file_ops(self): def test_with_open(self): for bufsize in (0, 100): - f = None with self.open(os_helper.TESTFN, "wb", bufsize) as f: f.write(b"xxx") self.assertEqual(f.closed, True) - f = None try: with self.open(os_helper.TESTFN, "wb", bufsize) as f: 1/0 diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 9fa4e406bd2468..307f0f11ddc33f 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3177,7 +3177,8 @@ class Win32NtTests(unittest.TestCase): def test_getfinalpathname_handles(self): nt = import_helper.import_module('nt') ctypes = import_helper.import_module('ctypes') - import ctypes.wintypes + # Ruff false positive -- it thinks we're redefining `ctypes` here + import ctypes.wintypes # noqa: F811 kernel = ctypes.WinDLL('Kernel32.dll', use_last_error=True) kernel.GetCurrentProcess.restype = ctypes.wintypes.HANDLE diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index e8c4ddf979e2ee..839cdec68d573e 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -171,7 +171,10 @@ def __exit__(self, *args): def shouldThrow(): ct = EnterThrows() self.foo = None - with ct as self.foo: + # Ruff complains that we're redefining `self.foo` here, + # but the whole point of the test is to check that `self.foo` + # is *not* redefined (because `__enter__` raises) + with ct as self.foo: # ruff: noqa: F811 pass self.assertRaises(RuntimeError, shouldThrow) self.assertEqual(self.foo, None) @@ -252,7 +255,6 @@ def testInlineGeneratorBoundSyntax(self): self.assertAfterWithGeneratorInvariantsNoError(foo) def testInlineGeneratorBoundToExistingVariable(self): - foo = None with mock_contextmanager_generator() as foo: self.assertInWithGeneratorInvariants(foo) self.assertAfterWithGeneratorInvariantsNoError(foo) From 27a62e7371f062a80704f6bf4d6e8f06568d37aa Mon Sep 17 00:00:00 2001 From: Wulian <1055917385@qq.com> Date: Tue, 24 Sep 2024 07:09:22 +0800 Subject: [PATCH 059/105] gh-124102: Update internal PCbuild docs to accurately list build dependencies (GH-124103) --- ...-09-16-09-42-05.gh-issue-124102.Ow254j.rst | 2 ++ PCbuild/build.bat | 2 +- PCbuild/env.bat | 11 ++++----- PCbuild/find_msbuild.bat | 10 -------- PCbuild/get_externals.bat | 2 +- PCbuild/python.props | 6 +---- PCbuild/pythoncore.vcxproj | 2 +- PCbuild/readme.txt | 8 +++---- PCbuild/tcltk.props | 3 --- Tools/msi/README.txt | 24 +++++++------------ Tools/msi/bundle/bootstrap/pythonba.vcxproj | 3 --- 11 files changed, 24 insertions(+), 49 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-09-16-09-42-05.gh-issue-124102.Ow254j.rst diff --git a/Misc/NEWS.d/next/Build/2024-09-16-09-42-05.gh-issue-124102.Ow254j.rst b/Misc/NEWS.d/next/Build/2024-09-16-09-42-05.gh-issue-124102.Ow254j.rst new file mode 100644 index 00000000000000..6edc9a6abbced4 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-09-16-09-42-05.gh-issue-124102.Ow254j.rst @@ -0,0 +1,2 @@ +Update internal documentation under PCbuild, so it now correctly states that +Windows requires VS2017 or later and Python 3.10 or later diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 6c76f09a071312..abe649553756a7 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -8,7 +8,7 @@ echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt). echo. echo.After the flags recognized by this script, up to 9 arguments to be passed echo.directly to MSBuild may be passed. If the argument contains an '=', the -echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`). +echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v141"`). echo.Alternatively you can put extra flags for MSBuild in a file named echo.`msbuild.rsp` in the `PCbuild` directory, one flag per line. This file echo.will be picked automatically by MSBuild. Flags put in this file does not diff --git a/PCbuild/env.bat b/PCbuild/env.bat index 2820e304582cff..cf4638b7aa63a7 100644 --- a/PCbuild/env.bat +++ b/PCbuild/env.bat @@ -4,8 +4,8 @@ rem command window. However, most builds of Python will ignore the version rem of the tools on PATH and use PlatformToolset instead. Ideally, both sets of rem tools should be the same version to avoid potential conflicts. rem -rem To build Python with an earlier toolset, pass "/p:PlatformToolset=v100" (or -rem 'v110', 'v120' or 'v140') to the build script. +rem To build Python with an earlier toolset, pass "/p:PlatformToolset=v141" (or +rem 'v142', 'v143') to the build script. echo Build environments: x86, amd64, x86_amd64 echo. @@ -20,8 +20,7 @@ call "%VSTOOLS%" %_ARGS% exit /B 0 :skip_vswhere -if not defined VSTOOLS set VSTOOLS=%VS140COMNTOOLS% -if not defined VSTOOLS set VSTOOLS=%VS120COMNTOOLS% -if not defined VSTOOLS set VSTOOLS=%VS110COMNTOOLS% -if not defined VSTOOLS set VSTOOLS=%VS100COMNTOOLS% +if not defined VSTOOLS set VSTOOLS=%VS143COMNTOOLS% +if not defined VSTOOLS set VSTOOLS=%VS142COMNTOOLS% +if not defined VSTOOLS set VSTOOLS=%VS141COMNTOOLS% call "%VSTOOLS%..\..\VC\vcvarsall.bat" %_ARGS% diff --git a/PCbuild/find_msbuild.bat b/PCbuild/find_msbuild.bat index ce7e71efa31f6c..82dd34beede6ee 100644 --- a/PCbuild/find_msbuild.bat +++ b/PCbuild/find_msbuild.bat @@ -39,16 +39,6 @@ @if defined MSBUILD @if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio installation) & goto :found :skip_vswhere -@rem VS 2015 and earlier register MSBuild separately, so we can find it. -@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul -@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @( - @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe") -) -@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found - - -@exit /b 1 - :found @pushd %MSBUILD% >nul 2>nul @if not ERRORLEVEL 1 @( diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index a1a67966182863..137c94789e1809 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -45,7 +45,7 @@ if "%ORG%"=="" (set ORG=python) call "%PCBUILD%\find_python.bat" "%PYTHON%" if NOT DEFINED PYTHON ( - where /Q git || echo Python 3.6 could not be found or installed, and git.exe is not on your PATH && exit /B 1 + where /Q git || echo Python 3.10 or later could not be found or installed, and git.exe is not on your PATH && exit /B 1 ) echo.Fetching external libraries... diff --git a/PCbuild/python.props b/PCbuild/python.props index c8ecdb4515ae9a..6e90178f4ea8ab 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -6,7 +6,7 @@ Release