From 23faf5f2dca009f7cc18840246de405dbb7fce9f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 27 Jan 2025 18:33:18 +0100 Subject: [PATCH 1/7] [3.12] gh-129346: Handle allocation errors for SQLite aggregate context (GH-129347) (#129373) (cherry picked from commit 379ab856f59423c570333403a7d5d72f3ea82d52) Co-authored-by: Erlend E. Aasland --- .../Library/2025-01-27-14-05-19.gh-issue-129346.gZRd3g.rst | 2 ++ Modules/_sqlite/connection.c | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-01-27-14-05-19.gh-issue-129346.gZRd3g.rst diff --git a/Misc/NEWS.d/next/Library/2025-01-27-14-05-19.gh-issue-129346.gZRd3g.rst b/Misc/NEWS.d/next/Library/2025-01-27-14-05-19.gh-issue-129346.gZRd3g.rst new file mode 100644 index 00000000000000..b5377277f6c51c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-27-14-05-19.gh-issue-129346.gZRd3g.rst @@ -0,0 +1,2 @@ +In :mod:`sqlite3`, handle out-of-memory when creating user-defined SQL +functions. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 12e5c135aafa50..1450037ca95c83 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -927,6 +927,11 @@ step_callback(sqlite3_context *context, int argc, sqlite3_value **params) assert(ctx != NULL); aggregate_instance = (PyObject**)sqlite3_aggregate_context(context, sizeof(PyObject*)); + if (aggregate_instance == NULL) { + (void)PyErr_NoMemory(); + set_sqlite_error(context, "unable to allocate SQLite aggregate context"); + goto error; + } if (*aggregate_instance == NULL) { *aggregate_instance = PyObject_CallNoArgs(ctx->callable); if (!*aggregate_instance) { From 2df8b395b00f01cf70a0cdad0203d784f8e62d9d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 27 Jan 2025 23:06:14 +0100 Subject: [PATCH 2/7] [3.12] gh-119511: Fix a potential denial of service in imaplib (GH-119514) (GH-129356) gh-119511: Fix a potential denial of service in imaplib (GH-119514) The IMAP4 client could consume an arbitrary amount of memory when trying to connect to a malicious server, because it read a "literal" data with a single read(size) call, and BufferedReader.read() allocates the bytes object of the specified size before reading. Now the IMAP4 client reads data by chunks, therefore the amount of used memory is limited by the amount of the data actually been sent by the server. (cherry picked from commit 735f25c5e3a0f74438c86468ec4dfbe219d93c91) Co-authored-by: Serhiy Storchaka Co-authored-by: Gregory P. Smith --- Lib/imaplib.py | 11 ++++++++++- Lib/test/test_imaplib.py | 14 ++++++++++++++ .../2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst | 7 +++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 577b4b9b03a88d..e337fe6471069f 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -52,6 +52,9 @@ # search command can be quite large, so we now use 1M. _MAXLINE = 1000000 +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_SAFE_BUF_SIZE = 1 << 20 # Commands @@ -315,7 +318,13 @@ def open(self, host='', port=IMAP4_PORT, timeout=None): def read(self, size): """Read 'size' bytes from remote.""" - return self.file.read(size) + cursize = min(size, _SAFE_BUF_SIZE) + data = self.file.read(cursize) + while cursize < size and len(data) == cursize: + delta = min(cursize, size - cursize) + data += self.file.read(delta) + cursize += delta + return data def readline(self): diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 1d701281b839e8..4429a90050a14d 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -901,6 +901,20 @@ def handle(self): self.assertRaises(imaplib.IMAP4.error, self.imap_class, *server.server_address) + def test_truncated_large_literal(self): + size = 0 + class BadHandler(SimpleIMAPHandler): + def handle(self): + self._send_textline('* OK {%d}' % size) + self._send_textline('IMAP4rev1') + + for exponent in range(15, 64): + size = 1 << exponent + with self.subTest(f"size=2e{size}"): + with self.reaped_server(BadHandler) as server: + with self.assertRaises(imaplib.IMAP4.abort): + self.imap_class(*server.server_address) + @threading_helper.reap_threads def test_simple_with_statement(self): # simplest call diff --git a/Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst b/Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst new file mode 100644 index 00000000000000..f7b4031120e643 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst @@ -0,0 +1,7 @@ +Fix a potential denial of service in the :mod:`imaplib` module. When connecting +to a malicious server, it could cause an arbitrary amount of memory to be +allocated. On many systems this is harmless as unused virtual memory is only a +mapping, but if this hit a virtual address size limit it could lead to a +:exc:`MemoryError` or other process crash. On unusual systems or builds where +all allocated memory is touched and backed by actual ram or storage it could've +consumed resources doing so until similarly crashing. From ea143e67b328d961e3eea9c99ee1131550ce23fc Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 28 Jan 2025 01:20:13 +0100 Subject: [PATCH 3/7] [3.12] gh-85046: Document errno constants (GH-126420) (#129384) (cherry picked from commit 1c3bb200da77f9df30af4717988dad17db086d1a) Co-authored-by: RUANG (James Roy) --- Doc/library/errno.rst | 165 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst index 4983b8961b1c3f..ffebe8ee485eb6 100644 --- a/Doc/library/errno.rst +++ b/Doc/library/errno.rst @@ -665,6 +665,171 @@ defined by the module. The specific list of defined symbols is available as .. versionadded:: 3.11 + +.. data:: ENOMEDIUM + + No medium found + + +.. data:: EMEDIUMTYPE + + Wrong medium type + + +.. data:: ENOKEY + + Required key not available + + +.. data:: EKEYEXPIRED + + Key has expired + + +.. data:: EKEYREVOKED + + Key has been revoked + + +.. data:: EKEYREJECTED + + Key was rejected by service + + +.. data:: ERFKILL + + Operation not possible due to RF-kill + + +.. data:: ELOCKUNMAPPED + + Locked lock was unmapped + + +.. data:: ENOTACTIVE + + Facility is not active + + +.. data:: EAUTH + + Authentication error + + .. versionadded:: 3.2 + + +.. data:: EBADARCH + + Bad CPU type in executable + + .. versionadded:: 3.2 + + +.. data:: EBADEXEC + + Bad executable (or shared library) + + .. versionadded:: 3.2 + + +.. data:: EBADMACHO + + Malformed Mach-o file + + .. versionadded:: 3.2 + + +.. data:: EDEVERR + + Device error + + .. versionadded:: 3.2 + + +.. data:: EFTYPE + + Inappropriate file type or format + + .. versionadded:: 3.2 + + +.. data:: ENEEDAUTH + + Need authenticator + + .. versionadded:: 3.2 + + +.. data:: ENOATTR + + Attribute not found + + .. versionadded:: 3.2 + + +.. data:: ENOPOLICY + + Policy not found + + .. versionadded:: 3.2 + + +.. data:: EPROCLIM + + Too many processes + + .. versionadded:: 3.2 + + +.. data:: EPROCUNAVAIL + + Bad procedure for program + + .. versionadded:: 3.2 + + +.. data:: EPROGMISMATCH + + Program version wrong + + .. versionadded:: 3.2 + + +.. data:: EPROGUNAVAIL + + RPC prog. not avail + + .. versionadded:: 3.2 + + +.. data:: EPWROFF + + Device power is off + + .. versionadded:: 3.2 + + +.. data:: EBADRPC + + RPC struct is bad + + .. versionadded:: 3.2 + + +.. data:: ERPCMISMATCH + + RPC version wrong + + .. versionadded:: 3.2 + + +.. data:: ESHLIBVERS + + Shared library version mismatch + + .. versionadded:: 3.2 + + .. data:: ENOTCAPABLE Capabilities insufficient. This error is mapped to the exception From e5ab9e37408e2470040407deeb87f618c5f053ee Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 28 Jan 2025 12:54:27 +0100 Subject: [PATCH 4/7] [3.12] gh-112064: Fix incorrect handling of negative read sizes in `HTTPResponse.read()` (GH-128270) (#129396) gh-112064: Fix incorrect handling of negative read sizes in `HTTPResponse.read()` (GH-128270) The parameter `amt` of `HTTPResponse.read()`, which could be a negative integer, has not been handled before and led to waiting for the connection to close for `keep-alive connections`. Now, this has been fixed, and passing negative values to `HTTPResponse().read()` works the same as passing `None` value. (cherry picked from commit 4d0d24f6e3dff2864007c3cfd1cf7d49c6ee5317) Co-authored-by: Yury Manushkin --- Lib/http/client.py | 4 +++- Lib/test/test_httplib.py | 19 +++++++++++++++++++ ...-12-26-11-00-03.gh-issue-112064.mCcw3B.rst | 2 ++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-26-11-00-03.gh-issue-112064.mCcw3B.rst diff --git a/Lib/http/client.py b/Lib/http/client.py index a353716a8506e6..fb29923d94274c 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -472,7 +472,7 @@ def read(self, amt=None): if self.chunked: return self._read_chunked(amt) - if amt is not None: + if amt is not None and amt >= 0: if self.length is not None and amt > self.length: # clip the read to the "end of response" amt = self.length @@ -590,6 +590,8 @@ def _get_chunk_left(self): def _read_chunked(self, amt=None): assert self.chunked != _UNKNOWN + if amt is not None and amt < 0: + amt = None value = [] try: while (chunk_left := self._get_chunk_left()) is not None: diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 6e63a8872d9c6e..01f5a10190194c 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1078,6 +1078,25 @@ def test_chunked(self): self.assertEqual(resp.read(), expected) resp.close() + # Explicit full read + for n in (-123, -1, None): + with self.subTest('full read', n=n): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertTrue(resp.chunked) + self.assertEqual(resp.read(n), expected) + resp.close() + + # Read first chunk + with self.subTest('read1(-1)'): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertTrue(resp.chunked) + self.assertEqual(resp.read1(-1), b"hello worl") + resp.close() + # Various read sizes for n in range(1, 12): sock = FakeSocket(chunked_start + last_chunk + chunked_end) diff --git a/Misc/NEWS.d/next/Library/2024-12-26-11-00-03.gh-issue-112064.mCcw3B.rst b/Misc/NEWS.d/next/Library/2024-12-26-11-00-03.gh-issue-112064.mCcw3B.rst new file mode 100644 index 00000000000000..e885add7b68c0f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-26-11-00-03.gh-issue-112064.mCcw3B.rst @@ -0,0 +1,2 @@ +Fix incorrect handling of negative read sizes in :meth:`HTTPResponse.read +`. Patch by Yury Manushkin. From cdaec318cb9189b7b74d46432532a3f88ec4c249 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 29 Jan 2025 12:32:54 +0100 Subject: [PATCH 5/7] [3.12] gh-129409: Fix Integer overflow - SEGV while writing data more than 2GB in CSV file (GH-129413) (#129437) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-129409: Fix Integer overflow - SEGV while writing data more than 2GB in CSV file (GH-129413) (cherry picked from commit 97b0ef05d987ebef354512b516a246feb411e815) Co-authored-by: Srinivas Reddy Thatiparthy (తాటిపర్తి శ్రీనివాస్ రెడ్డి) --- .../next/Library/2025-01-29-14-30-54.gh-issue-129409.JZbOE6.rst | 2 ++ Modules/_csv.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-29-14-30-54.gh-issue-129409.JZbOE6.rst diff --git a/Misc/NEWS.d/next/Library/2025-01-29-14-30-54.gh-issue-129409.JZbOE6.rst b/Misc/NEWS.d/next/Library/2025-01-29-14-30-54.gh-issue-129409.JZbOE6.rst new file mode 100644 index 00000000000000..7e00b44c0ef471 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-29-14-30-54.gh-issue-129409.JZbOE6.rst @@ -0,0 +1,2 @@ +Fix an integer overflow in the :mod:`csv` module when writing a data field +larger than 2GB. diff --git a/Modules/_csv.c b/Modules/_csv.c index 9a7b7d27c2ed39..df7207d0894af8 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1074,7 +1074,7 @@ join_append_data(WriterObj *self, int field_kind, const void *field_data, int copy_phase) { DialectObj *dialect = self->dialect; - int i; + Py_ssize_t i; Py_ssize_t rec_len; #define INCLEN \ From 0e54315c319386ad15a6941617858e58a849f657 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 29 Jan 2025 16:47:50 +0200 Subject: [PATCH 6/7] [3.12]: CI: Change job name to 'build arm64' on Windows (#129434) (#129444) CI: Change job name to 'build arm64' on Windows (#129434) (cherry picked from commit c67afb581eccb3ce20a4965c8f407fd2662b6bdf) --- .github/workflows/reusable-windows.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 35c75f4a0877dd..cd93f1b75817c4 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -20,9 +20,7 @@ env: jobs: build: - name: >- - build${{ inputs.arch != 'arm64' && ' and test' || '' }} - (${{ inputs.arch }}) + name: ${{ inputs.arch == 'arm64' && 'build' || 'build and test' }} (${{ inputs.arch }}) runs-on: windows-latest timeout-minutes: 60 env: From f65aa0d1bf7b636ab8f9d226429205854b24cd7a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 29 Jan 2025 17:48:48 +0100 Subject: [PATCH 7/7] [3.12] gh-118761: Improve import time of `subprocess` (GH-129427) (#129448) gh-118761: Improve import time of `subprocess` (GH-129427) * subprocess: lazy import signal and locale to improve module import time (cherry picked from commit 49f24650e4541456872490ec2b59d6d186204891) Co-authored-by: Taneli Hukkinen <3275109+hukkin@users.noreply.github.com> --- Lib/subprocess.py | 16 ++++++++++++++-- ...025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst | 2 ++ 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 881a9ce800ae99..3ec39ca3e61e7a 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -43,10 +43,8 @@ import builtins import errno import io -import locale import os import time -import signal import sys import threading import warnings @@ -138,6 +136,8 @@ def __init__(self, returncode, cmd, output=None, stderr=None): def __str__(self): if self.returncode and self.returncode < 0: + # Lazy import to improve module import time + import signal try: return "Command '%s' died with %r." % ( self.cmd, signal.Signals(-self.returncode)) @@ -375,6 +375,8 @@ def _text_encoding(): if sys.flags.utf8_mode: return "utf-8" else: + # Lazy import to improve module import time + import locale return locale.getencoding() @@ -1655,6 +1657,9 @@ def send_signal(self, sig): # Don't signal a process that we know has already died. if self.returncode is not None: return + + # Lazy import to improve module import time + import signal if sig == signal.SIGTERM: self.terminate() elif sig == signal.CTRL_C_EVENT: @@ -1759,6 +1764,9 @@ def _posix_spawn(self, args, executable, env, restore_signals, kwargs = {} if restore_signals: + # Lazy import to improve module import time + import signal + # See _Py_RestoreSignals() in Python/pylifecycle.c sigset = [] for signame in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'): @@ -2208,9 +2216,13 @@ def send_signal(self, sig): def terminate(self): """Terminate the process with SIGTERM """ + # Lazy import to improve module import time + import signal self.send_signal(signal.SIGTERM) def kill(self): """Kill the process with SIGKILL """ + # Lazy import to improve module import time + import signal self.send_signal(signal.SIGKILL) diff --git a/Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst b/Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst new file mode 100644 index 00000000000000..0762cbe5d63949 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst @@ -0,0 +1,2 @@ +Improve import time of :mod:`subprocess` by lazy importing ``locale`` and +``signal``. Patch by Taneli Hukkinen.