From f2584eade378910b9ea18072bb1dab3dd58e23bb Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 7 Sep 2023 08:56:43 -0600 Subject: [PATCH 01/66] gh-108732: include comprehension locals in frame.f_locals (#109026) Co-authored-by: Radislav Chugunov <52372310+chgnrdv@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- Lib/test/test_listcomps.py | 7 +++++++ .../2023-09-06-13-28-42.gh-issue-108732.I6DkEQ.rst | 2 ++ Objects/frameobject.c | 14 ++++++++++---- 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-06-13-28-42.gh-issue-108732.I6DkEQ.rst diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index bedd99b4a44fcb..c1089574d71b02 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -596,6 +596,13 @@ def test_exception_in_post_comp_call(self): """ self._check_in_scopes(code, {"value": [1, None]}) + def test_frame_locals(self): + code = """ + val = [sys._getframe().f_locals for a in [0]][0]["a"] + """ + import sys + self._check_in_scopes(code, {"val": 0}, ns={"sys": sys}) + __test__ = {'doctests' : doctests} diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-06-13-28-42.gh-issue-108732.I6DkEQ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-06-13-28-42.gh-issue-108732.I6DkEQ.rst new file mode 100644 index 00000000000000..94a143b86b6708 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-06-13-28-42.gh-issue-108732.I6DkEQ.rst @@ -0,0 +1,2 @@ +Make iteration variables of module- and class-scoped comprehensions visible +to pdb and other tools that use ``frame.f_locals`` again. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 7017cc6c0e295f..53764a41ca010a 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -25,10 +25,16 @@ static PyMemberDef frame_memberlist[] = { static PyObject * frame_getlocals(PyFrameObject *f, void *closure) { - if (PyFrame_FastToLocalsWithError(f) < 0) + if (f == NULL) { + PyErr_BadInternalCall(); return NULL; - PyObject *locals = f->f_frame->f_locals; - return Py_NewRef(locals); + } + assert(!_PyFrame_IsIncomplete(f->f_frame)); + PyObject *locals = _PyFrame_GetLocals(f->f_frame, 1); + if (locals) { + f->f_fast_as_locals = 1; + } + return locals; } int @@ -1342,11 +1348,11 @@ PyFrame_GetVarString(PyFrameObject *frame, const char *name) int PyFrame_FastToLocalsWithError(PyFrameObject *f) { - assert(!_PyFrame_IsIncomplete(f->f_frame)); if (f == NULL) { PyErr_BadInternalCall(); return -1; } + assert(!_PyFrame_IsIncomplete(f->f_frame)); int err = _PyFrame_FastToLocalsWithError(f->f_frame); if (err == 0) { f->f_fast_as_locals = 1; From 403ab1306a6e9860197bce57eadcb83418966f21 Mon Sep 17 00:00:00 2001 From: Christoph Anton Mitterer Date: Thu, 7 Sep 2023 17:50:10 +0200 Subject: [PATCH 02/66] gh-107924: re-order os.sendfile() flag documentation (#107926) Co-authored-by: Hugo van Kemenade --- Doc/library/os.rst | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 4f4d8c99023529..c67b966f777db8 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1618,25 +1618,6 @@ or `the MSDN `_ on Windo Parameters *out* and *in* was renamed to *out_fd* and *in_fd*. -.. function:: set_blocking(fd, blocking, /) - - Set the blocking mode of the specified file descriptor. Set the - :data:`O_NONBLOCK` flag if blocking is ``False``, clear the flag otherwise. - - See also :func:`get_blocking` and :meth:`socket.socket.setblocking`. - - .. availability:: Unix, Windows. - - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - - On Windows, this function is limited to pipes. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.12 - Added support for pipes on Windows. - .. data:: SF_NODISKIO SF_MNOWAIT SF_SYNC @@ -1658,6 +1639,26 @@ or `the MSDN `_ on Windo .. versionadded:: 3.11 +.. function:: set_blocking(fd, blocking, /) + + Set the blocking mode of the specified file descriptor. Set the + :data:`O_NONBLOCK` flag if blocking is ``False``, clear the flag otherwise. + + See also :func:`get_blocking` and :meth:`socket.socket.setblocking`. + + .. availability:: Unix, Windows. + + The function is limited on Emscripten and WASI, see + :ref:`wasm-availability` for more information. + + On Windows, this function is limited to pipes. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.12 + Added support for pipes on Windows. + + .. function:: splice(src, dst, count, offset_src=None, offset_dst=None) Transfer *count* bytes from file descriptor *src*, starting from offset From 96396962ce7a83c09bac5061121c779ca6b25ef5 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 7 Sep 2023 18:23:11 +0100 Subject: [PATCH 03/66] gh-109094: remove unnecessary updates of frame->prev_instr in instrumentation functions (#109076) --- Lib/test/test_sys_settrace.py | 9 +++++++++ Python/instrumentation.c | 8 ++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 232900876392ff..369a276ac33a12 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -317,6 +317,13 @@ def generator_example(): [(5, 'line'), (5, 'return')]) +def lineno_matches_lasti(frame): + last_line = None + for start, end, line in frame.f_code.co_lines(): + if start <= frame.f_lasti < end: + last_line = line + return last_line == frame.f_lineno + class Tracer: def __init__(self, trace_line_events=None, trace_opcode_events=None): self.trace_line_events = trace_line_events @@ -330,6 +337,7 @@ def _reconfigure_frame(self, frame): frame.f_trace_opcodes = self.trace_opcode_events def trace(self, frame, event, arg): + assert lineno_matches_lasti(frame) self._reconfigure_frame(frame) self.events.append((frame.f_lineno, event)) return self.trace @@ -1890,6 +1898,7 @@ def __init__(self, function, jumpFrom, jumpTo, event='line', def trace(self, frame, event, arg): if self.done: return + assert lineno_matches_lasti(frame) # frame.f_code.co_firstlineno is the first line of the decorator when # 'function' is decorated and the decorator may be written using # multiple physical lines when it is too long. Use the first line diff --git a/Python/instrumentation.c b/Python/instrumentation.c index acc3278d50a60a..74c3adf0c1f475 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1057,8 +1057,6 @@ _Py_call_instrumentation_jump( assert(event == PY_MONITORING_EVENT_JUMP || event == PY_MONITORING_EVENT_BRANCH); assert(frame->prev_instr == instr); - /* Event should occur after the jump */ - frame->prev_instr = target; PyCodeObject *code = _PyFrame_GetCode(frame); int to = (int)(target - _PyCode_CODE(code)); PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); @@ -1071,12 +1069,10 @@ _Py_call_instrumentation_jump( if (err) { return NULL; } - if (frame->prev_instr != target) { + if (frame->prev_instr != instr) { /* The callback has caused a jump (by setting the line number) */ return frame->prev_instr; } - /* Reset prev_instr for INSTRUMENTED_LINE */ - frame->prev_instr = instr; return target; } @@ -1125,7 +1121,7 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { - frame->prev_instr = instr; + assert(frame->prev_instr == instr); PyCodeObject *code = _PyFrame_GetCode(frame); assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); From f9f085c326cdaa34ebb3ca018228a63825b12122 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 7 Sep 2023 20:53:38 +0300 Subject: [PATCH 04/66] gh-103186: Make test_generated_cases less noisy by default (GH-109100) Print additional details only when tests are run with -vv. --- Lib/test/test_generated_cases.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index ed56f95af99417..be3dfd204ee143 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1,7 +1,9 @@ +import contextlib import tempfile import unittest import os +from test import support from test import test_tools test_tools.skip_if_missing('cases_generator') @@ -12,6 +14,12 @@ from parsing import StackEffect +def handle_stderr(): + if support.verbose > 1: + return contextlib.nullcontext() + else: + return support.captured_stderr() + class TestEffects(unittest.TestCase): def test_effect_sizes(self): input_effects = [ @@ -81,11 +89,12 @@ def run_cases_test(self, input: str, expected: str): temp_input.flush() a = generate_cases.Generator([self.temp_input_filename]) - a.parse() - a.analyze() - if a.errors: - raise RuntimeError(f"Found {a.errors} errors") - a.write_instructions(self.temp_output_filename, False) + with handle_stderr(): + a.parse() + a.analyze() + if a.errors: + raise RuntimeError(f"Found {a.errors} errors") + a.write_instructions(self.temp_output_filename, False) with open(self.temp_output_filename) as temp_output: lines = temp_output.readlines() From 1829a3c9a3712b6a68a3a449e4a08787c73da51d Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Thu, 7 Sep 2023 14:13:32 -0400 Subject: [PATCH 05/66] gh-75743: Restore test_timeout.testConnectTimeout() (#109087) This un-skips this test now that pythontest.net implements appropriate firewall rules for it. --- Lib/test/test_timeout.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_timeout.py b/Lib/test/test_timeout.py index 30e843a423a777..35ff56f1a5ee09 100644 --- a/Lib/test/test_timeout.py +++ b/Lib/test/test_timeout.py @@ -148,13 +148,12 @@ def setUp(self): def tearDown(self): self.sock.close() - @unittest.skipIf(True, 'need to replace these hosts; see bpo-35518') def testConnectTimeout(self): # Testing connect timeout is tricky: we need to have IP connectivity # to a host that silently drops our packets. We can't simulate this # from Python because it's a function of the underlying TCP/IP stack. - # So, the following Snakebite host has been defined: - blackhole = resolve_address('blackhole.snakebite.net', 56666) + # So, the following port on the pythontest.net host has been defined: + blackhole = resolve_address('pythontest.net', 56666) # Blackhole has been configured to silently drop any incoming packets. # No RSTs (for TCP) or ICMP UNREACH (for UDP/ICMP) will be sent back @@ -166,7 +165,7 @@ def testConnectTimeout(self): # to firewalling or general network configuration. In order to improve # our confidence in testing the blackhole, a corresponding 'whitehole' # has also been set up using one port higher: - whitehole = resolve_address('whitehole.snakebite.net', 56667) + whitehole = resolve_address('pythontest.net', 56667) # This address has been configured to immediately drop any incoming # packets as well, but it does it respectfully with regards to the @@ -180,20 +179,15 @@ def testConnectTimeout(self): # timeframe). # For the records, the whitehole/blackhole configuration has been set - # up using the 'pf' firewall (available on BSDs), using the following: + # up using the 'iptables' firewall, using the following rules: # - # ext_if="bge0" - # - # blackhole_ip="35.8.247.6" - # whitehole_ip="35.8.247.6" - # blackhole_port="56666" - # whitehole_port="56667" - # - # block return in log quick on $ext_if proto { tcp udp } \ - # from any to $whitehole_ip port $whitehole_port - # block drop in log quick on $ext_if proto { tcp udp } \ - # from any to $blackhole_ip port $blackhole_port + # -A INPUT -p tcp --destination-port 56666 -j DROP + # -A INPUT -p udp --destination-port 56666 -j DROP + # -A INPUT -p tcp --destination-port 56667 -j REJECT + # -A INPUT -p udp --destination-port 56667 -j REJECT # + # See https://github.com/python/psf-salt/blob/main/pillar/base/firewall/snakebite.sls + # for the current configuration. skip = True sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) From 7e1a7abb9831965cdec477e62dbe4f8415b8a582 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 7 Sep 2023 21:28:18 +0300 Subject: [PATCH 06/66] gh-68403: Fix test_coverage in test_trace (GH-108910) Its behavior no longer affected by test running options such as -m. --- Lib/test/test_trace.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index d1ef005a4314ed..c1e289bcaff9e5 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -360,9 +360,14 @@ def tearDown(self): rmtree(TESTFN) unlink(TESTFN) - def _coverage(self, tracer, - cmd='import test.support, test.test_pprint;' - 'test.support.run_unittest(test.test_pprint.QueryTestCase)'): + DEFAULT_SCRIPT = '''if True: + import unittest + from test.test_pprint import QueryTestCase + loader = unittest.TestLoader() + tests = loader.loadTestsFromTestCase(QueryTestCase) + tests(unittest.TestResult()) + ''' + def _coverage(self, tracer, cmd=DEFAULT_SCRIPT): tracer.run(cmd) r = tracer.results() r.write_results(show_missing=True, summary=True, coverdir=TESTFN) From 74fc96bc60f5c02bde50ff2f3516add99483e402 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 7 Sep 2023 11:34:18 -0700 Subject: [PATCH 07/66] Add version directives to ast docs (#108788) --- Doc/library/ast.rst | 46 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 8f5148feb809a3..10b3e2dae472e1 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -650,10 +650,10 @@ Expressions .. class:: NamedExpr(target, value) - A named expression. This AST node is produced by the assignment expressions - operator (also known as the walrus operator). As opposed to the :class:`Assign` - node in which the first argument can be multiple nodes, in this case both - ``target`` and ``value`` must be single nodes. + A named expression. This AST node is produced by the assignment expressions + operator (also known as the walrus operator). As opposed to the :class:`Assign` + node in which the first argument can be multiple nodes, in this case both + ``target`` and ``value`` must be single nodes. .. doctest:: @@ -663,6 +663,7 @@ Expressions target=Name(id='x', ctx=Store()), value=Constant(value=4))) + .. versionadded:: 3.8 Subscripting ~~~~~~~~~~~~ @@ -1036,6 +1037,7 @@ Statements value=Name(id='int', ctx=Load()))], type_ignores=[]) + .. versionadded:: 3.12 Other statements which are only applicable inside functions or loops are described in other sections. @@ -1318,6 +1320,7 @@ Control flow finalbody=[])], type_ignores=[]) + .. versionadded:: 3.11 .. class:: ExceptHandler(type, name, body) @@ -1407,6 +1410,8 @@ Pattern matching that is being matched against the cases) and ``cases`` contains an iterable of :class:`match_case` nodes with the different cases. + .. versionadded:: 3.10 + .. class:: match_case(pattern, guard, body) A single case pattern in a ``match`` statement. ``pattern`` contains the @@ -1458,6 +1463,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchValue(value) A match literal or value pattern that compares by equality. ``value`` is @@ -1485,6 +1492,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchSingleton(value) A match literal pattern that compares by identity. ``value`` is the @@ -1510,6 +1519,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchSequence(patterns) A match sequence pattern. ``patterns`` contains the patterns to be matched @@ -1541,6 +1552,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchStar(name) Matches the rest of the sequence in a variable length match sequence pattern. @@ -1581,6 +1594,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchMapping(keys, patterns, rest) A match mapping pattern. ``keys`` is a sequence of expression nodes. @@ -1627,6 +1642,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchClass(cls, patterns, kwd_attrs, kwd_patterns) A match class pattern. ``cls`` is an expression giving the nominal class to @@ -1691,6 +1708,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchAs(pattern, name) A match "as-pattern", capture pattern or wildcard pattern. ``pattern`` @@ -1732,6 +1751,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchOr(patterns) A match "or-pattern". An or-pattern matches each of its subpatterns in turn @@ -1764,6 +1785,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. _ast-type-params: Type parameters @@ -1795,6 +1818,8 @@ aliases. ctx=Load()))], type_ignores=[]) + .. versionadded:: 3.12 + .. class:: ParamSpec(name) A :class:`typing.ParamSpec`. ``name`` is the name of the parameter specification. @@ -1818,6 +1843,8 @@ aliases. ctx=Load()))], type_ignores=[]) + .. versionadded:: 3.12 + .. class:: TypeVarTuple(name) A :class:`typing.TypeVarTuple`. ``name`` is the name of the type variable tuple. @@ -1842,6 +1869,8 @@ aliases. ctx=Load()))], type_ignores=[]) + .. versionadded:: 3.12 + Function and class definitions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1861,6 +1890,9 @@ Function and class definitions ``type_comment`` is an optional string with the type annotation as a comment. + .. versionchanged:: 3.12 + Added ``type_params``. + .. class:: Lambda(args, body) @@ -2059,6 +2091,9 @@ Function and class definitions type_params=[])], type_ignores=[]) + .. versionchanged:: 3.12 + Added ``type_params``. + Async and await ^^^^^^^^^^^^^^^ @@ -2067,6 +2102,9 @@ Async and await An ``async def`` function definition. Has the same fields as :class:`FunctionDef`. + .. versionchanged:: 3.12 + Added ``type_params``. + .. class:: Await(value) From b9831e5c98de280870b6d932033b868ef56fa2fa Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 7 Sep 2023 23:08:55 +0300 Subject: [PATCH 08/66] Use unittest test runner for doctests in test_statistics (GH-108921) --- Lib/test/test_statistics.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 23a3973305303d..f9b0ac2ad7b116 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -698,14 +698,6 @@ def test_check_all(self): 'missing name "%s" in __all__' % name) -class DocTests(unittest.TestCase): - @unittest.skipIf(sys.flags.optimize >= 2, - "Docstrings are omitted with -OO and above") - def test_doc_tests(self): - failed, tried = doctest.testmod(statistics, optionflags=doctest.ELLIPSIS) - self.assertGreater(tried, 0) - self.assertEqual(failed, 0) - class StatisticsErrorTest(unittest.TestCase): def test_has_exception(self): errmsg = ( @@ -3145,6 +3137,7 @@ def tearDown(self): def load_tests(loader, tests, ignore): """Used for doctest/unittest integration.""" tests.addTests(doctest.DocTestSuite()) + tests.addTests(doctest.DocTestSuite(statistics)) return tests From c74e440168fab9bf91346471087a394af13fa2db Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Thu, 7 Sep 2023 18:19:03 -0700 Subject: [PATCH 09/66] gh-109022: [Enum] require `names=()` to create empty enum type (GH-109048) add guard so that ``Enum('bar')`` raises a TypeError instead of creating a new enum class called `bar`. To create the new but empty class, use: huh = Enum('bar', names=()) --- Lib/enum.py | 5 +++++ Lib/test/test_enum.py | 11 +++++++---- .../2023-09-06-19-33-41.gh-issue-108682.35Xnc5.rst | 2 ++ 3 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-06-19-33-41.gh-issue-108682.35Xnc5.rst diff --git a/Lib/enum.py b/Lib/enum.py index 4b99e7bda2cca5..994a7b9c73f9a7 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -730,6 +730,11 @@ def __call__(cls, value, names=None, *values, module=None, qualname=None, type=N value = (value, names) + values return cls.__new__(cls, value) # otherwise, functional API: we're creating a new Enum type + if names is None and type is None: + # no body? no data-type? possibly wrong usage + raise TypeError( + f"{cls} has no members; specify `names=()` if you meant to create a new, empty, enum" + ) return cls._create_( class_name=value, names=names, diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index a838b93341a608..8c1f285f7b3bc2 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -316,6 +316,7 @@ def __str__(self): return self.name.title() def __format__(self, spec): return ''.join(reversed(self.name)) + self.NewBaseEnum = NewBaseEnum class NewSubEnum(NewBaseEnum): first = auto() self.NewSubEnum = NewSubEnum @@ -382,10 +383,8 @@ def __str__(self): return self.name.title() def __format__(self, spec): return ''.join(reversed(self.name)) - NewBaseEnum = self.enum_type('NewBaseEnum', dict(__format__=__format__, __str__=__str__)) - class NewSubEnum(NewBaseEnum): - first = auto() - self.NewSubEnum = NewBaseEnum('NewSubEnum', 'first') + self.NewBaseEnum = self.enum_type('NewBaseEnum', dict(__format__=__format__, __str__=__str__)) + self.NewSubEnum = self.NewBaseEnum('NewSubEnum', 'first') # def _generate_next_value_(name, start, last, values): pass @@ -601,6 +600,10 @@ class SubEnum(SuperEnum): self.assertTrue('description' not in dir(SubEnum)) self.assertTrue('description' in dir(SubEnum.sample), dir(SubEnum.sample)) + def test_empty_enum_has_no_values(self): + with self.assertRaisesRegex(TypeError, "<.... 'NewBaseEnum'> has no members"): + self.NewBaseEnum(7) + def test_enum_in_enum_out(self): Main = self.MainEnum self.assertIs(Main(Main.first), Main.first) diff --git a/Misc/NEWS.d/next/Library/2023-09-06-19-33-41.gh-issue-108682.35Xnc5.rst b/Misc/NEWS.d/next/Library/2023-09-06-19-33-41.gh-issue-108682.35Xnc5.rst new file mode 100644 index 00000000000000..8c13d43ee9744b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-06-19-33-41.gh-issue-108682.35Xnc5.rst @@ -0,0 +1,2 @@ +Enum: require ``names=()`` or ``type=...`` to create an empty enum using +the functional syntax. From 00cf626cd41f806062c22a913b647b4efa84c476 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 7 Sep 2023 22:37:29 -0700 Subject: [PATCH 10/66] Update `CODEOWNERS` for `Tools/wasm/` (#109119) --- .github/CODEOWNERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 578cd71a7bd211..81c580eb778625 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -179,3 +179,6 @@ Doc/c-api/stable.rst @encukou /Tools/clinic/** @erlend-aasland @AlexWaygood /Lib/test/test_clinic.py @erlend-aasland @AlexWaygood Doc/howto/clinic.rst @erlend-aasland + +# WebAssembly +/Tools/wasm/ @brettcannon From 15d4c9fabce67b8a1b5bd9dec9612014ec18291a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 8 Sep 2023 10:34:40 +0100 Subject: [PATCH 11/66] GH-108716: Turn off deep-freezing of code objects. (GH-108722) --- Include/cpython/import.h | 1 - Include/internal/pycore_code.h | 2 - Include/internal/pycore_interp.h | 1 + Lib/test/test_ctypes/test_values.py | 1 - Makefile.pre.in | 1 - ...-08-28-03-38-28.gh-issue-108716.HJBPwt.rst | 2 + Objects/codeobject.c | 7 ++- Objects/funcobject.c | 9 +-- Programs/_bootstrap_python.c | 2 - Programs/_freeze_module.c | 2 - Python/frozen.c | 58 +++++++++---------- Python/import.c | 13 +---- Python/pylifecycle.c | 6 -- Python/pystate.c | 1 + Tools/build/freeze_modules.py | 30 +++------- 15 files changed, 50 insertions(+), 86 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-28-03-38-28.gh-issue-108716.HJBPwt.rst diff --git a/Include/cpython/import.h b/Include/cpython/import.h index cdfdd15bfa48d2..7daf0b84fcf71b 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -17,7 +17,6 @@ struct _frozen { const unsigned char *code; int size; int is_package; - PyObject *(*get_code)(void); }; /* Embedding apps may change this pointer to point to their favorite diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index b3f480c7204172..257b0583edae11 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -465,8 +465,6 @@ adaptive_counter_backoff(uint16_t counter) { return adaptive_counter_bits(value, backoff); } -extern uint32_t _Py_next_func_version; - /* Comparison bit masks. */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index e0b7a325929257..ba5764e943e676 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -186,6 +186,7 @@ struct _is { _PyOptimizerObject *optimizer; uint16_t optimizer_resume_threshold; uint16_t optimizer_backedge_threshold; + uint32_t next_func_version; _Py_GlobalMonitors monitors; bool f_opcode_trace_set; diff --git a/Lib/test/test_ctypes/test_values.py b/Lib/test/test_ctypes/test_values.py index 9f8b69409cb880..d0b4803dff8529 100644 --- a/Lib/test/test_ctypes/test_values.py +++ b/Lib/test/test_ctypes/test_values.py @@ -58,7 +58,6 @@ class struct_frozen(Structure): ("code", POINTER(c_ubyte)), ("size", c_int), ("is_package", c_int), - ("get_code", POINTER(c_ubyte)), # Function ptr ] FrozenTable = POINTER(struct_frozen) diff --git a/Makefile.pre.in b/Makefile.pre.in index aa3eaabc7559f6..19a802997838a4 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -505,7 +505,6 @@ LIBRARY_OBJS_OMIT_FROZEN= \ LIBRARY_OBJS= \ $(LIBRARY_OBJS_OMIT_FROZEN) \ - $(DEEPFREEZE_OBJS) \ Modules/getpath.o \ Python/frozen.o diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-28-03-38-28.gh-issue-108716.HJBPwt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-28-03-38-28.gh-issue-108716.HJBPwt.rst new file mode 100644 index 00000000000000..f63eb8689d63a3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-28-03-38-28.gh-issue-108716.HJBPwt.rst @@ -0,0 +1,2 @@ +Turn off deep-freezing of code objects. Modules are still frozen, so that a +file system search is not needed for common modules. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 1443027ff293ac..d00bd0422f004d 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -427,9 +427,10 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_framesize = nlocalsplus + con->stacksize + FRAME_SPECIALS_SIZE; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; - co->co_version = _Py_next_func_version; - if (_Py_next_func_version != 0) { - _Py_next_func_version++; + PyInterpreterState *interp = _PyInterpreterState_GET(); + co->co_version = interp->next_func_version; + if (interp->next_func_version != 0) { + interp->next_func_version++; } co->_co_monitoring = NULL; co->_co_instrumentation_version = 0; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 648b660859c04d..231a9c141d0c0b 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -3,7 +3,6 @@ #include "Python.h" #include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals() -#include "pycore_code.h" // _Py_next_func_version #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -252,11 +251,9 @@ When the function version is 0, the `CALL` bytecode is not specialized. Code object versions -------------------- -So where to code objects get their `co_version`? There is a single -static global counter, `_Py_next_func_version`. This is initialized in -the generated (!) file `Python/deepfreeze/deepfreeze.c`, to 1 plus the -number of deep-frozen function objects in that file. -(In `_bootstrap_python.c` and `freeze_module.c` it is initialized to 1.) +So where to code objects get their `co_version`? +There is a per-interpreter counter, `next_func_version`. +This is initialized to 1 when the interpreter is created. Code objects get a new `co_version` allocated from this counter upon creation. Since code objects are nominally immutable, `co_version` can diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index 6c388fc7033dd0..34f79191b4e8d7 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -15,8 +15,6 @@ #include "Python/frozen_modules/zipimport.h" /* End includes */ -uint32_t _Py_next_func_version = 1; - /* Empty initializer for deepfrozen modules */ int _Py_Deepfreeze_Init(void) { diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index f6c46fa629efba..3de6c6816c1e61 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -22,8 +22,6 @@ # include #endif -uint32_t _Py_next_func_version = 1; - /* Empty initializer for deepfrozen modules */ int _Py_Deepfreeze_Init(void) { diff --git a/Python/frozen.c b/Python/frozen.c index 6b977710e6e342..0fb38a11902f35 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -101,46 +101,46 @@ extern PyObject *_Py_get_frozen_only_toplevel(void); /* End extern declarations */ static const struct _frozen bootstrap_modules[] = { - {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), false, GET_CODE(importlib__bootstrap)}, - {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), false, GET_CODE(importlib__bootstrap_external)}, - {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport), false, GET_CODE(zipimport)}, + {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), false}, + {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), false}, + {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport), false}, {0, 0, 0} /* bootstrap sentinel */ }; static const struct _frozen stdlib_modules[] = { /* stdlib - startup, without site (python -S) */ - {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), false, GET_CODE(abc)}, - {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), false, GET_CODE(codecs)}, - {"io", _Py_M__io, (int)sizeof(_Py_M__io), false, GET_CODE(io)}, + {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), false}, + {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), false}, + {"io", _Py_M__io, (int)sizeof(_Py_M__io), false}, /* stdlib - startup, with site */ - {"_collections_abc", _Py_M___collections_abc, (int)sizeof(_Py_M___collections_abc), false, GET_CODE(_collections_abc)}, - {"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins), false, GET_CODE(_sitebuiltins)}, - {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath), false, GET_CODE(genericpath)}, - {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath), false, GET_CODE(ntpath)}, - {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), false, GET_CODE(posixpath)}, - {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), false, GET_CODE(posixpath)}, - {"os", _Py_M__os, (int)sizeof(_Py_M__os), false, GET_CODE(os)}, - {"site", _Py_M__site, (int)sizeof(_Py_M__site), false, GET_CODE(site)}, - {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat), false, GET_CODE(stat)}, + {"_collections_abc", _Py_M___collections_abc, (int)sizeof(_Py_M___collections_abc), false}, + {"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins), false}, + {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath), false}, + {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath), false}, + {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), false}, + {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), false}, + {"os", _Py_M__os, (int)sizeof(_Py_M__os), false}, + {"site", _Py_M__site, (int)sizeof(_Py_M__site), false}, + {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat), false}, /* runpy - run module with -m */ - {"importlib.util", _Py_M__importlib_util, (int)sizeof(_Py_M__importlib_util), false, GET_CODE(importlib_util)}, - {"importlib.machinery", _Py_M__importlib_machinery, (int)sizeof(_Py_M__importlib_machinery), false, GET_CODE(importlib_machinery)}, - {"runpy", _Py_M__runpy, (int)sizeof(_Py_M__runpy), false, GET_CODE(runpy)}, + {"importlib.util", _Py_M__importlib_util, (int)sizeof(_Py_M__importlib_util), false}, + {"importlib.machinery", _Py_M__importlib_machinery, (int)sizeof(_Py_M__importlib_machinery), false}, + {"runpy", _Py_M__runpy, (int)sizeof(_Py_M__runpy), false}, {0, 0, 0} /* stdlib sentinel */ }; static const struct _frozen test_modules[] = { - {"__hello__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false, GET_CODE(__hello__)}, - {"__hello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false, GET_CODE(__hello__)}, - {"__phello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), true, GET_CODE(__hello__)}, - {"__phello_alias__.spam", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false, GET_CODE(__hello__)}, - {"__phello__", _Py_M____phello__, (int)sizeof(_Py_M____phello__), true, GET_CODE(__phello__)}, - {"__phello__.__init__", _Py_M____phello__, (int)sizeof(_Py_M____phello__), false, GET_CODE(__phello__)}, - {"__phello__.ham", _Py_M____phello___ham, (int)sizeof(_Py_M____phello___ham), true, GET_CODE(__phello___ham)}, - {"__phello__.ham.__init__", _Py_M____phello___ham, (int)sizeof(_Py_M____phello___ham), false, GET_CODE(__phello___ham)}, - {"__phello__.ham.eggs", _Py_M____phello___ham_eggs, (int)sizeof(_Py_M____phello___ham_eggs), false, GET_CODE(__phello___ham_eggs)}, - {"__phello__.spam", _Py_M____phello___spam, (int)sizeof(_Py_M____phello___spam), false, GET_CODE(__phello___spam)}, - {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only), false, GET_CODE(frozen_only)}, + {"__hello__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false}, + {"__hello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false}, + {"__phello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), true}, + {"__phello_alias__.spam", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false}, + {"__phello__", _Py_M____phello__, (int)sizeof(_Py_M____phello__), true}, + {"__phello__.__init__", _Py_M____phello__, (int)sizeof(_Py_M____phello__), false}, + {"__phello__.ham", _Py_M____phello___ham, (int)sizeof(_Py_M____phello___ham), true}, + {"__phello__.ham.__init__", _Py_M____phello___ham, (int)sizeof(_Py_M____phello___ham), false}, + {"__phello__.ham.eggs", _Py_M____phello___ham_eggs, (int)sizeof(_Py_M____phello___ham_eggs), false}, + {"__phello__.spam", _Py_M____phello___spam, (int)sizeof(_Py_M____phello___spam), false}, + {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only), false}, {0, 0, 0} /* test sentinel */ }; const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules; diff --git a/Python/import.c b/Python/import.c index 48090d05240188..126eb5e162a9e1 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2006,7 +2006,6 @@ look_up_frozen(const char *name) struct frozen_info { PyObject *nameobj; const char *data; - PyObject *(*get_code)(void); Py_ssize_t size; bool is_package; bool is_alias; @@ -2040,7 +2039,6 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) if (info != NULL) { info->nameobj = nameobj; // borrowed info->data = (const char *)p->code; - info->get_code = p->get_code; info->size = p->size; info->is_package = p->is_package; if (p->size < 0) { @@ -2052,10 +2050,6 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) info->is_alias = resolve_module_alias(name, _PyImport_FrozenAliases, &info->origname); } - if (p->code == NULL && p->size == 0 && p->get_code != NULL) { - /* It is only deepfrozen. */ - return FROZEN_OKAY; - } if (p->code == NULL) { /* It is frozen but marked as un-importable. */ return FROZEN_EXCLUDED; @@ -2070,11 +2064,6 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) static PyObject * unmarshal_frozen_code(PyInterpreterState *interp, struct frozen_info *info) { - if (info->get_code && _Py_IsMainInterpreter(interp)) { - PyObject *code = info->get_code(); - assert(code != NULL); - return code; - } PyObject *co = PyMarshal_ReadObjectFromString(info->data, info->size); if (co == NULL) { /* Does not contain executable code. */ @@ -3567,7 +3556,7 @@ _imp_get_frozen_object_impl(PyObject *module, PyObject *name, if (info.nameobj == NULL) { info.nameobj = name; } - if (info.size == 0 && info.get_code == NULL) { + if (info.size == 0) { /* Does not contain executable code. */ set_frozen_error(FROZEN_INVALID, name); return NULL; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 92eef6d50712bb..480001538540bb 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -832,11 +832,6 @@ pycore_interp_init(PyThreadState *tstate) if (_PyStatus_EXCEPTION(status)) { return status; } - // Intern strings in deep-frozen modules first so that others - // can use it instead of creating a heap allocated string. - if (_Py_Deepfreeze_Init() < 0) { - return _PyStatus_ERR("failed to initialize deep-frozen modules"); - } status = pycore_init_types(interp); if (_PyStatus_EXCEPTION(status)) { @@ -1743,7 +1738,6 @@ finalize_interp_clear(PyThreadState *tstate) _Py_HashRandomization_Fini(); _PyArg_Fini(); _Py_ClearFileSystemEncoding(); - _Py_Deepfreeze_Fini(); _PyPerfTrampoline_Fini(); } diff --git a/Python/pystate.c b/Python/pystate.c index b2b9b9f8776c33..ed14f82688f401 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -697,6 +697,7 @@ init_interpreter(PyInterpreterState *interp, interp->optimizer = &_PyOptimizer_Default; interp->optimizer_backedge_threshold = _PyOptimizer_Default.backedge_threshold; interp->optimizer_resume_threshold = _PyOptimizer_Default.backedge_threshold; + interp->next_func_version = 1; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index 12200979fa4bc7..a07f4d9786ea65 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -467,15 +467,14 @@ def replace_block(lines, start_marker, end_marker, replacements, file): return lines[:start_pos + 1] + replacements + lines[end_pos:] -def regen_frozen(modules, frozen_modules: bool): +def regen_frozen(modules): headerlines = [] parentdir = os.path.dirname(FROZEN_FILE) - if frozen_modules: - for src in _iter_sources(modules): - # Adding a comment to separate sections here doesn't add much, - # so we don't. - header = relpath_for_posix_display(src.frozenfile, parentdir) - headerlines.append(f'#include "{header}"') + for src in _iter_sources(modules): + # Adding a comment to separate sections here doesn't add much, + # so we don't. + header = relpath_for_posix_display(src.frozenfile, parentdir) + headerlines.append(f'#include "{header}"') externlines = [] bootstraplines = [] @@ -504,14 +503,9 @@ def regen_frozen(modules, frozen_modules: bool): get_code_name = "_Py_get_%s_toplevel" % code_name externlines.append("extern PyObject *%s(void);" % get_code_name) - symbol = mod.symbol pkg = 'true' if mod.ispkg else 'false' - if not frozen_modules: - line = ('{"%s", NULL, 0, %s, GET_CODE(%s)},' - ) % (mod.name, pkg, code_name) - else: - line = ('{"%s", %s, (int)sizeof(%s), %s, GET_CODE(%s)},' - ) % (mod.name, symbol, symbol, pkg, code_name) + size = f"(int)sizeof({mod.symbol})" + line = f'{{"{mod.name}", {mod.symbol}, {size}, {pkg}}},' lines.append(line) if mod.isalias: @@ -718,20 +712,14 @@ def regen_pcbuild(modules): ####################################### # the script -parser = argparse.ArgumentParser() -parser.add_argument("--frozen-modules", action="store_true", - help="Use both frozen and deepfrozen modules. (default: uses only deepfrozen modules)") - def main(): - args = parser.parse_args() - frozen_modules: bool = args.frozen_modules # Expand the raw specs, preserving order. modules = list(parse_frozen_specs()) # Regen build-related files. regen_makefile(modules) regen_pcbuild(modules) - regen_frozen(modules, frozen_modules) + regen_frozen(modules) if __name__ == '__main__': From b0edf3b98e4b3e68a13776e034b9dd86ad7e529d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 8 Sep 2023 11:48:28 +0200 Subject: [PATCH 12/66] GH-91079: Rename C_RECURSION_LIMIT to Py_C_RECURSION_LIMIT (#108507) Symbols of the C API should be prefixed by "Py_" to avoid conflict with existing names in 3rd party C extensions on "#include ". test.pythoninfo now logs Py_C_RECURSION_LIMIT constant and other _testcapi and _testinternalcapi constants. --- Include/cpython/pystate.h | 19 +++++++++---------- Lib/test/list_tests.py | 4 ++-- Lib/test/mapping_tests.py | 4 ++-- Lib/test/pythoninfo.py | 23 +++++++++++++++++++++++ Lib/test/support/__init__.py | 4 ++-- Lib/test/test_compile.py | 12 ++++++------ Lib/test/test_dict.py | 4 ++-- Lib/test/test_dictviews.py | 4 ++-- Lib/test/test_exception_group.py | 4 ++-- Modules/_testcapimodule.c | 1 + Modules/_testinternalcapi.c | 5 +++++ Parser/asdl_c.py | 4 ++-- Python/Python-ast.c | 4 ++-- Python/ast.c | 4 ++-- Python/ast_opt.c | 4 ++-- Python/pystate.c | 2 +- Python/symtable.c | 4 ++-- 17 files changed, 67 insertions(+), 39 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index fc5f58db86dbe8..e1a15cddd3d723 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -194,18 +194,17 @@ struct _ts { }; -/* WASI has limited call stack. Python's recursion limit depends on code - layout, optimization, and WASI runtime. Wasmtime can handle about 700 - recursions, sometimes less. 500 is a more conservative limit. */ -#ifndef C_RECURSION_LIMIT -# ifdef __wasi__ -# define C_RECURSION_LIMIT 500 -# else - // This value is duplicated in Lib/test/support/__init__.py -# define C_RECURSION_LIMIT 1500 -# endif +#ifdef __wasi__ + // WASI has limited call stack. Python's recursion limit depends on code + // layout, optimization, and WASI runtime. Wasmtime can handle about 700 + // recursions, sometimes less. 500 is a more conservative limit. +# define Py_C_RECURSION_LIMIT 500 +#else + // This value is duplicated in Lib/test/support/__init__.py +# define Py_C_RECURSION_LIMIT 1500 #endif + /* other API */ /* Similar to PyThreadState_Get(), but don't issue a fatal error diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index b1ef332522d2ce..d9ab21d4941cdb 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ, C_RECURSION_LIMIT +from test.support import ALWAYS_EQ, NEVER_EQ, Py_C_RECURSION_LIMIT class CommonTest(seq_tests.CommonTest): @@ -61,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(C_RECURSION_LIMIT + 1): + for i in range(Py_C_RECURSION_LIMIT + 1): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 5492bbf86d1f87..b3e4192e65d957 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -2,7 +2,7 @@ import unittest import collections import sys -from test.support import C_RECURSION_LIMIT +from test.support import Py_C_RECURSION_LIMIT class BasicTestMappingProtocol(unittest.TestCase): @@ -625,7 +625,7 @@ def __repr__(self): def test_repr_deep(self): d = self._empty_mapping() - for i in range(C_RECURSION_LIMIT + 1): + for i in range(Py_C_RECURSION_LIMIT + 1): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index c628833478044e..b25def78e42be4 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -665,6 +665,22 @@ def collect_decimal(info_add): def collect_testcapi(info_add): + try: + import _testcapi + except ImportError: + return + + for name in ( + 'LONG_MAX', # always 32-bit on Windows, 64-bit on 64-bit Unix + 'PY_SSIZE_T_MAX', + 'Py_C_RECURSION_LIMIT', + 'SIZEOF_TIME_T', # 32-bit or 64-bit depending on the platform + 'SIZEOF_WCHAR_T', # 16-bit or 32-bit depending on the platform + ): + copy_attr(info_add, f'_testcapi.{name}', _testcapi, name) + + +def collect_testinternalcapi(info_add): try: import _testinternalcapi except ImportError: @@ -672,6 +688,12 @@ def collect_testcapi(info_add): call_func(info_add, 'pymem.allocator', _testinternalcapi, 'pymem_getallocatorsname') + for name in ( + 'SIZEOF_PYGC_HEAD', + 'SIZEOF_PYOBJECT', + ): + copy_attr(info_add, f'_testinternalcapi.{name}', _testinternalcapi, name) + def collect_resource(info_add): try: @@ -907,6 +929,7 @@ def collect_info(info): collect_sys, collect_sysconfig, collect_testcapi, + collect_testinternalcapi, collect_time, collect_tkinter, collect_windows, diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 38ad965e155302..84b74ee2c298e9 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -60,7 +60,7 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "C_RECURSION_LIMIT", + "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "Py_C_RECURSION_LIMIT", "skip_on_s390x", ] @@ -2531,7 +2531,7 @@ def adjust_int_max_str_digits(max_digits): EXCEEDS_RECURSION_LIMIT = 5000 # The default C recursion limit (from Include/cpython/pystate.h). -C_RECURSION_LIMIT = 1500 +Py_C_RECURSION_LIMIT = 1500 #Windows doesn't have os.uname() but it doesn't support s390x. skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index de513daf825d81..28b2c4686bbc89 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -11,7 +11,7 @@ import warnings from test import support from test.support import (script_helper, requires_debug_ranges, - requires_specialization, C_RECURSION_LIMIT) + requires_specialization, Py_C_RECURSION_LIMIT) from test.support.os_helper import FakePath class TestSpecifics(unittest.TestCase): @@ -111,7 +111,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - repeat = int(C_RECURSION_LIMIT * 0.9) + repeat = int(Py_C_RECURSION_LIMIT * 0.9) longexpr = 'x = x or ' + '-x' * repeat g = {} code = textwrap.dedent(''' @@ -557,12 +557,12 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is C_RECURSION_LIMIT * 2 + # Expected limit is Py_C_RECURSION_LIMIT * 2 # Duplicating the limit here is a little ugly. # Perhaps it should be exposed somewhere... - fail_depth = C_RECURSION_LIMIT * 2 + 1 - crash_depth = C_RECURSION_LIMIT * 100 - success_depth = int(C_RECURSION_LIMIT * 1.8) + fail_depth = Py_C_RECURSION_LIMIT * 2 + 1 + crash_depth = Py_C_RECURSION_LIMIT * 100 + success_depth = int(Py_C_RECURSION_LIMIT * 1.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index eab64b4f9106c1..620d0ca4f4c2da 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -8,7 +8,7 @@ import unittest import weakref from test import support -from test.support import import_helper, C_RECURSION_LIMIT +from test.support import import_helper, Py_C_RECURSION_LIMIT class DictTest(unittest.TestCase): @@ -596,7 +596,7 @@ def __repr__(self): def test_repr_deep(self): d = {} - for i in range(C_RECURSION_LIMIT + 1): + for i in range(Py_C_RECURSION_LIMIT + 1): d = {1: d} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 2bd9d6eef8cfc6..34918585513846 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -3,7 +3,7 @@ import pickle import sys import unittest -from test.support import C_RECURSION_LIMIT +from test.support import Py_C_RECURSION_LIMIT class DictSetTest(unittest.TestCase): @@ -280,7 +280,7 @@ def test_recursive_repr(self): def test_deeply_nested_repr(self): d = {} - for i in range(C_RECURSION_LIMIT//2 + 100): + for i in range(Py_C_RECURSION_LIMIT//2 + 100): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index a02d54da35e948..20122679223843 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest -from test.support import C_RECURSION_LIMIT +from test.support import Py_C_RECURSION_LIMIT class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(C_RECURSION_LIMIT + 1): + for i in range(Py_C_RECURSION_LIMIT + 1): e = ExceptionGroup('eg', [e]) return e diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 9bd963baf066a4..85d8401435e1b5 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3922,6 +3922,7 @@ PyInit__testcapi(void) PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); PyModule_AddIntConstant(m, "the_number_three", 3); + PyModule_AddIntMacro(m, Py_C_RECURSION_LIMIT); TestError = PyErr_NewException("_testcapi.error", NULL, NULL); Py_INCREF(TestError); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index b6792e38fa98c2..922672d1a9f915 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1552,6 +1552,11 @@ module_exec(PyObject *module) return 1; } + if (PyModule_Add(module, "SIZEOF_PYOBJECT", + PyLong_FromSsize_t(sizeof(PyObject))) < 0) { + return 1; + } + if (PyModule_Add(module, "SIZEOF_TIME_T", PyLong_FromSsize_t(sizeof(time_t))) < 0) { return 1; diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index ca259c8cd1f3ba..f61099b97055ad 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1401,8 +1401,8 @@ class PartingShots(StaticVisitor): if (!tstate) { return 0; } - state->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; - int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; + state->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; state->recursion_depth = starting_recursion_depth; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 77b23f7c5edf23..a197d44868b5aa 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -13081,8 +13081,8 @@ PyObject* PyAST_mod2obj(mod_ty t) if (!tstate) { return 0; } - state->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; - int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; + state->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; state->recursion_depth = starting_recursion_depth; diff --git a/Python/ast.c b/Python/ast.c index 74c97f948d15e6..21cb38f8cbfb65 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1046,10 +1046,10 @@ _PyAST_Validate(mod_ty mod) return 0; } /* Be careful here to prevent overflow. */ - int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; + int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; switch (mod->kind) { case Module_kind: diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 82e7559e5b629a..41f48eba08afc4 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1130,10 +1130,10 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) return 0; } /* Be careful here to prevent overflow. */ - int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; + int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; int ret = astfold_mod(mod, arena, &state); assert(ret || PyErr_Occurred()); diff --git a/Python/pystate.c b/Python/pystate.c index ed14f82688f401..89275fd7e025ca 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1334,7 +1334,7 @@ init_threadstate(PyThreadState *tstate, tstate->py_recursion_limit = interp->ceval.recursion_limit, tstate->py_recursion_remaining = interp->ceval.recursion_limit, - tstate->c_recursion_remaining = C_RECURSION_LIMIT; + tstate->c_recursion_remaining = Py_C_RECURSION_LIMIT; tstate->exc_info = &tstate->exc_state; diff --git a/Python/symtable.c b/Python/symtable.c index f157d4c170314a..217e6f59a61484 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -312,10 +312,10 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) return NULL; } /* Be careful here to prevent overflow. */ - int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; + int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; st->recursion_depth = starting_recursion_depth; - st->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + st->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; /* Make the initial symbol information gathering pass */ if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) { From f63d37877ad166041489a968233b57540f8456e8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 8 Sep 2023 11:50:46 +0200 Subject: [PATCH 13/66] gh-104690: thread_run() checks for tstate dangling pointer (#109056) thread_run() of _threadmodule.c now calls _PyThreadState_CheckConsistency() to check if tstate is a dangling pointer when Python is built in debug mode. Rename ceval_gil.c is_tstate_valid() to _PyThreadState_CheckConsistency() to reuse it in _threadmodule.c. --- Include/internal/pycore_pystate.h | 4 ++++ Modules/_threadmodule.c | 7 +++++-- Python/ceval_gil.c | 26 ++++++++------------------ Python/pystate.c | 18 ++++++++++++++++++ 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index a30036aeb57e05..9c0e42e7bad06c 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -67,6 +67,10 @@ _Py_ThreadCanHandleSignals(PyInterpreterState *interp) extern _Py_thread_local PyThreadState *_Py_tss_tstate; #endif +#ifndef NDEBUG +extern int _PyThreadState_CheckConsistency(PyThreadState *tstate); +#endif + // Export for most shared extensions, used via _PyThreadState_GET() static // inline function. PyAPI_FUNC(PyThreadState *) _PyThreadState_GetCurrent(void); diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 49f34fcb9feb70..05bb49756c9303 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1074,9 +1074,12 @@ static void thread_run(void *boot_raw) { struct bootstate *boot = (struct bootstate *) boot_raw; - PyThreadState *tstate; + PyThreadState *tstate = boot->tstate; + + // gh-104690: If Python is being finalized and PyInterpreterState_Delete() + // was called, tstate becomes a dangling pointer. + assert(_PyThreadState_CheckConsistency(tstate)); - tstate = boot->tstate; _PyThreadState_Bind(tstate); PyEval_AcquireThread(tstate); tstate->interp->threads.count++; diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index e53ffa76b1164b..cef5317b46bf8e 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -163,16 +163,6 @@ UNSIGNAL_ASYNC_EXC(PyInterpreterState *interp) COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); } -#ifndef NDEBUG -/* Ensure that tstate is valid */ -static int -is_tstate_valid(PyThreadState *tstate) -{ - assert(!_PyMem_IsPtrFreed(tstate)); - assert(!_PyMem_IsPtrFreed(tstate->interp)); - return 1; -} -#endif /* * Implementation of the Global Interpreter Lock (GIL). @@ -325,7 +315,7 @@ drop_gil(struct _ceval_state *ceval, PyThreadState *tstate) /* Not switched yet => wait */ if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate) { - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); RESET_GIL_DROP_REQUEST(tstate->interp); /* NOTE: if COND_WAIT does not atomically start waiting when releasing the mutex, another thread can run through, take @@ -386,7 +376,7 @@ take_gil(PyThreadState *tstate) PyThread_exit_thread(); } - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); PyInterpreterState *interp = tstate->interp; struct _ceval_state *ceval = &interp->ceval; struct _gil_runtime_state *gil = ceval->gil; @@ -427,7 +417,7 @@ take_gil(PyThreadState *tstate) } PyThread_exit_thread(); } - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); SET_GIL_DROP_REQUEST(interp); drop_requested = 1; @@ -466,7 +456,7 @@ take_gil(PyThreadState *tstate) drop_gil(ceval, tstate); PyThread_exit_thread(); } - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); if (_Py_atomic_load_relaxed(&ceval->gil_drop_request)) { RESET_GIL_DROP_REQUEST(interp); @@ -679,7 +669,7 @@ PyEval_AcquireThread(PyThreadState *tstate) void PyEval_ReleaseThread(PyThreadState *tstate) { - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); PyThreadState *new_tstate = _PyThreadState_SwapNoGIL(NULL); if (new_tstate != tstate) { @@ -877,7 +867,7 @@ Py_AddPendingCall(int (*func)(void *), void *arg) static int handle_signals(PyThreadState *tstate) { - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); if (!_Py_ThreadCanHandleSignals(tstate->interp)) { return 0; } @@ -983,7 +973,7 @@ void _Py_FinishPendingCalls(PyThreadState *tstate) { assert(PyGILState_Check()); - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); if (make_pending_calls(tstate->interp) < 0) { PyObject *exc = _PyErr_GetRaisedException(tstate); @@ -1024,7 +1014,7 @@ Py_MakePendingCalls(void) assert(PyGILState_Check()); PyThreadState *tstate = _PyThreadState_GET(); - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); /* Only execute pending calls on the main thread. */ if (!_Py_IsMainThread() || !_Py_IsMainInterpreter(tstate->interp)) { diff --git a/Python/pystate.c b/Python/pystate.c index 89275fd7e025ca..09c3538ad7b872 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2890,6 +2890,24 @@ _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame * frame) } +#ifndef NDEBUG +// Check that a Python thread state valid. In practice, this function is used +// on a Python debug build to check if 'tstate' is a dangling pointer, if the +// PyThreadState memory has been freed. +// +// Usage: +// +// assert(_PyThreadState_CheckConsistency(tstate)); +int +_PyThreadState_CheckConsistency(PyThreadState *tstate) +{ + assert(!_PyMem_IsPtrFreed(tstate)); + assert(!_PyMem_IsPtrFreed(tstate->interp)); + return 1; +} +#endif + + #ifdef __cplusplus } #endif From aa51182320f3c391195eb7d5bd970867e63bd978 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Fri, 8 Sep 2023 09:30:28 -0600 Subject: [PATCH 14/66] gh-109140: Rename duplicated tests in `test_binascii` (#109141) --- Lib/test/test_binascii.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index eb831b1a06fcb8..3d3e0746e9bfdf 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -233,7 +233,7 @@ def test_uu(self): binary=hypothesis.strategies.binary(), backtick=hypothesis.strategies.booleans(), ) - def test_hex_roundtrip(self, binary, backtick): + def test_b2a_roundtrip(self, binary, backtick): converted = binascii.b2a_uu(self.type2test(binary), backtick=backtick) restored = binascii.a2b_uu(self.type2test(converted)) self.assertConversion(binary, converted, restored, backtick=backtick) From 6275c67ea68645e5b296a80ea63b90707a0be792 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 8 Sep 2023 17:18:35 +0100 Subject: [PATCH 15/66] gh-106922: Fix error location for constructs with spaces and parentheses (#108959) --- Lib/test/test_traceback.py | 36 +++++++++++++++++++ Lib/traceback.py | 16 +++++++-- ...-09-05-20-52-17.gh-issue-108959.6z45Sy.rst | 2 ++ Python/traceback.c | 17 +++++++++ 4 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-05-20-52-17.gh-issue-108959.6z45Sy.rst diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 316ade2171e94f..be81082bb19472 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -596,6 +596,24 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) + def test_caret_for_binary_operators_with_spaces_and_parenthesis(self): + def f_with_binary_operator(): + a = 1 + b = "" + return ( a ) + b + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' return ( a ) + b\n' + ' ~~~~~~~~~~^~~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + def test_caret_for_subscript(self): def f_with_subscript(): some_dict = {'x': {'y': None}} @@ -630,6 +648,24 @@ def f_with_subscript(): result_lines = self.get_exception(f_with_subscript) self.assertEqual(result_lines, expected_error.splitlines()) + def test_caret_for_subscript_with_spaces_and_parenthesis(self): + def f_with_binary_operator(): + a = [] + b = c = 1 + return b [ a ] + c + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' return b [ a ] + c\n' + ' ~~~~~~^^^^^^^^^\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + def test_traceback_specialization_with_syntax_error(self): bytecode = compile("1 / 0 / 1 / 2\n", TESTFN, "exec") diff --git a/Lib/traceback.py b/Lib/traceback.py index 354754b9560a19..67941ff45988c2 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -608,11 +608,21 @@ def _extract_caret_anchors_from_line_segment(segment): and not operator_str[operator_offset + 1].isspace() ): right_anchor += 1 + + while left_anchor < len(segment) and ((ch := segment[left_anchor]).isspace() or ch in ")#"): + left_anchor += 1 + right_anchor += 1 return _Anchors(normalize(left_anchor), normalize(right_anchor)) case ast.Subscript(): - subscript_start = normalize(expr.value.end_col_offset) - subscript_end = normalize(expr.slice.end_col_offset + 1) - return _Anchors(subscript_start, subscript_end) + left_anchor = normalize(expr.value.end_col_offset) + right_anchor = normalize(expr.slice.end_col_offset + 1) + while left_anchor < len(segment) and ((ch := segment[left_anchor]).isspace() or ch != "["): + left_anchor += 1 + while right_anchor < len(segment) and ((ch := segment[right_anchor]).isspace() or ch != "]"): + right_anchor += 1 + if right_anchor < len(segment): + right_anchor += 1 + return _Anchors(left_anchor, right_anchor) return None diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-05-20-52-17.gh-issue-108959.6z45Sy.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-05-20-52-17.gh-issue-108959.6z45Sy.rst new file mode 100644 index 00000000000000..792bbc454f2b27 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-05-20-52-17.gh-issue-108959.6z45Sy.rst @@ -0,0 +1,2 @@ +Fix caret placement for error locations for subscript and binary operations +that involve non-semantic parentheses and spaces. Patch by Pablo Galindo diff --git a/Python/traceback.c b/Python/traceback.c index 2fcfa7ca56c140..a75b7833af4e05 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -616,6 +616,11 @@ extract_anchors_from_expr(const char *segment_str, expr_ty expr, Py_ssize_t *lef ++*right_anchor; } + // Keep going if the current char is not ')' + if (i+1 < right->col_offset && (segment_str[i] == ')')) { + continue; + } + // Set the error characters *primary_error_char = "~"; *secondary_error_char = "^"; @@ -626,6 +631,18 @@ extract_anchors_from_expr(const char *segment_str, expr_ty expr, Py_ssize_t *lef case Subscript_kind: { *left_anchor = expr->v.Subscript.value->end_col_offset; *right_anchor = expr->v.Subscript.slice->end_col_offset + 1; + Py_ssize_t str_len = strlen(segment_str); + + // Move right_anchor and left_anchor forward to the first non-whitespace character that is not ']' and '[' + while (*left_anchor < str_len && (IS_WHITESPACE(segment_str[*left_anchor]) || segment_str[*left_anchor] != '[')) { + ++*left_anchor; + } + while (*right_anchor < str_len && (IS_WHITESPACE(segment_str[*right_anchor]) || segment_str[*right_anchor] != ']')) { + ++*right_anchor; + } + if (*right_anchor < str_len){ + *right_anchor += 1; + } // Set the error characters *primary_error_char = "~"; From 52beebc856fedf507ac0eb9e45c2e2c9fed1e5b8 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 8 Sep 2023 12:23:58 -0400 Subject: [PATCH 16/66] gh-109136: Fix summarize_stats.py tool (#109137) --- Tools/scripts/summarize_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 2d198506fb5c6c..55b67643977a00 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -430,7 +430,7 @@ def emit_comparative_specialization_overview(base_opcode_stats, base_total, head ) def get_stats_defines(): - stats_path = os.path.join(os.path.dirname(__file__), "../../Include/pystats.h") + stats_path = os.path.join(os.path.dirname(__file__), "../../Include/cpython/pystats.h") with open(stats_path) as stats_src: defines = parse_kinds(stats_src, prefix="EVAL_CALL") return defines From ccd48623d4860e730a16f3f252d67bfea8c1e905 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Fri, 8 Sep 2023 21:57:58 +0530 Subject: [PATCH 17/66] GH-109067: fix randomly failing `test_async_gen_asyncio_gc_aclose_09` test (#109142) Use `asyncio.sleep(0)` instead of short sleeps. --- Lib/test/test_asyncgen.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 4f00558770dafd..a49630112af510 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1058,8 +1058,7 @@ async def gen(): while True: yield 1 finally: - await asyncio.sleep(0.01) - await asyncio.sleep(0.01) + await asyncio.sleep(0) DONE = 1 async def run(): @@ -1069,7 +1068,10 @@ async def run(): del g gc_collect() # For PyPy or other GCs. - await asyncio.sleep(0.1) + # Starts running the aclose task + await asyncio.sleep(0) + # For asyncio.sleep(0) in finally block + await asyncio.sleep(0) self.loop.run_until_complete(run()) self.assertEqual(DONE, 1) From 501f2dc527010a4fe17026b3f352d72b01228b6f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 8 Sep 2023 17:54:45 +0100 Subject: [PATCH 18/66] GH-108614: Unbreak emscripten build (GH-109132) --- Include/internal/pycore_emscripten_signal.h | 1 + Python/bytecodes.c | 4 ++-- Python/emscripten_signal.c | 8 ++++---- Python/executor_cases.c.h | 4 ++-- Python/generated_cases.c.h | 4 ++-- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_emscripten_signal.h b/Include/internal/pycore_emscripten_signal.h index d1bcb9a92c7726..754193e21dec5a 100644 --- a/Include/internal/pycore_emscripten_signal.h +++ b/Include/internal/pycore_emscripten_signal.h @@ -18,6 +18,7 @@ _Py_CheckEmscriptenSignalsPeriodically(void); #define _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY() _Py_CheckEmscriptenSignalsPeriodically() extern int Py_EMSCRIPTEN_SIGNAL_HANDLING; +extern int _Py_emscripten_signal_clock; #else diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8820b52774671b..21069204cf8693 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -154,8 +154,8 @@ dummy_func( inst(RESUME_CHECK, (--)) { #if defined(__EMSCRIPTEN__) - DEOPT_IF(emscripten_signal_clock == 0, RESUME); - emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version diff --git a/Python/emscripten_signal.c b/Python/emscripten_signal.c index 1a196385b8ce34..561b5b73cd6b70 100644 --- a/Python/emscripten_signal.c +++ b/Python/emscripten_signal.c @@ -39,16 +39,16 @@ _Py_CheckEmscriptenSignals(void) } #define PY_EMSCRIPTEN_SIGNAL_INTERVAL 50 -static int emscripten_signal_clock = PY_EMSCRIPTEN_SIGNAL_INTERVAL; +int _Py_emscripten_signal_clock = PY_EMSCRIPTEN_SIGNAL_INTERVAL; void _Py_CheckEmscriptenSignalsPeriodically(void) { - if (emscripten_signal_clock == 0) { - emscripten_signal_clock = PY_EMSCRIPTEN_SIGNAL_INTERVAL; + if (_Py_emscripten_signal_clock == 0) { + _Py_emscripten_signal_clock = PY_EMSCRIPTEN_SIGNAL_INTERVAL; _Py_CheckEmscriptenSignals(); } else if (Py_EMSCRIPTEN_SIGNAL_HANDLING) { - emscripten_signal_clock--; + _Py_emscripten_signal_clock--; } } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f4c526a5a8c0a2..fa7cb88e4a1e1f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -9,8 +9,8 @@ case RESUME_CHECK: { #if defined(__EMSCRIPTEN__) - DEOPT_IF(emscripten_signal_clock == 0, RESUME); - emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 84f83db128ea50..136b36c8260cb5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -28,8 +28,8 @@ TARGET(RESUME_CHECK) { #if defined(__EMSCRIPTEN__) - DEOPT_IF(emscripten_signal_clock == 0, RESUME); - emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version From 87a7faf6b68c8076e640a9a1347a255f132d8382 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 8 Sep 2023 19:57:41 +0300 Subject: [PATCH 19/66] Check the result of PySet_Contains() for error in Python/symtable.c (GH-109146) --- Python/symtable.c | 72 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/Python/symtable.c b/Python/symtable.c index 217e6f59a61484..7eef6f7231c035 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -523,6 +523,7 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, PyObject *bound, PyObject *local, PyObject *free, PyObject *global, PyObject *type_params, PySTEntryObject *class_entry) { + int contains; if (flags & DEF_GLOBAL) { if (flags & DEF_NONLOCAL) { PyErr_Format(PyExc_SyntaxError, @@ -543,14 +544,22 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, "nonlocal declaration not allowed at module level"); return error_at_directive(ste, name); } - if (!PySet_Contains(bound, name)) { + contains = PySet_Contains(bound, name); + if (contains < 0) { + return 0; + } + if (!contains) { PyErr_Format(PyExc_SyntaxError, "no binding for nonlocal '%U' found", name); return error_at_directive(ste, name); } - if (PySet_Contains(type_params, name)) { + contains = PySet_Contains(type_params, name); + if (contains < 0) { + return 0; + } + if (contains) { PyErr_Format(PyExc_SyntaxError, "nonlocal binding not allowed for type parameter '%U'", name); @@ -599,17 +608,29 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, Note that having a non-NULL bound implies that the block is nested. */ - if (bound && PySet_Contains(bound, name)) { - SET_SCOPE(scopes, name, FREE); - ste->ste_free = 1; - return PySet_Add(free, name) >= 0; + if (bound) { + contains = PySet_Contains(bound, name); + if (contains < 0) { + return 0; + } + if (contains) { + SET_SCOPE(scopes, name, FREE); + ste->ste_free = 1; + return PySet_Add(free, name) >= 0; + } } /* If a parent has a global statement, then call it global explicit? It could also be global implicit. */ - if (global && PySet_Contains(global, name)) { - SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); - return 1; + if (global) { + contains = PySet_Contains(global, name); + if (contains < 0) { + return 0; + } + if (contains) { + SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); + return 1; + } } if (ste->ste_nested) ste->ste_free = 1; @@ -712,8 +733,19 @@ analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells) scope = PyLong_AS_LONG(v); if (scope != LOCAL) continue; - if (!PySet_Contains(free, name) && !PySet_Contains(inlined_cells, name)) - continue; + int contains = PySet_Contains(free, name); + if (contains < 0) { + goto error; + } + if (!contains) { + contains = PySet_Contains(inlined_cells, name); + if (contains < 0) { + goto error; + } + if (!contains) { + continue; + } + } /* Replace LOCAL with CELL for this name, and remove from free. It is safe to replace the value of name in the dict, because it will not cause a resize. @@ -764,7 +796,11 @@ update_symbols(PyObject *symbols, PyObject *scopes, long scope, flags; assert(PyLong_Check(v)); flags = PyLong_AS_LONG(v); - if (PySet_Contains(inlined_cells, name)) { + int contains = PySet_Contains(inlined_cells, name); + if (contains < 0) { + return 0; + } + if (contains) { flags |= DEF_COMP_CELL; } v_scope = PyDict_GetItemWithError(scopes, name); @@ -822,9 +858,15 @@ update_symbols(PyObject *symbols, PyObject *scopes, goto error; } /* Handle global symbol */ - if (bound && !PySet_Contains(bound, name)) { - Py_DECREF(name); - continue; /* it's a global */ + if (bound) { + int contains = PySet_Contains(bound, name); + if (contains < 0) { + goto error; + } + if (!contains) { + Py_DECREF(name); + continue; /* it's a global */ + } } /* Propagate new free symbol up the lexical stack */ if (PyDict_SetItem(symbols, name, v_free) < 0) { From 5bda2f637e1cfbca45a83aa6e22db25498064b27 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 8 Sep 2023 18:00:23 +0100 Subject: [PATCH 20/66] gh-109114: Relax the check for invalid lambdas inside f-strings to avoid false positives (#109121) --- Grammar/python.gram | 2 +- Lib/test/test_fstring.py | 4 + ...-09-08-01-50-41.gh-issue-109114.adqgtb.rst | 3 + Parser/parser.c | 2557 ++++++++--------- 4 files changed, 1257 insertions(+), 1309 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-08-01-50-41.gh-issue-109114.adqgtb.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index e7c817856d514b..ae998f98076a0a 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -1170,7 +1170,7 @@ invalid_expression: _PyPegen_check_legacy_stmt(p, a) ? NULL : p->tokens[p->mark-1]->level == 0 ? NULL : RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Perhaps you forgot a comma?") } | a=disjunction 'if' b=disjunction !('else'|':') { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected 'else' after 'if' expression") } - | a='lambda' [lambda_params] b=':' &(FSTRING_MIDDLE | fstring_replacement_field) { + | a='lambda' [lambda_params] b=':' &FSTRING_MIDDLE { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "f-string: lambda expressions are not allowed without parentheses") } invalid_named_expression(memo): diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 16f01973f99f3e..4f05a149a901b2 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1027,6 +1027,10 @@ def test_lambda(self): "f'{lambda x:}'", "f'{lambda :}'", ]) + # Ensure the detection of invalid lambdas doesn't trigger detection + # for valid lambdas in the second error pass + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + compile("lambda name_3=f'{name_4}': {name_3}\n1 $ 1", "", "exec") # but don't emit the paren warning in general cases with self.assertRaisesRegex(SyntaxError, "f-string: expecting a valid expression after '{'"): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-08-01-50-41.gh-issue-109114.adqgtb.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-08-01-50-41.gh-issue-109114.adqgtb.rst new file mode 100644 index 00000000000000..3d95dd5d29450c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-08-01-50-41.gh-issue-109114.adqgtb.rst @@ -0,0 +1,3 @@ +Relax the detection of the error message for invalid lambdas inside +f-strings to not search for arbitrary replacement fields to avoid false +positives. Patch by Pablo Galindo diff --git a/Parser/parser.c b/Parser/parser.c index 95cbef4a64a10f..e5847e7d1bd920 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -483,65 +483,65 @@ static char *soft_keywords[] = { #define _tmp_157_type 1400 #define _tmp_158_type 1401 #define _tmp_159_type 1402 -#define _tmp_160_type 1403 +#define _loop0_160_type 1403 #define _loop0_161_type 1404 #define _loop0_162_type 1405 -#define _loop0_163_type 1406 +#define _tmp_163_type 1406 #define _tmp_164_type 1407 #define _tmp_165_type 1408 #define _tmp_166_type 1409 #define _tmp_167_type 1410 -#define _tmp_168_type 1411 +#define _loop0_168_type 1411 #define _loop0_169_type 1412 #define _loop0_170_type 1413 -#define _loop0_171_type 1414 -#define _loop1_172_type 1415 -#define _tmp_173_type 1416 -#define _loop0_174_type 1417 -#define _tmp_175_type 1418 -#define _loop0_176_type 1419 -#define _loop1_177_type 1420 +#define _loop1_171_type 1414 +#define _tmp_172_type 1415 +#define _loop0_173_type 1416 +#define _tmp_174_type 1417 +#define _loop0_175_type 1418 +#define _loop1_176_type 1419 +#define _tmp_177_type 1420 #define _tmp_178_type 1421 #define _tmp_179_type 1422 -#define _tmp_180_type 1423 -#define _loop0_181_type 1424 +#define _loop0_180_type 1423 +#define _tmp_181_type 1424 #define _tmp_182_type 1425 -#define _tmp_183_type 1426 -#define _loop1_184_type 1427 -#define _tmp_185_type 1428 +#define _loop1_183_type 1426 +#define _tmp_184_type 1427 +#define _loop0_185_type 1428 #define _loop0_186_type 1429 #define _loop0_187_type 1430 -#define _loop0_188_type 1431 -#define _loop0_190_type 1432 -#define _gather_189_type 1433 -#define _tmp_191_type 1434 -#define _loop0_192_type 1435 -#define _tmp_193_type 1436 -#define _loop0_194_type 1437 +#define _loop0_189_type 1431 +#define _gather_188_type 1432 +#define _tmp_190_type 1433 +#define _loop0_191_type 1434 +#define _tmp_192_type 1435 +#define _loop0_193_type 1436 +#define _loop1_194_type 1437 #define _loop1_195_type 1438 -#define _loop1_196_type 1439 +#define _tmp_196_type 1439 #define _tmp_197_type 1440 -#define _tmp_198_type 1441 -#define _loop0_199_type 1442 +#define _loop0_198_type 1441 +#define _tmp_199_type 1442 #define _tmp_200_type 1443 #define _tmp_201_type 1444 -#define _tmp_202_type 1445 -#define _loop0_204_type 1446 -#define _gather_203_type 1447 -#define _loop0_206_type 1448 -#define _gather_205_type 1449 -#define _loop0_208_type 1450 -#define _gather_207_type 1451 -#define _loop0_210_type 1452 -#define _gather_209_type 1453 -#define _loop0_212_type 1454 -#define _gather_211_type 1455 -#define _tmp_213_type 1456 -#define _loop0_214_type 1457 -#define _loop1_215_type 1458 -#define _tmp_216_type 1459 -#define _loop0_217_type 1460 -#define _loop1_218_type 1461 +#define _loop0_203_type 1445 +#define _gather_202_type 1446 +#define _loop0_205_type 1447 +#define _gather_204_type 1448 +#define _loop0_207_type 1449 +#define _gather_206_type 1450 +#define _loop0_209_type 1451 +#define _gather_208_type 1452 +#define _loop0_211_type 1453 +#define _gather_210_type 1454 +#define _tmp_212_type 1455 +#define _loop0_213_type 1456 +#define _loop1_214_type 1457 +#define _tmp_215_type 1458 +#define _loop0_216_type 1459 +#define _loop1_217_type 1460 +#define _tmp_218_type 1461 #define _tmp_219_type 1462 #define _tmp_220_type 1463 #define _tmp_221_type 1464 @@ -551,9 +551,9 @@ static char *soft_keywords[] = { #define _tmp_225_type 1468 #define _tmp_226_type 1469 #define _tmp_227_type 1470 -#define _tmp_228_type 1471 -#define _loop0_230_type 1472 -#define _gather_229_type 1473 +#define _loop0_229_type 1471 +#define _gather_228_type 1472 +#define _tmp_230_type 1473 #define _tmp_231_type 1474 #define _tmp_232_type 1475 #define _tmp_233_type 1476 @@ -566,8 +566,8 @@ static char *soft_keywords[] = { #define _tmp_240_type 1483 #define _tmp_241_type 1484 #define _tmp_242_type 1485 -#define _tmp_243_type 1486 -#define _loop0_244_type 1487 +#define _loop0_243_type 1486 +#define _tmp_244_type 1487 #define _tmp_245_type 1488 #define _tmp_246_type 1489 #define _tmp_247_type 1490 @@ -599,7 +599,6 @@ static char *soft_keywords[] = { #define _tmp_273_type 1516 #define _tmp_274_type 1517 #define _tmp_275_type 1518 -#define _tmp_276_type 1519 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -1004,65 +1003,65 @@ static void *_tmp_156_rule(Parser *p); static void *_tmp_157_rule(Parser *p); static void *_tmp_158_rule(Parser *p); static void *_tmp_159_rule(Parser *p); -static void *_tmp_160_rule(Parser *p); +static asdl_seq *_loop0_160_rule(Parser *p); static asdl_seq *_loop0_161_rule(Parser *p); static asdl_seq *_loop0_162_rule(Parser *p); -static asdl_seq *_loop0_163_rule(Parser *p); +static void *_tmp_163_rule(Parser *p); static void *_tmp_164_rule(Parser *p); static void *_tmp_165_rule(Parser *p); static void *_tmp_166_rule(Parser *p); static void *_tmp_167_rule(Parser *p); -static void *_tmp_168_rule(Parser *p); +static asdl_seq *_loop0_168_rule(Parser *p); static asdl_seq *_loop0_169_rule(Parser *p); static asdl_seq *_loop0_170_rule(Parser *p); -static asdl_seq *_loop0_171_rule(Parser *p); -static asdl_seq *_loop1_172_rule(Parser *p); -static void *_tmp_173_rule(Parser *p); -static asdl_seq *_loop0_174_rule(Parser *p); -static void *_tmp_175_rule(Parser *p); -static asdl_seq *_loop0_176_rule(Parser *p); -static asdl_seq *_loop1_177_rule(Parser *p); +static asdl_seq *_loop1_171_rule(Parser *p); +static void *_tmp_172_rule(Parser *p); +static asdl_seq *_loop0_173_rule(Parser *p); +static void *_tmp_174_rule(Parser *p); +static asdl_seq *_loop0_175_rule(Parser *p); +static asdl_seq *_loop1_176_rule(Parser *p); +static void *_tmp_177_rule(Parser *p); static void *_tmp_178_rule(Parser *p); static void *_tmp_179_rule(Parser *p); -static void *_tmp_180_rule(Parser *p); -static asdl_seq *_loop0_181_rule(Parser *p); +static asdl_seq *_loop0_180_rule(Parser *p); +static void *_tmp_181_rule(Parser *p); static void *_tmp_182_rule(Parser *p); -static void *_tmp_183_rule(Parser *p); -static asdl_seq *_loop1_184_rule(Parser *p); -static void *_tmp_185_rule(Parser *p); +static asdl_seq *_loop1_183_rule(Parser *p); +static void *_tmp_184_rule(Parser *p); +static asdl_seq *_loop0_185_rule(Parser *p); static asdl_seq *_loop0_186_rule(Parser *p); static asdl_seq *_loop0_187_rule(Parser *p); -static asdl_seq *_loop0_188_rule(Parser *p); -static asdl_seq *_loop0_190_rule(Parser *p); -static asdl_seq *_gather_189_rule(Parser *p); -static void *_tmp_191_rule(Parser *p); -static asdl_seq *_loop0_192_rule(Parser *p); -static void *_tmp_193_rule(Parser *p); -static asdl_seq *_loop0_194_rule(Parser *p); +static asdl_seq *_loop0_189_rule(Parser *p); +static asdl_seq *_gather_188_rule(Parser *p); +static void *_tmp_190_rule(Parser *p); +static asdl_seq *_loop0_191_rule(Parser *p); +static void *_tmp_192_rule(Parser *p); +static asdl_seq *_loop0_193_rule(Parser *p); +static asdl_seq *_loop1_194_rule(Parser *p); static asdl_seq *_loop1_195_rule(Parser *p); -static asdl_seq *_loop1_196_rule(Parser *p); +static void *_tmp_196_rule(Parser *p); static void *_tmp_197_rule(Parser *p); -static void *_tmp_198_rule(Parser *p); -static asdl_seq *_loop0_199_rule(Parser *p); +static asdl_seq *_loop0_198_rule(Parser *p); +static void *_tmp_199_rule(Parser *p); static void *_tmp_200_rule(Parser *p); static void *_tmp_201_rule(Parser *p); -static void *_tmp_202_rule(Parser *p); -static asdl_seq *_loop0_204_rule(Parser *p); -static asdl_seq *_gather_203_rule(Parser *p); -static asdl_seq *_loop0_206_rule(Parser *p); -static asdl_seq *_gather_205_rule(Parser *p); -static asdl_seq *_loop0_208_rule(Parser *p); -static asdl_seq *_gather_207_rule(Parser *p); -static asdl_seq *_loop0_210_rule(Parser *p); -static asdl_seq *_gather_209_rule(Parser *p); -static asdl_seq *_loop0_212_rule(Parser *p); -static asdl_seq *_gather_211_rule(Parser *p); -static void *_tmp_213_rule(Parser *p); -static asdl_seq *_loop0_214_rule(Parser *p); -static asdl_seq *_loop1_215_rule(Parser *p); -static void *_tmp_216_rule(Parser *p); -static asdl_seq *_loop0_217_rule(Parser *p); -static asdl_seq *_loop1_218_rule(Parser *p); +static asdl_seq *_loop0_203_rule(Parser *p); +static asdl_seq *_gather_202_rule(Parser *p); +static asdl_seq *_loop0_205_rule(Parser *p); +static asdl_seq *_gather_204_rule(Parser *p); +static asdl_seq *_loop0_207_rule(Parser *p); +static asdl_seq *_gather_206_rule(Parser *p); +static asdl_seq *_loop0_209_rule(Parser *p); +static asdl_seq *_gather_208_rule(Parser *p); +static asdl_seq *_loop0_211_rule(Parser *p); +static asdl_seq *_gather_210_rule(Parser *p); +static void *_tmp_212_rule(Parser *p); +static asdl_seq *_loop0_213_rule(Parser *p); +static asdl_seq *_loop1_214_rule(Parser *p); +static void *_tmp_215_rule(Parser *p); +static asdl_seq *_loop0_216_rule(Parser *p); +static asdl_seq *_loop1_217_rule(Parser *p); +static void *_tmp_218_rule(Parser *p); static void *_tmp_219_rule(Parser *p); static void *_tmp_220_rule(Parser *p); static void *_tmp_221_rule(Parser *p); @@ -1072,9 +1071,9 @@ static void *_tmp_224_rule(Parser *p); static void *_tmp_225_rule(Parser *p); static void *_tmp_226_rule(Parser *p); static void *_tmp_227_rule(Parser *p); -static void *_tmp_228_rule(Parser *p); -static asdl_seq *_loop0_230_rule(Parser *p); -static asdl_seq *_gather_229_rule(Parser *p); +static asdl_seq *_loop0_229_rule(Parser *p); +static asdl_seq *_gather_228_rule(Parser *p); +static void *_tmp_230_rule(Parser *p); static void *_tmp_231_rule(Parser *p); static void *_tmp_232_rule(Parser *p); static void *_tmp_233_rule(Parser *p); @@ -1087,8 +1086,8 @@ static void *_tmp_239_rule(Parser *p); static void *_tmp_240_rule(Parser *p); static void *_tmp_241_rule(Parser *p); static void *_tmp_242_rule(Parser *p); -static void *_tmp_243_rule(Parser *p); -static asdl_seq *_loop0_244_rule(Parser *p); +static asdl_seq *_loop0_243_rule(Parser *p); +static void *_tmp_244_rule(Parser *p); static void *_tmp_245_rule(Parser *p); static void *_tmp_246_rule(Parser *p); static void *_tmp_247_rule(Parser *p); @@ -1120,7 +1119,6 @@ static void *_tmp_272_rule(Parser *p); static void *_tmp_273_rule(Parser *p); static void *_tmp_274_rule(Parser *p); static void *_tmp_275_rule(Parser *p); -static void *_tmp_276_rule(Parser *p); // file: statements? $ @@ -20276,7 +20274,7 @@ invalid_legacy_expression_rule(Parser *p) // invalid_expression: // | !(NAME STRING | SOFT_KEYWORD) disjunction expression_without_invalid // | disjunction 'if' disjunction !('else' | ':') -// | 'lambda' lambda_params? ':' &(FSTRING_MIDDLE | fstring_replacement_field) +// | 'lambda' lambda_params? ':' &FSTRING_MIDDLE static void * invalid_expression_rule(Parser *p) { @@ -20350,12 +20348,12 @@ invalid_expression_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_expression[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "disjunction 'if' disjunction !('else' | ':')")); } - { // 'lambda' lambda_params? ':' &(FSTRING_MIDDLE | fstring_replacement_field) + { // 'lambda' lambda_params? ':' &FSTRING_MIDDLE if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'lambda' lambda_params? ':' &(FSTRING_MIDDLE | fstring_replacement_field)")); + D(fprintf(stderr, "%*c> invalid_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'lambda' lambda_params? ':' &FSTRING_MIDDLE")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; @@ -20367,10 +20365,10 @@ invalid_expression_rule(Parser *p) && (b = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_157_rule, p) + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, FSTRING_MIDDLE) // token=FSTRING_MIDDLE ) { - D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'lambda' lambda_params? ':' &(FSTRING_MIDDLE | fstring_replacement_field)")); + D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'lambda' lambda_params? ':' &FSTRING_MIDDLE")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "f-string: lambda expressions are not allowed without parentheses" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -20381,7 +20379,7 @@ invalid_expression_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_expression[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'lambda' lambda_params? ':' &(FSTRING_MIDDLE | fstring_replacement_field)")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'lambda' lambda_params? ':' &FSTRING_MIDDLE")); } _res = NULL; done: @@ -20455,7 +20453,7 @@ invalid_named_expression_rule(Parser *p) && (b = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, _tmp_158_rule, p) + _PyPegen_lookahead(0, _tmp_157_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' bitwise_or !('=' | ':=')")); @@ -20481,7 +20479,7 @@ invalid_named_expression_rule(Parser *p) Token * b; expr_ty bitwise_or_var; if ( - _PyPegen_lookahead(0, _tmp_159_rule, p) + _PyPegen_lookahead(0, _tmp_158_rule, p) && (a = bitwise_or_rule(p)) // bitwise_or && @@ -20489,7 +20487,7 @@ invalid_named_expression_rule(Parser *p) && (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, _tmp_160_rule, p) + _PyPegen_lookahead(0, _tmp_159_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(list | tuple | genexp | 'True' | 'None' | 'False') bitwise_or '=' bitwise_or !('=' | ':=')")); @@ -20569,7 +20567,7 @@ invalid_assignment_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions* ':' expression")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_161_var; + asdl_seq * _loop0_160_var; expr_ty a; expr_ty expression_var; if ( @@ -20577,7 +20575,7 @@ invalid_assignment_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_loop0_161_var = _loop0_161_rule(p)) // star_named_expressions* + (_loop0_160_var = _loop0_160_rule(p)) // star_named_expressions* && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -20634,10 +20632,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* star_expressions '='")); Token * _literal; - asdl_seq * _loop0_162_var; + asdl_seq * _loop0_161_var; expr_ty a; if ( - (_loop0_162_var = _loop0_162_rule(p)) // ((star_targets '='))* + (_loop0_161_var = _loop0_161_rule(p)) // ((star_targets '='))* && (a = star_expressions_rule(p)) // star_expressions && @@ -20664,10 +20662,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* yield_expr '='")); Token * _literal; - asdl_seq * _loop0_163_var; + asdl_seq * _loop0_162_var; expr_ty a; if ( - (_loop0_163_var = _loop0_163_rule(p)) // ((star_targets '='))* + (_loop0_162_var = _loop0_162_rule(p)) // ((star_targets '='))* && (a = yield_expr_rule(p)) // yield_expr && @@ -20693,7 +20691,7 @@ invalid_assignment_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions augassign (yield_expr | star_expressions)")); - void *_tmp_164_var; + void *_tmp_163_var; expr_ty a; AugOperator* augassign_var; if ( @@ -20701,7 +20699,7 @@ invalid_assignment_rule(Parser *p) && (augassign_var = augassign_rule(p)) // augassign && - (_tmp_164_var = _tmp_164_rule(p)) // yield_expr | star_expressions + (_tmp_163_var = _tmp_163_rule(p)) // yield_expr | star_expressions ) { D(fprintf(stderr, "%*c+ invalid_assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions augassign (yield_expr | star_expressions)")); @@ -20923,11 +20921,11 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '(' | '{') starred_expression for_if_clauses")); - void *_tmp_165_var; + void *_tmp_164_var; expr_ty a; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_165_var = _tmp_165_rule(p)) // '[' | '(' | '{' + (_tmp_164_var = _tmp_164_rule(p)) // '[' | '(' | '{' && (a = starred_expression_rule(p)) // starred_expression && @@ -20954,12 +20952,12 @@ invalid_comprehension_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' star_named_expressions for_if_clauses")); Token * _literal; - void *_tmp_166_var; + void *_tmp_165_var; expr_ty a; asdl_expr_seq* b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_166_var = _tmp_166_rule(p)) // '[' | '{' + (_tmp_165_var = _tmp_165_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -20989,12 +20987,12 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' for_if_clauses")); - void *_tmp_167_var; + void *_tmp_166_var; expr_ty a; Token * b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_167_var = _tmp_167_rule(p)) // '[' | '{' + (_tmp_166_var = _tmp_166_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -21129,13 +21127,13 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slash_no_default | slash_with_default) param_maybe_default* '/'")); - asdl_seq * _loop0_169_var; - void *_tmp_168_var; + asdl_seq * _loop0_168_var; + void *_tmp_167_var; Token * a; if ( - (_tmp_168_var = _tmp_168_rule(p)) // slash_no_default | slash_with_default + (_tmp_167_var = _tmp_167_rule(p)) // slash_no_default | slash_with_default && - (_loop0_169_var = _loop0_169_rule(p)) // param_maybe_default* + (_loop0_168_var = _loop0_168_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21159,7 +21157,7 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default? param_no_default* invalid_parameters_helper param_no_default")); - asdl_seq * _loop0_170_var; + asdl_seq * _loop0_169_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -21167,7 +21165,7 @@ invalid_parameters_rule(Parser *p) if ( (_opt_var = slash_no_default_rule(p), !p->error_indicator) // slash_no_default? && - (_loop0_170_var = _loop0_170_rule(p)) // param_no_default* + (_loop0_169_var = _loop0_169_rule(p)) // param_no_default* && (invalid_parameters_helper_var = invalid_parameters_helper_rule(p)) // invalid_parameters_helper && @@ -21193,18 +21191,18 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default* '(' param_no_default+ ','? ')'")); - asdl_seq * _loop0_171_var; - asdl_seq * _loop1_172_var; + asdl_seq * _loop0_170_var; + asdl_seq * _loop1_171_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_171_var = _loop0_171_rule(p)) // param_no_default* + (_loop0_170_var = _loop0_170_rule(p)) // param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_loop1_172_var = _loop1_172_rule(p)) // param_no_default+ + (_loop1_171_var = _loop1_171_rule(p)) // param_no_default+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -21231,22 +21229,22 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(slash_no_default | slash_with_default)] param_maybe_default* '*' (',' | param_no_default) param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_174_var; - asdl_seq * _loop0_176_var; + asdl_seq * _loop0_173_var; + asdl_seq * _loop0_175_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_175_var; + void *_tmp_174_var; Token * a; if ( - (_opt_var = _tmp_173_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] + (_opt_var = _tmp_172_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] && - (_loop0_174_var = _loop0_174_rule(p)) // param_maybe_default* + (_loop0_173_var = _loop0_173_rule(p)) // param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_175_var = _tmp_175_rule(p)) // ',' | param_no_default + (_tmp_174_var = _tmp_174_rule(p)) // ',' | param_no_default && - (_loop0_176_var = _loop0_176_rule(p)) // param_maybe_default* + (_loop0_175_var = _loop0_175_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21271,10 +21269,10 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_177_var; + asdl_seq * _loop1_176_var; Token * a; if ( - (_loop1_177_var = _loop1_177_rule(p)) // param_maybe_default+ + (_loop1_176_var = _loop1_176_rule(p)) // param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -21323,7 +21321,7 @@ invalid_default_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, _tmp_178_rule, p) + _PyPegen_lookahead(1, _tmp_177_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' &(')' | ',')")); @@ -21368,12 +21366,12 @@ invalid_star_etc_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); - void *_tmp_179_var; + void *_tmp_178_var; Token * a; if ( (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_179_var = _tmp_179_rule(p)) // ')' | ',' (')' | '**') + (_tmp_178_var = _tmp_178_rule(p)) // ')' | ',' (')' | '**') ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); @@ -21456,20 +21454,20 @@ invalid_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_181_var; - void *_tmp_180_var; - void *_tmp_182_var; + asdl_seq * _loop0_180_var; + void *_tmp_179_var; + void *_tmp_181_var; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_180_var = _tmp_180_rule(p)) // param_no_default | ',' + (_tmp_179_var = _tmp_179_rule(p)) // param_no_default | ',' && - (_loop0_181_var = _loop0_181_rule(p)) // param_maybe_default* + (_loop0_180_var = _loop0_180_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_182_var = _tmp_182_rule(p)) // param_no_default | ',' + (_tmp_181_var = _tmp_181_rule(p)) // param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); @@ -21584,7 +21582,7 @@ invalid_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_183_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_182_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param ',' ('*' | '**' | '/')")); @@ -21649,13 +21647,13 @@ invalid_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - asdl_seq * _loop1_184_var; + asdl_seq * _loop1_183_var; if ( - (_loop1_184_var = _loop1_184_rule(p)) // param_with_default+ + (_loop1_183_var = _loop1_183_rule(p)) // param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - _res = _loop1_184_var; + _res = _loop1_183_var; goto done; } p->mark = _mark; @@ -21720,13 +21718,13 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* '/'")); - asdl_seq * _loop0_186_var; - void *_tmp_185_var; + asdl_seq * _loop0_185_var; + void *_tmp_184_var; Token * a; if ( - (_tmp_185_var = _tmp_185_rule(p)) // lambda_slash_no_default | lambda_slash_with_default + (_tmp_184_var = _tmp_184_rule(p)) // lambda_slash_no_default | lambda_slash_with_default && - (_loop0_186_var = _loop0_186_rule(p)) // lambda_param_maybe_default* + (_loop0_185_var = _loop0_185_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21750,7 +21748,7 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default? lambda_param_no_default* invalid_lambda_parameters_helper lambda_param_no_default")); - asdl_seq * _loop0_187_var; + asdl_seq * _loop0_186_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -21758,7 +21756,7 @@ invalid_lambda_parameters_rule(Parser *p) if ( (_opt_var = lambda_slash_no_default_rule(p), !p->error_indicator) // lambda_slash_no_default? && - (_loop0_187_var = _loop0_187_rule(p)) // lambda_param_no_default* + (_loop0_186_var = _loop0_186_rule(p)) // lambda_param_no_default* && (invalid_lambda_parameters_helper_var = invalid_lambda_parameters_helper_rule(p)) // invalid_lambda_parameters_helper && @@ -21784,18 +21782,18 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default* '(' ','.lambda_param+ ','? ')'")); - asdl_seq * _gather_189_var; - asdl_seq * _loop0_188_var; + asdl_seq * _gather_188_var; + asdl_seq * _loop0_187_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_188_var = _loop0_188_rule(p)) // lambda_param_no_default* + (_loop0_187_var = _loop0_187_rule(p)) // lambda_param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_189_var = _gather_189_rule(p)) // ','.lambda_param+ + (_gather_188_var = _gather_188_rule(p)) // ','.lambda_param+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -21822,22 +21820,22 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(lambda_slash_no_default | lambda_slash_with_default)] lambda_param_maybe_default* '*' (',' | lambda_param_no_default) lambda_param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_192_var; - asdl_seq * _loop0_194_var; + asdl_seq * _loop0_191_var; + asdl_seq * _loop0_193_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_193_var; + void *_tmp_192_var; Token * a; if ( - (_opt_var = _tmp_191_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] + (_opt_var = _tmp_190_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] && - (_loop0_192_var = _loop0_192_rule(p)) // lambda_param_maybe_default* + (_loop0_191_var = _loop0_191_rule(p)) // lambda_param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_193_var = _tmp_193_rule(p)) // ',' | lambda_param_no_default + (_tmp_192_var = _tmp_192_rule(p)) // ',' | lambda_param_no_default && - (_loop0_194_var = _loop0_194_rule(p)) // lambda_param_maybe_default* + (_loop0_193_var = _loop0_193_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21862,10 +21860,10 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_195_var; + asdl_seq * _loop1_194_var; Token * a; if ( - (_loop1_195_var = _loop1_195_rule(p)) // lambda_param_maybe_default+ + (_loop1_194_var = _loop1_194_rule(p)) // lambda_param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -21936,13 +21934,13 @@ invalid_lambda_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - asdl_seq * _loop1_196_var; + asdl_seq * _loop1_195_var; if ( - (_loop1_196_var = _loop1_196_rule(p)) // lambda_param_with_default+ + (_loop1_195_var = _loop1_195_rule(p)) // lambda_param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - _res = _loop1_196_var; + _res = _loop1_195_var; goto done; } p->mark = _mark; @@ -21978,11 +21976,11 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); Token * _literal; - void *_tmp_197_var; + void *_tmp_196_var; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_197_var = _tmp_197_rule(p)) // ':' | ',' (':' | '**') + (_tmp_196_var = _tmp_196_rule(p)) // ':' | ',' (':' | '**') ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); @@ -22035,20 +22033,20 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_199_var; - void *_tmp_198_var; - void *_tmp_200_var; + asdl_seq * _loop0_198_var; + void *_tmp_197_var; + void *_tmp_199_var; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_198_var = _tmp_198_rule(p)) // lambda_param_no_default | ',' + (_tmp_197_var = _tmp_197_rule(p)) // lambda_param_no_default | ',' && - (_loop0_199_var = _loop0_199_rule(p)) // lambda_param_maybe_default* + (_loop0_198_var = _loop0_198_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_200_var = _tmp_200_rule(p)) // lambda_param_no_default | ',' + (_tmp_199_var = _tmp_199_rule(p)) // lambda_param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); @@ -22166,7 +22164,7 @@ invalid_lambda_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_201_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_200_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param ',' ('*' | '**' | '/')")); @@ -22272,7 +22270,7 @@ invalid_with_item_rule(Parser *p) && (a = expression_rule(p)) // expression && - _PyPegen_lookahead(1, _tmp_202_rule, p) + _PyPegen_lookahead(1, _tmp_201_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' expression &(',' | ')' | ':')")); @@ -22445,14 +22443,14 @@ invalid_import_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_import[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import' ','.dotted_name+ 'from' dotted_name")); - asdl_seq * _gather_203_var; + asdl_seq * _gather_202_var; Token * _keyword; Token * a; expr_ty dotted_name_var; if ( (a = _PyPegen_expect_token(p, 617)) // token='import' && - (_gather_203_var = _gather_203_rule(p)) // ','.dotted_name+ + (_gather_202_var = _gather_202_rule(p)) // ','.dotted_name+ && (_keyword = _PyPegen_expect_token(p, 618)) // token='from' && @@ -22548,7 +22546,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ NEWLINE")); - asdl_seq * _gather_205_var; + asdl_seq * _gather_204_var; Token * _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -22558,7 +22556,7 @@ invalid_with_stmt_rule(Parser *p) && (_keyword = _PyPegen_expect_token(p, 629)) // token='with' && - (_gather_205_var = _gather_205_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_204_var = _gather_204_rule(p)) // ','.(expression ['as' star_target])+ && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -22582,7 +22580,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE")); - asdl_seq * _gather_207_var; + asdl_seq * _gather_206_var; Token * _keyword; Token * _literal; Token * _literal_1; @@ -22598,7 +22596,7 @@ invalid_with_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_207_var = _gather_207_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_206_var = _gather_206_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22647,7 +22645,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT")); - asdl_seq * _gather_209_var; + asdl_seq * _gather_208_var; Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -22658,7 +22656,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (a = _PyPegen_expect_token(p, 629)) // token='with' && - (_gather_209_var = _gather_209_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_208_var = _gather_208_rule(p)) // ','.(expression ['as' star_target])+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -22686,7 +22684,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT")); - asdl_seq * _gather_211_var; + asdl_seq * _gather_210_var; Token * _literal; Token * _literal_1; Token * _literal_2; @@ -22703,7 +22701,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_211_var = _gather_211_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_210_var = _gather_210_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22800,7 +22798,7 @@ invalid_try_stmt_rule(Parser *p) && (block_var = block_rule(p)) // block && - _PyPegen_lookahead(0, _tmp_213_rule, p) + _PyPegen_lookahead(0, _tmp_212_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block !('except' | 'finally')")); @@ -22825,8 +22823,8 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_214_var; - asdl_seq * _loop1_215_var; + asdl_seq * _loop0_213_var; + asdl_seq * _loop1_214_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; @@ -22837,9 +22835,9 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_214_var = _loop0_214_rule(p)) // block* + (_loop0_213_var = _loop0_213_rule(p)) // block* && - (_loop1_215_var = _loop1_215_rule(p)) // except_block+ + (_loop1_214_var = _loop1_214_rule(p)) // except_block+ && (a = _PyPegen_expect_token(p, 651)) // token='except' && @@ -22847,7 +22845,7 @@ invalid_try_stmt_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_216_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_215_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -22874,8 +22872,8 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_217_var; - asdl_seq * _loop1_218_var; + asdl_seq * _loop0_216_var; + asdl_seq * _loop1_217_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; @@ -22884,13 +22882,13 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_217_var = _loop0_217_rule(p)) // block* + (_loop0_216_var = _loop0_216_rule(p)) // block* && - (_loop1_218_var = _loop1_218_rule(p)) // except_star_block+ + (_loop1_217_var = _loop1_217_rule(p)) // except_star_block+ && (a = _PyPegen_expect_token(p, 651)) // token='except' && - (_opt_var = _tmp_219_rule(p), !p->error_indicator) // [expression ['as' NAME]] + (_opt_var = _tmp_218_rule(p), !p->error_indicator) // [expression ['as' NAME]] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -22957,7 +22955,7 @@ invalid_except_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_opt_var_1 = _tmp_220_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_219_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -22995,7 +22993,7 @@ invalid_except_stmt_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var_1 = _tmp_221_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_220_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23047,14 +23045,14 @@ invalid_except_stmt_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_except_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); Token * _literal; - void *_tmp_222_var; + void *_tmp_221_var; Token * a; if ( (a = _PyPegen_expect_token(p, 651)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_222_var = _tmp_222_rule(p)) // NEWLINE | ':' + (_tmp_221_var = _tmp_221_rule(p)) // NEWLINE | ':' ) { D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); @@ -23159,7 +23157,7 @@ invalid_except_stmt_indent_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_223_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_222_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23253,7 +23251,7 @@ invalid_except_star_stmt_indent_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_224_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_223_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23617,7 +23615,7 @@ invalid_class_argument_pattern_rule(Parser *p) asdl_pattern_seq* a; asdl_seq* keyword_patterns_var; if ( - (_opt_var = _tmp_225_rule(p), !p->error_indicator) // [positional_patterns ','] + (_opt_var = _tmp_224_rule(p), !p->error_indicator) // [positional_patterns ','] && (keyword_patterns_var = keyword_patterns_rule(p)) // keyword_patterns && @@ -24105,7 +24103,7 @@ invalid_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_2 = _tmp_226_rule(p), !p->error_indicator) // ['->' expression] + (_opt_var_2 = _tmp_225_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24164,7 +24162,7 @@ invalid_class_def_raw_rule(Parser *p) && (name_var = _PyPegen_name_token(p)) // NAME && - (_opt_var = _tmp_227_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var = _tmp_226_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24199,7 +24197,7 @@ invalid_class_def_raw_rule(Parser *p) && (name_var = _PyPegen_name_token(p)) // NAME && - (_opt_var = _tmp_228_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var = _tmp_227_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24249,11 +24247,11 @@ invalid_double_starred_kvpairs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_double_starred_kvpairs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - asdl_seq * _gather_229_var; + asdl_seq * _gather_228_var; Token * _literal; void *invalid_kvpair_var; if ( - (_gather_229_var = _gather_229_rule(p)) // ','.double_starred_kvpair+ + (_gather_228_var = _gather_228_rule(p)) // ','.double_starred_kvpair+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -24261,7 +24259,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - _res = _PyPegen_dummy_name(p, _gather_229_var, _literal, invalid_kvpair_var); + _res = _PyPegen_dummy_name(p, _gather_228_var, _literal, invalid_kvpair_var); goto done; } p->mark = _mark; @@ -24314,7 +24312,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_231_rule, p) + _PyPegen_lookahead(1, _tmp_230_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24424,7 +24422,7 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_232_rule, p) + _PyPegen_lookahead(1, _tmp_231_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24640,7 +24638,7 @@ invalid_replacement_field_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - _PyPegen_lookahead(0, _tmp_233_rule, p) + _PyPegen_lookahead(0, _tmp_232_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' !(yield_expr | star_expressions)")); @@ -24663,13 +24661,13 @@ invalid_replacement_field_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}')")); Token * _literal; - void *_tmp_234_var; + void *_tmp_233_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_234_var = _tmp_234_rule(p)) // yield_expr | star_expressions + (_tmp_233_var = _tmp_233_rule(p)) // yield_expr | star_expressions && - _PyPegen_lookahead(0, _tmp_235_rule, p) + _PyPegen_lookahead(0, _tmp_234_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}')")); @@ -24693,15 +24691,15 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '=' !('!' | ':' | '}')")); Token * _literal; Token * _literal_1; - void *_tmp_236_var; + void *_tmp_235_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_236_var = _tmp_236_rule(p)) // yield_expr | star_expressions + (_tmp_235_var = _tmp_235_rule(p)) // yield_expr | star_expressions && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, _tmp_237_rule, p) + _PyPegen_lookahead(0, _tmp_236_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '=' !('!' | ':' | '}')")); @@ -24726,12 +24724,12 @@ invalid_replacement_field_rule(Parser *p) Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_238_var; + void *_tmp_237_var; void *invalid_conversion_character_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_238_var = _tmp_238_rule(p)) // yield_expr | star_expressions + (_tmp_237_var = _tmp_237_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && @@ -24739,7 +24737,7 @@ invalid_replacement_field_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? invalid_conversion_character")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_238_var, _opt_var, invalid_conversion_character_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_237_var, _opt_var, invalid_conversion_character_var); goto done; } p->mark = _mark; @@ -24757,17 +24755,17 @@ invalid_replacement_field_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_239_var; + void *_tmp_238_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_239_var = _tmp_239_rule(p)) // yield_expr | star_expressions + (_tmp_238_var = _tmp_238_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_240_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_239_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, _tmp_241_rule, p) + _PyPegen_lookahead(0, _tmp_240_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}')")); @@ -24791,24 +24789,24 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}'")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_244_var; + asdl_seq * _loop0_243_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_242_var; + void *_tmp_241_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_242_var = _tmp_242_rule(p)) // yield_expr | star_expressions + (_tmp_241_var = _tmp_241_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_243_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_242_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_244_var = _loop0_244_rule(p)) // fstring_format_spec* + (_loop0_243_var = _loop0_243_rule(p)) // fstring_format_spec* && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -24837,15 +24835,15 @@ invalid_replacement_field_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_245_var; + void *_tmp_244_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_245_var = _tmp_245_rule(p)) // yield_expr | star_expressions + (_tmp_244_var = _tmp_244_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_246_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_245_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -24892,7 +24890,7 @@ invalid_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, _tmp_247_rule, p) + _PyPegen_lookahead(1, _tmp_246_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -25822,12 +25820,12 @@ _loop1_15_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_15[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_248_var; + void *_tmp_247_var; while ( - (_tmp_248_var = _tmp_248_rule(p)) // star_targets '=' + (_tmp_247_var = _tmp_247_rule(p)) // star_targets '=' ) { - _res = _tmp_248_var; + _res = _tmp_247_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26391,12 +26389,12 @@ _loop0_25_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_249_var; + void *_tmp_248_var; while ( - (_tmp_249_var = _tmp_249_rule(p)) // '.' | '...' + (_tmp_248_var = _tmp_248_rule(p)) // '.' | '...' ) { - _res = _tmp_249_var; + _res = _tmp_248_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26458,12 +26456,12 @@ _loop1_26_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_26[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_250_var; + void *_tmp_249_var; while ( - (_tmp_250_var = _tmp_250_rule(p)) // '.' | '...' + (_tmp_249_var = _tmp_249_rule(p)) // '.' | '...' ) { - _res = _tmp_250_var; + _res = _tmp_249_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26856,12 +26854,12 @@ _loop1_33_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_33[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_251_var; + void *_tmp_250_var; while ( - (_tmp_251_var = _tmp_251_rule(p)) // '@' named_expression NEWLINE + (_tmp_250_var = _tmp_250_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_251_var; + _res = _tmp_250_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -29986,12 +29984,12 @@ _loop1_83_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); - void *_tmp_252_var; + void *_tmp_251_var; while ( - (_tmp_252_var = _tmp_252_rule(p)) // ',' expression + (_tmp_251_var = _tmp_251_rule(p)) // ',' expression ) { - _res = _tmp_252_var; + _res = _tmp_251_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30058,9 +30056,198 @@ _loop1_84_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_84[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); + void *_tmp_252_var; + while ( + (_tmp_252_var = _tmp_252_rule(p)) // ',' star_expression + ) + { + _res = _tmp_252_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop1_84[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_expression)")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _loop0_86: ',' star_named_expression +static asdl_seq * +_loop0_86_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ',' star_named_expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop0_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_named_expression")); + Token * _literal; + expr_ty elem; + while ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = star_named_expression_rule(p)) // star_named_expression + ) + { + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop0_86[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_named_expression")); + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _gather_85: star_named_expression _loop0_86 +static asdl_seq * +_gather_85_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + asdl_seq * _res = NULL; + int _mark = p->mark; + { // star_named_expression _loop0_86 + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _gather_85[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_86")); + expr_ty elem; + asdl_seq * seq; + if ( + (elem = star_named_expression_rule(p)) // star_named_expression + && + (seq = _loop0_86_rule(p)) // _loop0_86 + ) + { + D(fprintf(stderr, "%*c+ _gather_85[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_86")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _gather_85[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression _loop0_86")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _loop1_87: ('or' conjunction) +static asdl_seq * +_loop1_87_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ('or' conjunction) + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop1_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); void *_tmp_253_var; while ( - (_tmp_253_var = _tmp_253_rule(p)) // ',' star_expression + (_tmp_253_var = _tmp_253_rule(p)) // 'or' conjunction ) { _res = _tmp_253_var; @@ -30080,195 +30267,6 @@ _loop1_84_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_84[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_expression)")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_86: ',' star_named_expression -static asdl_seq * -_loop0_86_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' star_named_expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_named_expression")); - Token * _literal; - expr_ty elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = star_named_expression_rule(p)) // star_named_expression - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_86[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_named_expression")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_85: star_named_expression _loop0_86 -static asdl_seq * -_gather_85_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // star_named_expression _loop0_86 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_85[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_86")); - expr_ty elem; - asdl_seq * seq; - if ( - (elem = star_named_expression_rule(p)) // star_named_expression - && - (seq = _loop0_86_rule(p)) // _loop0_86 - ) - { - D(fprintf(stderr, "%*c+ _gather_85[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_86")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_85[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression _loop0_86")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop1_87: ('or' conjunction) -static asdl_seq * -_loop1_87_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ('or' conjunction) - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_254_var; - while ( - (_tmp_254_var = _tmp_254_rule(p)) // 'or' conjunction - ) - { - _res = _tmp_254_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; D(fprintf(stderr, "%*c%s _loop1_87[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('or' conjunction)")); } @@ -30319,12 +30317,12 @@ _loop1_88_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_88[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_255_var; + void *_tmp_254_var; while ( - (_tmp_255_var = _tmp_255_rule(p)) // 'and' inversion + (_tmp_254_var = _tmp_254_rule(p)) // 'and' inversion ) { - _res = _tmp_255_var; + _res = _tmp_254_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30511,7 +30509,7 @@ _loop0_92_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_256_rule(p)) // slice | starred_expression + (elem = _tmp_255_rule(p)) // slice | starred_expression ) { _res = elem; @@ -30576,7 +30574,7 @@ _gather_91_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_256_rule(p)) // slice | starred_expression + (elem = _tmp_255_rule(p)) // slice | starred_expression && (seq = _loop0_92_rule(p)) // _loop0_92 ) @@ -32108,12 +32106,12 @@ _loop1_115_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); - void *_tmp_257_var; + void *_tmp_256_var; while ( - (_tmp_257_var = _tmp_257_rule(p)) // fstring | string + (_tmp_256_var = _tmp_256_rule(p)) // fstring | string ) { - _res = _tmp_257_var; + _res = _tmp_256_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32418,12 +32416,12 @@ _loop0_120_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_258_var; + void *_tmp_257_var; while ( - (_tmp_258_var = _tmp_258_rule(p)) // 'if' disjunction + (_tmp_257_var = _tmp_257_rule(p)) // 'if' disjunction ) { - _res = _tmp_258_var; + _res = _tmp_257_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32485,12 +32483,12 @@ _loop0_121_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_259_var; + void *_tmp_258_var; while ( - (_tmp_259_var = _tmp_259_rule(p)) // 'if' disjunction + (_tmp_258_var = _tmp_258_rule(p)) // 'if' disjunction ) { - _res = _tmp_259_var; + _res = _tmp_258_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32616,7 +32614,7 @@ _loop0_124_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_260_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_259_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -32682,7 +32680,7 @@ _gather_123_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_260_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_259_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && (seq = _loop0_124_rule(p)) // _loop0_124 ) @@ -33243,12 +33241,12 @@ _loop0_134_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_261_var; + void *_tmp_260_var; while ( - (_tmp_261_var = _tmp_261_rule(p)) // ',' star_target + (_tmp_260_var = _tmp_260_rule(p)) // ',' star_target ) { - _res = _tmp_261_var; + _res = _tmp_260_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33427,12 +33425,12 @@ _loop1_137_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_262_var; + void *_tmp_261_var; while ( - (_tmp_262_var = _tmp_262_rule(p)) // ',' star_target + (_tmp_261_var = _tmp_261_rule(p)) // ',' star_target ) { - _res = _tmp_262_var; + _res = _tmp_261_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34529,66 +34527,9 @@ _tmp_156_rule(Parser *p) return _res; } -// _tmp_157: FSTRING_MIDDLE | fstring_replacement_field +// _tmp_157: '=' | ':=' static void * _tmp_157_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // FSTRING_MIDDLE - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "FSTRING_MIDDLE")); - Token * fstring_middle_var; - if ( - (fstring_middle_var = _PyPegen_expect_token(p, FSTRING_MIDDLE)) // token='FSTRING_MIDDLE' - ) - { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "FSTRING_MIDDLE")); - _res = fstring_middle_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "FSTRING_MIDDLE")); - } - { // fstring_replacement_field - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_replacement_field")); - expr_ty fstring_replacement_field_var; - if ( - (fstring_replacement_field_var = fstring_replacement_field_rule(p)) // fstring_replacement_field - ) - { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring_replacement_field")); - _res = fstring_replacement_field_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_replacement_field")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_158: '=' | ':=' -static void * -_tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34604,18 +34545,18 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -34623,18 +34564,18 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -34643,9 +34584,9 @@ _tmp_158_rule(Parser *p) return _res; } -// _tmp_159: list | tuple | genexp | 'True' | 'None' | 'False' +// _tmp_158: list | tuple | genexp | 'True' | 'None' | 'False' static void * -_tmp_159_rule(Parser *p) +_tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34661,18 +34602,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); expr_ty list_var; if ( (list_var = list_rule(p)) // list ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); _res = list_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); } { // tuple @@ -34680,18 +34621,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); expr_ty tuple_var; if ( (tuple_var = tuple_rule(p)) // tuple ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); _res = tuple_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); } { // genexp @@ -34699,18 +34640,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); expr_ty genexp_var; if ( (genexp_var = genexp_rule(p)) // genexp ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); _res = genexp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); } { // 'True' @@ -34718,18 +34659,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 610)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'None' @@ -34737,18 +34678,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 611)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } { // 'False' @@ -34756,18 +34697,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 612)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } _res = NULL; @@ -34776,9 +34717,9 @@ _tmp_159_rule(Parser *p) return _res; } -// _tmp_160: '=' | ':=' +// _tmp_159: '=' | ':=' static void * -_tmp_160_rule(Parser *p) +_tmp_159_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34794,18 +34735,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -34813,18 +34754,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -34833,9 +34774,9 @@ _tmp_160_rule(Parser *p) return _res; } -// _loop0_161: star_named_expressions +// _loop0_160: star_named_expressions static asdl_seq * -_loop0_161_rule(Parser *p) +_loop0_160_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34860,7 +34801,7 @@ _loop0_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); + D(fprintf(stderr, "%*c> _loop0_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); asdl_expr_seq* star_named_expressions_var; while ( (star_named_expressions_var = star_named_expressions_rule(p)) // star_named_expressions @@ -34883,7 +34824,7 @@ _loop0_161_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expressions")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34900,9 +34841,9 @@ _loop0_161_rule(Parser *p) return _seq; } -// _loop0_162: (star_targets '=') +// _loop0_161: (star_targets '=') static asdl_seq * -_loop0_162_rule(Parser *p) +_loop0_161_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34927,13 +34868,13 @@ _loop0_162_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_263_var; + D(fprintf(stderr, "%*c> _loop0_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_262_var; while ( - (_tmp_263_var = _tmp_263_rule(p)) // star_targets '=' + (_tmp_262_var = _tmp_262_rule(p)) // star_targets '=' ) { - _res = _tmp_263_var; + _res = _tmp_262_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34950,7 +34891,7 @@ _loop0_162_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_162[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_161[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34967,9 +34908,9 @@ _loop0_162_rule(Parser *p) return _seq; } -// _loop0_163: (star_targets '=') +// _loop0_162: (star_targets '=') static asdl_seq * -_loop0_163_rule(Parser *p) +_loop0_162_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34994,13 +34935,13 @@ _loop0_163_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_264_var; + D(fprintf(stderr, "%*c> _loop0_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_263_var; while ( - (_tmp_264_var = _tmp_264_rule(p)) // star_targets '=' + (_tmp_263_var = _tmp_263_rule(p)) // star_targets '=' ) { - _res = _tmp_264_var; + _res = _tmp_263_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -35017,7 +34958,7 @@ _loop0_163_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_163[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35034,9 +34975,9 @@ _loop0_163_rule(Parser *p) return _seq; } -// _tmp_164: yield_expr | star_expressions +// _tmp_163: yield_expr | star_expressions static void * -_tmp_164_rule(Parser *p) +_tmp_163_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35052,18 +34993,18 @@ _tmp_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -35071,18 +35012,18 @@ _tmp_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -35091,9 +35032,9 @@ _tmp_164_rule(Parser *p) return _res; } -// _tmp_165: '[' | '(' | '{' +// _tmp_164: '[' | '(' | '{' static void * -_tmp_165_rule(Parser *p) +_tmp_164_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35109,18 +35050,18 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '(' @@ -35128,18 +35069,18 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); } { // '{' @@ -35147,18 +35088,18 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35167,9 +35108,9 @@ _tmp_165_rule(Parser *p) return _res; } -// _tmp_166: '[' | '{' +// _tmp_165: '[' | '{' static void * -_tmp_166_rule(Parser *p) +_tmp_165_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35185,18 +35126,18 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '{' @@ -35204,18 +35145,18 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35224,9 +35165,9 @@ _tmp_166_rule(Parser *p) return _res; } -// _tmp_167: '[' | '{' +// _tmp_166: '[' | '{' static void * -_tmp_167_rule(Parser *p) +_tmp_166_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35242,18 +35183,18 @@ _tmp_167_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '{' @@ -35261,18 +35202,18 @@ _tmp_167_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35281,9 +35222,9 @@ _tmp_167_rule(Parser *p) return _res; } -// _tmp_168: slash_no_default | slash_with_default +// _tmp_167: slash_no_default | slash_with_default static void * -_tmp_168_rule(Parser *p) +_tmp_167_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35299,18 +35240,18 @@ _tmp_168_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); asdl_arg_seq* slash_no_default_var; if ( (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); _res = slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); } { // slash_with_default @@ -35318,18 +35259,18 @@ _tmp_168_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); SlashWithDefault* slash_with_default_var; if ( (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); _res = slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); } _res = NULL; @@ -35338,9 +35279,9 @@ _tmp_168_rule(Parser *p) return _res; } -// _loop0_169: param_maybe_default +// _loop0_168: param_maybe_default static asdl_seq * -_loop0_169_rule(Parser *p) +_loop0_168_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35365,7 +35306,7 @@ _loop0_169_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -35388,7 +35329,7 @@ _loop0_169_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_169[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_168[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35405,9 +35346,9 @@ _loop0_169_rule(Parser *p) return _seq; } -// _loop0_170: param_no_default +// _loop0_169: param_no_default static asdl_seq * -_loop0_170_rule(Parser *p) +_loop0_169_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35432,7 +35373,7 @@ _loop0_170_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop0_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -35455,7 +35396,7 @@ _loop0_170_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_170[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_169[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35472,9 +35413,9 @@ _loop0_170_rule(Parser *p) return _seq; } -// _loop0_171: param_no_default +// _loop0_170: param_no_default static asdl_seq * -_loop0_171_rule(Parser *p) +_loop0_170_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35499,7 +35440,7 @@ _loop0_171_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop0_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -35522,7 +35463,7 @@ _loop0_171_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_171[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_170[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35539,9 +35480,9 @@ _loop0_171_rule(Parser *p) return _seq; } -// _loop1_172: param_no_default +// _loop1_171: param_no_default static asdl_seq * -_loop1_172_rule(Parser *p) +_loop1_171_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35566,7 +35507,7 @@ _loop1_172_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop1_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -35589,7 +35530,7 @@ _loop1_172_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_171[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } if (_n == 0 || p->error_indicator) { @@ -35611,9 +35552,9 @@ _loop1_172_rule(Parser *p) return _seq; } -// _tmp_173: slash_no_default | slash_with_default +// _tmp_172: slash_no_default | slash_with_default static void * -_tmp_173_rule(Parser *p) +_tmp_172_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35629,18 +35570,18 @@ _tmp_173_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); asdl_arg_seq* slash_no_default_var; if ( (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_172[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); _res = slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_172[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); } { // slash_with_default @@ -35648,18 +35589,18 @@ _tmp_173_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); SlashWithDefault* slash_with_default_var; if ( (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_172[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); _res = slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_172[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); } _res = NULL; @@ -35668,9 +35609,9 @@ _tmp_173_rule(Parser *p) return _res; } -// _loop0_174: param_maybe_default +// _loop0_173: param_maybe_default static asdl_seq * -_loop0_174_rule(Parser *p) +_loop0_173_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35695,7 +35636,7 @@ _loop0_174_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -35718,7 +35659,7 @@ _loop0_174_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_174[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35735,9 +35676,9 @@ _loop0_174_rule(Parser *p) return _seq; } -// _tmp_175: ',' | param_no_default +// _tmp_174: ',' | param_no_default static void * -_tmp_175_rule(Parser *p) +_tmp_174_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35753,18 +35694,18 @@ _tmp_175_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // param_no_default @@ -35772,18 +35713,18 @@ _tmp_175_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } _res = NULL; @@ -35792,9 +35733,9 @@ _tmp_175_rule(Parser *p) return _res; } -// _loop0_176: param_maybe_default +// _loop0_175: param_maybe_default static asdl_seq * -_loop0_176_rule(Parser *p) +_loop0_175_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35819,7 +35760,7 @@ _loop0_176_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -35842,7 +35783,7 @@ _loop0_176_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_176[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_175[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35859,9 +35800,9 @@ _loop0_176_rule(Parser *p) return _seq; } -// _loop1_177: param_maybe_default +// _loop1_176: param_maybe_default static asdl_seq * -_loop1_177_rule(Parser *p) +_loop1_176_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35886,7 +35827,7 @@ _loop1_177_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -35909,7 +35850,7 @@ _loop1_177_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_177[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_176[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -35931,9 +35872,9 @@ _loop1_177_rule(Parser *p) return _seq; } -// _tmp_178: ')' | ',' +// _tmp_177: ')' | ',' static void * -_tmp_178_rule(Parser *p) +_tmp_177_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35949,18 +35890,18 @@ _tmp_178_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_177[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_177[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' @@ -35968,18 +35909,18 @@ _tmp_178_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_177[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_177[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -35988,9 +35929,9 @@ _tmp_178_rule(Parser *p) return _res; } -// _tmp_179: ')' | ',' (')' | '**') +// _tmp_178: ')' | ',' (')' | '**') static void * -_tmp_179_rule(Parser *p) +_tmp_178_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36006,18 +35947,18 @@ _tmp_179_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' (')' | '**') @@ -36025,21 +35966,21 @@ _tmp_179_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); Token * _literal; - void *_tmp_265_var; + void *_tmp_264_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_265_var = _tmp_265_rule(p)) // ')' | '**' + (_tmp_264_var = _tmp_264_rule(p)) // ')' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_265_var); + D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_264_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (')' | '**')")); } _res = NULL; @@ -36048,9 +35989,9 @@ _tmp_179_rule(Parser *p) return _res; } -// _tmp_180: param_no_default | ',' +// _tmp_179: param_no_default | ',' static void * -_tmp_180_rule(Parser *p) +_tmp_179_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36066,18 +36007,18 @@ _tmp_180_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } { // ',' @@ -36085,18 +36026,18 @@ _tmp_180_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36105,9 +36046,9 @@ _tmp_180_rule(Parser *p) return _res; } -// _loop0_181: param_maybe_default +// _loop0_180: param_maybe_default static asdl_seq * -_loop0_181_rule(Parser *p) +_loop0_180_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36132,7 +36073,7 @@ _loop0_181_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -36155,7 +36096,7 @@ _loop0_181_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_181[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_180[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36172,9 +36113,9 @@ _loop0_181_rule(Parser *p) return _seq; } -// _tmp_182: param_no_default | ',' +// _tmp_181: param_no_default | ',' static void * -_tmp_182_rule(Parser *p) +_tmp_181_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36190,18 +36131,18 @@ _tmp_182_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_181[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_181[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } { // ',' @@ -36209,18 +36150,18 @@ _tmp_182_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_181[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_181[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36229,9 +36170,9 @@ _tmp_182_rule(Parser *p) return _res; } -// _tmp_183: '*' | '**' | '/' +// _tmp_182: '*' | '**' | '/' static void * -_tmp_183_rule(Parser *p) +_tmp_182_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36247,18 +36188,18 @@ _tmp_183_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '**' @@ -36266,18 +36207,18 @@ _tmp_183_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } { // '/' @@ -36285,18 +36226,18 @@ _tmp_183_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } _res = NULL; @@ -36305,9 +36246,9 @@ _tmp_183_rule(Parser *p) return _res; } -// _loop1_184: param_with_default +// _loop1_183: param_with_default static asdl_seq * -_loop1_184_rule(Parser *p) +_loop1_183_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36332,7 +36273,7 @@ _loop1_184_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); + D(fprintf(stderr, "%*c> _loop1_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); NameDefaultPair* param_with_default_var; while ( (param_with_default_var = param_with_default_rule(p)) // param_with_default @@ -36355,7 +36296,7 @@ _loop1_184_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_184[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_183[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -36377,9 +36318,9 @@ _loop1_184_rule(Parser *p) return _seq; } -// _tmp_185: lambda_slash_no_default | lambda_slash_with_default +// _tmp_184: lambda_slash_no_default | lambda_slash_with_default static void * -_tmp_185_rule(Parser *p) +_tmp_184_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36395,18 +36336,18 @@ _tmp_185_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); asdl_arg_seq* lambda_slash_no_default_var; if ( (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_185[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_184[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); _res = lambda_slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_185[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_184[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); } { // lambda_slash_with_default @@ -36414,18 +36355,18 @@ _tmp_185_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); SlashWithDefault* lambda_slash_with_default_var; if ( (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_185[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_184[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); _res = lambda_slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_185[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_184[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); } _res = NULL; @@ -36434,9 +36375,9 @@ _tmp_185_rule(Parser *p) return _res; } -// _loop0_186: lambda_param_maybe_default +// _loop0_185: lambda_param_maybe_default static asdl_seq * -_loop0_186_rule(Parser *p) +_loop0_185_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36461,7 +36402,7 @@ _loop0_186_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -36484,7 +36425,7 @@ _loop0_186_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_186[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_185[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36501,9 +36442,9 @@ _loop0_186_rule(Parser *p) return _seq; } -// _loop0_187: lambda_param_no_default +// _loop0_186: lambda_param_no_default static asdl_seq * -_loop0_187_rule(Parser *p) +_loop0_186_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36528,7 +36469,7 @@ _loop0_187_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_187[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _loop0_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; while ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default @@ -36551,7 +36492,7 @@ _loop0_187_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_187[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_186[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36568,9 +36509,9 @@ _loop0_187_rule(Parser *p) return _seq; } -// _loop0_188: lambda_param_no_default +// _loop0_187: lambda_param_no_default static asdl_seq * -_loop0_188_rule(Parser *p) +_loop0_187_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36595,7 +36536,7 @@ _loop0_188_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_188[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _loop0_187[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; while ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default @@ -36618,7 +36559,7 @@ _loop0_188_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_188[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_187[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36635,9 +36576,9 @@ _loop0_188_rule(Parser *p) return _seq; } -// _loop0_190: ',' lambda_param +// _loop0_189: ',' lambda_param static asdl_seq * -_loop0_190_rule(Parser *p) +_loop0_189_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36662,7 +36603,7 @@ _loop0_190_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_190[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); + D(fprintf(stderr, "%*c> _loop0_189[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); Token * _literal; arg_ty elem; while ( @@ -36694,7 +36635,7 @@ _loop0_190_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_190[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_189[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' lambda_param")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36711,9 +36652,9 @@ _loop0_190_rule(Parser *p) return _seq; } -// _gather_189: lambda_param _loop0_190 +// _gather_188: lambda_param _loop0_189 static asdl_seq * -_gather_189_rule(Parser *p) +_gather_188_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36724,27 +36665,27 @@ _gather_189_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // lambda_param _loop0_190 + { // lambda_param _loop0_189 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_189[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_190")); + D(fprintf(stderr, "%*c> _gather_188[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_189")); arg_ty elem; asdl_seq * seq; if ( (elem = lambda_param_rule(p)) // lambda_param && - (seq = _loop0_190_rule(p)) // _loop0_190 + (seq = _loop0_189_rule(p)) // _loop0_189 ) { - D(fprintf(stderr, "%*c+ _gather_189[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_190")); + D(fprintf(stderr, "%*c+ _gather_188[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_189")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_189[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_190")); + D(fprintf(stderr, "%*c%s _gather_188[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_189")); } _res = NULL; done: @@ -36752,9 +36693,9 @@ _gather_189_rule(Parser *p) return _res; } -// _tmp_191: lambda_slash_no_default | lambda_slash_with_default +// _tmp_190: lambda_slash_no_default | lambda_slash_with_default static void * -_tmp_191_rule(Parser *p) +_tmp_190_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36770,18 +36711,18 @@ _tmp_191_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_190[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); asdl_arg_seq* lambda_slash_no_default_var; if ( (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_191[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_190[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); _res = lambda_slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_191[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_190[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); } { // lambda_slash_with_default @@ -36789,18 +36730,18 @@ _tmp_191_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_190[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); SlashWithDefault* lambda_slash_with_default_var; if ( (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_191[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_190[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); _res = lambda_slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_191[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_190[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); } _res = NULL; @@ -36809,9 +36750,9 @@ _tmp_191_rule(Parser *p) return _res; } -// _loop0_192: lambda_param_maybe_default +// _loop0_191: lambda_param_maybe_default static asdl_seq * -_loop0_192_rule(Parser *p) +_loop0_191_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36836,7 +36777,7 @@ _loop0_192_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -36859,7 +36800,7 @@ _loop0_192_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_192[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_191[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36876,9 +36817,9 @@ _loop0_192_rule(Parser *p) return _seq; } -// _tmp_193: ',' | lambda_param_no_default +// _tmp_192: ',' | lambda_param_no_default static void * -_tmp_193_rule(Parser *p) +_tmp_192_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36894,18 +36835,18 @@ _tmp_193_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_193[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_192[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_193[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_192[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // lambda_param_no_default @@ -36913,18 +36854,18 @@ _tmp_193_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_193[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_192[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_193[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_192[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } _res = NULL; @@ -36933,9 +36874,9 @@ _tmp_193_rule(Parser *p) return _res; } -// _loop0_194: lambda_param_maybe_default +// _loop0_193: lambda_param_maybe_default static asdl_seq * -_loop0_194_rule(Parser *p) +_loop0_193_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36960,7 +36901,7 @@ _loop0_194_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -36983,7 +36924,7 @@ _loop0_194_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_194[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_193[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37000,9 +36941,9 @@ _loop0_194_rule(Parser *p) return _seq; } -// _loop1_195: lambda_param_maybe_default +// _loop1_194: lambda_param_maybe_default static asdl_seq * -_loop1_195_rule(Parser *p) +_loop1_194_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37027,7 +36968,7 @@ _loop1_195_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_195[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37050,7 +36991,7 @@ _loop1_195_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_195[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_194[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -37072,9 +37013,9 @@ _loop1_195_rule(Parser *p) return _seq; } -// _loop1_196: lambda_param_with_default +// _loop1_195: lambda_param_with_default static asdl_seq * -_loop1_196_rule(Parser *p) +_loop1_195_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37099,7 +37040,7 @@ _loop1_196_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); + D(fprintf(stderr, "%*c> _loop1_195[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); NameDefaultPair* lambda_param_with_default_var; while ( (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default @@ -37122,7 +37063,7 @@ _loop1_196_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_196[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_195[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -37144,9 +37085,9 @@ _loop1_196_rule(Parser *p) return _seq; } -// _tmp_197: ':' | ',' (':' | '**') +// _tmp_196: ':' | ',' (':' | '**') static void * -_tmp_197_rule(Parser *p) +_tmp_196_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37162,18 +37103,18 @@ _tmp_197_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_196[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_196[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // ',' (':' | '**') @@ -37181,21 +37122,21 @@ _tmp_197_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + D(fprintf(stderr, "%*c> _tmp_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; - void *_tmp_266_var; + void *_tmp_265_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_266_var = _tmp_266_rule(p)) // ':' | '**' + (_tmp_265_var = _tmp_265_rule(p)) // ':' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_266_var); + D(fprintf(stderr, "%*c+ _tmp_196[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_265_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_196[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (':' | '**')")); } _res = NULL; @@ -37204,9 +37145,9 @@ _tmp_197_rule(Parser *p) return _res; } -// _tmp_198: lambda_param_no_default | ',' +// _tmp_197: lambda_param_no_default | ',' static void * -_tmp_198_rule(Parser *p) +_tmp_197_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37222,18 +37163,18 @@ _tmp_198_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } { // ',' @@ -37241,18 +37182,18 @@ _tmp_198_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -37261,9 +37202,9 @@ _tmp_198_rule(Parser *p) return _res; } -// _loop0_199: lambda_param_maybe_default +// _loop0_198: lambda_param_maybe_default static asdl_seq * -_loop0_199_rule(Parser *p) +_loop0_198_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37288,7 +37229,7 @@ _loop0_199_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37311,7 +37252,7 @@ _loop0_199_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_199[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_198[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37328,9 +37269,9 @@ _loop0_199_rule(Parser *p) return _seq; } -// _tmp_200: lambda_param_no_default | ',' +// _tmp_199: lambda_param_no_default | ',' static void * -_tmp_200_rule(Parser *p) +_tmp_199_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37346,18 +37287,18 @@ _tmp_200_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_199[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_199[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } { // ',' @@ -37365,18 +37306,18 @@ _tmp_200_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_199[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_199[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -37385,9 +37326,9 @@ _tmp_200_rule(Parser *p) return _res; } -// _tmp_201: '*' | '**' | '/' +// _tmp_200: '*' | '**' | '/' static void * -_tmp_201_rule(Parser *p) +_tmp_200_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37403,18 +37344,18 @@ _tmp_201_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '**' @@ -37422,18 +37363,18 @@ _tmp_201_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } { // '/' @@ -37441,18 +37382,18 @@ _tmp_201_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } _res = NULL; @@ -37461,9 +37402,9 @@ _tmp_201_rule(Parser *p) return _res; } -// _tmp_202: ',' | ')' | ':' +// _tmp_201: ',' | ')' | ':' static void * -_tmp_202_rule(Parser *p) +_tmp_201_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37479,18 +37420,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -37498,18 +37439,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ':' @@ -37517,18 +37458,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -37537,9 +37478,9 @@ _tmp_202_rule(Parser *p) return _res; } -// _loop0_204: ',' dotted_name +// _loop0_203: ',' dotted_name static asdl_seq * -_loop0_204_rule(Parser *p) +_loop0_203_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37564,7 +37505,7 @@ _loop0_204_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); + D(fprintf(stderr, "%*c> _loop0_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); Token * _literal; expr_ty elem; while ( @@ -37596,7 +37537,7 @@ _loop0_204_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_204[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_203[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' dotted_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37613,9 +37554,9 @@ _loop0_204_rule(Parser *p) return _seq; } -// _gather_203: dotted_name _loop0_204 +// _gather_202: dotted_name _loop0_203 static asdl_seq * -_gather_203_rule(Parser *p) +_gather_202_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37626,27 +37567,27 @@ _gather_203_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // dotted_name _loop0_204 + { // dotted_name _loop0_203 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_204")); + D(fprintf(stderr, "%*c> _gather_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_203")); expr_ty elem; asdl_seq * seq; if ( (elem = dotted_name_rule(p)) // dotted_name && - (seq = _loop0_204_rule(p)) // _loop0_204 + (seq = _loop0_203_rule(p)) // _loop0_203 ) { - D(fprintf(stderr, "%*c+ _gather_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_204")); + D(fprintf(stderr, "%*c+ _gather_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_203")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_203[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_204")); + D(fprintf(stderr, "%*c%s _gather_202[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_203")); } _res = NULL; done: @@ -37654,9 +37595,9 @@ _gather_203_rule(Parser *p) return _res; } -// _loop0_206: ',' (expression ['as' star_target]) +// _loop0_205: ',' (expression ['as' star_target]) static asdl_seq * -_loop0_206_rule(Parser *p) +_loop0_205_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37681,13 +37622,13 @@ _loop0_206_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_267_rule(p)) // expression ['as' star_target] + (elem = _tmp_266_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -37713,7 +37654,7 @@ _loop0_206_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_206[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_205[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37730,9 +37671,9 @@ _loop0_206_rule(Parser *p) return _seq; } -// _gather_205: (expression ['as' star_target]) _loop0_206 +// _gather_204: (expression ['as' star_target]) _loop0_205 static asdl_seq * -_gather_205_rule(Parser *p) +_gather_204_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37743,27 +37684,27 @@ _gather_205_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_206 + { // (expression ['as' star_target]) _loop0_205 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c> _gather_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_205")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_267_rule(p)) // expression ['as' star_target] + (elem = _tmp_266_rule(p)) // expression ['as' star_target] && - (seq = _loop0_206_rule(p)) // _loop0_206 + (seq = _loop0_205_rule(p)) // _loop0_205 ) { - D(fprintf(stderr, "%*c+ _gather_205[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c+ _gather_204[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_205")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_205[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c%s _gather_204[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_205")); } _res = NULL; done: @@ -37771,9 +37712,9 @@ _gather_205_rule(Parser *p) return _res; } -// _loop0_208: ',' (expressions ['as' star_target]) +// _loop0_207: ',' (expressions ['as' star_target]) static asdl_seq * -_loop0_208_rule(Parser *p) +_loop0_207_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37798,13 +37739,13 @@ _loop0_208_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_268_rule(p)) // expressions ['as' star_target] + (elem = _tmp_267_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -37830,7 +37771,7 @@ _loop0_208_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_208[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_207[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37847,9 +37788,9 @@ _loop0_208_rule(Parser *p) return _seq; } -// _gather_207: (expressions ['as' star_target]) _loop0_208 +// _gather_206: (expressions ['as' star_target]) _loop0_207 static asdl_seq * -_gather_207_rule(Parser *p) +_gather_206_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37860,27 +37801,27 @@ _gather_207_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_208 + { // (expressions ['as' star_target]) _loop0_207 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c> _gather_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_207")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_268_rule(p)) // expressions ['as' star_target] + (elem = _tmp_267_rule(p)) // expressions ['as' star_target] && - (seq = _loop0_208_rule(p)) // _loop0_208 + (seq = _loop0_207_rule(p)) // _loop0_207 ) { - D(fprintf(stderr, "%*c+ _gather_207[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c+ _gather_206[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_207")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_207[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c%s _gather_206[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_207")); } _res = NULL; done: @@ -37888,9 +37829,9 @@ _gather_207_rule(Parser *p) return _res; } -// _loop0_210: ',' (expression ['as' star_target]) +// _loop0_209: ',' (expression ['as' star_target]) static asdl_seq * -_loop0_210_rule(Parser *p) +_loop0_209_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37915,13 +37856,13 @@ _loop0_210_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_269_rule(p)) // expression ['as' star_target] + (elem = _tmp_268_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -37947,7 +37888,7 @@ _loop0_210_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_210[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_209[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37964,9 +37905,9 @@ _loop0_210_rule(Parser *p) return _seq; } -// _gather_209: (expression ['as' star_target]) _loop0_210 +// _gather_208: (expression ['as' star_target]) _loop0_209 static asdl_seq * -_gather_209_rule(Parser *p) +_gather_208_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37977,27 +37918,27 @@ _gather_209_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_210 + { // (expression ['as' star_target]) _loop0_209 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c> _gather_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_209")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_269_rule(p)) // expression ['as' star_target] + (elem = _tmp_268_rule(p)) // expression ['as' star_target] && - (seq = _loop0_210_rule(p)) // _loop0_210 + (seq = _loop0_209_rule(p)) // _loop0_209 ) { - D(fprintf(stderr, "%*c+ _gather_209[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c+ _gather_208[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_209")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_209[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c%s _gather_208[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_209")); } _res = NULL; done: @@ -38005,9 +37946,9 @@ _gather_209_rule(Parser *p) return _res; } -// _loop0_212: ',' (expressions ['as' star_target]) +// _loop0_211: ',' (expressions ['as' star_target]) static asdl_seq * -_loop0_212_rule(Parser *p) +_loop0_211_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38032,13 +37973,13 @@ _loop0_212_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_270_rule(p)) // expressions ['as' star_target] + (elem = _tmp_269_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -38064,7 +38005,7 @@ _loop0_212_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_212[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_211[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38081,9 +38022,9 @@ _loop0_212_rule(Parser *p) return _seq; } -// _gather_211: (expressions ['as' star_target]) _loop0_212 +// _gather_210: (expressions ['as' star_target]) _loop0_211 static asdl_seq * -_gather_211_rule(Parser *p) +_gather_210_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38094,27 +38035,27 @@ _gather_211_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_212 + { // (expressions ['as' star_target]) _loop0_211 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + D(fprintf(stderr, "%*c> _gather_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_211")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_270_rule(p)) // expressions ['as' star_target] + (elem = _tmp_269_rule(p)) // expressions ['as' star_target] && - (seq = _loop0_212_rule(p)) // _loop0_212 + (seq = _loop0_211_rule(p)) // _loop0_211 ) { - D(fprintf(stderr, "%*c+ _gather_211[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + D(fprintf(stderr, "%*c+ _gather_210[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_211")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_211[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + D(fprintf(stderr, "%*c%s _gather_210[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_211")); } _res = NULL; done: @@ -38122,9 +38063,9 @@ _gather_211_rule(Parser *p) return _res; } -// _tmp_213: 'except' | 'finally' +// _tmp_212: 'except' | 'finally' static void * -_tmp_213_rule(Parser *p) +_tmp_212_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38140,18 +38081,18 @@ _tmp_213_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c> _tmp_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 651)) // token='except' ) { - D(fprintf(stderr, "%*c+ _tmp_213[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c+ _tmp_212[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_213[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_212[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except'")); } { // 'finally' @@ -38159,18 +38100,18 @@ _tmp_213_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c> _tmp_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 647)) // token='finally' ) { - D(fprintf(stderr, "%*c+ _tmp_213[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c+ _tmp_212[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_213[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_212[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'finally'")); } _res = NULL; @@ -38179,9 +38120,9 @@ _tmp_213_rule(Parser *p) return _res; } -// _loop0_214: block +// _loop0_213: block static asdl_seq * -_loop0_214_rule(Parser *p) +_loop0_213_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38206,7 +38147,7 @@ _loop0_214_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -38229,7 +38170,7 @@ _loop0_214_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_214[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_213[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38246,9 +38187,9 @@ _loop0_214_rule(Parser *p) return _seq; } -// _loop1_215: except_block +// _loop1_214: except_block static asdl_seq * -_loop1_215_rule(Parser *p) +_loop1_214_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38273,7 +38214,7 @@ _loop1_215_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); + D(fprintf(stderr, "%*c> _loop1_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); excepthandler_ty except_block_var; while ( (except_block_var = except_block_rule(p)) // except_block @@ -38296,7 +38237,7 @@ _loop1_215_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_215[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_214[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); } if (_n == 0 || p->error_indicator) { @@ -38318,9 +38259,9 @@ _loop1_215_rule(Parser *p) return _seq; } -// _tmp_216: 'as' NAME +// _tmp_215: 'as' NAME static void * -_tmp_216_rule(Parser *p) +_tmp_215_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38336,7 +38277,7 @@ _tmp_216_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38345,12 +38286,12 @@ _tmp_216_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_216[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_215[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_216[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_215[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38359,9 +38300,9 @@ _tmp_216_rule(Parser *p) return _res; } -// _loop0_217: block +// _loop0_216: block static asdl_seq * -_loop0_217_rule(Parser *p) +_loop0_216_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38386,7 +38327,7 @@ _loop0_217_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -38409,7 +38350,7 @@ _loop0_217_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_217[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_216[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38426,9 +38367,9 @@ _loop0_217_rule(Parser *p) return _seq; } -// _loop1_218: except_star_block +// _loop1_217: except_star_block static asdl_seq * -_loop1_218_rule(Parser *p) +_loop1_217_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38453,7 +38394,7 @@ _loop1_218_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); + D(fprintf(stderr, "%*c> _loop1_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); excepthandler_ty except_star_block_var; while ( (except_star_block_var = except_star_block_rule(p)) // except_star_block @@ -38476,7 +38417,7 @@ _loop1_218_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_218[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_217[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); } if (_n == 0 || p->error_indicator) { @@ -38498,9 +38439,9 @@ _loop1_218_rule(Parser *p) return _seq; } -// _tmp_219: expression ['as' NAME] +// _tmp_218: expression ['as' NAME] static void * -_tmp_219_rule(Parser *p) +_tmp_218_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38516,22 +38457,22 @@ _tmp_219_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c> _tmp_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_271_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_270_rule(p), !p->error_indicator) // ['as' NAME] ) { - D(fprintf(stderr, "%*c+ _tmp_219[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c+ _tmp_218[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_219[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_218[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' NAME]")); } _res = NULL; @@ -38540,9 +38481,9 @@ _tmp_219_rule(Parser *p) return _res; } -// _tmp_220: 'as' NAME +// _tmp_219: 'as' NAME static void * -_tmp_220_rule(Parser *p) +_tmp_219_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38558,7 +38499,7 @@ _tmp_220_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38567,12 +38508,12 @@ _tmp_220_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_219[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_219[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38581,9 +38522,9 @@ _tmp_220_rule(Parser *p) return _res; } -// _tmp_221: 'as' NAME +// _tmp_220: 'as' NAME static void * -_tmp_221_rule(Parser *p) +_tmp_220_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38599,7 +38540,7 @@ _tmp_221_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38608,12 +38549,12 @@ _tmp_221_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38622,9 +38563,9 @@ _tmp_221_rule(Parser *p) return _res; } -// _tmp_222: NEWLINE | ':' +// _tmp_221: NEWLINE | ':' static void * -_tmp_222_rule(Parser *p) +_tmp_221_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38640,18 +38581,18 @@ _tmp_222_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } { // ':' @@ -38659,18 +38600,18 @@ _tmp_222_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -38679,9 +38620,9 @@ _tmp_222_rule(Parser *p) return _res; } -// _tmp_223: 'as' NAME +// _tmp_222: 'as' NAME static void * -_tmp_223_rule(Parser *p) +_tmp_222_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38697,7 +38638,7 @@ _tmp_223_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38706,12 +38647,12 @@ _tmp_223_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38720,9 +38661,9 @@ _tmp_223_rule(Parser *p) return _res; } -// _tmp_224: 'as' NAME +// _tmp_223: 'as' NAME static void * -_tmp_224_rule(Parser *p) +_tmp_223_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38738,7 +38679,7 @@ _tmp_224_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38747,12 +38688,12 @@ _tmp_224_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38761,9 +38702,9 @@ _tmp_224_rule(Parser *p) return _res; } -// _tmp_225: positional_patterns ',' +// _tmp_224: positional_patterns ',' static void * -_tmp_225_rule(Parser *p) +_tmp_224_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38779,7 +38720,7 @@ _tmp_225_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); Token * _literal; asdl_pattern_seq* positional_patterns_var; if ( @@ -38788,12 +38729,12 @@ _tmp_225_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); _res = _PyPegen_dummy_name(p, positional_patterns_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "positional_patterns ','")); } _res = NULL; @@ -38802,9 +38743,9 @@ _tmp_225_rule(Parser *p) return _res; } -// _tmp_226: '->' expression +// _tmp_225: '->' expression static void * -_tmp_226_rule(Parser *p) +_tmp_225_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38820,7 +38761,7 @@ _tmp_226_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); Token * _literal; expr_ty expression_var; if ( @@ -38829,12 +38770,12 @@ _tmp_226_rule(Parser *p) (expression_var = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); _res = _PyPegen_dummy_name(p, _literal, expression_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); } _res = NULL; @@ -38843,9 +38784,9 @@ _tmp_226_rule(Parser *p) return _res; } -// _tmp_227: '(' arguments? ')' +// _tmp_226: '(' arguments? ')' static void * -_tmp_227_rule(Parser *p) +_tmp_226_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38861,7 +38802,7 @@ _tmp_227_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -38874,12 +38815,12 @@ _tmp_227_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -38888,9 +38829,9 @@ _tmp_227_rule(Parser *p) return _res; } -// _tmp_228: '(' arguments? ')' +// _tmp_227: '(' arguments? ')' static void * -_tmp_228_rule(Parser *p) +_tmp_227_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38906,7 +38847,7 @@ _tmp_228_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -38919,12 +38860,12 @@ _tmp_228_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -38933,9 +38874,9 @@ _tmp_228_rule(Parser *p) return _res; } -// _loop0_230: ',' double_starred_kvpair +// _loop0_229: ',' double_starred_kvpair static asdl_seq * -_loop0_230_rule(Parser *p) +_loop0_229_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38960,7 +38901,7 @@ _loop0_230_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); + D(fprintf(stderr, "%*c> _loop0_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); Token * _literal; KeyValuePair* elem; while ( @@ -38992,7 +38933,7 @@ _loop0_230_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_230[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_229[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -39009,9 +38950,9 @@ _loop0_230_rule(Parser *p) return _seq; } -// _gather_229: double_starred_kvpair _loop0_230 +// _gather_228: double_starred_kvpair _loop0_229 static asdl_seq * -_gather_229_rule(Parser *p) +_gather_228_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39022,27 +38963,27 @@ _gather_229_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // double_starred_kvpair _loop0_230 + { // double_starred_kvpair _loop0_229 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_230")); + D(fprintf(stderr, "%*c> _gather_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_229")); KeyValuePair* elem; asdl_seq * seq; if ( (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair && - (seq = _loop0_230_rule(p)) // _loop0_230 + (seq = _loop0_229_rule(p)) // _loop0_229 ) { - D(fprintf(stderr, "%*c+ _gather_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_230")); + D(fprintf(stderr, "%*c+ _gather_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_229")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_229[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_230")); + D(fprintf(stderr, "%*c%s _gather_228[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_229")); } _res = NULL; done: @@ -39050,9 +38991,9 @@ _gather_229_rule(Parser *p) return _res; } -// _tmp_231: '}' | ',' +// _tmp_230: '}' | ',' static void * -_tmp_231_rule(Parser *p) +_tmp_230_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39068,18 +39009,18 @@ _tmp_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39087,18 +39028,18 @@ _tmp_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39107,9 +39048,9 @@ _tmp_231_rule(Parser *p) return _res; } -// _tmp_232: '}' | ',' +// _tmp_231: '}' | ',' static void * -_tmp_232_rule(Parser *p) +_tmp_231_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39125,18 +39066,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39144,18 +39085,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39164,9 +39105,9 @@ _tmp_232_rule(Parser *p) return _res; } -// _tmp_233: yield_expr | star_expressions +// _tmp_232: yield_expr | star_expressions static void * -_tmp_233_rule(Parser *p) +_tmp_232_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39182,18 +39123,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39201,18 +39142,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39221,9 +39162,9 @@ _tmp_233_rule(Parser *p) return _res; } -// _tmp_234: yield_expr | star_expressions +// _tmp_233: yield_expr | star_expressions static void * -_tmp_234_rule(Parser *p) +_tmp_233_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39239,18 +39180,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39258,18 +39199,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39278,9 +39219,9 @@ _tmp_234_rule(Parser *p) return _res; } -// _tmp_235: '=' | '!' | ':' | '}' +// _tmp_234: '=' | '!' | ':' | '}' static void * -_tmp_235_rule(Parser *p) +_tmp_234_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39296,18 +39237,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // '!' @@ -39315,18 +39256,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39334,18 +39275,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39353,18 +39294,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39373,9 +39314,9 @@ _tmp_235_rule(Parser *p) return _res; } -// _tmp_236: yield_expr | star_expressions +// _tmp_235: yield_expr | star_expressions static void * -_tmp_236_rule(Parser *p) +_tmp_235_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39391,18 +39332,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39410,18 +39351,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39430,9 +39371,9 @@ _tmp_236_rule(Parser *p) return _res; } -// _tmp_237: '!' | ':' | '}' +// _tmp_236: '!' | ':' | '}' static void * -_tmp_237_rule(Parser *p) +_tmp_236_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39448,18 +39389,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39467,18 +39408,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39486,18 +39427,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39506,9 +39447,9 @@ _tmp_237_rule(Parser *p) return _res; } -// _tmp_238: yield_expr | star_expressions +// _tmp_237: yield_expr | star_expressions static void * -_tmp_238_rule(Parser *p) +_tmp_237_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39524,18 +39465,18 @@ _tmp_238_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39543,18 +39484,18 @@ _tmp_238_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39563,9 +39504,9 @@ _tmp_238_rule(Parser *p) return _res; } -// _tmp_239: yield_expr | star_expressions +// _tmp_238: yield_expr | star_expressions static void * -_tmp_239_rule(Parser *p) +_tmp_238_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39581,18 +39522,18 @@ _tmp_239_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39600,18 +39541,18 @@ _tmp_239_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39620,9 +39561,9 @@ _tmp_239_rule(Parser *p) return _res; } -// _tmp_240: '!' NAME +// _tmp_239: '!' NAME static void * -_tmp_240_rule(Parser *p) +_tmp_239_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39638,7 +39579,7 @@ _tmp_240_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39647,12 +39588,12 @@ _tmp_240_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -39661,9 +39602,9 @@ _tmp_240_rule(Parser *p) return _res; } -// _tmp_241: ':' | '}' +// _tmp_240: ':' | '}' static void * -_tmp_241_rule(Parser *p) +_tmp_240_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39679,18 +39620,18 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39698,18 +39639,18 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39718,9 +39659,9 @@ _tmp_241_rule(Parser *p) return _res; } -// _tmp_242: yield_expr | star_expressions +// _tmp_241: yield_expr | star_expressions static void * -_tmp_242_rule(Parser *p) +_tmp_241_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39736,18 +39677,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39755,18 +39696,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39775,9 +39716,9 @@ _tmp_242_rule(Parser *p) return _res; } -// _tmp_243: '!' NAME +// _tmp_242: '!' NAME static void * -_tmp_243_rule(Parser *p) +_tmp_242_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39793,7 +39734,7 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39802,12 +39743,12 @@ _tmp_243_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -39816,9 +39757,9 @@ _tmp_243_rule(Parser *p) return _res; } -// _loop0_244: fstring_format_spec +// _loop0_243: fstring_format_spec static asdl_seq * -_loop0_244_rule(Parser *p) +_loop0_243_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39843,7 +39784,7 @@ _loop0_244_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); + D(fprintf(stderr, "%*c> _loop0_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); expr_ty fstring_format_spec_var; while ( (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec @@ -39866,7 +39807,7 @@ _loop0_244_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_244[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -39883,9 +39824,9 @@ _loop0_244_rule(Parser *p) return _seq; } -// _tmp_245: yield_expr | star_expressions +// _tmp_244: yield_expr | star_expressions static void * -_tmp_245_rule(Parser *p) +_tmp_244_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39901,18 +39842,18 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39920,18 +39861,18 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39940,9 +39881,9 @@ _tmp_245_rule(Parser *p) return _res; } -// _tmp_246: '!' NAME +// _tmp_245: '!' NAME static void * -_tmp_246_rule(Parser *p) +_tmp_245_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39958,7 +39899,7 @@ _tmp_246_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39967,12 +39908,12 @@ _tmp_246_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -39981,9 +39922,9 @@ _tmp_246_rule(Parser *p) return _res; } -// _tmp_247: ':' | '}' +// _tmp_246: ':' | '}' static void * -_tmp_247_rule(Parser *p) +_tmp_246_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39999,18 +39940,18 @@ _tmp_247_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -40018,18 +39959,18 @@ _tmp_247_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -40038,9 +39979,9 @@ _tmp_247_rule(Parser *p) return _res; } -// _tmp_248: star_targets '=' +// _tmp_247: star_targets '=' static void * -_tmp_248_rule(Parser *p) +_tmp_247_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40056,7 +39997,7 @@ _tmp_248_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty z; if ( @@ -40065,7 +40006,7 @@ _tmp_248_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40075,7 +40016,7 @@ _tmp_248_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40084,9 +40025,9 @@ _tmp_248_rule(Parser *p) return _res; } -// _tmp_249: '.' | '...' +// _tmp_248: '.' | '...' static void * -_tmp_249_rule(Parser *p) +_tmp_248_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40102,18 +40043,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40121,18 +40062,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40141,9 +40082,9 @@ _tmp_249_rule(Parser *p) return _res; } -// _tmp_250: '.' | '...' +// _tmp_249: '.' | '...' static void * -_tmp_250_rule(Parser *p) +_tmp_249_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40159,18 +40100,18 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40178,18 +40119,18 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40198,9 +40139,9 @@ _tmp_250_rule(Parser *p) return _res; } -// _tmp_251: '@' named_expression NEWLINE +// _tmp_250: '@' named_expression NEWLINE static void * -_tmp_251_rule(Parser *p) +_tmp_250_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40216,7 +40157,7 @@ _tmp_251_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); Token * _literal; expr_ty f; Token * newline_var; @@ -40228,7 +40169,7 @@ _tmp_251_rule(Parser *p) (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); _res = f; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40238,7 +40179,7 @@ _tmp_251_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); } _res = NULL; @@ -40247,9 +40188,9 @@ _tmp_251_rule(Parser *p) return _res; } -// _tmp_252: ',' expression +// _tmp_251: ',' expression static void * -_tmp_252_rule(Parser *p) +_tmp_251_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40265,7 +40206,7 @@ _tmp_252_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty c; if ( @@ -40274,7 +40215,7 @@ _tmp_252_rule(Parser *p) (c = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40284,7 +40225,7 @@ _tmp_252_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } _res = NULL; @@ -40293,9 +40234,9 @@ _tmp_252_rule(Parser *p) return _res; } -// _tmp_253: ',' star_expression +// _tmp_252: ',' star_expression static void * -_tmp_253_rule(Parser *p) +_tmp_252_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40311,7 +40252,7 @@ _tmp_253_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); Token * _literal; expr_ty c; if ( @@ -40320,7 +40261,7 @@ _tmp_253_rule(Parser *p) (c = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40330,7 +40271,7 @@ _tmp_253_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); } _res = NULL; @@ -40339,9 +40280,9 @@ _tmp_253_rule(Parser *p) return _res; } -// _tmp_254: 'or' conjunction +// _tmp_253: 'or' conjunction static void * -_tmp_254_rule(Parser *p) +_tmp_253_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40357,7 +40298,7 @@ _tmp_254_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); Token * _keyword; expr_ty c; if ( @@ -40366,7 +40307,7 @@ _tmp_254_rule(Parser *p) (c = conjunction_rule(p)) // conjunction ) { - D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40376,7 +40317,7 @@ _tmp_254_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); } _res = NULL; @@ -40385,9 +40326,9 @@ _tmp_254_rule(Parser *p) return _res; } -// _tmp_255: 'and' inversion +// _tmp_254: 'and' inversion static void * -_tmp_255_rule(Parser *p) +_tmp_254_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40403,7 +40344,7 @@ _tmp_255_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); Token * _keyword; expr_ty c; if ( @@ -40412,7 +40353,7 @@ _tmp_255_rule(Parser *p) (c = inversion_rule(p)) // inversion ) { - D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40422,7 +40363,7 @@ _tmp_255_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); } _res = NULL; @@ -40431,9 +40372,9 @@ _tmp_255_rule(Parser *p) return _res; } -// _tmp_256: slice | starred_expression +// _tmp_255: slice | starred_expression static void * -_tmp_256_rule(Parser *p) +_tmp_255_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40449,18 +40390,18 @@ _tmp_256_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); expr_ty slice_var; if ( (slice_var = slice_rule(p)) // slice ) { - D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); _res = slice_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); } { // starred_expression @@ -40468,18 +40409,18 @@ _tmp_256_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } _res = NULL; @@ -40488,9 +40429,9 @@ _tmp_256_rule(Parser *p) return _res; } -// _tmp_257: fstring | string +// _tmp_256: fstring | string static void * -_tmp_257_rule(Parser *p) +_tmp_256_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40506,18 +40447,18 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); expr_ty fstring_var; if ( (fstring_var = fstring_rule(p)) // fstring ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); _res = fstring_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); } { // string @@ -40525,18 +40466,18 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); expr_ty string_var; if ( (string_var = string_rule(p)) // string ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); _res = string_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); } _res = NULL; @@ -40545,9 +40486,9 @@ _tmp_257_rule(Parser *p) return _res; } -// _tmp_258: 'if' disjunction +// _tmp_257: 'if' disjunction static void * -_tmp_258_rule(Parser *p) +_tmp_257_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40563,7 +40504,7 @@ _tmp_258_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -40572,7 +40513,7 @@ _tmp_258_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40582,7 +40523,7 @@ _tmp_258_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -40591,9 +40532,9 @@ _tmp_258_rule(Parser *p) return _res; } -// _tmp_259: 'if' disjunction +// _tmp_258: 'if' disjunction static void * -_tmp_259_rule(Parser *p) +_tmp_258_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40609,7 +40550,7 @@ _tmp_259_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -40618,7 +40559,7 @@ _tmp_259_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40628,7 +40569,7 @@ _tmp_259_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -40637,9 +40578,9 @@ _tmp_259_rule(Parser *p) return _res; } -// _tmp_260: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_259: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_260_rule(Parser *p) +_tmp_259_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40655,18 +40596,18 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -40674,20 +40615,20 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_272_var; + D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_271_var; if ( - (_tmp_272_var = _tmp_272_rule(p)) // assignment_expression | expression !':=' + (_tmp_271_var = _tmp_271_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_272_var; + D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_271_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -40696,9 +40637,9 @@ _tmp_260_rule(Parser *p) return _res; } -// _tmp_261: ',' star_target +// _tmp_260: ',' star_target static void * -_tmp_261_rule(Parser *p) +_tmp_260_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40714,7 +40655,7 @@ _tmp_261_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -40723,7 +40664,7 @@ _tmp_261_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40733,7 +40674,7 @@ _tmp_261_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -40742,9 +40683,9 @@ _tmp_261_rule(Parser *p) return _res; } -// _tmp_262: ',' star_target +// _tmp_261: ',' star_target static void * -_tmp_262_rule(Parser *p) +_tmp_261_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40760,7 +40701,7 @@ _tmp_262_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -40769,7 +40710,7 @@ _tmp_262_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40779,7 +40720,7 @@ _tmp_262_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -40788,9 +40729,9 @@ _tmp_262_rule(Parser *p) return _res; } -// _tmp_263: star_targets '=' +// _tmp_262: star_targets '=' static void * -_tmp_263_rule(Parser *p) +_tmp_262_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40806,7 +40747,7 @@ _tmp_263_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -40815,12 +40756,12 @@ _tmp_263_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40829,9 +40770,9 @@ _tmp_263_rule(Parser *p) return _res; } -// _tmp_264: star_targets '=' +// _tmp_263: star_targets '=' static void * -_tmp_264_rule(Parser *p) +_tmp_263_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40847,7 +40788,7 @@ _tmp_264_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -40856,12 +40797,12 @@ _tmp_264_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40870,9 +40811,9 @@ _tmp_264_rule(Parser *p) return _res; } -// _tmp_265: ')' | '**' +// _tmp_264: ')' | '**' static void * -_tmp_265_rule(Parser *p) +_tmp_264_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40888,18 +40829,18 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // '**' @@ -40907,18 +40848,18 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -40927,9 +40868,9 @@ _tmp_265_rule(Parser *p) return _res; } -// _tmp_266: ':' | '**' +// _tmp_265: ':' | '**' static void * -_tmp_266_rule(Parser *p) +_tmp_265_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40945,18 +40886,18 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '**' @@ -40964,18 +40905,18 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -40984,9 +40925,9 @@ _tmp_266_rule(Parser *p) return _res; } -// _tmp_267: expression ['as' star_target] +// _tmp_266: expression ['as' star_target] static void * -_tmp_267_rule(Parser *p) +_tmp_266_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41002,22 +40943,22 @@ _tmp_267_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_273_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_272_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41026,9 +40967,9 @@ _tmp_267_rule(Parser *p) return _res; } -// _tmp_268: expressions ['as' star_target] +// _tmp_267: expressions ['as' star_target] static void * -_tmp_268_rule(Parser *p) +_tmp_267_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41044,22 +40985,22 @@ _tmp_268_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_274_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_273_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41068,9 +41009,9 @@ _tmp_268_rule(Parser *p) return _res; } -// _tmp_269: expression ['as' star_target] +// _tmp_268: expression ['as' star_target] static void * -_tmp_269_rule(Parser *p) +_tmp_268_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41086,22 +41027,22 @@ _tmp_269_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_275_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_274_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41110,9 +41051,9 @@ _tmp_269_rule(Parser *p) return _res; } -// _tmp_270: expressions ['as' star_target] +// _tmp_269: expressions ['as' star_target] static void * -_tmp_270_rule(Parser *p) +_tmp_269_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41128,22 +41069,22 @@ _tmp_270_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_276_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_275_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41152,9 +41093,9 @@ _tmp_270_rule(Parser *p) return _res; } -// _tmp_271: 'as' NAME +// _tmp_270: 'as' NAME static void * -_tmp_271_rule(Parser *p) +_tmp_270_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41170,7 +41111,7 @@ _tmp_271_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -41179,12 +41120,12 @@ _tmp_271_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -41193,9 +41134,9 @@ _tmp_271_rule(Parser *p) return _res; } -// _tmp_272: assignment_expression | expression !':=' +// _tmp_271: assignment_expression | expression !':=' static void * -_tmp_272_rule(Parser *p) +_tmp_271_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41211,18 +41152,18 @@ _tmp_272_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -41230,7 +41171,7 @@ _tmp_272_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -41238,12 +41179,12 @@ _tmp_272_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -41252,9 +41193,9 @@ _tmp_272_rule(Parser *p) return _res; } -// _tmp_273: 'as' star_target +// _tmp_272: 'as' star_target static void * -_tmp_273_rule(Parser *p) +_tmp_272_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41270,7 +41211,7 @@ _tmp_273_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41279,12 +41220,12 @@ _tmp_273_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41293,9 +41234,9 @@ _tmp_273_rule(Parser *p) return _res; } -// _tmp_274: 'as' star_target +// _tmp_273: 'as' star_target static void * -_tmp_274_rule(Parser *p) +_tmp_273_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41311,7 +41252,7 @@ _tmp_274_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41320,12 +41261,12 @@ _tmp_274_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_274[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41334,9 +41275,9 @@ _tmp_274_rule(Parser *p) return _res; } -// _tmp_275: 'as' star_target +// _tmp_274: 'as' star_target static void * -_tmp_275_rule(Parser *p) +_tmp_274_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41352,7 +41293,7 @@ _tmp_275_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41361,12 +41302,12 @@ _tmp_275_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_275[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_275[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_274[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41375,9 +41316,9 @@ _tmp_275_rule(Parser *p) return _res; } -// _tmp_276: 'as' star_target +// _tmp_275: 'as' star_target static void * -_tmp_276_rule(Parser *p) +_tmp_275_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41393,7 +41334,7 @@ _tmp_276_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_276[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41402,12 +41343,12 @@ _tmp_276_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_276[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_275[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_276[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_275[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; From 15d659f929bb2a79fa7211947ef4f2c43818fd31 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 8 Sep 2023 22:06:34 +0200 Subject: [PATCH 21/66] gh-91960: FreeBSD Cirrus CI runs configure separately (#109127) Run configure and make in separated steps to have more readable logs. --- .cirrus.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 823b1f921d66d1..ca41c2e9092fa6 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -10,13 +10,16 @@ freebsd_task: sysctl_script: - sysctl net.inet.tcp.blackhole=0 - sysctl net.inet.udp.blackhole=0 - build_script: + configure_script: - mkdir build - cd build - ../configure --with-pydebug + build_script: + - cd build - make -j$(sysctl -n hw.ncpu) pythoninfo_script: - - cd build && make pythoninfo + - cd build + - make pythoninfo test_script: - cd build # dtrace fails to build on FreeBSD - see gh-73263 From 697c9dcf8fc746636c6187e4f110e0e6e865b710 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 8 Sep 2023 22:05:40 +0100 Subject: [PATCH 22/66] gh-108455: peg_generator: enable mypy's `--warn-unreachable` setting and `redundant-expr` error code (#109160) --- Tools/peg_generator/mypy.ini | 14 +++++++++++--- Tools/peg_generator/pegen/ast_dump.py | 2 -- Tools/peg_generator/pegen/grammar.py | 6 ++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Tools/peg_generator/mypy.ini b/Tools/peg_generator/mypy.ini index 8820d77b36f646..f38f21bed004b2 100644 --- a/Tools/peg_generator/mypy.ini +++ b/Tools/peg_generator/mypy.ini @@ -8,8 +8,16 @@ python_version = 3.10 # Be strict... strict = True -enable_error_code = truthy-bool,ignore-without-code +warn_unreachable = True +enable_error_code = truthy-bool,ignore-without-code,redundant-expr -# except for a few settings that can't yet be enabled: +# This causes *many* false positives on the peg_generator +# due to pegen.grammar.GrammarVisitor returning Any from visit() and generic_visit(). +# It would be possible to workaround the false positives using asserts, +# but it would be pretty tedious, and probably isn't worth it. warn_return_any = False -warn_unreachable = False + +# Not all of the strictest settings can be enabled +# on generated Python code yet: +[mypy-pegen.grammar_parser.*] +disable_error_code = redundant-expr diff --git a/Tools/peg_generator/pegen/ast_dump.py b/Tools/peg_generator/pegen/ast_dump.py index 2c57d0932fda37..07f8799c114f5d 100644 --- a/Tools/peg_generator/pegen/ast_dump.py +++ b/Tools/peg_generator/pegen/ast_dump.py @@ -66,6 +66,4 @@ def _format(node: Any, level: int = 0) -> Tuple[str, bool]: if all(cls.__name__ != "AST" for cls in node.__class__.__mro__): raise TypeError("expected AST, got %r" % node.__class__.__name__) - if indent is not None and not isinstance(indent, str): - indent = " " * indent return _format(node)[0] diff --git a/Tools/peg_generator/pegen/grammar.py b/Tools/peg_generator/pegen/grammar.py index fcf868eb1753e5..065894e7fe66ab 100644 --- a/Tools/peg_generator/pegen/grammar.py +++ b/Tools/peg_generator/pegen/grammar.py @@ -112,8 +112,7 @@ def __str__(self) -> str: return self.value def __iter__(self) -> Iterable[str]: - if False: - yield + yield from () class NameLeaf(Leaf): @@ -335,8 +334,7 @@ def __str__(self) -> str: return f"~" def __iter__(self) -> Iterator[Tuple[str, str]]: - if False: - yield + yield from () def __eq__(self, other: object) -> bool: if not isinstance(other, Cut): From 1f7e42131d2800f0fbb89bfd91fafa8a073e066d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 8 Sep 2023 23:14:33 +0200 Subject: [PATCH 23/66] gh-109054: configure checks if libatomic is needed (#109101) Fix building the _testcapi extension on Linux AArch64 which requires linking to libatomic when is used: the _Py_atomic_or_uint64() function requires libatomic __atomic_fetch_or_8() on this platform. The configure script now checks if linking to libatomic is needed and generates a new LIBATOMIC variable used to build the _testcapi extension. Building the _testcapi extension now uses the LIBATOMIC variable in its LDFLAGS, since Modules/_testcapi/pyatomic.c uses . Co-authored-by: Erlend E. Aasland --- ...-09-07-19-58-05.gh-issue-109054.5r3S3l.rst | 6 ++ configure | 85 ++++++++++++++++++- configure.ac | 62 +++++++++++++- 3 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-09-07-19-58-05.gh-issue-109054.5r3S3l.rst diff --git a/Misc/NEWS.d/next/Build/2023-09-07-19-58-05.gh-issue-109054.5r3S3l.rst b/Misc/NEWS.d/next/Build/2023-09-07-19-58-05.gh-issue-109054.5r3S3l.rst new file mode 100644 index 00000000000000..d86a110e0de68c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-09-07-19-58-05.gh-issue-109054.5r3S3l.rst @@ -0,0 +1,6 @@ +Fix building the ``_testcapi`` extension on Linux AArch64 which requires +linking to libatomic when ```` is used: the +``_Py_atomic_or_uint64()`` function requires libatomic +``__atomic_fetch_or_8()`` on this platform. The configure script now checks +if linking to libatomic is needed and generates a new LIBATOMIC variable +used to build the _testcapi extension. Patch by Victor Stinner. diff --git a/configure b/configure index d73b4b271ac719..c78c45d11260f6 100755 --- a/configure +++ b/configure @@ -27752,6 +27752,88 @@ printf "%s\n" "#define Py_NOGIL 1" >>confdefs.h fi +# gh-109054: Check if -latomic is needed to get atomic functions. +# On Linux aarch64, GCC may require programs and libraries to be linked +# explicitly to libatomic. Call _Py_atomic_or_uint64() which may require +# libatomic __atomic_fetch_or_8(), or not, depending on the C compiler and the +# compiler flags. +# +# Avoid #include or #include . The header +# requires header which is only written below by AC_OUTPUT below. +# If the check is done after AC_OUTPUT, modifying LIBATOMIC has no effect +# anymore. cannot be included alone, it's designed to be included +# by : it expects other includes and macros to be defined. +save_CPPFLAGS=$CPPFLAGS +CPPFLAGS="${BASECPPFLAGS} -I. -I${srcdir}/Include ${CPPFLAGS}" + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether libatomic is needed by " >&5 +printf %s "checking whether libatomic is needed by ... " >&6; } +if test ${ac_cv_libatomic_needed+y} +then : + printf %s "(cached) " >&6 +else $as_nop + if test "$cross_compiling" = yes +then : + ac_cv_libatomic_needed=yes +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +// pyatomic.h needs uint64_t and Py_ssize_t types +#include // int64_t, intptr_t +#ifdef HAVE_SYS_TYPES_H +# include // ssize_t +#endif +// Code adapted from Include/pyport.h +#if HAVE_SSIZE_T +typedef ssize_t Py_ssize_t; +#elif SIZEOF_VOID_P == SIZEOF_SIZE_T +typedef intptr_t Py_ssize_t; +#else +# error "unable to define Py_ssize_t" +#endif + +#include "cpython/pyatomic.h" + +int main() +{ + uint64_t byte; + _Py_atomic_store_uint64(&byte, 2); + if (_Py_atomic_or_uint64(&byte, 8) != 2) { + return 1; // error + } + if (_Py_atomic_load_uint64(&byte) != 10) { + return 1; // error + } + return 0; // all good +} + +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + + ac_cv_libatomic_needed=no + +else $as_nop + ac_cv_libatomic_needed=yes +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_libatomic_needed" >&5 +printf "%s\n" "$ac_cv_libatomic_needed" >&6; } + +if test "x$ac_cv_libatomic_needed" = xyes +then : + LIBATOMIC=${LIBATOMIC-"-latomic"} +fi +CPPFLAGS=$save_CPPFLAGS + + +# stdlib # stdlib not available @@ -29900,7 +29982,7 @@ fi then : - + as_fn_append MODULE_BLOCK "MODULE__TESTCAPI_LDFLAGS=$LIBATOMIC$as_nl" fi if test "$py_cv_module__testcapi" = yes; then @@ -30344,6 +30426,7 @@ ac_config_files="$ac_config_files Modules/Setup.bootstrap Modules/Setup.stdlib" ac_config_files="$ac_config_files Modules/ld_so_aix" +# Generate files like pyconfig.h cat >confcache <<\_ACEOF # This file is a shell script that caches the results of configure # tests run on this system so they can be shared between configure diff --git a/configure.ac b/configure.ac index 612c072af329d4..f833755a466012 100644 --- a/configure.ac +++ b/configure.ac @@ -6962,6 +6962,62 @@ then [Define if you want to disable the GIL]) fi +# gh-109054: Check if -latomic is needed to get atomic functions. +# On Linux aarch64, GCC may require programs and libraries to be linked +# explicitly to libatomic. Call _Py_atomic_or_uint64() which may require +# libatomic __atomic_fetch_or_8(), or not, depending on the C compiler and the +# compiler flags. +# +# Avoid #include or #include . The header +# requires header which is only written below by AC_OUTPUT below. +# If the check is done after AC_OUTPUT, modifying LIBATOMIC has no effect +# anymore. cannot be included alone, it's designed to be included +# by : it expects other includes and macros to be defined. +_SAVE_VAR([CPPFLAGS]) +CPPFLAGS="${BASECPPFLAGS} -I. -I${srcdir}/Include ${CPPFLAGS}" + +AC_CACHE_CHECK([whether libatomic is needed by ], + [ac_cv_libatomic_needed], +[AC_RUN_IFELSE([AC_LANG_SOURCE([[ +// pyatomic.h needs uint64_t and Py_ssize_t types +#include // int64_t, intptr_t +#ifdef HAVE_SYS_TYPES_H +# include // ssize_t +#endif +// Code adapted from Include/pyport.h +#if HAVE_SSIZE_T +typedef ssize_t Py_ssize_t; +#elif SIZEOF_VOID_P == SIZEOF_SIZE_T +typedef intptr_t Py_ssize_t; +#else +# error "unable to define Py_ssize_t" +#endif + +#include "cpython/pyatomic.h" + +int main() +{ + uint64_t byte; + _Py_atomic_store_uint64(&byte, 2); + if (_Py_atomic_or_uint64(&byte, 8) != 2) { + return 1; // error + } + if (_Py_atomic_load_uint64(&byte) != 10) { + return 1; // error + } + return 0; // all good +} +]])],[ + ac_cv_libatomic_needed=no +],[ac_cv_libatomic_needed=yes],[ac_cv_libatomic_needed=yes]) +]) + +AS_VAR_IF([ac_cv_libatomic_needed], [yes], + [LIBATOMIC=${LIBATOMIC-"-latomic"}]) +_RESTORE_VAR([CPPFLAGS]) + + +# stdlib AC_DEFUN([PY_STDLIB_MOD_SET_NA], [ m4_foreach([mod], [$@], [ AS_VAR_SET([py_cv_module_]mod, [n/a])]) @@ -7229,7 +7285,10 @@ PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) dnl test modules -PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testcapi], + [test "$TEST_MODULES" = yes], [] + dnl Modules/_testcapi/pyatomic.c uses header + [], [], [$LIBATOMIC]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic_limited], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) @@ -7262,6 +7321,7 @@ AC_CONFIG_FILES(m4_normalize([ Modules/Setup.stdlib ])) AC_CONFIG_FILES([Modules/ld_so_aix], [chmod +x Modules/ld_so_aix]) +# Generate files like pyconfig.h AC_OUTPUT AC_MSG_NOTICE([creating Modules/Setup.local]) From bcb2ab5ef8c646565b09c860fb14e415d7b374bd Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 9 Sep 2023 06:38:38 +0800 Subject: [PATCH 24/66] gh-108996: add tests for msvcrt (#109004) Co-authored-by: Terry Jan Reedy Co-authored-by: Steve Dower --- Lib/test/test_msvcrt.py | 111 ++++++++++++++++++ ...-09-06-22-06-22.gh-issue-108996.IBhR3U.rst | 1 + 2 files changed, 112 insertions(+) create mode 100644 Lib/test/test_msvcrt.py create mode 100644 Misc/NEWS.d/next/Tests/2023-09-06-22-06-22.gh-issue-108996.IBhR3U.rst diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py new file mode 100644 index 00000000000000..adf4e26b98ac55 --- /dev/null +++ b/Lib/test/test_msvcrt.py @@ -0,0 +1,111 @@ +import os +import sys +import unittest + +from test.support import os_helper +from test.support.os_helper import TESTFN, TESTFN_ASCII + +if sys.platform != "win32": + raise unittest.SkipTest("windows related tests") + +import _winapi +import msvcrt; + +from _testconsole import write_input + + +class TestFileOperations(unittest.TestCase): + def test_locking(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + self.assertRaises(OSError, msvcrt.locking, f.fileno(), msvcrt.LK_NBLCK, 1) + + def test_unlockfile(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1) + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + + def test_setmode(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.setmode(f.fileno(), os.O_BINARY) + msvcrt.setmode(f.fileno(), os.O_TEXT) + + def test_open_osfhandle(self): + h = _winapi.CreateFile(TESTFN_ASCII, _winapi.GENERIC_WRITE, 0, 0, 1, 128, 0) + self.addCleanup(os_helper.unlink, TESTFN_ASCII) + + try: + fd = msvcrt.open_osfhandle(h, os.O_RDONLY) + h = None + os.close(fd) + finally: + if h: + _winapi.CloseHandle(h) + + def test_get_osfhandle(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.get_osfhandle(f.fileno()) + + +c = '\u5b57' # unicode CJK char (meaning 'character') for 'wide-char' tests +c_encoded = b'\x57\x5b' # utf-16-le (which windows internally used) encoded char for this CJK char + + +class TestConsoleIO(unittest.TestCase): + def test_kbhit(self): + self.assertEqual(msvcrt.kbhit(), 0) + + def test_getch(self): + msvcrt.ungetch(b'c') + self.assertEqual(msvcrt.getch(), b'c') + + def test_getwch(self): + stdin = open('CONIN$', 'r') + old_stdin = sys.stdin + try: + sys.stdin = stdin + write_input(stdin.buffer.raw, c_encoded) + self.assertEqual(msvcrt.getwch(), c) + finally: + sys.stdin = old_stdin + + def test_getche(self): + msvcrt.ungetch(b'c') + self.assertEqual(msvcrt.getche(), b'c') + + def test_getwche(self): + stdin = open('CONIN$', 'r') + old_stdin = sys.stdin + try: + sys.stdin = stdin + write_input(stdin.buffer.raw, c_encoded) + self.assertEqual(msvcrt.getwche(), c) + finally: + sys.stdin = old_stdin + + def test_putch(self): + msvcrt.putch(b'c') + + def test_putwch(self): + msvcrt.putwch(c) + + +class TestOther(unittest.TestCase): + def test_heap_min(self): + try: + msvcrt.heapmin() + except OSError: + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2023-09-06-22-06-22.gh-issue-108996.IBhR3U.rst b/Misc/NEWS.d/next/Tests/2023-09-06-22-06-22.gh-issue-108996.IBhR3U.rst new file mode 100644 index 00000000000000..887f8b74bcfa30 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-06-22-06-22.gh-issue-108996.IBhR3U.rst @@ -0,0 +1 @@ +Add tests for ``msvcrt``. From 5b7303e2653a0723a3e4c767d03dd02681206ca8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 00:41:26 +0200 Subject: [PATCH 25/66] gh-109162: Refactor Regrtest.main() (#109163) * main() now calls _parse_args() and pass 'ns' to Regrtest constructor. Remove kwargs argument from Regrtest.main(). * _parse_args() checks ns.huntrleaks. * set_temp_dir() is now responsible to call expanduser(). * Regrtest.main() sets self.tests earlier. * Add TestTuple and TestList types. * Rename MatchTests to FilterTuple and rename MatchTestsDict to FilterTestDict. * TestResult.get_rerun_match_tests() return type is now FilterTuple: return a tuple instead of a list. RunTests.tests type becomes TestTuple. --- Lib/test/libregrtest/cmdline.py | 9 ++++ Lib/test/libregrtest/main.py | 87 +++++++++++++----------------- Lib/test/libregrtest/runtest.py | 26 +++++---- Lib/test/libregrtest/runtest_mp.py | 4 +- 4 files changed, 64 insertions(+), 62 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index d1a590d8c1a5b3..bbac980a0bf5d8 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -448,4 +448,13 @@ def _parse_args(args, **kwargs): # --forever implies --failfast ns.failfast = True + if ns.huntrleaks: + warmup, repetitions, _ = ns.huntrleaks + if warmup < 1 or repetitions < 1: + msg = ("Invalid values for the --huntrleaks/-R parameters. The " + "number of warmups and repetitions must be at least 1 " + "each (1:1).") + print(msg, file=sys.stderr, flush=True) + sys.exit(2) + return ns diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index ab03647ca5802f..f973f03ab26512 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -9,10 +9,10 @@ import tempfile import time import unittest -from test.libregrtest.cmdline import _parse_args +from test.libregrtest.cmdline import _parse_args, Namespace from test.libregrtest.runtest import ( findtests, split_test_packages, runtest, abs_module_name, - PROGRESS_MIN_TIME, State, MatchTestsDict, RunTests) + PROGRESS_MIN_TIME, State, FilterDict, RunTests, TestResult, TestList) from test.libregrtest.setup import setup_tests from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.utils import (strip_py_suffix, count, format_duration, @@ -58,9 +58,9 @@ class Regrtest: directly to set the values that would normally be set by flags on the command line. """ - def __init__(self): + def __init__(self, ns: Namespace): # Namespace of command line options - self.ns = None + self.ns: Namespace = ns # tests self.tests = [] @@ -68,14 +68,14 @@ def __init__(self): self.all_runtests: list[RunTests] = [] # test results - self.good: list[str] = [] - self.bad: list[str] = [] - self.rerun_bad: list[str] = [] - self.skipped: list[str] = [] - self.resource_denied: list[str] = [] - self.environment_changed: list[str] = [] - self.run_no_tests: list[str] = [] - self.rerun: list[str] = [] + self.good: TestList = [] + self.bad: TestList = [] + self.rerun_bad: TestList = [] + self.skipped: TestList = [] + self.resource_denied: TestList = [] + self.environment_changed: TestList = [] + self.run_no_tests: TestList = [] + self.rerun: TestList = [] self.need_rerun: list[TestResult] = [] self.first_state: str | None = None @@ -184,29 +184,7 @@ def display_progress(self, test_index, text): line = f"{line}/{fails}" self.log(f"[{line}] {text}") - def parse_args(self, kwargs): - ns = _parse_args(sys.argv[1:], **kwargs) - - if ns.xmlpath: - support.junit_xml_list = self.testsuite_xml = [] - - strip_py_suffix(ns.args) - - if ns.huntrleaks: - warmup, repetitions, _ = ns.huntrleaks - if warmup < 1 or repetitions < 1: - msg = ("Invalid values for the --huntrleaks/-R parameters. The " - "number of warmups and repetitions must be at least 1 " - "each (1:1).") - print(msg, file=sys.stderr, flush=True) - sys.exit(2) - - if ns.tempdir: - ns.tempdir = os.path.expanduser(ns.tempdir) - - self.ns = ns - - def find_tests(self, tests): + def find_tests(self): ns = self.ns single = ns.single fromfile = ns.fromfile @@ -216,8 +194,6 @@ def find_tests(self, tests): starting_test = ns.start randomize = ns.randomize - self.tests = tests - if single: self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') try: @@ -321,7 +297,7 @@ def list_cases(self): print(count(len(skipped), "test"), "skipped:", file=stderr) printlist(skipped, file=stderr) - def get_rerun_match(self, rerun_list) -> MatchTestsDict: + def get_rerun_match(self, rerun_list) -> FilterDict: rerun_match_tests = {} for result in rerun_list: match_tests = result.get_rerun_match_tests() @@ -352,7 +328,7 @@ def _rerun_failed_tests(self, need_rerun): # Re-run failed tests self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") - runtests = RunTests(tests, match_tests=match_tests, rerun=True) + runtests = RunTests(tuple(tests), match_tests=match_tests, rerun=True) self.all_runtests.append(runtests) self._run_tests_mp(runtests) @@ -624,7 +600,7 @@ def run_tests(self): tests = self.selected self.set_tests(tests) - runtests = RunTests(tests, forever=self.ns.forever) + runtests = RunTests(tuple(tests), forever=self.ns.forever) self.all_runtests.append(runtests) if self.ns.use_mp: self._run_tests_mp(runtests) @@ -737,8 +713,12 @@ def fix_umask(self): os.umask(old_mask) def set_temp_dir(self): - if self.ns.tempdir: - self.tmp_dir = self.ns.tempdir + ns = self.ns + if ns.tempdir: + ns.tempdir = os.path.expanduser(ns.tempdir) + + if ns.tempdir: + self.tmp_dir = ns.tempdir if not self.tmp_dir: # When tests are run from the Python build directory, it is best practice @@ -795,14 +775,20 @@ def cleanup(self): print("Remove file: %s" % name) os_helper.unlink(name) - def main(self, tests=None, **kwargs): - self.parse_args(kwargs) + def main(self, tests: TestList | None = None): + ns = self.ns + self.tests = tests + + if ns.xmlpath: + support.junit_xml_list = self.testsuite_xml = [] + + strip_py_suffix(ns.args) self.set_temp_dir() self.fix_umask() - if self.ns.cleanup: + if ns.cleanup: self.cleanup() sys.exit(0) @@ -817,9 +803,9 @@ def main(self, tests=None, **kwargs): # When using multiprocessing, worker processes will use test_cwd # as their parent temporary directory. So when the main process # exit, it removes also subdirectories of worker processes. - self.ns.tempdir = test_cwd + ns.tempdir = test_cwd - self._main(tests, kwargs) + self._main() except SystemExit as exc: # bpo-38203: Python can hang at exit in Py_Finalize(), especially # on threading._shutdown() call: put a timeout @@ -862,7 +848,7 @@ def action_run_tests(self): self.display_summary() self.finalize() - def _main(self, tests, kwargs): + def _main(self): if self.is_worker(): from test.libregrtest.runtest_mp import run_tests_worker run_tests_worker(self.ns.worker_args) @@ -872,7 +858,7 @@ def _main(self, tests, kwargs): input("Press any key to continue...") setup_tests(self.ns) - self.find_tests(tests) + self.find_tests() exitcode = 0 if self.ns.list_tests: @@ -888,4 +874,5 @@ def _main(self, tests, kwargs): def main(tests=None, **kwargs): """Run the Python suite.""" - Regrtest().main(tests=tests, **kwargs) + ns = _parse_args(sys.argv[1:], **kwargs) + Regrtest(ns).main(tests=tests) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 16ae04191da768..7e4b2e6a36b452 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -19,8 +19,13 @@ from test.libregrtest.utils import clear_caches, format_duration, print_warning -MatchTests = list[str] -MatchTestsDict = dict[str, MatchTests] +TestTuple = list[str] +TestList = list[str] + +# --match and --ignore options: list of patterns +# ('*' joker character can be used) +FilterTuple = tuple[str, ...] +FilterDict = dict[str, FilterTuple] # Avoid enum.Enum to reduce the number of imports when tests are run @@ -174,7 +179,7 @@ def must_stop(self, fail_fast: bool, fail_env_changed: bool) -> bool: return True return False - def get_rerun_match_tests(self): + def get_rerun_match_tests(self) -> FilterTuple | None: match_tests = [] errors = self.errors or [] @@ -195,29 +200,30 @@ def get_rerun_match_tests(self): return None match_tests.append(match_name) - return match_tests + if not match_tests: + return None + return tuple(match_tests) @dataclasses.dataclass(slots=True, frozen=True) class RunTests: - tests: list[str] - match_tests: MatchTestsDict | None = None + tests: TestTuple + match_tests: FilterDict | None = None rerun: bool = False forever: bool = False - def get_match_tests(self, test_name) -> MatchTests | None: + def get_match_tests(self, test_name) -> FilterTuple | None: if self.match_tests is not None: return self.match_tests.get(test_name, None) else: return None def iter_tests(self): - tests = tuple(self.tests) if self.forever: while True: - yield from tests + yield from self.tests else: - yield from tests + yield from self.tests # Minimum duration of a test to display its duration or to mention that diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 60089554cab5dd..2ecdfca0e77010 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -20,7 +20,7 @@ from test.libregrtest.main import Regrtest from test.libregrtest.runtest import ( runtest, TestResult, State, PROGRESS_MIN_TIME, - MatchTests, RunTests) + FilterTuple, RunTests) from test.libregrtest.setup import setup_tests from test.libregrtest.utils import format_duration, print_warning @@ -49,7 +49,7 @@ class WorkerJob: test_name: str namespace: Namespace rerun: bool = False - match_tests: MatchTests | None = None + match_tests: FilterTuple | None = None class _EncodeWorkerJob(json.JSONEncoder): From ac8409b38b5a11d88b2cfe4e38a712e8967ec843 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 01:48:54 +0200 Subject: [PATCH 26/66] gh-109162: Regrtest copies 'ns' attributes (#109168) * Regrtest.__init__() now copies 'ns' namespace attributes to Regrtest attributes. Regrtest match_tests and ignore_tests attributes have type FilterTuple (tuple), instead of a list. * Add RunTests.copy(). Regrtest._rerun_failed_tests() now uses RunTests.copy(). * Replace Regrtest.all_tests (list) with Regrtest.first_runtests (RunTests). * Make random_seed maximum 10x larger (9 digits, instead of 8). --- Lib/test/libregrtest/main.py | 114 +++++++++++++++++++------------- Lib/test/libregrtest/runtest.py | 5 ++ Lib/test/test_regrtest.py | 2 +- 3 files changed, 73 insertions(+), 48 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f973f03ab26512..bf71ec3b1b08be 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -12,7 +12,8 @@ from test.libregrtest.cmdline import _parse_args, Namespace from test.libregrtest.runtest import ( findtests, split_test_packages, runtest, abs_module_name, - PROGRESS_MIN_TIME, State, FilterDict, RunTests, TestResult, TestList) + PROGRESS_MIN_TIME, State, RunTests, TestResult, + FilterTuple, FilterDict, TestList) from test.libregrtest.setup import setup_tests from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.utils import (strip_py_suffix, count, format_duration, @@ -62,10 +63,35 @@ def __init__(self, ns: Namespace): # Namespace of command line options self.ns: Namespace = ns + # Actions + self.want_header = ns.header + self.want_list_tests = ns.list_tests + self.want_list_cases = ns.list_cases + self.want_wait = ns.wait + self.want_cleanup = ns.cleanup + + # Select tests + if ns.match_tests: + self.match_tests: FilterTuple = tuple(ns.match_tests) + else: + self.match_tests = None + if ns.ignore_tests: + self.ignore_tests: FilterTuple = tuple(ns.ignore_tests) + else: + self.ignore_tests = None + self.exclude = ns.exclude + self.fromfile = ns.fromfile + self.starting_test = ns.start + + # Options to run tests + self.forever = ns.forever + self.randomize = ns.randomize + self.random_seed = ns.random_seed + # tests self.tests = [] self.selected = [] - self.all_runtests: list[RunTests] = [] + self.first_runtests: RunTests | None = None # test results self.good: TestList = [] @@ -187,12 +213,8 @@ def display_progress(self, test_index, text): def find_tests(self): ns = self.ns single = ns.single - fromfile = ns.fromfile pgo = ns.pgo - exclude = ns.exclude test_dir = ns.testdir - starting_test = ns.start - randomize = ns.randomize if single: self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') @@ -203,12 +225,12 @@ def find_tests(self): except OSError: pass - if fromfile: + if self.fromfile: self.tests = [] # regex to match 'test_builtin' in line: # '0:00:00 [ 4/400] test_builtin -- test_dict took 1 sec' regex = re.compile(r'\btest_[a-zA-Z0-9_]+\b') - with open(os.path.join(os_helper.SAVEDCWD, fromfile)) as fp: + with open(os.path.join(os_helper.SAVEDCWD, self.fromfile)) as fp: for line in fp: line = line.split('#', 1)[0] line = line.strip() @@ -223,14 +245,14 @@ def find_tests(self): setup_pgo_tests(ns) exclude_tests = set() - if exclude: + if self.exclude: for arg in ns.args: exclude_tests.add(arg) ns.args = [] alltests = findtests(testdir=test_dir, exclude=exclude_tests) - if not fromfile: + if not self.fromfile: self.selected = self.tests or ns.args if self.selected: self.selected = split_test_packages(self.selected) @@ -248,17 +270,17 @@ def find_tests(self): pass # Remove all the selected tests that precede start if it's set. - if starting_test: + if self.starting_test: try: - del self.selected[:self.selected.index(starting_test)] + del self.selected[:self.selected.index(self.starting_test)] except ValueError: - print(f"Cannot find starting test: {starting_test}") + print(f"Cannot find starting test: {self.starting_test}") sys.exit(1) - if randomize: - if ns.random_seed is None: - ns.random_seed = random.randrange(10000000) - random.seed(ns.random_seed) + if self.randomize: + if self.random_seed is None: + self.random_seed = random.randrange(100_000_000) + random.seed(self.random_seed) random.shuffle(self.selected) def list_tests(self): @@ -279,7 +301,7 @@ def list_cases(self): ns = self.ns test_dir = ns.testdir support.verbose = False - support.set_match_tests(ns.match_tests, ns.ignore_tests) + support.set_match_tests(self.match_tests, self.ignore_tests) skipped = [] for test_name in self.selected: @@ -306,20 +328,18 @@ def get_rerun_match(self, rerun_list) -> FilterDict: rerun_match_tests[result.test_name] = match_tests return rerun_match_tests - def _rerun_failed_tests(self, need_rerun): + def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Configure the runner to re-run tests ns = self.ns ns.verbose = True ns.failfast = False ns.verbose3 = False - ns.forever = False if ns.use_mp is None: ns.use_mp = 1 # Get tests to re-run tests = [result.test_name for result in need_rerun] match_tests = self.get_rerun_match(need_rerun) - self.set_tests(tests) # Clear previously failed tests self.rerun_bad.extend(self.bad) @@ -328,11 +348,14 @@ def _rerun_failed_tests(self, need_rerun): # Re-run failed tests self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") - runtests = RunTests(tuple(tests), match_tests=match_tests, rerun=True) - self.all_runtests.append(runtests) + runtests = runtests.copy(tests=tuple(tests), + match_tests=match_tests, + rerun=True, + forever=False) + self.set_tests(runtests) self._run_tests_mp(runtests) - def rerun_failed_tests(self, need_rerun): + def rerun_failed_tests(self, need_rerun, runtests: RunTests): if self.ns.python: # Temp patch for https://github.com/python/cpython/issues/94052 self.log( @@ -344,7 +367,7 @@ def rerun_failed_tests(self, need_rerun): self.first_state = self.get_tests_state() print() - self._rerun_failed_tests(need_rerun) + self._rerun_failed_tests(need_rerun, runtests) if self.bad: print(count(len(self.bad), 'test'), "failed again:") @@ -572,9 +595,9 @@ def _run_tests_mp(self, runtests: RunTests) -> None: self.win_load_tracker.close() self.win_load_tracker = None - def set_tests(self, tests): - self.tests = tests - if self.ns.forever: + def set_tests(self, runtests: RunTests): + self.tests = runtests.tests + if runtests.forever: self.test_count_text = '' self.test_count_width = 3 else: @@ -583,7 +606,7 @@ def set_tests(self, tests): def run_tests(self): # For a partial run, we do not need to clutter the output. - if (self.ns.header + if (self.want_header or not(self.ns.pgo or self.ns.quiet or self.ns.single or self.tests or self.ns.args)): self.display_header() @@ -595,17 +618,18 @@ def run_tests(self): "3 warmup repetitions can give false positives!") print(msg, file=sys.stdout, flush=True) - if self.ns.randomize: - print("Using random seed", self.ns.random_seed) + if self.randomize: + print("Using random seed", self.random_seed) tests = self.selected - self.set_tests(tests) - runtests = RunTests(tuple(tests), forever=self.ns.forever) - self.all_runtests.append(runtests) + runtests = RunTests(tuple(tests), forever=self.forever) + self.first_runtests = runtests + self.set_tests(runtests) if self.ns.use_mp: self._run_tests_mp(runtests) else: self.run_tests_sequentially(runtests) + return runtests def finalize(self): if self.next_single_filename: @@ -627,11 +651,7 @@ def finalize(self): def display_summary(self): duration = time.perf_counter() - self.start_time - first_runtests = self.all_runtests[0] - # the second runtests (re-run failed tests) disables forever, - # use the first runtests - forever = first_runtests.forever - filtered = bool(self.ns.match_tests) or bool(self.ns.ignore_tests) + filtered = bool(self.match_tests) or bool(self.ignore_tests) # Total duration print() @@ -655,8 +675,8 @@ def display_summary(self): self.environment_changed, self.run_no_tests] run = sum(map(len, all_tests)) text = f'run={run}' - if not forever: - ntest = len(first_runtests.tests) + if not self.first_runtests.forever: + ntest = len(self.first_runtests.tests) text = f"{text}/{ntest}" if filtered: text = f"{text} (filtered)" @@ -788,7 +808,7 @@ def main(self, tests: TestList | None = None): self.fix_umask() - if ns.cleanup: + if self.want_cleanup: self.cleanup() sys.exit(0) @@ -838,12 +858,12 @@ def get_exitcode(self): return exitcode def action_run_tests(self): - self.run_tests() + runtests = self.run_tests() self.display_result() need_rerun = self.need_rerun if self.ns.rerun and need_rerun: - self.rerun_failed_tests(need_rerun) + self.rerun_failed_tests(need_rerun, runtests) self.display_summary() self.finalize() @@ -854,16 +874,16 @@ def _main(self): run_tests_worker(self.ns.worker_args) return - if self.ns.wait: + if self.want_wait: input("Press any key to continue...") setup_tests(self.ns) self.find_tests() exitcode = 0 - if self.ns.list_tests: + if self.want_list_tests: self.list_tests() - elif self.ns.list_cases: + elif self.want_list_cases: self.list_cases() else: self.action_run_tests() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 7e4b2e6a36b452..5fcbb777bb829b 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -212,6 +212,11 @@ class RunTests: rerun: bool = False forever: bool = False + def copy(self, **override): + state = dataclasses.asdict(self) + state.update(override) + return RunTests(**state) + def get_match_tests(self, test_name) -> FilterTuple | None: if self.match_tests is not None: return self.match_tests.get(test_name, None) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index aff5404408f8d0..ece8c1f5efdff6 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -589,7 +589,7 @@ def list_regex(line_format, tests): def parse_random_seed(self, output): match = self.regex_search(r'Using random seed ([0-9]+)', output) randseed = int(match.group(1)) - self.assertTrue(0 <= randseed <= 10000000, randseed) + self.assertTrue(0 <= randseed <= 100_000_000, randseed) return randseed def run_command(self, args, input=None, exitcode=0, **kw): From 2fafc3d5c6d720c9b9a4803dde60607fa44b89ce Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 01:56:53 +0200 Subject: [PATCH 27/66] gh-108996: Skip broken test_msvcrt for now (#109169) --- Lib/test/test_msvcrt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py index adf4e26b98ac55..3a63de351e095d 100644 --- a/Lib/test/test_msvcrt.py +++ b/Lib/test/test_msvcrt.py @@ -2,6 +2,8 @@ import sys import unittest +raise unittest.SkipTest("FIXME! broken test see: https://github.com/python/cpython/pull/109004") + from test.support import os_helper from test.support.os_helper import TESTFN, TESTFN_ASCII From 489ca0acf00bb87ae63ab9199787843fdec5a0c8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 02:30:28 +0200 Subject: [PATCH 28/66] gh-109162: Refactor Regrtest.action_run_tests() (#109170) Refator Regrtest class: * Rename finalize() finalize_tests(). * Pass tracer to run_test() and finalize_tests(). Remove Regrtest.tracer. * run_test() does less things: move code to its caller. --- Lib/test/libregrtest/main.py | 98 ++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index bf71ec3b1b08be..13500fae50c8e2 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -111,9 +111,6 @@ def __init__(self, ns: Namespace): # used by --slow self.test_times = [] - # used by --coverage, trace.Trace instance - self.tracer = None - # used to display the progress bar "[ 3/100]" self.start_time = time.perf_counter() self.test_count_text = '' @@ -443,28 +440,18 @@ def display_result(self): print(count(len(self.run_no_tests), "test"), "run no tests:") printlist(self.run_no_tests) - def run_test(self, test_index, test_name, previous_test, save_modules): - text = test_name - if previous_test: - text = '%s -- %s' % (text, previous_test) - self.display_progress(test_index, text) - - if self.tracer: + def run_test(self, test_name: str, runtests: RunTests, tracer): + if tracer is not None: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - cmd = ('result = runtest(self.ns, test_name); ' - 'self.accumulate_result(result)') + cmd = ('result = runtest(self.ns, test_name)') ns = dict(locals()) - self.tracer.runctx(cmd, globals=globals(), locals=ns) + tracer.runctx(cmd, globals=globals(), locals=ns) result = ns['result'] else: result = runtest(self.ns, test_name) - self.accumulate_result(result) - # Unload the newly imported modules (best effort finalization) - for module in sys.modules.keys(): - if module not in save_modules and module.startswith("test."): - support.unload(module) + self.accumulate_result(result) return result @@ -477,7 +464,9 @@ def run_tests_sequentially(self, runtests): if coverage: import trace - self.tracer = trace.Trace(trace=False, count=True) + tracer = trace.Trace(trace=False, count=True) + else: + tracer = None save_modules = sys.modules.keys() @@ -491,8 +480,17 @@ def run_tests_sequentially(self, runtests): for test_index, test_name in enumerate(tests_iter, 1): start_time = time.perf_counter() - result = self.run_test(test_index, test_name, - previous_test, save_modules) + text = test_name + if previous_test: + text = '%s -- %s' % (text, previous_test) + self.display_progress(test_index, text) + + result = self.run_test(test_name, runtests, tracer) + + # Unload the newly imported modules (best effort finalization) + for module in sys.modules.keys(): + if module not in save_modules and module.startswith("test."): + support.unload(module) if result.must_stop(fail_fast, fail_env_changed): break @@ -508,6 +506,8 @@ def run_tests_sequentially(self, runtests): if previous_test: print(previous_test) + return tracer + def display_header(self): # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) @@ -604,34 +604,17 @@ def set_tests(self, runtests: RunTests): self.test_count_text = '/{}'.format(len(self.tests)) self.test_count_width = len(self.test_count_text) - 1 - def run_tests(self): - # For a partial run, we do not need to clutter the output. - if (self.want_header - or not(self.ns.pgo or self.ns.quiet or self.ns.single - or self.tests or self.ns.args)): - self.display_header() - - if self.ns.huntrleaks: - warmup, repetitions, _ = self.ns.huntrleaks - if warmup < 3: - msg = ("WARNING: Running tests with --huntrleaks/-R and less than " - "3 warmup repetitions can give false positives!") - print(msg, file=sys.stdout, flush=True) - - if self.randomize: - print("Using random seed", self.random_seed) - - tests = self.selected - runtests = RunTests(tuple(tests), forever=self.forever) + def run_tests(self, runtests: RunTests): self.first_runtests = runtests self.set_tests(runtests) if self.ns.use_mp: self._run_tests_mp(runtests) + tracer = None else: - self.run_tests_sequentially(runtests) - return runtests + tracer = self.run_tests_sequentially(runtests) + return tracer - def finalize(self): + def finalize_tests(self, tracer): if self.next_single_filename: if self.next_single_test: with open(self.next_single_filename, 'w') as fp: @@ -639,10 +622,10 @@ def finalize(self): else: os.unlink(self.next_single_filename) - if self.tracer: - r = self.tracer.results() - r.write_results(show_missing=True, summary=True, - coverdir=self.ns.coverdir) + if tracer is not None: + results = tracer.results() + results.write_results(show_missing=True, summary=True, + coverdir=self.ns.coverdir) if self.ns.runleaks: os.system("leaks %d" % os.getpid()) @@ -858,7 +841,24 @@ def get_exitcode(self): return exitcode def action_run_tests(self): - runtests = self.run_tests() + if self.ns.huntrleaks: + warmup, repetitions, _ = self.ns.huntrleaks + if warmup < 3: + msg = ("WARNING: Running tests with --huntrleaks/-R and less than " + "3 warmup repetitions can give false positives!") + print(msg, file=sys.stdout, flush=True) + + # For a partial run, we do not need to clutter the output. + if (self.want_header + or not(self.ns.pgo or self.ns.quiet or self.ns.single + or self.tests or self.ns.args)): + self.display_header() + + if self.randomize: + print("Using random seed", self.random_seed) + + runtests = RunTests(tuple(self.selected), forever=self.forever) + tracer = self.run_tests(runtests) self.display_result() need_rerun = self.need_rerun @@ -866,7 +866,7 @@ def action_run_tests(self): self.rerun_failed_tests(need_rerun, runtests) self.display_summary() - self.finalize() + self.finalize_tests(tracer) def _main(self): if self.is_worker(): From a56c92875699c2ba92ed49e72f6abbf363a5c537 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 03:03:39 +0200 Subject: [PATCH 29/66] gh-109162: Refactor libregrtest WorkerJob (#109171) * Rename --worker-args command line option to --worker-json. * Rename _parse_worker_args() to _parse_worker_json(). * WorkerJob: * Add runtests attribute * Remove test_name and rerun attribute * Rename run_test_in_subprocess() to create_worker_process(). * Rename run_tests_worker() to worker_process(). * create_worker_process() uses json.dump(): write directly JSON to stdout. * Convert MultiprocessResult to a frozen dataclass. * Rename RunTests.match_tests to RunTests.match_tests_dict. --- Lib/test/libregrtest/cmdline.py | 3 +- Lib/test/libregrtest/main.py | 10 ++--- Lib/test/libregrtest/runtest.py | 6 +-- Lib/test/libregrtest/runtest_mp.py | 62 ++++++++++++++++-------------- Lib/test/test_regrtest.py | 8 ++-- 5 files changed, 48 insertions(+), 41 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index bbac980a0bf5d8..c08b3be9285d9f 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -170,6 +170,7 @@ def __init__(self, **kwargs) -> None: self.ignore_tests = None self.pgo = False self.pgo_extended = False + self.worker_json = None super().__init__(**kwargs) @@ -205,7 +206,7 @@ def _create_parser(): group.add_argument('--wait', action='store_true', help='wait for user input, e.g., allow a debugger ' 'to be attached') - group.add_argument('--worker-args', metavar='ARGS') + group.add_argument('--worker-json', metavar='ARGS') group.add_argument('-S', '--start', metavar='START', help='the name of the test at which to start.' + more_details) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 13500fae50c8e2..cd053c59a98548 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -336,7 +336,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Get tests to re-run tests = [result.test_name for result in need_rerun] - match_tests = self.get_rerun_match(need_rerun) + match_tests_dict = self.get_rerun_match(need_rerun) # Clear previously failed tests self.rerun_bad.extend(self.bad) @@ -346,7 +346,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Re-run failed tests self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") runtests = runtests.copy(tests=tuple(tests), - match_tests=match_tests, + match_tests_dict=match_tests_dict, rerun=True, forever=False) self.set_tests(runtests) @@ -742,7 +742,7 @@ def set_temp_dir(self): self.tmp_dir = os.path.abspath(self.tmp_dir) def is_worker(self): - return (self.ns.worker_args is not None) + return (self.ns.worker_json is not None) def create_temp_dir(self): os.makedirs(self.tmp_dir, exist_ok=True) @@ -870,8 +870,8 @@ def action_run_tests(self): def _main(self): if self.is_worker(): - from test.libregrtest.runtest_mp import run_tests_worker - run_tests_worker(self.ns.worker_args) + from test.libregrtest.runtest_mp import worker_process + worker_process(self.ns.worker_json) return if self.want_wait: diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 5fcbb777bb829b..ea5d0564b14aea 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -208,7 +208,7 @@ def get_rerun_match_tests(self) -> FilterTuple | None: @dataclasses.dataclass(slots=True, frozen=True) class RunTests: tests: TestTuple - match_tests: FilterDict | None = None + match_tests_dict: FilterDict | None = None rerun: bool = False forever: bool = False @@ -218,8 +218,8 @@ def copy(self, **override): return RunTests(**state) def get_match_tests(self, test_name) -> FilterTuple | None: - if self.match_tests is not None: - return self.match_tests.get(test_name, None) + if self.match_tests_dict is not None: + return self.match_tests_dict.get(test_name, None) else: return None diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 2ecdfca0e77010..45db24f255fcf7 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -46,9 +46,8 @@ @dataclasses.dataclass(slots=True) class WorkerJob: - test_name: str + runtests: RunTests namespace: Namespace - rerun: bool = False match_tests: FilterTuple | None = None @@ -70,6 +69,7 @@ def default(self, o: Any) -> dict[str, Any]: def _decode_worker_job(d: dict[str, Any]) -> WorkerJob | dict[str, Any]: if "__worker_job__" in d: d.pop('__worker_job__') + d['runtests'] = RunTests(**d['runtests']) return WorkerJob(**d) if "__namespace__" in d: d.pop('__namespace__') @@ -78,17 +78,16 @@ def _decode_worker_job(d: dict[str, Any]) -> WorkerJob | dict[str, Any]: return d -def _parse_worker_args(worker_json: str) -> tuple[Namespace, str]: - return json.loads(worker_json, - object_hook=_decode_worker_job) +def _parse_worker_json(worker_json: str) -> tuple[Namespace, str]: + return json.loads(worker_json, object_hook=_decode_worker_job) -def run_test_in_subprocess(worker_job: WorkerJob, - output_file: TextIO, - tmp_dir: str | None = None) -> subprocess.Popen: +def create_worker_process(worker_job: WorkerJob, + output_file: TextIO, + tmp_dir: str | None = None) -> subprocess.Popen: ns = worker_job.namespace python = ns.python - worker_args = json.dumps(worker_job, cls=_EncodeWorkerJob) + worker_json = json.dumps(worker_job, cls=_EncodeWorkerJob) if python is not None: executable = python @@ -97,7 +96,7 @@ def run_test_in_subprocess(worker_job: WorkerJob, cmd = [*executable, *support.args_from_interpreter_flags(), '-u', # Unbuffered stdout and stderr '-m', 'test.regrtest', - '--worker-args', worker_args] + '--worker-json', worker_json] env = dict(os.environ) if tmp_dir is not None: @@ -122,16 +121,16 @@ def run_test_in_subprocess(worker_job: WorkerJob, return subprocess.Popen(cmd, **kw) -def run_tests_worker(worker_json: str) -> NoReturn: - worker_job = _parse_worker_args(worker_json) +def worker_process(worker_json: str) -> NoReturn: + worker_job = _parse_worker_json(worker_json) + runtests = worker_job.runtests ns = worker_job.namespace - test_name = worker_job.test_name - rerun = worker_job.rerun - match_tests = worker_job.match_tests + test_name = runtests.tests[0] + match_tests: FilterTuple | None = worker_job.match_tests setup_tests(ns) - if rerun: + if runtests.rerun: if match_tests: matching = "matching: " + ", ".join(match_tests) print(f"Re-running {test_name} in verbose mode ({matching})", flush=True) @@ -139,14 +138,15 @@ def run_tests_worker(worker_json: str) -> NoReturn: print(f"Re-running {test_name} in verbose mode", flush=True) ns.verbose = True - if match_tests is not None: - ns.match_tests = match_tests + if match_tests is not None: + ns.match_tests = match_tests result = runtest(ns, test_name) print() # Force a newline (just in case) # Serialize TestResult as dict in JSON - print(json.dumps(result, cls=EncodeTestResult), flush=True) + json.dump(result, sys.stdout, cls=EncodeTestResult) + sys.stdout.flush() sys.exit(0) @@ -173,7 +173,8 @@ def stop(self): self.tests_iter = None -class MultiprocessResult(NamedTuple): +@dataclasses.dataclass(slots=True, frozen=True) +class MultiprocessResult: result: TestResult # bpo-45410: stderr is written into stdout to keep messages order worker_stdout: str | None = None @@ -198,7 +199,6 @@ def __init__(self, worker_id: int, runner: "MultiprocessTestRunner") -> None: self.ns = runner.ns self.timeout = runner.worker_timeout self.regrtest = runner.regrtest - self.rerun = runner.rerun self.current_test_name = None self.start_time = None self._popen = None @@ -264,9 +264,8 @@ def mp_result_error( def _run_process(self, worker_job, output_file: TextIO, tmp_dir: str | None = None) -> int: - self.current_test_name = worker_job.test_name try: - popen = run_test_in_subprocess(worker_job, output_file, tmp_dir) + popen = create_worker_process(worker_job, output_file, tmp_dir) self._killed = False self._popen = popen @@ -316,6 +315,8 @@ def _run_process(self, worker_job, output_file: TextIO, self.current_test_name = None def _runtest(self, test_name: str) -> MultiprocessResult: + self.current_test_name = test_name + if sys.platform == 'win32': # gh-95027: When stdout is not a TTY, Python uses the ANSI code # page for the sys.stdout encoding. If the main process runs in a @@ -324,15 +325,20 @@ def _runtest(self, test_name: str) -> MultiprocessResult: else: encoding = sys.stdout.encoding - match_tests = self.runtests.get_match_tests(test_name) + tests = (test_name,) + if self.runtests.rerun: + match_tests = self.runtests.get_match_tests(test_name) + else: + match_tests = None + worker_runtests = self.runtests.copy(tests=tests) + worker_job = WorkerJob( + worker_runtests, + namespace=self.ns, + match_tests=match_tests) # gh-94026: Write stdout+stderr to a tempfile as workaround for # non-blocking pipes on Emscripten with NodeJS. with tempfile.TemporaryFile('w+', encoding=encoding) as stdout_file: - worker_job = WorkerJob(test_name, - namespace=self.ns, - rerun=self.rerun, - match_tests=match_tests) # gh-93353: Check for leaked temporary files in the parent process, # since the deletion of temporary files can happen late during # Python finalization: too late for libregrtest. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ece8c1f5efdff6..8cced1f5185c2f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -75,10 +75,10 @@ def test_wait(self): ns = libregrtest._parse_args(['--wait']) self.assertTrue(ns.wait) - def test_worker_args(self): - ns = libregrtest._parse_args(['--worker-args', '[[], {}]']) - self.assertEqual(ns.worker_args, '[[], {}]') - self.checkError(['--worker-args'], 'expected one argument') + def test_worker_json(self): + ns = libregrtest._parse_args(['--worker-json', '[[], {}]']) + self.assertEqual(ns.worker_json, '[[], {}]') + self.checkError(['--worker-json'], 'expected one argument') def test_start(self): for opt in '-S', '--start': From 057bc7249073066ed8087b548ee06f0eabfa9e7c Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 8 Sep 2023 18:24:49 -0700 Subject: [PATCH 30/66] gh-109052: Use the base opcode when comparing code objects (gh-109107) --- Lib/test/test_code.py | 19 +++++++++++++++++++ ...-09-07-18-49-01.gh-issue-109052.TBU4nC.rst | 1 + Objects/codeobject.c | 10 ++++------ 3 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-49-01.gh-issue-109052.TBU4nC.rst diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 812c0682569207..a961ddbe17a3d3 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -505,6 +505,25 @@ def test_code_hash_uses_bytecode(self): self.assertNotEqual(c, c1) self.assertNotEqual(hash(c), hash(c1)) + @cpython_only + def test_code_equal_with_instrumentation(self): + """ GH-109052 + + Make sure the instrumentation doesn't affect the code equality + The validity of this test relies on the fact that "x is x" and + "x in x" have only one different instruction and the instructions + have the same argument. + + """ + code1 = compile("x is x", "example.py", "eval") + code2 = compile("x in x", "example.py", "eval") + sys._getframe().f_trace_opcodes = True + sys.settrace(lambda *args: None) + exec(code1, {'x': []}) + exec(code2, {'x': []}) + self.assertNotEqual(code1, code2) + sys.settrace(None) + def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-49-01.gh-issue-109052.TBU4nC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-49-01.gh-issue-109052.TBU4nC.rst new file mode 100644 index 00000000000000..175046c771cdf3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-49-01.gh-issue-109052.TBU4nC.rst @@ -0,0 +1 @@ +Use the base opcode when comparing code objects to avoid interference from instrumentation diff --git a/Objects/codeobject.c b/Objects/codeobject.c index d00bd0422f004d..20e5dedb22826f 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1798,28 +1798,26 @@ code_richcompare(PyObject *self, PyObject *other, int op) for (int i = 0; i < Py_SIZE(co); i++) { _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; _Py_CODEUNIT cp_instr = _PyCode_CODE(cp)[i]; - uint8_t co_code = co_instr.op.code; + uint8_t co_code = _Py_GetBaseOpcode(co, i); uint8_t co_arg = co_instr.op.arg; - uint8_t cp_code = cp_instr.op.code; + uint8_t cp_code = _Py_GetBaseOpcode(cp, i); uint8_t cp_arg = cp_instr.op.arg; if (co_code == ENTER_EXECUTOR) { const int exec_index = co_arg; _PyExecutorObject *exec = co->co_executors->executors[exec_index]; - co_code = exec->vm_data.opcode; + co_code = _PyOpcode_Deopt[exec->vm_data.opcode]; co_arg = exec->vm_data.oparg; } assert(co_code != ENTER_EXECUTOR); - co_code = _PyOpcode_Deopt[co_code]; if (cp_code == ENTER_EXECUTOR) { const int exec_index = cp_arg; _PyExecutorObject *exec = cp->co_executors->executors[exec_index]; - cp_code = exec->vm_data.opcode; + cp_code = _PyOpcode_Deopt[exec->vm_data.opcode]; cp_arg = exec->vm_data.oparg; } assert(cp_code != ENTER_EXECUTOR); - cp_code = _PyOpcode_Deopt[cp_code]; if (co_code != cp_code || co_arg != cp_arg) { goto unequal; From e9e2ca7a7b4b4320009cdf85c84ec5bd6c4923c3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 03:37:48 +0200 Subject: [PATCH 31/66] gh-109162: Refactor libregrtest.runtest (#109172) * Rename runtest() to run_single_test(). * Pass runtests to run_single_test(). * Add type annotation to Regrtest attributes. Add missing attributes to Namespace. * Add attributes to Regrtest and RunTests: * fail_fast * ignore_tests * match_tests * output_on_failure * pgo * pgo_extended * timeout * Get pgo from 'runtests', rather than from 'ns'. * Remove WorkerJob.match_tests. * setup_support() now gets pgo_extended from runtests. * save_env(): change parameter order, pass test_name first. * Add setup_test_dir() function. * Pass runtests to setup_tests(). --- Lib/test/libregrtest/cmdline.py | 6 ++ Lib/test/libregrtest/main.py | 91 +++++++++++++++++------------- Lib/test/libregrtest/runtest.py | 60 ++++++++++++-------- Lib/test/libregrtest/runtest_mp.py | 52 ++++++++--------- Lib/test/libregrtest/setup.py | 29 +++++----- 5 files changed, 134 insertions(+), 104 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index c08b3be9285d9f..71e7396d71d4a5 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -149,6 +149,10 @@ def __init__(self, **kwargs) -> None: self.verbose = 0 self.quiet = False self.exclude = False + self.cleanup = False + self.wait = False + self.list_cases = False + self.list_tests = False self.single = False self.randomize = False self.fromfile = None @@ -171,6 +175,8 @@ def __init__(self, **kwargs) -> None: self.pgo = False self.pgo_extended = False self.worker_json = None + self.start = None + self.timeout = None super().__init__(**kwargs) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index cd053c59a98548..f7d28a859213f5 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -11,10 +11,10 @@ import unittest from test.libregrtest.cmdline import _parse_args, Namespace from test.libregrtest.runtest import ( - findtests, split_test_packages, runtest, abs_module_name, + findtests, split_test_packages, run_single_test, abs_module_name, PROGRESS_MIN_TIME, State, RunTests, TestResult, FilterTuple, FilterDict, TestList) -from test.libregrtest.setup import setup_tests +from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.utils import (strip_py_suffix, count, format_duration, printlist, get_build_info) @@ -64,11 +64,11 @@ def __init__(self, ns: Namespace): self.ns: Namespace = ns # Actions - self.want_header = ns.header - self.want_list_tests = ns.list_tests - self.want_list_cases = ns.list_cases - self.want_wait = ns.wait - self.want_cleanup = ns.cleanup + self.want_header: bool = ns.header + self.want_list_tests: bool = ns.list_tests + self.want_list_cases: bool = ns.list_cases + self.want_wait: bool = ns.wait + self.want_cleanup: bool = ns.cleanup # Select tests if ns.match_tests: @@ -79,14 +79,19 @@ def __init__(self, ns: Namespace): self.ignore_tests: FilterTuple = tuple(ns.ignore_tests) else: self.ignore_tests = None - self.exclude = ns.exclude - self.fromfile = ns.fromfile - self.starting_test = ns.start + self.exclude: bool = ns.exclude + self.fromfile: str | None = ns.fromfile + self.starting_test: str | None = ns.start # Options to run tests - self.forever = ns.forever - self.randomize = ns.randomize - self.random_seed = ns.random_seed + self.fail_fast: bool = ns.failfast + self.forever: bool = ns.forever + self.randomize: bool = ns.randomize + self.random_seed: int | None = ns.random_seed + self.pgo: bool = ns.pgo + self.pgo_extended: bool = ns.pgo_extended + self.output_on_failure: bool = ns.verbose3 + self.timeout: float | None = ns.timeout # tests self.tests = [] @@ -196,21 +201,19 @@ def log(self, line=''): def display_progress(self, test_index, text): quiet = self.ns.quiet - pgo = self.ns.pgo if quiet: return # "[ 51/405/1] test_tcl passed" line = f"{test_index:{self.test_count_width}}{self.test_count_text}" fails = len(self.bad) + len(self.environment_changed) - if fails and not pgo: + if fails and not self.pgo: line = f"{line}/{fails}" self.log(f"[{line}] {text}") def find_tests(self): ns = self.ns single = ns.single - pgo = ns.pgo test_dir = ns.testdir if single: @@ -237,7 +240,7 @@ def find_tests(self): strip_py_suffix(self.tests) - if pgo: + if self.pgo: # add default PGO tests if no tests are specified setup_pgo_tests(ns) @@ -329,8 +332,6 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Configure the runner to re-run tests ns = self.ns ns.verbose = True - ns.failfast = False - ns.verbose3 = False if ns.use_mp is None: ns.use_mp = 1 @@ -345,12 +346,16 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Re-run failed tests self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") - runtests = runtests.copy(tests=tuple(tests), - match_tests_dict=match_tests_dict, - rerun=True, - forever=False) + runtests = runtests.copy( + tests=tuple(tests), + rerun=True, + forever=False, + fail_fast=False, + match_tests_dict=match_tests_dict, + output_on_failure=False) self.set_tests(runtests) self._run_tests_mp(runtests) + return runtests def rerun_failed_tests(self, need_rerun, runtests: RunTests): if self.ns.python: @@ -364,16 +369,16 @@ def rerun_failed_tests(self, need_rerun, runtests: RunTests): self.first_state = self.get_tests_state() print() - self._rerun_failed_tests(need_rerun, runtests) + rerun_runtests = self._rerun_failed_tests(need_rerun, runtests) if self.bad: print(count(len(self.bad), 'test'), "failed again:") printlist(self.bad) - self.display_result() + self.display_result(rerun_runtests) - def display_result(self): - pgo = self.ns.pgo + def display_result(self, runtests): + pgo = runtests.pgo quiet = self.ns.quiet print_slow = self.ns.print_slow @@ -444,12 +449,12 @@ def run_test(self, test_name: str, runtests: RunTests, tracer): if tracer is not None: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - cmd = ('result = runtest(self.ns, test_name)') + cmd = ('result = run_single_test(test_name, runtests, self.ns)') ns = dict(locals()) tracer.runctx(cmd, globals=globals(), locals=ns) result = ns['result'] else: - result = runtest(self.ns, test_name) + result = run_single_test(test_name, runtests, self.ns) self.accumulate_result(result) @@ -458,9 +463,7 @@ def run_test(self, test_name: str, runtests: RunTests, tracer): def run_tests_sequentially(self, runtests): ns = self.ns coverage = ns.trace - fail_fast = ns.failfast fail_env_changed = ns.fail_env_changed - timeout = ns.timeout if coverage: import trace @@ -471,8 +474,8 @@ def run_tests_sequentially(self, runtests): save_modules = sys.modules.keys() msg = "Run tests sequentially" - if timeout: - msg += " (timeout: %s)" % format_duration(timeout) + if runtests.timeout: + msg += " (timeout: %s)" % format_duration(runtests.timeout) self.log(msg) previous_test = None @@ -492,7 +495,7 @@ def run_tests_sequentially(self, runtests): if module not in save_modules and module.startswith("test."): support.unload(module) - if result.must_stop(fail_fast, fail_env_changed): + if result.must_stop(self.fail_fast, fail_env_changed): break previous_test = str(result) @@ -850,16 +853,28 @@ def action_run_tests(self): # For a partial run, we do not need to clutter the output. if (self.want_header - or not(self.ns.pgo or self.ns.quiet or self.ns.single + or not(self.pgo or self.ns.quiet or self.ns.single or self.tests or self.ns.args)): self.display_header() if self.randomize: print("Using random seed", self.random_seed) - runtests = RunTests(tuple(self.selected), forever=self.forever) + runtests = RunTests( + tuple(self.selected), + fail_fast=self.fail_fast, + match_tests=self.match_tests, + ignore_tests=self.ignore_tests, + forever=self.forever, + pgo=self.pgo, + pgo_extended=self.pgo_extended, + output_on_failure=self.output_on_failure, + timeout=self.timeout) + + setup_tests(runtests, self.ns) + tracer = self.run_tests(runtests) - self.display_result() + self.display_result(runtests) need_rerun = self.need_rerun if self.ns.rerun and need_rerun: @@ -877,7 +892,7 @@ def _main(self): if self.want_wait: input("Press any key to continue...") - setup_tests(self.ns) + setup_test_dir(self.ns.testdir) self.find_tests() exitcode = 0 diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index ea5d0564b14aea..bfb0718aa56c32 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -208,9 +208,16 @@ def get_rerun_match_tests(self) -> FilterTuple | None: @dataclasses.dataclass(slots=True, frozen=True) class RunTests: tests: TestTuple + fail_fast: bool = False + match_tests: FilterTuple | None = None + ignore_tests: FilterTuple | None = None match_tests_dict: FilterDict | None = None rerun: bool = False forever: bool = False + pgo: bool = False + pgo_extended: bool = False + output_on_failure: bool = False + timeout: float | None = None def copy(self, **override): state = dataclasses.asdict(self) @@ -295,11 +302,11 @@ def abs_module_name(test_name: str, test_dir: str | None) -> str: return 'test.' + test_name -def setup_support(ns: Namespace): - support.PGO = ns.pgo - support.PGO_EXTENDED = ns.pgo_extended - support.set_match_tests(ns.match_tests, ns.ignore_tests) - support.failfast = ns.failfast +def setup_support(runtests: RunTests, ns: Namespace): + support.PGO = runtests.pgo + support.PGO_EXTENDED = runtests.pgo_extended + support.set_match_tests(runtests.match_tests, runtests.ignore_tests) + support.failfast = runtests.fail_fast support.verbose = ns.verbose if ns.xmlpath: support.junit_xml_list = [] @@ -307,12 +314,12 @@ def setup_support(ns: Namespace): support.junit_xml_list = None -def _runtest(result: TestResult, ns: Namespace) -> None: +def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: # Capture stdout and stderr, set faulthandler timeout, # and create JUnit XML report. verbose = ns.verbose - output_on_failure = ns.verbose3 - timeout = ns.timeout + output_on_failure = runtests.output_on_failure + timeout = runtests.timeout use_timeout = ( timeout is not None and threading_helper.can_start_thread @@ -321,7 +328,7 @@ def _runtest(result: TestResult, ns: Namespace) -> None: faulthandler.dump_traceback_later(timeout, exit=True) try: - setup_support(ns) + setup_support(runtests, ns) if output_on_failure: support.verbose = True @@ -341,7 +348,7 @@ def _runtest(result: TestResult, ns: Namespace) -> None: # warnings will be written to sys.stderr below. print_warning.orig_stderr = stream - _runtest_env_changed_exc(result, ns, display_failure=False) + _runtest_env_changed_exc(result, runtests, ns, display_failure=False) # Ignore output if the test passed successfully if result.state != State.PASSED: output = stream.getvalue() @@ -356,7 +363,7 @@ def _runtest(result: TestResult, ns: Namespace) -> None: else: # Tell tests to be moderately quiet support.verbose = verbose - _runtest_env_changed_exc(result, ns, display_failure=not verbose) + _runtest_env_changed_exc(result, runtests, ns, display_failure=not verbose) xml_list = support.junit_xml_list if xml_list: @@ -369,7 +376,7 @@ def _runtest(result: TestResult, ns: Namespace) -> None: support.junit_xml_list = None -def runtest(ns: Namespace, test_name: str) -> TestResult: +def run_single_test(test_name: str, runtests: RunTests, ns: Namespace) -> TestResult: """Run a single test. ns -- regrtest namespace of options @@ -382,10 +389,11 @@ def runtest(ns: Namespace, test_name: str) -> TestResult: """ start_time = time.perf_counter() result = TestResult(test_name) + pgo = runtests.pgo try: - _runtest(result, ns) + _runtest(result, runtests, ns) except: - if not ns.pgo: + if not pgo: msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", file=sys.stderr, flush=True) @@ -404,8 +412,8 @@ def run_unittest(test_mod): return support.run_unittest(tests) -def save_env(ns: Namespace, test_name: str): - return saved_test_environment(test_name, ns.verbose, ns.quiet, pgo=ns.pgo) +def save_env(test_name: str, runtests: RunTests, ns: Namespace): + return saved_test_environment(test_name, ns.verbose, ns.quiet, pgo=runtests.pgo) def regrtest_runner(result, test_func, ns) -> None: @@ -442,7 +450,7 @@ def regrtest_runner(result, test_func, ns) -> None: FOUND_GARBAGE = [] -def _load_run_test(result: TestResult, ns: Namespace) -> None: +def _load_run_test(result: TestResult, runtests: RunTests, ns: Namespace) -> None: # Load the test function, run the test function. module_name = abs_module_name(result.test_name, ns.testdir) @@ -458,7 +466,7 @@ def test_func(): return run_unittest(test_mod) try: - with save_env(ns, result.test_name): + with save_env(result.test_name, runtests, ns): regrtest_runner(result, test_func, ns) finally: # First kill any dangling references to open files etc. @@ -482,7 +490,8 @@ def test_func(): support.reap_children() -def _runtest_env_changed_exc(result: TestResult, ns: Namespace, +def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, + ns: Namespace, display_failure: bool = True) -> None: # Detect environment changes, handle exceptions. @@ -490,7 +499,8 @@ def _runtest_env_changed_exc(result: TestResult, ns: Namespace, # the environment support.environment_altered = False - if ns.pgo: + pgo = runtests.pgo + if pgo: display_failure = False test_name = result.test_name @@ -498,15 +508,15 @@ def _runtest_env_changed_exc(result: TestResult, ns: Namespace, clear_caches() support.gc_collect() - with save_env(ns, test_name): - _load_run_test(result, ns) + with save_env(test_name, runtests, ns): + _load_run_test(result, runtests, ns) except support.ResourceDenied as msg: - if not ns.quiet and not ns.pgo: + if not ns.quiet and not pgo: print(f"{test_name} skipped -- {msg}", flush=True) result.state = State.RESOURCE_DENIED return except unittest.SkipTest as msg: - if not ns.quiet and not ns.pgo: + if not ns.quiet and not pgo: print(f"{test_name} skipped -- {msg}", flush=True) result.state = State.SKIPPED return @@ -536,7 +546,7 @@ def _runtest_env_changed_exc(result: TestResult, ns: Namespace, result.state = State.INTERRUPTED return except: - if not ns.pgo: + if not pgo: msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", file=sys.stderr, flush=True) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 45db24f255fcf7..ecdde3aa523098 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -10,7 +10,7 @@ import threading import time import traceback -from typing import NamedTuple, NoReturn, Literal, Any, TextIO +from typing import NoReturn, Literal, Any, TextIO from test import support from test.support import os_helper @@ -19,9 +19,9 @@ from test.libregrtest.cmdline import Namespace from test.libregrtest.main import Regrtest from test.libregrtest.runtest import ( - runtest, TestResult, State, PROGRESS_MIN_TIME, + run_single_test, TestResult, State, PROGRESS_MIN_TIME, FilterTuple, RunTests) -from test.libregrtest.setup import setup_tests +from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.utils import format_duration, print_warning if sys.platform == 'win32': @@ -48,7 +48,6 @@ class WorkerJob: runtests: RunTests namespace: Namespace - match_tests: FilterTuple | None = None class _EncodeWorkerJob(json.JSONEncoder): @@ -126,9 +125,10 @@ def worker_process(worker_json: str) -> NoReturn: runtests = worker_job.runtests ns = worker_job.namespace test_name = runtests.tests[0] - match_tests: FilterTuple | None = worker_job.match_tests + match_tests: FilterTuple | None = runtests.match_tests - setup_tests(ns) + setup_test_dir(ns.testdir) + setup_tests(runtests, ns) if runtests.rerun: if match_tests: @@ -138,10 +138,7 @@ def worker_process(worker_json: str) -> NoReturn: print(f"Re-running {test_name} in verbose mode", flush=True) ns.verbose = True - if match_tests is not None: - ns.match_tests = match_tests - - result = runtest(ns, test_name) + result = run_single_test(test_name, runtests, ns) print() # Force a newline (just in case) # Serialize TestResult as dict in JSON @@ -330,11 +327,13 @@ def _runtest(self, test_name: str) -> MultiprocessResult: match_tests = self.runtests.get_match_tests(test_name) else: match_tests = None - worker_runtests = self.runtests.copy(tests=tests) + kwargs = {} + if match_tests: + kwargs['match_tests'] = match_tests + worker_runtests = self.runtests.copy(tests=tests, **kwargs) worker_job = WorkerJob( worker_runtests, - namespace=self.ns, - match_tests=match_tests) + namespace=self.ns) # gh-94026: Write stdout+stderr to a tempfile as workaround for # non-blocking pipes on Emscripten with NodeJS. @@ -401,7 +400,7 @@ def _runtest(self, test_name: str) -> MultiprocessResult: return MultiprocessResult(result, stdout) def run(self) -> None: - fail_fast = self.ns.failfast + fail_fast = self.runtests.fail_fast fail_env_changed = self.ns.fail_env_changed while not self._stopped: try: @@ -473,7 +472,6 @@ def get_running(workers: list[TestWorkerProcess]) -> list[TestWorkerProcess]: class MultiprocessTestRunner: def __init__(self, regrtest: Regrtest, runtests: RunTests) -> None: ns = regrtest.ns - timeout = ns.timeout self.regrtest = regrtest self.runtests = runtests @@ -483,24 +481,24 @@ def __init__(self, regrtest: Regrtest, runtests: RunTests) -> None: self.output: queue.Queue[QueueOutput] = queue.Queue() tests_iter = runtests.iter_tests() self.pending = MultiprocessIterator(tests_iter) - if timeout is not None: + self.timeout = runtests.timeout + if self.timeout is not None: # Rely on faulthandler to kill a worker process. This timouet is # when faulthandler fails to kill a worker process. Give a maximum # of 5 minutes to faulthandler to kill the worker. - self.worker_timeout = min(timeout * 1.5, timeout + 5 * 60) + self.worker_timeout = min(self.timeout * 1.5, self.timeout + 5 * 60) else: self.worker_timeout = None self.workers = None def start_workers(self) -> None: use_mp = self.ns.use_mp - timeout = self.ns.timeout self.workers = [TestWorkerProcess(index, self) for index in range(1, use_mp + 1)] msg = f"Run tests in parallel using {len(self.workers)} child processes" - if timeout: + if self.timeout: msg += (" (timeout: %s, worker timeout: %s)" - % (format_duration(timeout), + % (format_duration(self.timeout), format_duration(self.worker_timeout))) self.log(msg) for worker in self.workers: @@ -514,9 +512,8 @@ def stop_workers(self) -> None: worker.wait_stopped(start_time) def _get_result(self) -> QueueOutput | None: - pgo = self.ns.pgo - use_faulthandler = (self.ns.timeout is not None) - timeout = PROGRESS_UPDATE + pgo = self.runtests.pgo + use_faulthandler = (self.timeout is not None) # bpo-46205: check the status of workers every iteration to avoid # waiting forever on an empty queue. @@ -527,7 +524,7 @@ def _get_result(self) -> QueueOutput | None: # wait for a thread try: - return self.output.get(timeout=timeout) + return self.output.get(timeout=PROGRESS_UPDATE) except queue.Empty: pass @@ -544,7 +541,7 @@ def _get_result(self) -> QueueOutput | None: def display_result(self, mp_result: MultiprocessResult) -> None: result = mp_result.result - pgo = self.ns.pgo + pgo = self.runtests.pgo text = str(result) if mp_result.err_msg: @@ -580,9 +577,8 @@ def _process_result(self, item: QueueOutput) -> bool: return result def run_tests(self) -> None: - fail_fast = self.ns.failfast + fail_fast = self.runtests.fail_fast fail_env_changed = self.ns.fail_env_changed - timeout = self.ns.timeout self.start_workers() @@ -600,7 +596,7 @@ def run_tests(self) -> None: print() self.regrtest.interrupted = True finally: - if timeout is not None: + if self.timeout is not None: faulthandler.cancel_dump_traceback_later() # Always ensure that all worker processes are no longer diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index b76bece7ca08b5..f640362cb2df7f 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -18,7 +18,14 @@ UNICODE_GUARD_ENV = "PYTHONREGRTEST_UNICODE_GUARD" -def setup_tests(ns): +def setup_test_dir(testdir): + if testdir: + # Prepend test directory to sys.path, so runtest() will be able + # to locate tests + sys.path.insert(0, os.path.abspath(testdir)) + + +def setup_tests(runtests, ns): try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): @@ -44,11 +51,6 @@ def setup_tests(ns): replace_stdout() support.record_original_stdout(sys.stdout) - if ns.testdir: - # Prepend test directory to sys.path, so runtest() will be able - # to locate tests - sys.path.insert(0, os.path.abspath(ns.testdir)) - # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). @@ -88,16 +90,17 @@ def _test_audit_hook(name, args): setup_unraisable_hook() setup_threading_excepthook() - if ns.timeout is not None: + timeout = runtests.timeout + if timeout is not None: # For a slow buildbot worker, increase SHORT_TIMEOUT and LONG_TIMEOUT - support.SHORT_TIMEOUT = max(support.SHORT_TIMEOUT, ns.timeout / 40) - support.LONG_TIMEOUT = max(support.LONG_TIMEOUT, ns.timeout / 4) + support.SHORT_TIMEOUT = max(support.SHORT_TIMEOUT, timeout / 40) + support.LONG_TIMEOUT = max(support.LONG_TIMEOUT, timeout / 4) # If --timeout is short: reduce timeouts - support.LOOPBACK_TIMEOUT = min(support.LOOPBACK_TIMEOUT, ns.timeout) - support.INTERNET_TIMEOUT = min(support.INTERNET_TIMEOUT, ns.timeout) - support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, ns.timeout) - support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, ns.timeout) + support.LOOPBACK_TIMEOUT = min(support.LOOPBACK_TIMEOUT, timeout) + support.INTERNET_TIMEOUT = min(support.INTERNET_TIMEOUT, timeout) + support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, timeout) + support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, timeout) if ns.xmlpath: from test.support.testresult import RegressionTestResult From 17f994174de9211b2baaff217eeb1033343230fc Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 8 Sep 2023 19:49:20 -0700 Subject: [PATCH 32/66] gh-109118: Fix runtime crash when NameError happens in PEP 695 function (#109123) --- Include/internal/pycore_opcode_metadata.h | 90 ++++++++----------- Lib/test/test_type_params.py | 40 +++++++++ ...-09-07-18-24-42.gh-issue-109118.yPXRAe.rst | 2 + Python/abstract_interp_cases.c.h | 10 ++- Python/bytecodes.c | 44 +++++++-- Python/executor_cases.c.h | 42 ++++++++- Python/generated_cases.c.h | 67 ++++++-------- 7 files changed, 189 insertions(+), 106 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-24-42.gh-issue-109118.yPXRAe.rst diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index fb5c0465a10021..f70b75a56c150c 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -39,40 +39,38 @@ #define _BINARY_OP_ADD_UNICODE 311 #define _BINARY_OP_INPLACE_ADD_UNICODE 312 #define _POP_FRAME 313 -#define _LOAD_LOCALS 314 -#define _LOAD_FROM_DICT_OR_GLOBALS 315 -#define _GUARD_GLOBALS_VERSION 316 -#define _GUARD_BUILTINS_VERSION 317 -#define _LOAD_GLOBAL_MODULE 318 -#define _LOAD_GLOBAL_BUILTINS 319 -#define _GUARD_TYPE_VERSION 320 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 321 -#define _LOAD_ATTR_INSTANCE_VALUE 322 -#define IS_NONE 323 -#define _ITER_CHECK_LIST 324 -#define _ITER_JUMP_LIST 325 -#define _IS_ITER_EXHAUSTED_LIST 326 -#define _ITER_NEXT_LIST 327 -#define _ITER_CHECK_TUPLE 328 -#define _ITER_JUMP_TUPLE 329 -#define _IS_ITER_EXHAUSTED_TUPLE 330 -#define _ITER_NEXT_TUPLE 331 -#define _ITER_CHECK_RANGE 332 -#define _ITER_JUMP_RANGE 333 -#define _IS_ITER_EXHAUSTED_RANGE 334 -#define _ITER_NEXT_RANGE 335 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 336 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 337 -#define _CHECK_PEP_523 338 -#define _CHECK_FUNCTION_EXACT_ARGS 339 -#define _CHECK_STACK_SPACE 340 -#define _INIT_CALL_PY_EXACT_ARGS 341 -#define _PUSH_FRAME 342 -#define _POP_JUMP_IF_FALSE 343 -#define _POP_JUMP_IF_TRUE 344 -#define JUMP_TO_TOP 345 -#define SAVE_CURRENT_IP 346 -#define INSERT 347 +#define _GUARD_GLOBALS_VERSION 314 +#define _GUARD_BUILTINS_VERSION 315 +#define _LOAD_GLOBAL_MODULE 316 +#define _LOAD_GLOBAL_BUILTINS 317 +#define _GUARD_TYPE_VERSION 318 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 319 +#define _LOAD_ATTR_INSTANCE_VALUE 320 +#define IS_NONE 321 +#define _ITER_CHECK_LIST 322 +#define _ITER_JUMP_LIST 323 +#define _IS_ITER_EXHAUSTED_LIST 324 +#define _ITER_NEXT_LIST 325 +#define _ITER_CHECK_TUPLE 326 +#define _ITER_JUMP_TUPLE 327 +#define _IS_ITER_EXHAUSTED_TUPLE 328 +#define _ITER_NEXT_TUPLE 329 +#define _ITER_CHECK_RANGE 330 +#define _ITER_JUMP_RANGE 331 +#define _IS_ITER_EXHAUSTED_RANGE 332 +#define _ITER_NEXT_RANGE 333 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 334 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 335 +#define _CHECK_PEP_523 336 +#define _CHECK_FUNCTION_EXACT_ARGS 337 +#define _CHECK_STACK_SPACE 338 +#define _INIT_CALL_PY_EXACT_ARGS 339 +#define _PUSH_FRAME 340 +#define _POP_JUMP_IF_FALSE 341 +#define _POP_JUMP_IF_TRUE 342 +#define JUMP_TO_TOP 343 +#define SAVE_CURRENT_IP 344 +#define INSERT 345 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -268,16 +266,12 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case DELETE_GLOBAL: return 0; - case _LOAD_LOCALS: - return 0; case LOAD_LOCALS: return 0; - case _LOAD_FROM_DICT_OR_GLOBALS: + case LOAD_FROM_DICT_OR_GLOBALS: return 1; case LOAD_NAME: return 0; - case LOAD_FROM_DICT_OR_GLOBALS: - return 1; case LOAD_GLOBAL: return 0; case _GUARD_GLOBALS_VERSION: @@ -802,16 +796,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case DELETE_GLOBAL: return 0; - case _LOAD_LOCALS: - return 1; case LOAD_LOCALS: return 1; - case _LOAD_FROM_DICT_OR_GLOBALS: + case LOAD_FROM_DICT_OR_GLOBALS: return 1; case LOAD_NAME: return 1; - case LOAD_FROM_DICT_OR_GLOBALS: - return 1; case LOAD_GLOBAL: return ((oparg & 1) ? 1 : 0) + 1; case _GUARD_GLOBALS_VERSION: @@ -1305,11 +1295,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [_LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [_LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [_GUARD_GLOBALS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [_GUARD_BUILTINS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, @@ -1546,9 +1534,9 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [DELETE_ATTR] = { .nuops = 1, .uops = { { DELETE_ATTR, 0, 0 } } }, [STORE_GLOBAL] = { .nuops = 1, .uops = { { STORE_GLOBAL, 0, 0 } } }, [DELETE_GLOBAL] = { .nuops = 1, .uops = { { DELETE_GLOBAL, 0, 0 } } }, - [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } }, - [LOAD_NAME] = { .nuops = 2, .uops = { { _LOAD_LOCALS, 0, 0 }, { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, - [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, + [LOAD_LOCALS] = { .nuops = 1, .uops = { { LOAD_LOCALS, 0, 0 } } }, + [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, + [LOAD_NAME] = { .nuops = 1, .uops = { { LOAD_NAME, 0, 0 } } }, [LOAD_GLOBAL] = { .nuops = 1, .uops = { { LOAD_GLOBAL, 0, 0 } } }, [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, @@ -1633,8 +1621,6 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_POP_FRAME] = "_POP_FRAME", - [_LOAD_LOCALS] = "_LOAD_LOCALS", - [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 0045057f181e1c..f93d088ea758a9 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -956,3 +956,43 @@ class NewStyle[T]: for case in cases: with self.subTest(case=case): weakref.ref(case) + + +class TypeParamsRuntimeTest(unittest.TestCase): + def test_name_error(self): + # gh-109118: This crashed the interpreter due to a refcounting bug + code = """ + class name_2[name_5]: + class name_4[name_5](name_0): + pass + """ + with self.assertRaises(NameError): + run_code(code) + + # Crashed with a slightly different stack trace + code = """ + class name_2[name_5]: + class name_4[name_5: name_5](name_0): + pass + """ + with self.assertRaises(NameError): + run_code(code) + + def test_broken_class_namespace(self): + code = """ + class WeirdMapping(dict): + def __missing__(self, key): + if key == "T": + raise RuntimeError + raise KeyError(key) + + class Meta(type): + def __prepare__(name, bases): + return WeirdMapping() + + class MyClass[V](metaclass=Meta): + class Inner[U](T): + pass + """ + with self.assertRaises(RuntimeError): + run_code(code) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-24-42.gh-issue-109118.yPXRAe.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-24-42.gh-issue-109118.yPXRAe.rst new file mode 100644 index 00000000000000..f14fce4423896f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-24-42.gh-issue-109118.yPXRAe.rst @@ -0,0 +1,2 @@ +Fix interpreter crash when a NameError is raised inside the type parameters +of a generic class. diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 11fbf4443dda16..398c04616bc091 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -291,13 +291,19 @@ break; } - case _LOAD_LOCALS: { + case LOAD_LOCALS: { STACK_GROW(1); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); break; } - case _LOAD_FROM_DICT_OR_GLOBALS: { + case LOAD_FROM_DICT_OR_GLOBALS: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LOAD_NAME: { + STACK_GROW(1); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); break; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 21069204cf8693..ff95f37b900da6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1286,7 +1286,7 @@ dummy_func( } } - op(_LOAD_LOCALS, ( -- locals)) { + inst(LOAD_LOCALS, ( -- locals)) { locals = LOCALS(); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1296,15 +1296,11 @@ dummy_func( Py_INCREF(locals); } - macro(LOAD_LOCALS) = _LOAD_LOCALS; - - op(_LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { + inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - Py_DECREF(mod_or_class_dict); goto error; } - Py_DECREF(mod_or_class_dict); if (v == NULL) { v = PyDict_GetItemWithError(GLOBALS(), name); if (v != NULL) { @@ -1325,11 +1321,41 @@ dummy_func( } } } + DECREF_INPUTS(); } - macro(LOAD_NAME) = _LOAD_LOCALS + _LOAD_FROM_DICT_OR_GLOBALS; - - macro(LOAD_FROM_DICT_OR_GLOBALS) = _LOAD_FROM_DICT_OR_GLOBALS; + inst(LOAD_NAME, (-- v)) { + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + ERROR_IF(true, error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + goto error; + } + if (v == NULL) { + v = PyDict_GetItemWithError(GLOBALS(), name); + if (v != NULL) { + Py_INCREF(v); + } + else if (_PyErr_Occurred(tstate)) { + goto error; + } + else { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + goto error; + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + goto error; + } + } + } + } family(LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL) = { LOAD_GLOBAL_MODULE, diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index fa7cb88e4a1e1f..918991dca7dd25 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1046,7 +1046,7 @@ break; } - case _LOAD_LOCALS: { + case LOAD_LOCALS: { PyObject *locals; locals = LOCALS(); if (locals == NULL) { @@ -1060,16 +1060,51 @@ break; } - case _LOAD_FROM_DICT_OR_GLOBALS: { + case LOAD_FROM_DICT_OR_GLOBALS: { PyObject *mod_or_class_dict; PyObject *v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - Py_DECREF(mod_or_class_dict); goto error; } + if (v == NULL) { + v = PyDict_GetItemWithError(GLOBALS(), name); + if (v != NULL) { + Py_INCREF(v); + } + else if (_PyErr_Occurred(tstate)) { + goto error; + } + else { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + goto error; + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + goto error; + } + } + } Py_DECREF(mod_or_class_dict); + stack_pointer[-1] = v; + break; + } + + case LOAD_NAME: { + PyObject *v; + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + goto error; + } if (v == NULL) { v = PyDict_GetItemWithError(GLOBALS(), name); if (v != NULL) { @@ -1090,6 +1125,7 @@ } } } + STACK_GROW(1); stack_pointer[-1] = v; break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 136b36c8260cb5..e84599d87a6968 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1675,65 +1675,51 @@ DISPATCH(); } - TARGET(LOAD_NAME) { - PyObject *locals; + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { PyObject *mod_or_class_dict; PyObject *v; - // _LOAD_LOCALS - { - locals = LOCALS(); - if (locals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - Py_INCREF(locals); + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + goto error; } - // _LOAD_FROM_DICT_OR_GLOBALS - mod_or_class_dict = locals; - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - Py_DECREF(mod_or_class_dict); + if (v == NULL) { + v = PyDict_GetItemWithError(GLOBALS(), name); + if (v != NULL) { + Py_INCREF(v); + } + else if (_PyErr_Occurred(tstate)) { goto error; } - Py_DECREF(mod_or_class_dict); - if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { + else { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { goto error; } - else { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - goto error; - } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + goto error; } } } - STACK_GROW(1); + Py_DECREF(mod_or_class_dict); stack_pointer[-1] = v; DISPATCH(); } - TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - PyObject *mod_or_class_dict; + TARGET(LOAD_NAME) { PyObject *v; - mod_or_class_dict = stack_pointer[-1]; + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - Py_DECREF(mod_or_class_dict); goto error; } - Py_DECREF(mod_or_class_dict); if (v == NULL) { v = PyDict_GetItemWithError(GLOBALS(), name); if (v != NULL) { @@ -1754,6 +1740,7 @@ } } } + STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); } From b4131a13cb41d0f397776683c3b99500db9e2cfd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 9 Sep 2023 08:44:46 +0300 Subject: [PATCH 33/66] gh-109050: Remove remaining tests for legacy Unicode C API (GH-109068) --- Lib/test/support/__init__.py | 9 ----- Lib/test/test_capi/test_getargs.py | 64 ------------------------------ Lib/test/test_csv.py | 12 ------ Lib/test/test_decimal.py | 31 +-------------- Lib/test/test_str.py | 30 -------------- Modules/_testcapi/getargs.c | 52 ------------------------ 6 files changed, 1 insertion(+), 197 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 84b74ee2c298e9..f40b73b0a799f1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -506,15 +506,6 @@ def has_no_debug_ranges(): def requires_debug_ranges(reason='requires co_positions / debug_ranges'): return unittest.skipIf(has_no_debug_ranges(), reason) -def requires_legacy_unicode_capi(): - try: - from _testcapi import unicode_legacy_string - except ImportError: - unicode_legacy_string = None - - return unittest.skipUnless(unicode_legacy_string, - 'requires legacy Unicode C API') - # Is not actually used in tests, but is kept for compatibility. is_jython = sys.platform.startswith('java') diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 246206af86101c..e10f679eeb71c8 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -1004,70 +1004,6 @@ def test_et_hash(self): buf = bytearray() self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) - @support.requires_legacy_unicode_capi() - def test_u(self): - from _testcapi import getargs_u - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_u('abc\xe9'), 'abc\xe9') - with self.assertWarns(DeprecationWarning): - self.assertRaises(ValueError, getargs_u, 'nul:\0') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u, b'bytes') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u, bytearray(b'bytearray')) - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u, memoryview(b'memoryview')) - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u, None) - - @support.requires_legacy_unicode_capi() - def test_u_hash(self): - from _testcapi import getargs_u_hash - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_u_hash('abc\xe9'), 'abc\xe9') - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_u_hash('nul:\0'), 'nul:\0') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u_hash, b'bytes') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u_hash, bytearray(b'bytearray')) - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u_hash, memoryview(b'memoryview')) - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u_hash, None) - - @support.requires_legacy_unicode_capi() - def test_Z(self): - from _testcapi import getargs_Z - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_Z('abc\xe9'), 'abc\xe9') - with self.assertWarns(DeprecationWarning): - self.assertRaises(ValueError, getargs_Z, 'nul:\0') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_Z, b'bytes') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_Z, bytearray(b'bytearray')) - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_Z, memoryview(b'memoryview')) - with self.assertWarns(DeprecationWarning): - self.assertIsNone(getargs_Z(None)) - - @support.requires_legacy_unicode_capi() - def test_Z_hash(self): - from _testcapi import getargs_Z_hash - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_Z_hash('abc\xe9'), 'abc\xe9') - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_Z_hash('nul:\0'), 'nul:\0') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_Z_hash, b'bytes') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_Z_hash, bytearray(b'bytearray')) - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_Z_hash, memoryview(b'memoryview')) - with self.assertWarns(DeprecationWarning): - self.assertIsNone(getargs_Z_hash(None)) - def test_gh_99240_clear_args(self): from _testcapi import gh_99240_clear_args self.assertRaises(TypeError, gh_99240_clear_args, 'a', '\0b') diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index bc6879176cd85e..27f4978ca66a88 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -281,18 +281,6 @@ def test_writerows_errors(self): self.assertRaises(TypeError, writer.writerows, None) self.assertRaises(OSError, writer.writerows, BadIterable()) - @support.cpython_only - @support.requires_legacy_unicode_capi() - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_writerows_legacy_strings(self): - import _testcapi - c = _testcapi.unicode_legacy_string('a') - with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: - writer = csv.writer(fileobj) - writer.writerows([[c]]) - fileobj.seek(0) - self.assertEqual(fileobj.read(), "a\r\n") - def _read_test(self, input, expect, **kwargs): reader = csv.reader(input, **kwargs) result = list(reader) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index abfd71c868d009..d806eeac25fb2f 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -34,7 +34,7 @@ import locale from test.support import (is_resource_enabled, requires_IEEE_754, requires_docstrings, - requires_legacy_unicode_capi, check_sanitizer, + check_sanitizer, check_disallow_instantiation) from test.support import (TestFailed, run_with_locale, cpython_only, @@ -587,18 +587,6 @@ def test_explicit_from_string(self): # underscores don't prevent errors self.assertRaises(InvalidOperation, Decimal, "1_2_\u00003") - @cpython_only - @requires_legacy_unicode_capi() - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_from_legacy_strings(self): - import _testcapi - Decimal = self.decimal.Decimal - context = self.decimal.Context() - - s = _testcapi.unicode_legacy_string('9.999999') - self.assertEqual(str(Decimal(s)), '9.999999') - self.assertEqual(str(context.create_decimal(s)), '9.999999') - def test_explicit_from_tuples(self): Decimal = self.decimal.Decimal @@ -2919,23 +2907,6 @@ def test_none_args(self): assert_signals(self, c, 'traps', [InvalidOperation, DivisionByZero, Overflow]) - @cpython_only - @requires_legacy_unicode_capi() - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_from_legacy_strings(self): - import _testcapi - c = self.decimal.Context() - - for rnd in RoundingModes: - c.rounding = _testcapi.unicode_legacy_string(rnd) - self.assertEqual(c.rounding, rnd) - - s = _testcapi.unicode_legacy_string('') - self.assertRaises(TypeError, setattr, c, 'rounding', s) - - s = _testcapi.unicode_legacy_string('ROUND_\x00UP') - self.assertRaises(TypeError, setattr, c, 'rounding', s) - def test_pickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index 3ae2f45ef6bddc..814ef111c5bec8 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -812,16 +812,6 @@ def test_isidentifier(self): self.assertFalse("©".isidentifier()) self.assertFalse("0".isidentifier()) - @support.cpython_only - @support.requires_legacy_unicode_capi() - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_isidentifier_legacy(self): - u = '𝖀𝖓𝖎𝖈𝖔𝖉𝖊' - self.assertTrue(u.isidentifier()) - with warnings_helper.check_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.assertTrue(_testcapi.unicode_legacy_string(u).isidentifier()) - def test_isprintable(self): self.assertTrue("".isprintable()) self.assertTrue(" ".isprintable()) @@ -2489,26 +2479,6 @@ def test_getnewargs(self): self.assertEqual(args[0], text) self.assertEqual(len(args), 1) - @support.cpython_only - @support.requires_legacy_unicode_capi() - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_resize(self): - for length in range(1, 100, 7): - # generate a fresh string (refcount=1) - text = 'a' * length + 'b' - - # fill wstr internal field - with self.assertWarns(DeprecationWarning): - abc = _testcapi.getargs_u(text) - self.assertEqual(abc, text) - - # resize text: wstr field must be cleared and then recomputed - text += 'c' - with self.assertWarns(DeprecationWarning): - abcdef = _testcapi.getargs_u(text) - self.assertNotEqual(abc, abcdef) - self.assertEqual(abcdef, text) - def test_compare(self): # Issue #17615 N = 10 diff --git a/Modules/_testcapi/getargs.c b/Modules/_testcapi/getargs.c index 10a1c1dd05253d..5f4a6dc8ca7672 100644 --- a/Modules/_testcapi/getargs.c +++ b/Modules/_testcapi/getargs.c @@ -589,54 +589,6 @@ getargs_y_hash(PyObject *self, PyObject *args) return PyBytes_FromStringAndSize(str, size); } -static PyObject * -getargs_u(PyObject *self, PyObject *args) -{ - wchar_t *str; - if (!PyArg_ParseTuple(args, "u", &str)) { - return NULL; - } - return PyUnicode_FromWideChar(str, -1); -} - -static PyObject * -getargs_u_hash(PyObject *self, PyObject *args) -{ - wchar_t *str; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "u#", &str, &size)) { - return NULL; - } - return PyUnicode_FromWideChar(str, size); -} - -static PyObject * -getargs_Z(PyObject *self, PyObject *args) -{ - wchar_t *str; - if (!PyArg_ParseTuple(args, "Z", &str)) { - return NULL; - } - if (str != NULL) { - return PyUnicode_FromWideChar(str, -1); - } - Py_RETURN_NONE; -} - -static PyObject * -getargs_Z_hash(PyObject *self, PyObject *args) -{ - wchar_t *str; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Z#", &str, &size)) { - return NULL; - } - if (str != NULL) { - return PyUnicode_FromWideChar(str, size); - } - Py_RETURN_NONE; -} - static PyObject * getargs_es(PyObject *self, PyObject *args) { @@ -845,8 +797,6 @@ static PyMethodDef test_methods[] = { {"getargs_S", getargs_S, METH_VARARGS}, {"getargs_U", getargs_U, METH_VARARGS}, {"getargs_Y", getargs_Y, METH_VARARGS}, - {"getargs_Z", getargs_Z, METH_VARARGS}, - {"getargs_Z_hash", getargs_Z_hash, METH_VARARGS}, {"getargs_b", getargs_b, METH_VARARGS}, {"getargs_c", getargs_c, METH_VARARGS}, {"getargs_d", getargs_d, METH_VARARGS}, @@ -868,8 +818,6 @@ static PyMethodDef test_methods[] = { {"getargs_s_hash", getargs_s_hash, METH_VARARGS}, {"getargs_s_star", getargs_s_star, METH_VARARGS}, {"getargs_tuple", getargs_tuple, METH_VARARGS}, - {"getargs_u", getargs_u, METH_VARARGS}, - {"getargs_u_hash", getargs_u_hash, METH_VARARGS}, {"getargs_w_star", getargs_w_star, METH_VARARGS}, {"getargs_y", getargs_y, METH_VARARGS}, {"getargs_y_hash", getargs_y_hash, METH_VARARGS}, From e21c89f9841488a1d4ca1024dc97c1aba44e0c87 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 11:18:14 +0200 Subject: [PATCH 34/66] gh-109162: Refactor libregrtest.RunTests (#109177) * Rename dash_R() runtest_refleak(). The function now gets huntrleaks and quiet arguments, instead of 'ns' argument. * Add attributes to Regrtest and RunTests: * verbose * quiet * huntrleaks * test_dir * Add HuntRefleak class. --- Lib/test/libregrtest/main.py | 57 +++++++++++++++++------------- Lib/test/libregrtest/refleak.py | 23 +++++++----- Lib/test/libregrtest/runtest.py | 57 +++++++++++++++++++----------- Lib/test/libregrtest/runtest_mp.py | 3 +- Lib/test/libregrtest/setup.py | 8 ++--- 5 files changed, 88 insertions(+), 60 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f7d28a859213f5..bb02101d97a2d2 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -12,7 +12,7 @@ from test.libregrtest.cmdline import _parse_args, Namespace from test.libregrtest.runtest import ( findtests, split_test_packages, run_single_test, abs_module_name, - PROGRESS_MIN_TIME, State, RunTests, TestResult, + PROGRESS_MIN_TIME, State, RunTests, TestResult, HuntRefleak, FilterTuple, FilterDict, TestList) from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.pgo import setup_pgo_tests @@ -92,6 +92,14 @@ def __init__(self, ns: Namespace): self.pgo_extended: bool = ns.pgo_extended self.output_on_failure: bool = ns.verbose3 self.timeout: float | None = ns.timeout + self.verbose: bool = ns.verbose + self.quiet: bool = ns.quiet + if ns.huntrleaks: + self.hunt_refleak: HuntRefleak = HuntRefleak(*ns.huntrleaks) + else: + self.hunt_refleak = None + self.test_dir: str | None = ns.testdir + self.junit_filename: str | None = ns.xmlpath # tests self.tests = [] @@ -200,8 +208,7 @@ def log(self, line=''): print(line, flush=True) def display_progress(self, test_index, text): - quiet = self.ns.quiet - if quiet: + if self.quiet: return # "[ 51/405/1] test_tcl passed" @@ -214,7 +221,6 @@ def display_progress(self, test_index, text): def find_tests(self): ns = self.ns single = ns.single - test_dir = ns.testdir if single: self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') @@ -250,7 +256,8 @@ def find_tests(self): exclude_tests.add(arg) ns.args = [] - alltests = findtests(testdir=test_dir, exclude=exclude_tests) + alltests = findtests(testdir=self.test_dir, + exclude=exclude_tests) if not self.fromfile: self.selected = self.tests or ns.args @@ -298,14 +305,12 @@ def _list_cases(self, suite): print(test.id()) def list_cases(self): - ns = self.ns - test_dir = ns.testdir support.verbose = False support.set_match_tests(self.match_tests, self.ignore_tests) skipped = [] for test_name in self.selected: - module_name = abs_module_name(test_name, test_dir) + module_name = abs_module_name(test_name, self.test_dir) try: suite = unittest.defaultTestLoader.loadTestsFromName(module_name) self._list_cases(suite) @@ -331,7 +336,6 @@ def get_rerun_match(self, rerun_list) -> FilterDict: def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Configure the runner to re-run tests ns = self.ns - ns.verbose = True if ns.use_mp is None: ns.use_mp = 1 @@ -349,6 +353,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): runtests = runtests.copy( tests=tuple(tests), rerun=True, + verbose=True, forever=False, fail_fast=False, match_tests_dict=match_tests_dict, @@ -379,7 +384,6 @@ def rerun_failed_tests(self, need_rerun, runtests: RunTests): def display_result(self, runtests): pgo = runtests.pgo - quiet = self.ns.quiet print_slow = self.ns.print_slow # If running the test suite for PGO then no one cares about results. @@ -398,7 +402,7 @@ def display_result(self, runtests): print(count(len(omitted), "test"), "omitted:") printlist(omitted) - if self.good and not quiet: + if self.good and not self.quiet: print() if (not self.bad and not self.skipped @@ -425,12 +429,12 @@ def display_result(self, runtests): count(len(self.environment_changed), "test"))) printlist(self.environment_changed) - if self.skipped and not quiet: + if self.skipped and not self.quiet: print() print(count(len(self.skipped), "test"), "skipped:") printlist(self.skipped) - if self.resource_denied and not quiet: + if self.resource_denied and not self.quiet: print() print(count(len(self.resource_denied), "test"), "skipped (resource denied):") printlist(self.resource_denied) @@ -684,7 +688,7 @@ def display_summary(self): print(f"Result: {result}") def save_xml_result(self): - if not self.ns.xmlpath and not self.testsuite_xml: + if not self.junit_filename and not self.testsuite_xml: return import xml.etree.ElementTree as ET @@ -703,7 +707,7 @@ def save_xml_result(self): for k, v in totals.items(): root.set(k, str(v)) - xmlpath = os.path.join(os_helper.SAVEDCWD, self.ns.xmlpath) + xmlpath = os.path.join(os_helper.SAVEDCWD, self.junit_filename) with open(xmlpath, 'wb') as f: for s in ET.tostringlist(root): f.write(s) @@ -785,7 +789,7 @@ def main(self, tests: TestList | None = None): ns = self.ns self.tests = tests - if ns.xmlpath: + if self.junit_filename: support.junit_xml_list = self.testsuite_xml = [] strip_py_suffix(ns.args) @@ -844,16 +848,14 @@ def get_exitcode(self): return exitcode def action_run_tests(self): - if self.ns.huntrleaks: - warmup, repetitions, _ = self.ns.huntrleaks - if warmup < 3: - msg = ("WARNING: Running tests with --huntrleaks/-R and less than " - "3 warmup repetitions can give false positives!") - print(msg, file=sys.stdout, flush=True) + if self.hunt_refleak and self.hunt_refleak.warmups < 3: + msg = ("WARNING: Running tests with --huntrleaks/-R and " + "less than 3 warmup repetitions can give false positives!") + print(msg, file=sys.stdout, flush=True) # For a partial run, we do not need to clutter the output. if (self.want_header - or not(self.pgo or self.ns.quiet or self.ns.single + or not(self.pgo or self.quiet or self.ns.single or self.tests or self.ns.args)): self.display_header() @@ -869,7 +871,12 @@ def action_run_tests(self): pgo=self.pgo, pgo_extended=self.pgo_extended, output_on_failure=self.output_on_failure, - timeout=self.timeout) + timeout=self.timeout, + verbose=self.verbose, + quiet=self.quiet, + hunt_refleak=self.hunt_refleak, + test_dir=self.test_dir, + junit_filename=self.junit_filename) setup_tests(runtests, self.ns) @@ -892,7 +899,7 @@ def _main(self): if self.want_wait: input("Press any key to continue...") - setup_test_dir(self.ns.testdir) + setup_test_dir(self.test_dir) self.find_tests() exitcode = 0 diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 206802b60ddcd0..2e9f17e1c1eee6 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -4,6 +4,7 @@ from inspect import isabstract from test import support from test.support import os_helper +from test.libregrtest.runtest import HuntRefleak from test.libregrtest.utils import clear_caches try: @@ -19,7 +20,9 @@ def _get_dump(cls): cls._abc_negative_cache, cls._abc_negative_cache_version) -def dash_R(ns, test_name, test_func): +def runtest_refleak(test_name, test_func, + hunt_refleak: HuntRefleak, + quiet: bool): """Run a test multiple times, looking for reference leaks. Returns: @@ -62,9 +65,11 @@ def dash_R(ns, test_name, test_func): def get_pooled_int(value): return int_pool.setdefault(value, value) - nwarmup, ntracked, fname = ns.huntrleaks - fname = os.path.join(os_helper.SAVEDCWD, fname) - repcount = nwarmup + ntracked + warmups = hunt_refleak.warmups + runs = hunt_refleak.runs + filename = hunt_refleak.filename + filename = os.path.join(os_helper.SAVEDCWD, filename) + repcount = warmups + runs # Pre-allocate to ensure that the loop doesn't allocate anything new rep_range = list(range(repcount)) @@ -78,7 +83,7 @@ def get_pooled_int(value): # initialize variables to make pyflakes quiet rc_before = alloc_before = fd_before = interned_before = 0 - if not ns.quiet: + if not quiet: print("beginning", repcount, "repetitions", file=sys.stderr) print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, flush=True) @@ -102,7 +107,7 @@ def get_pooled_int(value): rc_after = gettotalrefcount() - interned_after * 2 fd_after = fd_count() - if not ns.quiet: + if not quiet: print('.', end='', file=sys.stderr, flush=True) rc_deltas[i] = get_pooled_int(rc_after - rc_before) @@ -114,7 +119,7 @@ def get_pooled_int(value): fd_before = fd_after interned_before = interned_after - if not ns.quiet: + if not quiet: print(file=sys.stderr) # These checkers return False on success, True on failure @@ -143,12 +148,12 @@ def check_fd_deltas(deltas): (fd_deltas, 'file descriptors', check_fd_deltas) ]: # ignore warmup runs - deltas = deltas[nwarmup:] + deltas = deltas[warmups:] if checker(deltas): msg = '%s leaked %s %s, sum=%s' % ( test_name, deltas, item_name, sum(deltas)) print(msg, file=sys.stderr, flush=True) - with open(fname, "a", encoding="utf-8") as refrep: + with open(filename, "a", encoding="utf-8") as refrep: print(msg, file=refrep) refrep.flush() failed = True diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index bfb0718aa56c32..4f12176ced5c87 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -28,6 +28,13 @@ FilterDict = dict[str, FilterTuple] +@dataclasses.dataclass(slots=True, frozen=True) +class HuntRefleak: + warmups: int + runs: int + filename: str + + # Avoid enum.Enum to reduce the number of imports when tests are run class State: PASSED = "PASSED" @@ -218,6 +225,11 @@ class RunTests: pgo_extended: bool = False output_on_failure: bool = False timeout: float | None = None + verbose: bool = False + quiet: bool = False + hunt_refleak: HuntRefleak | None = None + test_dir: str | None = None + junit_filename: str | None = None def copy(self, **override): state = dataclasses.asdict(self) @@ -260,7 +272,7 @@ def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(*, testdir=None, exclude=(), +def findtests(*, testdir: str | None =None, exclude=(), split_test_dirs=SPLITTESTDIRS, base_mod=""): """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) @@ -279,7 +291,7 @@ def findtests(*, testdir=None, exclude=(), return sorted(tests) -def split_test_packages(tests, *, testdir=None, exclude=(), +def split_test_packages(tests, *, testdir: str | None = None, exclude=(), split_test_dirs=SPLITTESTDIRS): testdir = findtestdir(testdir) splitted = [] @@ -307,8 +319,8 @@ def setup_support(runtests: RunTests, ns: Namespace): support.PGO_EXTENDED = runtests.pgo_extended support.set_match_tests(runtests.match_tests, runtests.ignore_tests) support.failfast = runtests.fail_fast - support.verbose = ns.verbose - if ns.xmlpath: + support.verbose = runtests.verbose + if runtests.junit_filename: support.junit_xml_list = [] else: support.junit_xml_list = None @@ -317,7 +329,7 @@ def setup_support(runtests: RunTests, ns: Namespace): def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: # Capture stdout and stderr, set faulthandler timeout, # and create JUnit XML report. - verbose = ns.verbose + verbose = runtests.verbose output_on_failure = runtests.output_on_failure timeout = runtests.timeout @@ -363,7 +375,8 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: else: # Tell tests to be moderately quiet support.verbose = verbose - _runtest_env_changed_exc(result, runtests, ns, display_failure=not verbose) + _runtest_env_changed_exc(result, runtests, ns, + display_failure=not verbose) xml_list = support.junit_xml_list if xml_list: @@ -384,7 +397,7 @@ def run_single_test(test_name: str, runtests: RunTests, ns: Namespace) -> TestRe Returns a TestResult. - If ns.xmlpath is not None, xml_data is a list containing each + If runtests.junit_filename is not None, xml_data is a list containing each generated testsuite element. """ start_time = time.perf_counter() @@ -412,16 +425,19 @@ def run_unittest(test_mod): return support.run_unittest(tests) -def save_env(test_name: str, runtests: RunTests, ns: Namespace): - return saved_test_environment(test_name, ns.verbose, ns.quiet, pgo=runtests.pgo) +def save_env(test_name: str, runtests: RunTests): + return saved_test_environment(test_name, runtests.verbose, runtests.quiet, + pgo=runtests.pgo) -def regrtest_runner(result, test_func, ns) -> None: +def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: # Run test_func(), collect statistics, and detect reference and memory # leaks. - if ns.huntrleaks: - from test.libregrtest.refleak import dash_R - refleak, test_result = dash_R(ns, result.test_name, test_func) + if runtests.hunt_refleak: + from test.libregrtest.refleak import runtest_refleak + refleak, test_result = runtest_refleak(result.test_name, test_func, + runtests.hunt_refleak, + runtests.quiet) else: test_result = test_func() refleak = False @@ -452,7 +468,7 @@ def regrtest_runner(result, test_func, ns) -> None: def _load_run_test(result: TestResult, runtests: RunTests, ns: Namespace) -> None: # Load the test function, run the test function. - module_name = abs_module_name(result.test_name, ns.testdir) + module_name = abs_module_name(result.test_name, runtests.test_dir) # Remove the module from sys.module to reload it if it was already imported sys.modules.pop(module_name, None) @@ -466,8 +482,8 @@ def test_func(): return run_unittest(test_mod) try: - with save_env(result.test_name, runtests, ns): - regrtest_runner(result, test_func, ns) + with save_env(result.test_name, runtests): + regrtest_runner(result, test_func, runtests) finally: # First kill any dangling references to open files etc. # This can also issue some ResourceWarnings which would otherwise get @@ -475,7 +491,7 @@ def test_func(): # failures. support.gc_collect() - remove_testfn(result.test_name, ns.verbose) + remove_testfn(result.test_name, runtests.verbose) if gc.garbage: support.environment_altered = True @@ -502,21 +518,22 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, pgo = runtests.pgo if pgo: display_failure = False + quiet = runtests.quiet test_name = result.test_name try: clear_caches() support.gc_collect() - with save_env(test_name, runtests, ns): + with save_env(test_name, runtests): _load_run_test(result, runtests, ns) except support.ResourceDenied as msg: - if not ns.quiet and not pgo: + if not quiet and not pgo: print(f"{test_name} skipped -- {msg}", flush=True) result.state = State.RESOURCE_DENIED return except unittest.SkipTest as msg: - if not ns.quiet and not pgo: + if not quiet and not pgo: print(f"{test_name} skipped -- {msg}", flush=True) result.state = State.SKIPPED return diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index ecdde3aa523098..4bff7f99964119 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -127,7 +127,7 @@ def worker_process(worker_json: str) -> NoReturn: test_name = runtests.tests[0] match_tests: FilterTuple | None = runtests.match_tests - setup_test_dir(ns.testdir) + setup_test_dir(runtests.test_dir) setup_tests(runtests, ns) if runtests.rerun: @@ -136,7 +136,6 @@ def worker_process(worker_json: str) -> NoReturn: print(f"Re-running {test_name} in verbose mode ({matching})", flush=True) else: print(f"Re-running {test_name} in verbose mode", flush=True) - ns.verbose = True result = run_single_test(test_name, runtests, ns) print() # Force a newline (just in case) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index f640362cb2df7f..59bbf2c81a167a 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -18,7 +18,7 @@ UNICODE_GUARD_ENV = "PYTHONREGRTEST_UNICODE_GUARD" -def setup_test_dir(testdir): +def setup_test_dir(testdir: str | None) -> None: if testdir: # Prepend test directory to sys.path, so runtest() will be able # to locate tests @@ -68,7 +68,7 @@ def setup_tests(runtests, ns): if getattr(module, '__file__', None): module.__file__ = os.path.abspath(module.__file__) - if ns.huntrleaks: + if runtests.hunt_refleak: unittest.BaseTestSuite._cleanup = False if ns.memlimit is not None: @@ -77,7 +77,7 @@ def setup_tests(runtests, ns): if ns.threshold is not None: gc.set_threshold(ns.threshold) - support.suppress_msvcrt_asserts(ns.verbose and ns.verbose >= 2) + support.suppress_msvcrt_asserts(runtests.verbose and runtests.verbose >= 2) support.use_resources = ns.use_resources @@ -102,7 +102,7 @@ def _test_audit_hook(name, args): support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, timeout) support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, timeout) - if ns.xmlpath: + if runtests.junit_filename: from test.support.testresult import RegressionTestResult RegressionTestResult.USE_XML = True From 75cd86599bad05cb372aed9fccc3ff884cd38b70 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 9 Sep 2023 10:21:42 -0500 Subject: [PATCH 35/66] Fix an ironic typo in a code comment. (gh-109186) --- Lib/random.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/random.py b/Lib/random.py index 586c3f7f9da938..84bbfc5df1bf23 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -827,7 +827,7 @@ def binomialvariate(self, n=1, p=0.5): return k # Acceptance-rejection test. - # Note, the original paper errorneously omits the call to log(v) + # Note, the original paper erroneously omits the call to log(v) # when comparing to the log of the rescaled binomial distribution. if not setup_complete: alpha = (2.83 + 5.1 / b) * spq From d3ed9921cdd8ac291fbfe3adf42f7730d3a14dbc Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 9 Sep 2023 17:50:04 -0500 Subject: [PATCH 36/66] Improve the sieve() recipe in the itertools docs (gh-109199) Lazier sieve --- Doc/library/itertools.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 42243715c2d93b..3cfc2602fe0694 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1030,13 +1030,16 @@ The following recipes have a more mathematical flavor: def sieve(n): "Primes less than n." # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 + if n > 2: + yield 2 + start = 3 data = bytearray((0, 1)) * (n // 2) - data[:3] = 0, 0, 0 limit = math.isqrt(n) + 1 - for p in compress(range(limit), data): + for p in iter_index(data, 1, start, limit): + yield from iter_index(data, 1, start, p*p) data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) - data[2] = 1 - return iter_index(data, 1) if n > 2 else iter([]) + start = p*p + yield from iter_index(data, 1, start) def factor(n): "Prime factors of n." From 24fa8f2046965b46c70a750a5a004708a63ac770 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 00:51:24 +0200 Subject: [PATCH 37/66] gh-109162: libregrtest: fix _decode_worker_job() (#109202) Decode also HuntRefleak() object inside the RunTests object. Add an unit test on huntrleaks with multiprocessing (-R -jN). --- Lib/test/libregrtest/runtest.py | 6 ++++++ Lib/test/libregrtest/runtest_mp.py | 2 +- Lib/test/test_regrtest.py | 18 ++++++++++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 4f12176ced5c87..ab59d8123013da 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -249,6 +249,12 @@ def iter_tests(self): else: yield from self.tests + @staticmethod + def from_json_dict(json_dict): + if json_dict['hunt_refleak']: + json_dict['hunt_refleak'] = HuntRefleak(**json_dict['hunt_refleak']) + return RunTests(**json_dict) + # Minimum duration of a test to display its duration or to mention that # the test is running in background diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 4bff7f99964119..4e3148fa8e2e67 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -68,7 +68,7 @@ def default(self, o: Any) -> dict[str, Any]: def _decode_worker_job(d: dict[str, Any]) -> WorkerJob | dict[str, Any]: if "__worker_job__" in d: d.pop('__worker_job__') - d['runtests'] = RunTests(**d['runtests']) + d['runtests'] = RunTests.from_json_dict(d['runtests']) return WorkerJob(**d) if "__namespace__" in d: d.pop('__namespace__') diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 8cced1f5185c2f..23896fdedaccac 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1018,12 +1018,16 @@ def test_run(self): stats=TestStats(4, 1), forever=True) - def check_leak(self, code, what): + def check_leak(self, code, what, *, multiprocessing=False): test = self.create_test('huntrleaks', code=code) filename = 'reflog.txt' self.addCleanup(os_helper.unlink, filename) - output = self.run_tests('--huntrleaks', '6:3:', test, + cmd = ['--huntrleaks', '6:3:'] + if multiprocessing: + cmd.append('-j1') + cmd.append(test) + output = self.run_tests(*cmd, exitcode=EXITCODE_BAD_TEST, stderr=subprocess.STDOUT) self.check_executed_tests(output, [test], failed=test, stats=1) @@ -1039,7 +1043,7 @@ def check_leak(self, code, what): self.assertIn(line2, reflog) @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') - def test_huntrleaks(self): + def check_huntrleaks(self, *, multiprocessing: bool): # test --huntrleaks code = textwrap.dedent(""" import unittest @@ -1050,7 +1054,13 @@ class RefLeakTest(unittest.TestCase): def test_leak(self): GLOBAL_LIST.append(object()) """) - self.check_leak(code, 'references') + self.check_leak(code, 'references', multiprocessing=multiprocessing) + + def test_huntrleaks(self): + self.check_huntrleaks(multiprocessing=False) + + def test_huntrleaks_mp(self): + self.check_huntrleaks(multiprocessing=True) @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): From 0c0f254230bbc9532a7e78077a639a3ac940953c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 01:41:21 +0200 Subject: [PATCH 38/66] gh-109162: libregrtest: remove WorkerJob class (#109204) * Add attributes to Regrtest and RunTests: * gc_threshold * memory_limit * python_cmd * use_resources * Remove WorkerJob class. Add as_json() and from_json() methods to RunTests. A worker process now only uses RunTests for all parameters. * Add tests on support.set_memlimit() in test_support. Create _parse_memlimit() and also adds tests on it. * Remove 'ns' parameter from runtest.py. --- Lib/test/libregrtest/cmdline.py | 2 ++ Lib/test/libregrtest/main.py | 21 +++++++---- Lib/test/libregrtest/runtest.py | 56 ++++++++++++++++++++--------- Lib/test/libregrtest/runtest_mp.py | 58 ++++++------------------------ Lib/test/libregrtest/setup.py | 12 +++---- Lib/test/support/__init__.py | 24 +++++++------ Lib/test/test_support.py | 41 +++++++++++++++++++-- 7 files changed, 126 insertions(+), 88 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 71e7396d71d4a5..9afb13224975ee 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -177,6 +177,8 @@ def __init__(self, **kwargs) -> None: self.worker_json = None self.start = None self.timeout = None + self.memlimit = None + self.threshold = None super().__init__(**kwargs) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index bb02101d97a2d2..4066d06c98600e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -100,6 +100,10 @@ def __init__(self, ns: Namespace): self.hunt_refleak = None self.test_dir: str | None = ns.testdir self.junit_filename: str | None = ns.xmlpath + self.memory_limit: str | None = ns.memlimit + self.gc_threshold: int | None = ns.threshold + self.use_resources: list[str] = ns.use_resources + self.python_cmd: list[str] | None = ns.python # tests self.tests = [] @@ -363,7 +367,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): return runtests def rerun_failed_tests(self, need_rerun, runtests: RunTests): - if self.ns.python: + if self.python_cmd: # Temp patch for https://github.com/python/cpython/issues/94052 self.log( "Re-running failed tests is not supported with --python " @@ -453,12 +457,12 @@ def run_test(self, test_name: str, runtests: RunTests, tracer): if tracer is not None: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - cmd = ('result = run_single_test(test_name, runtests, self.ns)') + cmd = ('result = run_single_test(test_name, runtests)') ns = dict(locals()) tracer.runctx(cmd, globals=globals(), locals=ns) result = ns['result'] else: - result = run_single_test(test_name, runtests, self.ns) + result = run_single_test(test_name, runtests) self.accumulate_result(result) @@ -876,9 +880,14 @@ def action_run_tests(self): quiet=self.quiet, hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, - junit_filename=self.junit_filename) - - setup_tests(runtests, self.ns) + junit_filename=self.junit_filename, + memory_limit=self.memory_limit, + gc_threshold=self.gc_threshold, + use_resources=self.use_resources, + python_cmd=self.python_cmd, + ) + + setup_tests(runtests) tracer = self.run_tests(runtests) self.display_result(runtests) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index ab59d8123013da..dc574eda8b99f5 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -4,17 +4,18 @@ import gc import importlib import io +import json import os import sys import time import traceback import unittest +from typing import Any from test import support from test.support import TestStats from test.support import os_helper from test.support import threading_helper -from test.libregrtest.cmdline import Namespace from test.libregrtest.save_env import saved_test_environment from test.libregrtest.utils import clear_caches, format_duration, print_warning @@ -230,6 +231,10 @@ class RunTests: hunt_refleak: HuntRefleak | None = None test_dir: str | None = None junit_filename: str | None = None + memory_limit: str | None = None + gc_threshold: int | None = None + use_resources: list[str] = None + python_cmd: list[str] | None = None def copy(self, **override): state = dataclasses.asdict(self) @@ -249,11 +254,32 @@ def iter_tests(self): else: yield from self.tests + def as_json(self): + return json.dumps(self, cls=_EncodeRunTests) + @staticmethod - def from_json_dict(json_dict): - if json_dict['hunt_refleak']: - json_dict['hunt_refleak'] = HuntRefleak(**json_dict['hunt_refleak']) - return RunTests(**json_dict) + def from_json(worker_json): + return json.loads(worker_json, object_hook=_decode_runtests) + + +class _EncodeRunTests(json.JSONEncoder): + def default(self, o: Any) -> dict[str, Any]: + if isinstance(o, RunTests): + result = dataclasses.asdict(o) + result["__runtests__"] = True + return result + else: + return super().default(o) + + +def _decode_runtests(data: dict[str, Any]) -> RunTests | dict[str, Any]: + if "__runtests__" in data: + data.pop('__runtests__') + if data['hunt_refleak']: + data['hunt_refleak'] = HuntRefleak(**data['hunt_refleak']) + return RunTests(**data) + else: + return data # Minimum duration of a test to display its duration or to mention that @@ -320,7 +346,7 @@ def abs_module_name(test_name: str, test_dir: str | None) -> str: return 'test.' + test_name -def setup_support(runtests: RunTests, ns: Namespace): +def setup_support(runtests: RunTests): support.PGO = runtests.pgo support.PGO_EXTENDED = runtests.pgo_extended support.set_match_tests(runtests.match_tests, runtests.ignore_tests) @@ -332,7 +358,7 @@ def setup_support(runtests: RunTests, ns: Namespace): support.junit_xml_list = None -def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: +def _runtest(result: TestResult, runtests: RunTests) -> None: # Capture stdout and stderr, set faulthandler timeout, # and create JUnit XML report. verbose = runtests.verbose @@ -346,7 +372,7 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: faulthandler.dump_traceback_later(timeout, exit=True) try: - setup_support(runtests, ns) + setup_support(runtests) if output_on_failure: support.verbose = True @@ -366,7 +392,7 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: # warnings will be written to sys.stderr below. print_warning.orig_stderr = stream - _runtest_env_changed_exc(result, runtests, ns, display_failure=False) + _runtest_env_changed_exc(result, runtests, display_failure=False) # Ignore output if the test passed successfully if result.state != State.PASSED: output = stream.getvalue() @@ -381,7 +407,7 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: else: # Tell tests to be moderately quiet support.verbose = verbose - _runtest_env_changed_exc(result, runtests, ns, + _runtest_env_changed_exc(result, runtests, display_failure=not verbose) xml_list = support.junit_xml_list @@ -395,10 +421,9 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: support.junit_xml_list = None -def run_single_test(test_name: str, runtests: RunTests, ns: Namespace) -> TestResult: +def run_single_test(test_name: str, runtests: RunTests) -> TestResult: """Run a single test. - ns -- regrtest namespace of options test_name -- the name of the test Returns a TestResult. @@ -410,7 +435,7 @@ def run_single_test(test_name: str, runtests: RunTests, ns: Namespace) -> TestRe result = TestResult(test_name) pgo = runtests.pgo try: - _runtest(result, runtests, ns) + _runtest(result, runtests) except: if not pgo: msg = traceback.format_exc() @@ -472,7 +497,7 @@ def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: FOUND_GARBAGE = [] -def _load_run_test(result: TestResult, runtests: RunTests, ns: Namespace) -> None: +def _load_run_test(result: TestResult, runtests: RunTests) -> None: # Load the test function, run the test function. module_name = abs_module_name(result.test_name, runtests.test_dir) @@ -513,7 +538,6 @@ def test_func(): def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, - ns: Namespace, display_failure: bool = True) -> None: # Detect environment changes, handle exceptions. @@ -532,7 +556,7 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, support.gc_collect() with save_env(test_name, runtests): - _load_run_test(result, runtests, ns) + _load_run_test(result, runtests) except support.ResourceDenied as msg: if not quiet and not pgo: print(f"{test_name} skipped -- {msg}", flush=True) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 4e3148fa8e2e67..e4a9301656436b 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -47,49 +47,16 @@ @dataclasses.dataclass(slots=True) class WorkerJob: runtests: RunTests - namespace: Namespace -class _EncodeWorkerJob(json.JSONEncoder): - def default(self, o: Any) -> dict[str, Any]: - match o: - case WorkerJob(): - result = dataclasses.asdict(o) - result["__worker_job__"] = True - return result - case Namespace(): - result = vars(o) - result["__namespace__"] = True - return result - case _: - return super().default(o) - - -def _decode_worker_job(d: dict[str, Any]) -> WorkerJob | dict[str, Any]: - if "__worker_job__" in d: - d.pop('__worker_job__') - d['runtests'] = RunTests.from_json_dict(d['runtests']) - return WorkerJob(**d) - if "__namespace__" in d: - d.pop('__namespace__') - return Namespace(**d) - else: - return d - - -def _parse_worker_json(worker_json: str) -> tuple[Namespace, str]: - return json.loads(worker_json, object_hook=_decode_worker_job) - - -def create_worker_process(worker_job: WorkerJob, +def create_worker_process(runtests: RunTests, output_file: TextIO, tmp_dir: str | None = None) -> subprocess.Popen: - ns = worker_job.namespace - python = ns.python - worker_json = json.dumps(worker_job, cls=_EncodeWorkerJob) + python_cmd = runtests.python_cmd + worker_json = runtests.as_json() - if python is not None: - executable = python + if python_cmd is not None: + executable = python_cmd else: executable = [sys.executable] cmd = [*executable, *support.args_from_interpreter_flags(), @@ -121,14 +88,12 @@ def create_worker_process(worker_job: WorkerJob, def worker_process(worker_json: str) -> NoReturn: - worker_job = _parse_worker_json(worker_json) - runtests = worker_job.runtests - ns = worker_job.namespace + runtests = RunTests.from_json(worker_json) test_name = runtests.tests[0] match_tests: FilterTuple | None = runtests.match_tests setup_test_dir(runtests.test_dir) - setup_tests(runtests, ns) + setup_tests(runtests) if runtests.rerun: if match_tests: @@ -137,7 +102,7 @@ def worker_process(worker_json: str) -> NoReturn: else: print(f"Re-running {test_name} in verbose mode", flush=True) - result = run_single_test(test_name, runtests, ns) + result = run_single_test(test_name, runtests) print() # Force a newline (just in case) # Serialize TestResult as dict in JSON @@ -330,9 +295,6 @@ def _runtest(self, test_name: str) -> MultiprocessResult: if match_tests: kwargs['match_tests'] = match_tests worker_runtests = self.runtests.copy(tests=tests, **kwargs) - worker_job = WorkerJob( - worker_runtests, - namespace=self.ns) # gh-94026: Write stdout+stderr to a tempfile as workaround for # non-blocking pipes on Emscripten with NodeJS. @@ -347,12 +309,12 @@ def _runtest(self, test_name: str) -> MultiprocessResult: tmp_dir = tempfile.mkdtemp(prefix="test_python_") tmp_dir = os.path.abspath(tmp_dir) try: - retcode = self._run_process(worker_job, stdout_file, tmp_dir) + retcode = self._run_process(worker_runtests, stdout_file, tmp_dir) finally: tmp_files = os.listdir(tmp_dir) os_helper.rmtree(tmp_dir) else: - retcode = self._run_process(worker_job, stdout_file) + retcode = self._run_process(worker_runtests, stdout_file) tmp_files = () stdout_file.seek(0) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 59bbf2c81a167a..594498333fe792 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -25,7 +25,7 @@ def setup_test_dir(testdir: str | None) -> None: sys.path.insert(0, os.path.abspath(testdir)) -def setup_tests(runtests, ns): +def setup_tests(runtests): try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): @@ -71,15 +71,15 @@ def setup_tests(runtests, ns): if runtests.hunt_refleak: unittest.BaseTestSuite._cleanup = False - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) + if runtests.memory_limit is not None: + support.set_memlimit(runtests.memory_limit) - if ns.threshold is not None: - gc.set_threshold(ns.threshold) + if runtests.gc_threshold is not None: + gc.set_threshold(runtests.gc_threshold) support.suppress_msvcrt_asserts(runtests.verbose and runtests.verbose >= 2) - support.use_resources = ns.use_resources + support.use_resources = runtests.use_resources if hasattr(sys, 'addaudithook'): # Add an auditing hook for all tests to ensure PySys_Audit is tested diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f40b73b0a799f1..6fc85ff5d704e1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -878,27 +878,31 @@ def inner(*args, **kwds): MAX_Py_ssize_t = sys.maxsize -def set_memlimit(limit): - global max_memuse - global real_max_memuse +def _parse_memlimit(limit: str) -> int: sizes = { 'k': 1024, 'm': _1M, 'g': _1G, 't': 1024*_1G, } - m = re.match(r'(\d+(\.\d+)?) (K|M|G|T)b?$', limit, + m = re.match(r'(\d+(?:\.\d+)?) (K|M|G|T)b?$', limit, re.IGNORECASE | re.VERBOSE) if m is None: - raise ValueError('Invalid memory limit %r' % (limit,)) - memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()]) - real_max_memuse = memlimit - if memlimit > MAX_Py_ssize_t: - memlimit = MAX_Py_ssize_t + raise ValueError(f'Invalid memory limit: {limit!r}') + return int(float(m.group(1)) * sizes[m.group(2).lower()]) + +def set_memlimit(limit: str) -> None: + global max_memuse + global real_max_memuse + memlimit = _parse_memlimit(limit) if memlimit < _2G - 1: - raise ValueError('Memory limit %r too low to be useful' % (limit,)) + raise ValueError('Memory limit {limit!r} too low to be useful') + + real_max_memuse = memlimit + memlimit = min(memlimit, MAX_Py_ssize_t) max_memuse = memlimit + class _MemoryWatchdog: """An object which periodically watches the process' memory consumption and prints it out. diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 64280739f00946..5b57c5fd54a68d 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -760,7 +760,45 @@ def recursive_function(depth): else: self.fail("RecursionError was not raised") - #self.assertEqual(available, 2) + def test_parse_memlimit(self): + parse = support._parse_memlimit + KiB = 1024 + MiB = KiB * 1024 + GiB = MiB * 1024 + TiB = GiB * 1024 + self.assertEqual(parse('0k'), 0) + self.assertEqual(parse('3k'), 3 * KiB) + self.assertEqual(parse('2.4m'), int(2.4 * MiB)) + self.assertEqual(parse('4g'), int(4 * GiB)) + self.assertEqual(parse('1t'), TiB) + + for limit in ('', '3', '3.5.10k', '10x'): + with self.subTest(limit=limit): + with self.assertRaises(ValueError): + parse(limit) + + def test_set_memlimit(self): + _4GiB = 4 * 1024 ** 3 + TiB = 1024 ** 4 + old_max_memuse = support.max_memuse + old_real_max_memuse = support.real_max_memuse + try: + if sys.maxsize > 2**32: + support.set_memlimit('4g') + self.assertEqual(support.max_memuse, _4GiB) + self.assertEqual(support.real_max_memuse, _4GiB) + + big = 2**100 // TiB + support.set_memlimit(f'{big}t') + self.assertEqual(support.max_memuse, sys.maxsize) + self.assertEqual(support.real_max_memuse, big * TiB) + else: + support.set_memlimit('4g') + self.assertEqual(support.max_memuse, sys.maxsize) + self.assertEqual(support.real_max_memuse, _4GiB) + finally: + support.max_memuse = old_max_memuse + support.real_max_memuse = old_real_max_memuse # XXX -follows a list of untested API # make_legacy_pyc @@ -773,7 +811,6 @@ def recursive_function(depth): # EnvironmentVarGuard # transient_internet # run_with_locale - # set_memlimit # bigmemtest # precisionbigmemtest # bigaddrspacetest From 0553fdfe3040073307e8c53273041130148541d5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 02:24:38 +0200 Subject: [PATCH 39/66] gh-109162: Refactor libregrtest.runtest_mp (#109205) * Add attributes to Regrtest and RunTests: * fail_env_changed * num_workers * Rename MultiprocessTestRunner to RunWorkers. Add num_workers parameters to RunWorkers constructor. Remove RunWorkers.ns attribute. * Rename TestWorkerProcess to WorkerThread. * get_running() now returns a string like: "running (...): ...". * Regrtest.action_run_tests() now selects the number of worker processes, instead of the command line parser. --- Lib/test/libregrtest/cmdline.py | 6 +-- Lib/test/libregrtest/main.py | 44 +++++++++++++-------- Lib/test/libregrtest/runtest.py | 1 + Lib/test/libregrtest/runtest_mp.py | 62 ++++++++++++++---------------- 4 files changed, 57 insertions(+), 56 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 9afb13224975ee..2835546fc713cf 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,5 +1,5 @@ import argparse -import os +import os.path import shlex import sys from test.support import os_helper @@ -410,10 +410,6 @@ def _parse_args(args, **kwargs): if ns.timeout is not None: if ns.timeout <= 0: ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) if ns.use: for a in ns.use: for r in a: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 4066d06c98600e..1fa7b07a09d701 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -83,8 +83,18 @@ def __init__(self, ns: Namespace): self.fromfile: str | None = ns.fromfile self.starting_test: str | None = ns.start + # Run tests + if ns.use_mp is None: + num_workers = 0 # run sequentially + elif ns.use_mp <= 0: + num_workers = -1 # use the number of CPUs + else: + num_workers = ns.use_mp + self.num_workers: int = num_workers + # Options to run tests self.fail_fast: bool = ns.failfast + self.fail_env_changed: bool = ns.fail_env_changed self.forever: bool = ns.forever self.randomize: bool = ns.randomize self.random_seed: int | None = ns.random_seed @@ -150,7 +160,6 @@ def get_executed(self): | set(self.run_no_tests)) def accumulate_result(self, result, rerun=False): - fail_env_changed = self.ns.fail_env_changed test_name = result.test_name match result.state: @@ -167,7 +176,7 @@ def accumulate_result(self, result, rerun=False): case State.DID_NOT_RUN: self.run_no_tests.append(test_name) case _: - if result.is_failed(fail_env_changed): + if result.is_failed(self.fail_env_changed): self.bad.append(test_name) self.need_rerun.append(result) else: @@ -339,9 +348,8 @@ def get_rerun_match(self, rerun_list) -> FilterDict: def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Configure the runner to re-run tests - ns = self.ns - if ns.use_mp is None: - ns.use_mp = 1 + if self.num_workers == 0: + self.num_workers = 1 # Get tests to re-run tests = [result.test_name for result in need_rerun] @@ -363,7 +371,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): match_tests_dict=match_tests_dict, output_on_failure=False) self.set_tests(runtests) - self._run_tests_mp(runtests) + self._run_tests_mp(runtests, self.num_workers) return runtests def rerun_failed_tests(self, need_rerun, runtests: RunTests): @@ -471,7 +479,6 @@ def run_test(self, test_name: str, runtests: RunTests, tracer): def run_tests_sequentially(self, runtests): ns = self.ns coverage = ns.trace - fail_env_changed = ns.fail_env_changed if coverage: import trace @@ -503,7 +510,7 @@ def run_tests_sequentially(self, runtests): if module not in save_modules and module.startswith("test."): support.unload(module) - if result.must_stop(self.fail_fast, fail_env_changed): + if result.must_stop(self.fail_fast, self.fail_env_changed): break previous_test = str(result) @@ -564,12 +571,10 @@ def no_tests_run(self): self.environment_changed)) def get_tests_state(self): - fail_env_changed = self.ns.fail_env_changed - result = [] if self.bad: result.append("FAILURE") - elif fail_env_changed and self.environment_changed: + elif self.fail_env_changed and self.environment_changed: result.append("ENV CHANGED") elif self.no_tests_run(): result.append("NO TESTS RAN") @@ -585,8 +590,9 @@ def get_tests_state(self): result = '%s then %s' % (self.first_state, result) return result - def _run_tests_mp(self, runtests: RunTests) -> None: - from test.libregrtest.runtest_mp import run_tests_multiprocess + def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: + from test.libregrtest.runtest_mp import RunWorkers + # If we're on windows and this is the parent runner (not a worker), # track the load average. if sys.platform == 'win32': @@ -600,7 +606,7 @@ def _run_tests_mp(self, runtests: RunTests) -> None: print(f'Failed to create WindowsLoadTracker: {error}') try: - run_tests_multiprocess(self, runtests) + RunWorkers(self, runtests, num_workers).run() finally: if self.win_load_tracker is not None: self.win_load_tracker.close() @@ -618,8 +624,8 @@ def set_tests(self, runtests: RunTests): def run_tests(self, runtests: RunTests): self.first_runtests = runtests self.set_tests(runtests) - if self.ns.use_mp: - self._run_tests_mp(runtests) + if self.num_workers: + self._run_tests_mp(runtests, self.num_workers) tracer = None else: tracer = self.run_tests_sequentially(runtests) @@ -843,7 +849,7 @@ def get_exitcode(self): exitcode = EXITCODE_BAD_TEST elif self.interrupted: exitcode = EXITCODE_INTERRUPTED - elif self.ns.fail_env_changed and self.environment_changed: + elif self.fail_env_changed and self.environment_changed: exitcode = EXITCODE_ENV_CHANGED elif self.no_tests_run(): exitcode = EXITCODE_NO_TESTS_RAN @@ -866,6 +872,10 @@ def action_run_tests(self): if self.randomize: print("Using random seed", self.random_seed) + if self.num_workers < 0: + # Use all cores + extras for tests that like to sleep + self.num_workers = 2 + (os.cpu_count() or 1) + runtests = RunTests( tuple(self.selected), fail_fast=self.fail_fast, diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index dc574eda8b99f5..667701778d9a79 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -217,6 +217,7 @@ def get_rerun_match_tests(self) -> FilterTuple | None: class RunTests: tests: TestTuple fail_fast: bool = False + fail_env_changed: bool = False match_tests: FilterTuple | None = None ignore_tests: FilterTuple | None = None match_tests_dict: FilterDict | None = None diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index e4a9301656436b..28c05b5976e159 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -16,7 +16,6 @@ from test.support import os_helper from test.support import TestStats -from test.libregrtest.cmdline import Namespace from test.libregrtest.main import Regrtest from test.libregrtest.runtest import ( run_single_test, TestResult, State, PROGRESS_MIN_TIME, @@ -150,14 +149,13 @@ class ExitThread(Exception): pass -class TestWorkerProcess(threading.Thread): - def __init__(self, worker_id: int, runner: "MultiprocessTestRunner") -> None: +class WorkerThread(threading.Thread): + def __init__(self, worker_id: int, runner: "RunWorkers") -> None: super().__init__() self.worker_id = worker_id self.runtests = runner.runtests self.pending = runner.pending self.output = runner.output - self.ns = runner.ns self.timeout = runner.worker_timeout self.regrtest = runner.regrtest self.current_test_name = None @@ -167,7 +165,7 @@ def __init__(self, worker_id: int, runner: "MultiprocessTestRunner") -> None: self._stopped = False def __repr__(self) -> str: - info = [f'TestWorkerProcess #{self.worker_id}'] + info = [f'WorkerThread #{self.worker_id}'] if self.is_alive(): info.append("running") else: @@ -203,7 +201,7 @@ def _kill(self) -> None: else: popen.kill() except ProcessLookupError: - # popen.kill(): the process completed, the TestWorkerProcess thread + # popen.kill(): the process completed, the WorkerThread thread # read its exit status, but Popen.send_signal() read the returncode # just before Popen.wait() set returncode. pass @@ -362,7 +360,7 @@ def _runtest(self, test_name: str) -> MultiprocessResult: def run(self) -> None: fail_fast = self.runtests.fail_fast - fail_env_changed = self.ns.fail_env_changed + fail_env_changed = self.runtests.fail_env_changed while not self._stopped: try: try: @@ -394,10 +392,10 @@ def _wait_completed(self) -> None: f"{exc!r}") def wait_stopped(self, start_time: float) -> None: - # bpo-38207: MultiprocessTestRunner.stop_workers() called self.stop() + # bpo-38207: RunWorkers.stop_workers() called self.stop() # which killed the process. Sometimes, killing the process from the # main thread does not interrupt popen.communicate() in - # TestWorkerProcess thread. This loop with a timeout is a workaround + # WorkerThread thread. This loop with a timeout is a workaround # for that. # # Moreover, if this method fails to join the thread, it is likely @@ -417,7 +415,7 @@ def wait_stopped(self, start_time: float) -> None: break -def get_running(workers: list[TestWorkerProcess]) -> list[TestWorkerProcess]: +def get_running(workers: list[WorkerThread]) -> list[str]: running = [] for worker in workers: current_test_name = worker.current_test_name @@ -427,18 +425,17 @@ def get_running(workers: list[TestWorkerProcess]) -> list[TestWorkerProcess]: if dt >= PROGRESS_MIN_TIME: text = '%s (%s)' % (current_test_name, format_duration(dt)) running.append(text) - return running + if not running: + return None + return f"running ({len(running)}): {', '.join(running)}" -class MultiprocessTestRunner: - def __init__(self, regrtest: Regrtest, runtests: RunTests) -> None: - ns = regrtest.ns - +class RunWorkers: + def __init__(self, regrtest: Regrtest, runtests: RunTests, num_workers: int) -> None: self.regrtest = regrtest + self.log = regrtest.log + self.num_workers = num_workers self.runtests = runtests - self.rerun = runtests.rerun - self.log = self.regrtest.log - self.ns = ns self.output: queue.Queue[QueueOutput] = queue.Queue() tests_iter = runtests.iter_tests() self.pending = MultiprocessIterator(tests_iter) @@ -453,9 +450,8 @@ def __init__(self, regrtest: Regrtest, runtests: RunTests) -> None: self.workers = None def start_workers(self) -> None: - use_mp = self.ns.use_mp - self.workers = [TestWorkerProcess(index, self) - for index in range(1, use_mp + 1)] + self.workers = [WorkerThread(index, self) + for index in range(1, self.num_workers + 1)] msg = f"Run tests in parallel using {len(self.workers)} child processes" if self.timeout: msg += (" (timeout: %s, worker timeout: %s)" @@ -489,10 +485,11 @@ def _get_result(self) -> QueueOutput | None: except queue.Empty: pass - # display progress - running = get_running(self.workers) - if running and not pgo: - self.log('running: %s' % ', '.join(running)) + if not pgo: + # display progress + running = get_running(self.workers) + if running: + self.log(running) # all worker threads are done: consume pending results try: @@ -510,9 +507,10 @@ def display_result(self, mp_result: MultiprocessResult) -> None: text += ' (%s)' % mp_result.err_msg elif (result.duration >= PROGRESS_MIN_TIME and not pgo): text += ' (%s)' % format_duration(result.duration) - running = get_running(self.workers) - if running and not pgo: - text += ' -- running: %s' % ', '.join(running) + if not pgo: + running = get_running(self.workers) + if running: + text += f' -- {running}' self.regrtest.display_progress(self.test_index, text) def _process_result(self, item: QueueOutput) -> bool: @@ -537,9 +535,9 @@ def _process_result(self, item: QueueOutput) -> bool: return result - def run_tests(self) -> None: + def run(self) -> None: fail_fast = self.runtests.fail_fast - fail_env_changed = self.ns.fail_env_changed + fail_env_changed = self.runtests.fail_env_changed self.start_workers() @@ -566,10 +564,6 @@ def run_tests(self) -> None: self.stop_workers() -def run_tests_multiprocess(regrtest: Regrtest, runtests: RunTests) -> None: - MultiprocessTestRunner(regrtest, runtests).run_tests() - - class EncodeTestResult(json.JSONEncoder): """Encode a TestResult (sub)class object into a JSON dict.""" From a341750078e6de5206560adcf53337e0172ae1c6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 03:07:05 +0200 Subject: [PATCH 40/66] gh-109162: Refactor libregrtest.Regrtest (#109206) * Add type hint types: TestName, StrPath, StrJSON. * Add attributes to Regrtest: * cmdline_args * coverage * coverage_dir * fail_rerun * next_single_filename * print_slowest * tmp_dir * want_rerun * want_run_leaks * Remove Regrtest.ns attribute. * Rename Regrtest methods: * cleanup() => cleanup_temp_dir() * create_temp_dir() => make_temp_dir() * set_temp_dir() => select_temp_dir() * Convert Regrtest methods to static methods: * cleanup_temp_dir() * display_header() * fix_umask() * get_rerun_match_tests() * list_tests() * make_temp_dir() * select_temp_dir() * Remove display_sanitizers() method: move code into display_header(). * Rename 'test_cwd' variable to 'work_dir'. --- Lib/test/libregrtest/cmdline.py | 2 + Lib/test/libregrtest/main.py | 174 ++++++++++++++--------------- Lib/test/libregrtest/pgo.py | 6 +- Lib/test/libregrtest/runtest.py | 43 +++---- Lib/test/libregrtest/runtest_mp.py | 10 +- 5 files changed, 120 insertions(+), 115 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 2835546fc713cf..41d969625d04d2 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -179,6 +179,8 @@ def __init__(self, **kwargs) -> None: self.timeout = None self.memlimit = None self.threshold = None + self.fail_rerun = False + self.tempdir = None super().__init__(**kwargs) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 1fa7b07a09d701..e2e732ebd69e1c 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -13,7 +13,7 @@ from test.libregrtest.runtest import ( findtests, split_test_packages, run_single_test, abs_module_name, PROGRESS_MIN_TIME, State, RunTests, TestResult, HuntRefleak, - FilterTuple, FilterDict, TestList) + FilterTuple, FilterDict, TestList, StrPath, StrJSON, TestName) from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.utils import (strip_py_suffix, count, format_duration, @@ -60,15 +60,14 @@ class Regrtest: on the command line. """ def __init__(self, ns: Namespace): - # Namespace of command line options - self.ns: Namespace = ns - # Actions self.want_header: bool = ns.header self.want_list_tests: bool = ns.list_tests self.want_list_cases: bool = ns.list_cases self.want_wait: bool = ns.wait self.want_cleanup: bool = ns.cleanup + self.want_rerun: bool = ns.rerun + self.want_run_leaks: bool = ns.runleaks # Select tests if ns.match_tests: @@ -80,10 +79,11 @@ def __init__(self, ns: Namespace): else: self.ignore_tests = None self.exclude: bool = ns.exclude - self.fromfile: str | None = ns.fromfile - self.starting_test: str | None = ns.start + self.fromfile: StrPath | None = ns.fromfile + self.starting_test: TestName | None = ns.start + self.cmdline_args: TestList = ns.args - # Run tests + # Workers if ns.use_mp is None: num_workers = 0 # run sequentially elif ns.use_mp <= 0: @@ -91,10 +91,12 @@ def __init__(self, ns: Namespace): else: num_workers = ns.use_mp self.num_workers: int = num_workers + self.worker_json: StrJSON | None = ns.worker_json # Options to run tests self.fail_fast: bool = ns.failfast self.fail_env_changed: bool = ns.fail_env_changed + self.fail_rerun: bool = ns.fail_rerun self.forever: bool = ns.forever self.randomize: bool = ns.randomize self.random_seed: int | None = ns.random_seed @@ -108,12 +110,15 @@ def __init__(self, ns: Namespace): self.hunt_refleak: HuntRefleak = HuntRefleak(*ns.huntrleaks) else: self.hunt_refleak = None - self.test_dir: str | None = ns.testdir - self.junit_filename: str | None = ns.xmlpath + self.test_dir: StrPath | None = ns.testdir + self.junit_filename: StrPath | None = ns.xmlpath self.memory_limit: str | None = ns.memlimit self.gc_threshold: int | None = ns.threshold self.use_resources: list[str] = ns.use_resources self.python_cmd: list[str] | None = ns.python + self.coverage: bool = ns.trace + self.coverage_dir: StrPath | None = ns.coverdir + self.tmp_dir: StrPath | None = ns.tempdir # tests self.tests = [] @@ -135,8 +140,9 @@ def __init__(self, ns: Namespace): self.interrupted = False self.total_stats = TestStats() - # used by --slow - self.test_times = [] + # used by --slowest + self.test_times: list[tuple[float, TestName]] = [] + self.print_slowest: bool = ns.print_slow # used to display the progress bar "[ 3/100]" self.start_time = time.perf_counter() @@ -144,15 +150,15 @@ def __init__(self, ns: Namespace): self.test_count_width = 1 # used by --single - self.next_single_test = None - self.next_single_filename = None + self.single_test_run: bool = ns.single + self.next_single_test: TestName | None = None + self.next_single_filename: StrPath | None = None # used by --junit-xml self.testsuite_xml = None # misc self.win_load_tracker = None - self.tmp_dir = None def get_executed(self): return (set(self.good) | set(self.bad) | set(self.skipped) @@ -232,10 +238,7 @@ def display_progress(self, test_index, text): self.log(f"[{line}] {text}") def find_tests(self): - ns = self.ns - single = ns.single - - if single: + if self.single_test_run: self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') try: with open(self.next_single_filename, 'r') as fp: @@ -261,19 +264,19 @@ def find_tests(self): if self.pgo: # add default PGO tests if no tests are specified - setup_pgo_tests(ns) + setup_pgo_tests(self.cmdline_args, self.pgo_extended) exclude_tests = set() if self.exclude: - for arg in ns.args: + for arg in self.cmdline_args: exclude_tests.add(arg) - ns.args = [] + self.cmdline_args = [] alltests = findtests(testdir=self.test_dir, exclude=exclude_tests) if not self.fromfile: - self.selected = self.tests or ns.args + self.selected = self.tests or self.cmdline_args if self.selected: self.selected = split_test_packages(self.selected) else: @@ -281,7 +284,7 @@ def find_tests(self): else: self.selected = self.tests - if single: + if self.single_test_run: self.selected = self.selected[:1] try: pos = alltests.index(self.selected[0]) @@ -303,8 +306,9 @@ def find_tests(self): random.seed(self.random_seed) random.shuffle(self.selected) - def list_tests(self): - for name in self.selected: + @staticmethod + def list_tests(tests: TestList): + for name in tests: print(name) def _list_cases(self, suite): @@ -337,7 +341,8 @@ def list_cases(self): print(count(len(skipped), "test"), "skipped:", file=stderr) printlist(skipped, file=stderr) - def get_rerun_match(self, rerun_list) -> FilterDict: + @staticmethod + def get_rerun_match(rerun_list) -> FilterDict: rerun_match_tests = {} for result in rerun_list: match_tests = result.get_rerun_match_tests() @@ -396,7 +401,6 @@ def rerun_failed_tests(self, need_rerun, runtests: RunTests): def display_result(self, runtests): pgo = runtests.pgo - print_slow = self.ns.print_slow # If running the test suite for PGO then no one cares about results. if pgo: @@ -423,7 +427,7 @@ def display_result(self, runtests): print("All", end=' ') print(count(len(self.good), "test"), "OK.") - if print_slow: + if self.print_slowest: self.test_times.sort(reverse=True) print() print("10 slowest tests:") @@ -461,14 +465,14 @@ def display_result(self, runtests): print(count(len(self.run_no_tests), "test"), "run no tests:") printlist(self.run_no_tests) - def run_test(self, test_name: str, runtests: RunTests, tracer): + def run_test(self, test_name: TestName, runtests: RunTests, tracer): if tracer is not None: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. cmd = ('result = run_single_test(test_name, runtests)') - ns = dict(locals()) - tracer.runctx(cmd, globals=globals(), locals=ns) - result = ns['result'] + namespace = dict(locals()) + tracer.runctx(cmd, globals=globals(), locals=namespace) + result = namespace['result'] else: result = run_single_test(test_name, runtests) @@ -477,10 +481,7 @@ def run_test(self, test_name: str, runtests: RunTests, tracer): return result def run_tests_sequentially(self, runtests): - ns = self.ns - coverage = ns.trace - - if coverage: + if self.coverage: import trace tracer = trace.Trace(trace=False, count=True) else: @@ -526,7 +527,8 @@ def run_tests_sequentially(self, runtests): return tracer - def display_header(self): + @staticmethod + def display_header(): # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) print("==", platform.platform(aliased=True), @@ -538,9 +540,7 @@ def display_header(self): print("== CPU count:", cpu_count) print("== encodings: locale=%s, FS=%s" % (locale.getencoding(), sys.getfilesystemencoding())) - self.display_sanitizers() - def display_sanitizers(self): # This makes it easier to remember what to set in your local # environment when trying to reproduce a sanitizer failure. asan = support.check_sanitizer(address=True) @@ -642,9 +642,9 @@ def finalize_tests(self, tracer): if tracer is not None: results = tracer.results() results.write_results(show_missing=True, summary=True, - coverdir=self.ns.coverdir) + coverdir=self.coverage_dir) - if self.ns.runleaks: + if self.want_run_leaks: os.system("leaks %d" % os.getpid()) self.save_xml_result() @@ -722,7 +722,8 @@ def save_xml_result(self): for s in ET.tostringlist(root): f.write(s) - def fix_umask(self): + @staticmethod + def fix_umask(): if support.is_emscripten: # Emscripten has default umask 0o777, which breaks some tests. # see https://github.com/emscripten-core/emscripten/issues/17269 @@ -732,37 +733,34 @@ def fix_umask(self): else: os.umask(old_mask) - def set_temp_dir(self): - ns = self.ns - if ns.tempdir: - ns.tempdir = os.path.expanduser(ns.tempdir) - - if ns.tempdir: - self.tmp_dir = ns.tempdir - - if not self.tmp_dir: + @staticmethod + def select_temp_dir(tmp_dir): + if tmp_dir: + tmp_dir = os.path.expanduser(tmp_dir) + else: # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. if sysconfig.is_python_build(): - self.tmp_dir = sysconfig.get_config_var('abs_builddir') - if self.tmp_dir is None: + tmp_dir = sysconfig.get_config_var('abs_builddir') + if tmp_dir is None: # bpo-30284: On Windows, only srcdir is available. Using # abs_builddir mostly matters on UNIX when building Python # out of the source tree, especially when the source tree # is read only. - self.tmp_dir = sysconfig.get_config_var('srcdir') - self.tmp_dir = os.path.join(self.tmp_dir, 'build') + tmp_dir = sysconfig.get_config_var('srcdir') + tmp_dir = os.path.join(tmp_dir, 'build') else: - self.tmp_dir = tempfile.gettempdir() + tmp_dir = tempfile.gettempdir() - self.tmp_dir = os.path.abspath(self.tmp_dir) + return os.path.abspath(tmp_dir) def is_worker(self): - return (self.ns.worker_json is not None) + return (self.worker_json is not None) - def create_temp_dir(self): - os.makedirs(self.tmp_dir, exist_ok=True) + @staticmethod + def make_temp_dir(tmp_dir: StrPath, is_worker: bool): + os.makedirs(tmp_dir, exist_ok=True) # Define a writable temp dir that will be used as cwd while running # the tests. The name of the dir includes the pid to allow parallel @@ -774,19 +772,20 @@ def create_temp_dir(self): else: nounce = os.getpid() - if self.is_worker(): - test_cwd = 'test_python_worker_{}'.format(nounce) + if is_worker: + work_dir = 'test_python_worker_{}'.format(nounce) else: - test_cwd = 'test_python_{}'.format(nounce) - test_cwd += os_helper.FS_NONASCII - test_cwd = os.path.join(self.tmp_dir, test_cwd) - return test_cwd + work_dir = 'test_python_{}'.format(nounce) + work_dir += os_helper.FS_NONASCII + work_dir = os.path.join(tmp_dir, work_dir) + return work_dir - def cleanup(self): + @staticmethod + def cleanup_temp_dir(tmp_dir: StrPath): import glob - path = os.path.join(glob.escape(self.tmp_dir), 'test_python_*') - print("Cleanup %s directory" % self.tmp_dir) + path = os.path.join(glob.escape(tmp_dir), 'test_python_*') + print("Cleanup %s directory" % tmp_dir) for name in glob.glob(path): if os.path.isdir(name): print("Remove directory: %s" % name) @@ -796,34 +795,33 @@ def cleanup(self): os_helper.unlink(name) def main(self, tests: TestList | None = None): - ns = self.ns self.tests = tests if self.junit_filename: support.junit_xml_list = self.testsuite_xml = [] - strip_py_suffix(ns.args) + strip_py_suffix(self.cmdline_args) - self.set_temp_dir() + self.tmp_dir = self.select_temp_dir(self.tmp_dir) self.fix_umask() if self.want_cleanup: - self.cleanup() + self.cleanup_temp_dir(self.tmp_dir) sys.exit(0) - test_cwd = self.create_temp_dir() + work_dir = self.make_temp_dir(self.tmp_dir, self.is_worker()) try: - # Run the tests in a context manager that temporarily changes the CWD - # to a temporary and writable directory. If it's not possible to - # create or change the CWD, the original CWD will be used. + # Run the tests in a context manager that temporarily changes the + # CWD to a temporary and writable directory. If it's not possible + # to create or change the CWD, the original CWD will be used. # The original CWD is available from os_helper.SAVEDCWD. - with os_helper.temp_cwd(test_cwd, quiet=True): - # When using multiprocessing, worker processes will use test_cwd - # as their parent temporary directory. So when the main process - # exit, it removes also subdirectories of worker processes. - ns.tempdir = test_cwd + with os_helper.temp_cwd(work_dir, quiet=True): + # When using multiprocessing, worker processes will use + # work_dir as their parent temporary directory. So when the + # main process exit, it removes also subdirectories of worker + # processes. self._main() except SystemExit as exc: @@ -853,7 +851,7 @@ def get_exitcode(self): exitcode = EXITCODE_ENV_CHANGED elif self.no_tests_run(): exitcode = EXITCODE_NO_TESTS_RAN - elif self.rerun and self.ns.fail_rerun: + elif self.rerun and self.fail_rerun: exitcode = EXITCODE_RERUN_FAIL return exitcode @@ -865,8 +863,8 @@ def action_run_tests(self): # For a partial run, we do not need to clutter the output. if (self.want_header - or not(self.pgo or self.quiet or self.ns.single - or self.tests or self.ns.args)): + or not(self.pgo or self.quiet or self.single_test_run + or self.tests or self.cmdline_args)): self.display_header() if self.randomize: @@ -903,7 +901,7 @@ def action_run_tests(self): self.display_result(runtests) need_rerun = self.need_rerun - if self.ns.rerun and need_rerun: + if self.want_rerun and need_rerun: self.rerun_failed_tests(need_rerun, runtests) self.display_summary() @@ -912,7 +910,7 @@ def action_run_tests(self): def _main(self): if self.is_worker(): from test.libregrtest.runtest_mp import worker_process - worker_process(self.ns.worker_json) + worker_process(self.worker_json) return if self.want_wait: @@ -923,7 +921,7 @@ def _main(self): exitcode = 0 if self.want_list_tests: - self.list_tests() + self.list_tests(self.selected) elif self.want_list_cases: self.list_cases() else: diff --git a/Lib/test/libregrtest/pgo.py b/Lib/test/libregrtest/pgo.py index 42ce5fba7a97c3..cabbba73d5eff5 100644 --- a/Lib/test/libregrtest/pgo.py +++ b/Lib/test/libregrtest/pgo.py @@ -50,7 +50,7 @@ 'test_xml_etree_c', ] -def setup_pgo_tests(ns): - if not ns.args and not ns.pgo_extended: +def setup_pgo_tests(cmdline_args, pgo_extended: bool): + if not cmdline_args and not pgo_extended: # run default set of tests for PGO training - ns.args = PGO_TESTS[:] + cmdline_args[:] = PGO_TESTS[:] diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 667701778d9a79..6607e912330b52 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -20,20 +20,23 @@ from test.libregrtest.utils import clear_caches, format_duration, print_warning -TestTuple = list[str] -TestList = list[str] +StrJSON = str +StrPath = str +TestName = str +TestTuple = tuple[TestName, ...] +TestList = list[TestName] # --match and --ignore options: list of patterns # ('*' joker character can be used) -FilterTuple = tuple[str, ...] -FilterDict = dict[str, FilterTuple] +FilterTuple = tuple[TestName, ...] +FilterDict = dict[TestName, FilterTuple] @dataclasses.dataclass(slots=True, frozen=True) class HuntRefleak: warmups: int runs: int - filename: str + filename: StrPath # Avoid enum.Enum to reduce the number of imports when tests are run @@ -110,7 +113,7 @@ def normalize_test_name(test_full_name, *, is_error=False): @dataclasses.dataclass(slots=True) class TestResult: - test_name: str + test_name: TestName state: str | None = None # Test duration in seconds duration: float | None = None @@ -230,8 +233,8 @@ class RunTests: verbose: bool = False quiet: bool = False hunt_refleak: HuntRefleak | None = None - test_dir: str | None = None - junit_filename: str | None = None + test_dir: StrPath | None = None + junit_filename: StrPath | None = None memory_limit: str | None = None gc_threshold: int | None = None use_resources: list[str] = None @@ -255,11 +258,11 @@ def iter_tests(self): else: yield from self.tests - def as_json(self): + def as_json(self) -> StrJSON: return json.dumps(self, cls=_EncodeRunTests) @staticmethod - def from_json(worker_json): + def from_json(worker_json: StrJSON) -> 'RunTests': return json.loads(worker_json, object_hook=_decode_runtests) @@ -292,7 +295,7 @@ def _decode_runtests(data: dict[str, Any]) -> RunTests | dict[str, Any]: # Beware this can't generally be done for any directory with sub-tests as the # __init__.py may do things which alter what tests are to be run. -SPLITTESTDIRS = { +SPLITTESTDIRS: set[TestName] = { "test_asyncio", "test_concurrent_futures", "test_multiprocessing_fork", @@ -305,8 +308,9 @@ def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(*, testdir: str | None =None, exclude=(), - split_test_dirs=SPLITTESTDIRS, base_mod=""): +def findtests(*, testdir: StrPath | None = None, exclude=(), + split_test_dirs: set[TestName] = SPLITTESTDIRS, + base_mod: str = "") -> TestList: """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) tests = [] @@ -318,13 +322,14 @@ def findtests(*, testdir: str | None =None, exclude=(), subdir = os.path.join(testdir, mod) mod = f"{base_mod or 'test'}.{mod}" tests.extend(findtests(testdir=subdir, exclude=exclude, - split_test_dirs=split_test_dirs, base_mod=mod)) + split_test_dirs=split_test_dirs, + base_mod=mod)) elif ext in (".py", ""): tests.append(f"{base_mod}.{mod}" if base_mod else mod) return sorted(tests) -def split_test_packages(tests, *, testdir: str | None = None, exclude=(), +def split_test_packages(tests, *, testdir: StrPath | None = None, exclude=(), split_test_dirs=SPLITTESTDIRS): testdir = findtestdir(testdir) splitted = [] @@ -339,7 +344,7 @@ def split_test_packages(tests, *, testdir: str | None = None, exclude=(), return splitted -def abs_module_name(test_name: str, test_dir: str | None) -> str: +def abs_module_name(test_name: TestName, test_dir: StrPath | None) -> TestName: if test_name.startswith('test.') or test_dir: return test_name else: @@ -422,7 +427,7 @@ def _runtest(result: TestResult, runtests: RunTests) -> None: support.junit_xml_list = None -def run_single_test(test_name: str, runtests: RunTests) -> TestResult: +def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: """Run a single test. test_name -- the name of the test @@ -457,7 +462,7 @@ def run_unittest(test_mod): return support.run_unittest(tests) -def save_env(test_name: str, runtests: RunTests): +def save_env(test_name: TestName, runtests: RunTests): return saved_test_environment(test_name, runtests.verbose, runtests.quiet, pgo=runtests.pgo) @@ -608,7 +613,7 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, result.state = State.PASSED -def remove_testfn(test_name: str, verbose: int) -> None: +def remove_testfn(test_name: TestName, verbose: int) -> None: # Try to clean up os_helper.TESTFN if left behind. # # While tests shouldn't leave any files or directories behind, when a test diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 28c05b5976e159..2575f1811a4930 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -19,7 +19,7 @@ from test.libregrtest.main import Regrtest from test.libregrtest.runtest import ( run_single_test, TestResult, State, PROGRESS_MIN_TIME, - FilterTuple, RunTests) + FilterTuple, RunTests, StrPath, StrJSON, TestName) from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.utils import format_duration, print_warning @@ -50,7 +50,7 @@ class WorkerJob: def create_worker_process(runtests: RunTests, output_file: TextIO, - tmp_dir: str | None = None) -> subprocess.Popen: + tmp_dir: StrPath | None = None) -> subprocess.Popen: python_cmd = runtests.python_cmd worker_json = runtests.as_json() @@ -86,7 +86,7 @@ def create_worker_process(runtests: RunTests, return subprocess.Popen(cmd, **kw) -def worker_process(worker_json: str) -> NoReturn: +def worker_process(worker_json: StrJSON) -> NoReturn: runtests = RunTests.from_json(worker_json) test_name = runtests.tests[0] match_tests: FilterTuple | None = runtests.match_tests @@ -222,7 +222,7 @@ def mp_result_error( return MultiprocessResult(test_result, stdout, err_msg) def _run_process(self, worker_job, output_file: TextIO, - tmp_dir: str | None = None) -> int: + tmp_dir: StrPath | None = None) -> int: try: popen = create_worker_process(worker_job, output_file, tmp_dir) @@ -273,7 +273,7 @@ def _run_process(self, worker_job, output_file: TextIO, self._popen = None self.current_test_name = None - def _runtest(self, test_name: str) -> MultiprocessResult: + def _runtest(self, test_name: TestName) -> MultiprocessResult: self.current_test_name = test_name if sys.platform == 'win32': From db5bfe91f822bb06b4c8fe7c58e649694c854bf2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 04:30:43 +0200 Subject: [PATCH 41/66] gh-109162: libregrtest: add TestResults class (#109208) * Add TestResults class. * Move Regrtest methods to TestResults: * accumulate_result(): now takes a RunTests parameter * get_executed() * no_tests_run() * Add methods to TestResults: * add_junit() * display_result() * display_summary() * need_rerun() * prepare_rerun() * write_junit() * Rename 'need_rerun' attribute to 'bad_results'. * Rename 'total_stats' attribute to 'stats'. --- Lib/test/libregrtest/main.py | 303 ++++------------------------- Lib/test/libregrtest/results.py | 259 ++++++++++++++++++++++++ Lib/test/libregrtest/runtest_mp.py | 18 +- 3 files changed, 306 insertions(+), 274 deletions(-) create mode 100644 Lib/test/libregrtest/results.py diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e2e732ebd69e1c..75c3d0e8350ade 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -12,14 +12,14 @@ from test.libregrtest.cmdline import _parse_args, Namespace from test.libregrtest.runtest import ( findtests, split_test_packages, run_single_test, abs_module_name, - PROGRESS_MIN_TIME, State, RunTests, TestResult, HuntRefleak, - FilterTuple, FilterDict, TestList, StrPath, StrJSON, TestName) + PROGRESS_MIN_TIME, State, RunTests, HuntRefleak, + FilterTuple, TestList, StrPath, StrJSON, TestName) from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.pgo import setup_pgo_tests +from test.libregrtest.results import TestResults from test.libregrtest.utils import (strip_py_suffix, count, format_duration, printlist, get_build_info) from test import support -from test.support import TestStats from test.support import os_helper from test.support import threading_helper @@ -29,12 +29,6 @@ # Must be smaller than buildbot "1200 seconds without output" limit. EXIT_TIMEOUT = 120.0 -EXITCODE_BAD_TEST = 2 -EXITCODE_ENV_CHANGED = 3 -EXITCODE_NO_TESTS_RAN = 4 -EXITCODE_RERUN_FAIL = 5 -EXITCODE_INTERRUPTED = 130 - class Regrtest: """Execute a test suite. @@ -122,26 +116,15 @@ def __init__(self, ns: Namespace): # tests self.tests = [] - self.selected = [] + self.selected: TestList = [] self.first_runtests: RunTests | None = None # test results - self.good: TestList = [] - self.bad: TestList = [] - self.rerun_bad: TestList = [] - self.skipped: TestList = [] - self.resource_denied: TestList = [] - self.environment_changed: TestList = [] - self.run_no_tests: TestList = [] - self.rerun: TestList = [] - - self.need_rerun: list[TestResult] = [] + self.results: TestResults = TestResults() + self.first_state: str | None = None - self.interrupted = False - self.total_stats = TestStats() # used by --slowest - self.test_times: list[tuple[float, TestName]] = [] self.print_slowest: bool = ns.print_slow # used to display the progress bar "[ 3/100]" @@ -154,57 +137,9 @@ def __init__(self, ns: Namespace): self.next_single_test: TestName | None = None self.next_single_filename: StrPath | None = None - # used by --junit-xml - self.testsuite_xml = None - # misc self.win_load_tracker = None - def get_executed(self): - return (set(self.good) | set(self.bad) | set(self.skipped) - | set(self.resource_denied) | set(self.environment_changed) - | set(self.run_no_tests)) - - def accumulate_result(self, result, rerun=False): - test_name = result.test_name - - match result.state: - case State.PASSED: - self.good.append(test_name) - case State.ENV_CHANGED: - self.environment_changed.append(test_name) - case State.SKIPPED: - self.skipped.append(test_name) - case State.RESOURCE_DENIED: - self.resource_denied.append(test_name) - case State.INTERRUPTED: - self.interrupted = True - case State.DID_NOT_RUN: - self.run_no_tests.append(test_name) - case _: - if result.is_failed(self.fail_env_changed): - self.bad.append(test_name) - self.need_rerun.append(result) - else: - raise ValueError(f"invalid test state: {result.state!r}") - - if result.has_meaningful_duration() and not rerun: - self.test_times.append((result.duration, test_name)) - if result.stats is not None: - self.total_stats.accumulate(result.stats) - if rerun: - self.rerun.append(test_name) - - xml_data = result.xml_data - if xml_data: - import xml.etree.ElementTree as ET - for e in xml_data: - try: - self.testsuite_xml.append(ET.fromstring(e)) - except ET.ParseError: - print(xml_data, file=sys.__stderr__) - raise - def log(self, line=''): empty = not line @@ -232,7 +167,7 @@ def display_progress(self, test_index, text): # "[ 51/405/1] test_tcl passed" line = f"{test_index:{self.test_count_width}}{self.test_count_text}" - fails = len(self.bad) + len(self.environment_changed) + fails = len(self.results.bad) + len(self.results.env_changed) if fails and not self.pgo: line = f"{line}/{fails}" self.log(f"[{line}] {text}") @@ -341,34 +276,17 @@ def list_cases(self): print(count(len(skipped), "test"), "skipped:", file=stderr) printlist(skipped, file=stderr) - @staticmethod - def get_rerun_match(rerun_list) -> FilterDict: - rerun_match_tests = {} - for result in rerun_list: - match_tests = result.get_rerun_match_tests() - # ignore empty match list - if match_tests: - rerun_match_tests[result.test_name] = match_tests - return rerun_match_tests - - def _rerun_failed_tests(self, need_rerun, runtests: RunTests): + def _rerun_failed_tests(self, runtests: RunTests): # Configure the runner to re-run tests if self.num_workers == 0: self.num_workers = 1 - # Get tests to re-run - tests = [result.test_name for result in need_rerun] - match_tests_dict = self.get_rerun_match(need_rerun) - - # Clear previously failed tests - self.rerun_bad.extend(self.bad) - self.bad.clear() - self.need_rerun.clear() + tests, match_tests_dict = self.results.prepare_rerun() # Re-run failed tests self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") runtests = runtests.copy( - tests=tuple(tests), + tests=tests, rerun=True, verbose=True, forever=False, @@ -379,7 +297,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): self._run_tests_mp(runtests, self.num_workers) return runtests - def rerun_failed_tests(self, need_rerun, runtests: RunTests): + def rerun_failed_tests(self, runtests: RunTests): if self.python_cmd: # Temp patch for https://github.com/python/cpython/issues/94052 self.log( @@ -388,82 +306,27 @@ def rerun_failed_tests(self, need_rerun, runtests: RunTests): ) return - self.first_state = self.get_tests_state() + self.first_state = self.get_state() print() - rerun_runtests = self._rerun_failed_tests(need_rerun, runtests) + rerun_runtests = self._rerun_failed_tests(runtests) - if self.bad: - print(count(len(self.bad), 'test'), "failed again:") - printlist(self.bad) + if self.results.bad: + print(count(len(self.results.bad), 'test'), "failed again:") + printlist(self.results.bad) self.display_result(rerun_runtests) def display_result(self, runtests): - pgo = runtests.pgo - # If running the test suite for PGO then no one cares about results. - if pgo: + if runtests.pgo: return + state = self.get_state() print() - print("== Tests result: %s ==" % self.get_tests_state()) - - if self.interrupted: - print("Test suite interrupted by signal SIGINT.") - - omitted = set(self.selected) - self.get_executed() - if omitted: - print() - print(count(len(omitted), "test"), "omitted:") - printlist(omitted) - - if self.good and not self.quiet: - print() - if (not self.bad - and not self.skipped - and not self.interrupted - and len(self.good) > 1): - print("All", end=' ') - print(count(len(self.good), "test"), "OK.") - - if self.print_slowest: - self.test_times.sort(reverse=True) - print() - print("10 slowest tests:") - for test_time, test in self.test_times[:10]: - print("- %s: %s" % (test, format_duration(test_time))) - - if self.bad: - print() - print(count(len(self.bad), "test"), "failed:") - printlist(self.bad) - - if self.environment_changed: - print() - print("{} altered the execution environment:".format( - count(len(self.environment_changed), "test"))) - printlist(self.environment_changed) - - if self.skipped and not self.quiet: - print() - print(count(len(self.skipped), "test"), "skipped:") - printlist(self.skipped) - - if self.resource_denied and not self.quiet: - print() - print(count(len(self.resource_denied), "test"), "skipped (resource denied):") - printlist(self.resource_denied) - - if self.rerun: - print() - print("%s:" % count(len(self.rerun), "re-run test")) - printlist(self.rerun) - - if self.run_no_tests: - print() - print(count(len(self.run_no_tests), "test"), "run no tests:") - printlist(self.run_no_tests) + print(f"== Tests result: {state} ==") + + self.results.display_result(self.selected, self.quiet, self.print_slowest) def run_test(self, test_name: TestName, runtests: RunTests, tracer): if tracer is not None: @@ -476,7 +339,7 @@ def run_test(self, test_name: TestName, runtests: RunTests, tracer): else: result = run_single_test(test_name, runtests) - self.accumulate_result(result) + self.results.accumulate_result(result, runtests) return result @@ -566,29 +429,11 @@ def display_header(): if sanitizer and options is not None: print(f"== {env_var}={options!r}") - def no_tests_run(self): - return not any((self.good, self.bad, self.skipped, self.interrupted, - self.environment_changed)) - - def get_tests_state(self): - result = [] - if self.bad: - result.append("FAILURE") - elif self.fail_env_changed and self.environment_changed: - result.append("ENV CHANGED") - elif self.no_tests_run(): - result.append("NO TESTS RAN") - - if self.interrupted: - result.append("INTERRUPTED") - - if not result: - result.append("SUCCESS") - - result = ', '.join(result) + def get_state(self): + state = self.results.get_state(self.fail_env_changed) if self.first_state: - result = '%s then %s' % (self.first_state, result) - return result + state = f'{self.first_state} then {state}' + return state def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from test.libregrtest.runtest_mp import RunWorkers @@ -647,7 +492,8 @@ def finalize_tests(self, tracer): if self.want_run_leaks: os.system("leaks %d" % os.getpid()) - self.save_xml_result() + if self.junit_filename: + self.results.write_junit(self.junit_filename) def display_summary(self): duration = time.perf_counter() - self.start_time @@ -657,70 +503,11 @@ def display_summary(self): print() print("Total duration: %s" % format_duration(duration)) - # Total tests - total = self.total_stats - text = f'run={total.tests_run:,}' - if filtered: - text = f"{text} (filtered)" - stats = [text] - if total.failures: - stats.append(f'failures={total.failures:,}') - if total.skipped: - stats.append(f'skipped={total.skipped:,}') - print(f"Total tests: {' '.join(stats)}") - - # Total test files - all_tests = [self.good, self.bad, self.rerun, - self.skipped, - self.environment_changed, self.run_no_tests] - run = sum(map(len, all_tests)) - text = f'run={run}' - if not self.first_runtests.forever: - ntest = len(self.first_runtests.tests) - text = f"{text}/{ntest}" - if filtered: - text = f"{text} (filtered)" - report = [text] - for name, tests in ( - ('failed', self.bad), - ('env_changed', self.environment_changed), - ('skipped', self.skipped), - ('resource_denied', self.resource_denied), - ('rerun', self.rerun), - ('run_no_tests', self.run_no_tests), - ): - if tests: - report.append(f'{name}={len(tests)}') - print(f"Total test files: {' '.join(report)}") + self.results.display_summary(self.first_runtests, filtered) # Result - result = self.get_tests_state() - print(f"Result: {result}") - - def save_xml_result(self): - if not self.junit_filename and not self.testsuite_xml: - return - - import xml.etree.ElementTree as ET - root = ET.Element("testsuites") - - # Manually count the totals for the overall summary - totals = {'tests': 0, 'errors': 0, 'failures': 0} - for suite in self.testsuite_xml: - root.append(suite) - for k in totals: - try: - totals[k] += int(suite.get(k, 0)) - except ValueError: - pass - - for k, v in totals.items(): - root.set(k, str(v)) - - xmlpath = os.path.join(os_helper.SAVEDCWD, self.junit_filename) - with open(xmlpath, 'wb') as f: - for s in ET.tostringlist(root): - f.write(s) + state = self.get_state() + print(f"Result: {state}") @staticmethod def fix_umask(): @@ -795,10 +582,10 @@ def cleanup_temp_dir(tmp_dir: StrPath): os_helper.unlink(name) def main(self, tests: TestList | None = None): - self.tests = tests + if self.junit_filename and not os.path.isabs(self.junit_filename): + self.junit_filename = os.path.abspath(self.junit_filename) - if self.junit_filename: - support.junit_xml_list = self.testsuite_xml = [] + self.tests = tests strip_py_suffix(self.cmdline_args) @@ -841,20 +628,6 @@ def getloadavg(self): return None - def get_exitcode(self): - exitcode = 0 - if self.bad: - exitcode = EXITCODE_BAD_TEST - elif self.interrupted: - exitcode = EXITCODE_INTERRUPTED - elif self.fail_env_changed and self.environment_changed: - exitcode = EXITCODE_ENV_CHANGED - elif self.no_tests_run(): - exitcode = EXITCODE_NO_TESTS_RAN - elif self.rerun and self.fail_rerun: - exitcode = EXITCODE_RERUN_FAIL - return exitcode - def action_run_tests(self): if self.hunt_refleak and self.hunt_refleak.warmups < 3: msg = ("WARNING: Running tests with --huntrleaks/-R and " @@ -900,9 +673,8 @@ def action_run_tests(self): tracer = self.run_tests(runtests) self.display_result(runtests) - need_rerun = self.need_rerun - if self.want_rerun and need_rerun: - self.rerun_failed_tests(need_rerun, runtests) + if self.want_rerun and self.results.need_rerun(): + self.rerun_failed_tests(runtests) self.display_summary() self.finalize_tests(tracer) @@ -926,7 +698,8 @@ def _main(self): self.list_cases() else: self.action_run_tests() - exitcode = self.get_exitcode() + exitcode = self.results.get_exitcode(self.fail_env_changed, + self.fail_rerun) sys.exit(exitcode) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py new file mode 100644 index 00000000000000..1df15c23770cc1 --- /dev/null +++ b/Lib/test/libregrtest/results.py @@ -0,0 +1,259 @@ +import sys +from test.support import TestStats + +from test.libregrtest.runtest import ( + TestName, TestTuple, TestList, FilterDict, StrPath, State, + TestResult, RunTests) +from test.libregrtest.utils import printlist, count, format_duration + + +EXITCODE_BAD_TEST = 2 +EXITCODE_ENV_CHANGED = 3 +EXITCODE_NO_TESTS_RAN = 4 +EXITCODE_RERUN_FAIL = 5 +EXITCODE_INTERRUPTED = 130 + + +class TestResults: + def __init__(self): + self.bad: TestList = [] + self.good: TestList = [] + self.rerun_bad: TestList = [] + self.skipped: TestList = [] + self.resource_denied: TestList = [] + self.env_changed: TestList = [] + self.run_no_tests: TestList = [] + self.rerun: TestList = [] + self.bad_results: list[TestResult] = [] + + self.interrupted: bool = False + self.test_times: list[tuple[float, TestName]] = [] + self.stats = TestStats() + # used by --junit-xml + self.testsuite_xml: list[str] = [] + + def get_executed(self): + return (set(self.good) | set(self.bad) | set(self.skipped) + | set(self.resource_denied) | set(self.env_changed) + | set(self.run_no_tests)) + + def no_tests_run(self): + return not any((self.good, self.bad, self.skipped, self.interrupted, + self.env_changed)) + + def get_state(self, fail_env_changed): + state = [] + if self.bad: + state.append("FAILURE") + elif fail_env_changed and self.env_changed: + state.append("ENV CHANGED") + elif self.no_tests_run(): + state.append("NO TESTS RAN") + + if self.interrupted: + state.append("INTERRUPTED") + if not state: + state.append("SUCCESS") + + return ', '.join(state) + + def get_exitcode(self, fail_env_changed, fail_rerun): + exitcode = 0 + if self.bad: + exitcode = EXITCODE_BAD_TEST + elif self.interrupted: + exitcode = EXITCODE_INTERRUPTED + elif fail_env_changed and self.env_changed: + exitcode = EXITCODE_ENV_CHANGED + elif self.no_tests_run(): + exitcode = EXITCODE_NO_TESTS_RAN + elif fail_rerun and self.rerun: + exitcode = EXITCODE_RERUN_FAIL + return exitcode + + def accumulate_result(self, result: TestResult, runtests: RunTests): + test_name = result.test_name + rerun = runtests.rerun + fail_env_changed = runtests.fail_env_changed + + match result.state: + case State.PASSED: + self.good.append(test_name) + case State.ENV_CHANGED: + self.env_changed.append(test_name) + case State.SKIPPED: + self.skipped.append(test_name) + case State.RESOURCE_DENIED: + self.resource_denied.append(test_name) + case State.INTERRUPTED: + self.interrupted = True + case State.DID_NOT_RUN: + self.run_no_tests.append(test_name) + case _: + if result.is_failed(fail_env_changed): + self.bad.append(test_name) + self.bad_results.append(result) + else: + raise ValueError(f"invalid test state: {result.state!r}") + + if result.has_meaningful_duration() and not rerun: + self.test_times.append((result.duration, test_name)) + if result.stats is not None: + self.stats.accumulate(result.stats) + if rerun: + self.rerun.append(test_name) + + xml_data = result.xml_data + if xml_data: + self.add_junit(result.xml_data) + + def need_rerun(self): + return bool(self.bad_results) + + def prepare_rerun(self) -> (TestTuple, FilterDict): + tests: TestList = [] + match_tests_dict = {} + for result in self.bad_results: + tests.append(result.test_name) + + match_tests = result.get_rerun_match_tests() + # ignore empty match list + if match_tests: + match_tests_dict[result.test_name] = match_tests + + # Clear previously failed tests + self.rerun_bad.extend(self.bad) + self.bad.clear() + self.bad_results.clear() + + return (tuple(tests), match_tests_dict) + + def add_junit(self, xml_data: list[str]): + import xml.etree.ElementTree as ET + for e in xml_data: + try: + self.testsuite_xml.append(ET.fromstring(e)) + except ET.ParseError: + print(xml_data, file=sys.__stderr__) + raise + + def write_junit(self, filename: StrPath): + if not self.testsuite_xml: + # Don't create empty XML file + return + + import xml.etree.ElementTree as ET + root = ET.Element("testsuites") + + # Manually count the totals for the overall summary + totals = {'tests': 0, 'errors': 0, 'failures': 0} + for suite in self.testsuite_xml: + root.append(suite) + for k in totals: + try: + totals[k] += int(suite.get(k, 0)) + except ValueError: + pass + + for k, v in totals.items(): + root.set(k, str(v)) + + with open(filename, 'wb') as f: + for s in ET.tostringlist(root): + f.write(s) + + def display_result(self, tests: TestList, quiet: bool, print_slowest: bool): + if self.interrupted: + print("Test suite interrupted by signal SIGINT.") + + omitted = set(tests) - self.get_executed() + if omitted: + print() + print(count(len(omitted), "test"), "omitted:") + printlist(omitted) + + if self.good and not quiet: + print() + if (not self.bad + and not self.skipped + and not self.interrupted + and len(self.good) > 1): + print("All", end=' ') + print(count(len(self.good), "test"), "OK.") + + if print_slowest: + self.test_times.sort(reverse=True) + print() + print("10 slowest tests:") + for test_time, test in self.test_times[:10]: + print("- %s: %s" % (test, format_duration(test_time))) + + if self.bad: + print() + print(count(len(self.bad), "test"), "failed:") + printlist(self.bad) + + if self.env_changed: + print() + print("{} altered the execution environment:".format( + count(len(self.env_changed), "test"))) + printlist(self.env_changed) + + if self.skipped and not quiet: + print() + print(count(len(self.skipped), "test"), "skipped:") + printlist(self.skipped) + + if self.resource_denied and not quiet: + print() + print(count(len(self.resource_denied), "test"), "skipped (resource denied):") + printlist(self.resource_denied) + + if self.rerun: + print() + print("%s:" % count(len(self.rerun), "re-run test")) + printlist(self.rerun) + + if self.run_no_tests: + print() + print(count(len(self.run_no_tests), "test"), "run no tests:") + printlist(self.run_no_tests) + + def display_summary(self, first_runtests: RunTests, filtered: bool): + # Total tests + stats = self.stats + text = f'run={stats.tests_run:,}' + if filtered: + text = f"{text} (filtered)" + report = [text] + if stats.failures: + report.append(f'failures={stats.failures:,}') + if stats.skipped: + report.append(f'skipped={stats.skipped:,}') + report = ' '.join(report) + print(f"Total tests: {report}") + + # Total test files + all_tests = [self.good, self.bad, self.rerun, + self.skipped, + self.env_changed, self.run_no_tests] + run = sum(map(len, all_tests)) + text = f'run={run}' + if not first_runtests.forever: + ntest = len(first_runtests.tests) + text = f"{text}/{ntest}" + if filtered: + text = f"{text} (filtered)" + report = [text] + for name, tests in ( + ('failed', self.bad), + ('env_changed', self.env_changed), + ('skipped', self.skipped), + ('resource_denied', self.resource_denied), + ('rerun', self.rerun), + ('run_no_tests', self.run_no_tests), + ): + if tests: + report.append(f'{name}={len(tests)}') + report = ' '.join(report) + print(f"Total test files: {report}") diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 2575f1811a4930..179b6104a8f79a 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -21,6 +21,7 @@ run_single_test, TestResult, State, PROGRESS_MIN_TIME, FilterTuple, RunTests, StrPath, StrJSON, TestName) from test.libregrtest.setup import setup_tests, setup_test_dir +from test.libregrtest.results import TestResults from test.libregrtest.utils import format_duration, print_warning if sys.platform == 'win32': @@ -157,7 +158,7 @@ def __init__(self, worker_id: int, runner: "RunWorkers") -> None: self.pending = runner.pending self.output = runner.output self.timeout = runner.worker_timeout - self.regrtest = runner.regrtest + self.log = runner.log self.current_test_name = None self.start_time = None self._popen = None @@ -408,8 +409,7 @@ def wait_stopped(self, start_time: float) -> None: if not self.is_alive(): break dt = time.monotonic() - start_time - self.regrtest.log(f"Waiting for {self} thread " - f"for {format_duration(dt)}") + self.log(f"Waiting for {self} thread for {format_duration(dt)}") if dt > JOIN_TIMEOUT: print_warning(f"Failed to join {self} in {format_duration(dt)}") break @@ -432,8 +432,9 @@ def get_running(workers: list[WorkerThread]) -> list[str]: class RunWorkers: def __init__(self, regrtest: Regrtest, runtests: RunTests, num_workers: int) -> None: - self.regrtest = regrtest + self.results: TestResults = regrtest.results self.log = regrtest.log + self.display_progress = regrtest.display_progress self.num_workers = num_workers self.runtests = runtests self.output: queue.Queue[QueueOutput] = queue.Queue() @@ -511,23 +512,22 @@ def display_result(self, mp_result: MultiprocessResult) -> None: running = get_running(self.workers) if running: text += f' -- {running}' - self.regrtest.display_progress(self.test_index, text) + self.display_progress(self.test_index, text) def _process_result(self, item: QueueOutput) -> bool: """Returns True if test runner must stop.""" - rerun = self.runtests.rerun if item[0]: # Thread got an exception format_exc = item[1] print_warning(f"regrtest worker thread failed: {format_exc}") result = TestResult("", state=State.MULTIPROCESSING_ERROR) - self.regrtest.accumulate_result(result, rerun=rerun) + self.results.accumulate_result(result, self.runtests) return result self.test_index += 1 mp_result = item[1] result = mp_result.result - self.regrtest.accumulate_result(result, rerun=rerun) + self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) if mp_result.worker_stdout: @@ -553,7 +553,7 @@ def run(self) -> None: break except KeyboardInterrupt: print() - self.regrtest.interrupted = True + self.results.interrupted = True finally: if self.timeout is not None: faulthandler.cancel_dump_traceback_later() From 0eab2427b149cd46e0dee3efbb6b2cfca2a4f723 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 05:04:26 +0200 Subject: [PATCH 42/66] gh-109162: libregrtest: add Logger class (#109212) * Add Logger class in a new logger.py file. * Move Regrtest attributes to Logger: * start_time * test_count_text * test_count_width * win_load_tracker * Move Regrtest method to Logger: * log() * getloadavg(): rename to get_load_avg() * set_tests() * Add methods to the Logger class: * start_load_tracker() * stop_load_tracker() --- Lib/test/libregrtest/logger.py | 69 ++++++++++++++ Lib/test/libregrtest/main.py | 148 ++++++++++------------------- Lib/test/libregrtest/runtest_mp.py | 2 +- Lib/test/libregrtest/utils.py | 3 + 4 files changed, 123 insertions(+), 99 deletions(-) create mode 100644 Lib/test/libregrtest/logger.py diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py new file mode 100644 index 00000000000000..c4498a43545545 --- /dev/null +++ b/Lib/test/libregrtest/logger.py @@ -0,0 +1,69 @@ +import os +import time + +from test.libregrtest.runtest import RunTests +from test.libregrtest.utils import print_warning, MS_WINDOWS + +if MS_WINDOWS: + from test.libregrtest.win_utils import WindowsLoadTracker + + +class Logger: + def __init__(self): + self.start_time = time.perf_counter() + self.test_count_text = '' + self.test_count_width = 3 + self.win_load_tracker = None + + def log(self, line: str = '') -> None: + empty = not line + + # add the system load prefix: "load avg: 1.80 " + load_avg = self.get_load_avg() + if load_avg is not None: + line = f"load avg: {load_avg:.2f} {line}" + + # add the timestamp prefix: "0:01:05 " + test_time = time.perf_counter() - self.start_time + + mins, secs = divmod(int(test_time), 60) + hours, mins = divmod(mins, 60) + test_time = "%d:%02d:%02d" % (hours, mins, secs) + + line = f"{test_time} {line}" + if empty: + line = line[:-1] + + print(line, flush=True) + + def get_load_avg(self) -> float | None: + if hasattr(os, 'getloadavg'): + return os.getloadavg()[0] + if self.win_load_tracker is not None: + return self.win_load_tracker.getloadavg() + return None + + def set_tests(self, runtests: RunTests) -> None: + if runtests.forever: + self.test_count_text = '' + self.test_count_width = 3 + else: + self.test_count_text = '/{}'.format(len(runtests.tests)) + self.test_count_width = len(self.test_count_text) - 1 + + def start_load_tracker(self) -> None: + if not MS_WINDOWS: + return + + try: + self.win_load_tracker = WindowsLoadTracker() + except PermissionError as error: + # Standard accounts may not have access to the performance + # counters. + print_warning(f'Failed to create WindowsLoadTracker: {error}') + + def stop_load_tracker(self) -> None: + if self.win_load_tracker is None: + return + self.win_load_tracker.close() + self.win_load_tracker = None diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 75c3d0e8350ade..74ef69b7c65307 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -10,6 +10,7 @@ import time import unittest from test.libregrtest.cmdline import _parse_args, Namespace +from test.libregrtest.logger import Logger from test.libregrtest.runtest import ( findtests, split_test_packages, run_single_test, abs_module_name, PROGRESS_MIN_TIME, State, RunTests, HuntRefleak, @@ -54,6 +55,8 @@ class Regrtest: on the command line. """ def __init__(self, ns: Namespace): + self.logger = Logger() + # Actions self.want_header: bool = ns.header self.want_list_tests: bool = ns.list_tests @@ -137,29 +140,8 @@ def __init__(self, ns: Namespace): self.next_single_test: TestName | None = None self.next_single_filename: StrPath | None = None - # misc - self.win_load_tracker = None - def log(self, line=''): - empty = not line - - # add the system load prefix: "load avg: 1.80 " - load_avg = self.getloadavg() - if load_avg is not None: - line = f"load avg: {load_avg:.2f} {line}" - - # add the timestamp prefix: "0:01:05 " - test_time = time.perf_counter() - self.start_time - - mins, secs = divmod(int(test_time), 60) - hours, mins = divmod(mins, 60) - test_time = "%d:%02d:%02d" % (hours, mins, secs) - - line = f"{test_time} {line}" - if empty: - line = line[:-1] - - print(line, flush=True) + self.logger.log(line) def display_progress(self, test_index, text): if self.quiet: @@ -293,7 +275,7 @@ def _rerun_failed_tests(self, runtests: RunTests): fail_fast=False, match_tests_dict=match_tests_dict, output_on_failure=False) - self.set_tests(runtests) + self.logger.set_tests(runtests) self._run_tests_mp(runtests, self.num_workers) return runtests @@ -437,44 +419,7 @@ def get_state(self): def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from test.libregrtest.runtest_mp import RunWorkers - - # If we're on windows and this is the parent runner (not a worker), - # track the load average. - if sys.platform == 'win32': - from test.libregrtest.win_utils import WindowsLoadTracker - - try: - self.win_load_tracker = WindowsLoadTracker() - except PermissionError as error: - # Standard accounts may not have access to the performance - # counters. - print(f'Failed to create WindowsLoadTracker: {error}') - - try: - RunWorkers(self, runtests, num_workers).run() - finally: - if self.win_load_tracker is not None: - self.win_load_tracker.close() - self.win_load_tracker = None - - def set_tests(self, runtests: RunTests): - self.tests = runtests.tests - if runtests.forever: - self.test_count_text = '' - self.test_count_width = 3 - else: - self.test_count_text = '/{}'.format(len(self.tests)) - self.test_count_width = len(self.test_count_text) - 1 - - def run_tests(self, runtests: RunTests): - self.first_runtests = runtests - self.set_tests(runtests) - if self.num_workers: - self._run_tests_mp(runtests, self.num_workers) - tracer = None - else: - tracer = self.run_tests_sequentially(runtests) - return tracer + RunWorkers(self, runtests, num_workers).run() def finalize_tests(self, tracer): if self.next_single_filename: @@ -496,7 +441,7 @@ def finalize_tests(self, tracer): self.results.write_junit(self.junit_filename) def display_summary(self): - duration = time.perf_counter() - self.start_time + duration = time.perf_counter() - self.logger.start_time filtered = bool(self.match_tests) or bool(self.ignore_tests) # Total duration @@ -619,35 +564,8 @@ def main(self, tests: TestList | None = None): sys.exit(exc.code) - def getloadavg(self): - if self.win_load_tracker is not None: - return self.win_load_tracker.getloadavg() - - if hasattr(os, 'getloadavg'): - return os.getloadavg()[0] - - return None - - def action_run_tests(self): - if self.hunt_refleak and self.hunt_refleak.warmups < 3: - msg = ("WARNING: Running tests with --huntrleaks/-R and " - "less than 3 warmup repetitions can give false positives!") - print(msg, file=sys.stdout, flush=True) - - # For a partial run, we do not need to clutter the output. - if (self.want_header - or not(self.pgo or self.quiet or self.single_test_run - or self.tests or self.cmdline_args)): - self.display_header() - - if self.randomize: - print("Using random seed", self.random_seed) - - if self.num_workers < 0: - # Use all cores + extras for tests that like to sleep - self.num_workers = 2 + (os.cpu_count() or 1) - - runtests = RunTests( + def create_run_tests(self): + return RunTests( tuple(self.selected), fail_fast=self.fail_fast, match_tests=self.match_tests, @@ -668,17 +586,53 @@ def action_run_tests(self): python_cmd=self.python_cmd, ) + def run_tests(self) -> int: + if self.hunt_refleak and self.hunt_refleak.warmups < 3: + msg = ("WARNING: Running tests with --huntrleaks/-R and " + "less than 3 warmup repetitions can give false positives!") + print(msg, file=sys.stdout, flush=True) + + if self.num_workers < 0: + # Use all CPUs + 2 extra worker processes for tests + # that like to sleep + self.num_workers = (os.cpu_count() or 1) + 2 + + # For a partial run, we do not need to clutter the output. + if (self.want_header + or not(self.pgo or self.quiet or self.single_test_run + or self.tests or self.cmdline_args)): + self.display_header() + + if self.randomize: + print("Using random seed", self.random_seed) + + runtests = self.create_run_tests() + self.first_runtests = runtests + self.logger.set_tests(runtests) + setup_tests(runtests) - tracer = self.run_tests(runtests) - self.display_result(runtests) + self.logger.start_load_tracker() + try: + if self.num_workers: + self._run_tests_mp(runtests, self.num_workers) + tracer = None + else: + tracer = self.run_tests_sequentially(runtests) - if self.want_rerun and self.results.need_rerun(): - self.rerun_failed_tests(runtests) + self.display_result(runtests) + + if self.want_rerun and self.results.need_rerun(): + self.rerun_failed_tests(runtests) + finally: + self.logger.stop_load_tracker() self.display_summary() self.finalize_tests(tracer) + return self.results.get_exitcode(self.fail_env_changed, + self.fail_rerun) + def _main(self): if self.is_worker(): from test.libregrtest.runtest_mp import worker_process @@ -697,9 +651,7 @@ def _main(self): elif self.want_list_cases: self.list_cases() else: - self.action_run_tests() - exitcode = self.results.get_exitcode(self.fail_env_changed, - self.fail_rerun) + exitcode = self.run_tests() sys.exit(exitcode) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 179b6104a8f79a..c4cffff57b14c4 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -433,7 +433,7 @@ def get_running(workers: list[WorkerThread]) -> list[str]: class RunWorkers: def __init__(self, regrtest: Regrtest, runtests: RunTests, num_workers: int) -> None: self.results: TestResults = regrtest.results - self.log = regrtest.log + self.log = regrtest.logger.log self.display_progress = regrtest.display_progress self.num_workers = num_workers self.runtests = runtests diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 9a60a3d40b4c2c..57d85432bbfc95 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -6,6 +6,9 @@ from test import support +MS_WINDOWS = (sys.platform == 'win32') + + def format_duration(seconds): ms = math.ceil(seconds * 1e3) seconds, ms = divmod(ms, 1000) From 92578919a60ebe2b8d6d42377f1e27479c156d65 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 10 Sep 2023 08:09:25 +0300 Subject: [PATCH 43/66] gh-109174: Add support of SimpleNamespace in copy.replace() (GH-109175) --- Doc/library/types.rst | 2 ++ Lib/test/test_types.py | 27 ++++++++++++++++++ ...-09-09-09-05-41.gh-issue-109174.OJea5s.rst | 1 + Objects/namespaceobject.c | 28 +++++++++++++++++++ 4 files changed, 58 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-09-09-09-05-41.gh-issue-109174.OJea5s.rst diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 82300afef0641e..875916be1049a3 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -504,6 +504,8 @@ Additional Utility Classes and Functions However, for a structured record type use :func:`~collections.namedtuple` instead. + :class:`!SimpleNamespace` objects are supported by :func:`copy.replace`. + .. versionadded:: 3.3 .. versionchanged:: 3.9 diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index f2efee90dc0240..c6bff79f903828 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1900,6 +1900,33 @@ def test_pickle(self): self.assertEqual(ns, ns_roundtrip, pname) + def test_replace(self): + ns = types.SimpleNamespace(x=11, y=22) + + ns2 = copy.replace(ns) + self.assertEqual(ns2, ns) + self.assertIsNot(ns2, ns) + self.assertIs(type(ns2), types.SimpleNamespace) + self.assertEqual(vars(ns2), {'x': 11, 'y': 22}) + ns2.x = 3 + self.assertEqual(ns.x, 11) + ns.x = 4 + self.assertEqual(ns2.x, 3) + + self.assertEqual(vars(copy.replace(ns, x=1)), {'x': 1, 'y': 22}) + self.assertEqual(vars(copy.replace(ns, y=2)), {'x': 4, 'y': 2}) + self.assertEqual(vars(copy.replace(ns, x=1, y=2)), {'x': 1, 'y': 2}) + + def test_replace_subclass(self): + class Spam(types.SimpleNamespace): + pass + + spam = Spam(ham=8, eggs=9) + spam2 = copy.replace(spam, ham=5) + + self.assertIs(type(spam2), Spam) + self.assertEqual(vars(spam2), {'ham': 5, 'eggs': 9}) + def test_fake_namespace_compare(self): # Issue #24257: Incorrect use of PyObject_IsInstance() caused # SystemError. diff --git a/Misc/NEWS.d/next/Library/2023-09-09-09-05-41.gh-issue-109174.OJea5s.rst b/Misc/NEWS.d/next/Library/2023-09-09-09-05-41.gh-issue-109174.OJea5s.rst new file mode 100644 index 00000000000000..63461fac3b96f7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-09-09-05-41.gh-issue-109174.OJea5s.rst @@ -0,0 +1 @@ +Add support of :class:`types.SimpleNamespace` in :func:`copy.replace`. diff --git a/Objects/namespaceobject.c b/Objects/namespaceobject.c index 11cf859add3ab8..204c114fd9de2d 100644 --- a/Objects/namespaceobject.c +++ b/Objects/namespaceobject.c @@ -189,9 +189,37 @@ namespace_reduce(_PyNamespaceObject *ns, PyObject *Py_UNUSED(ignored)) } +static PyObject * +namespace_replace(PyObject *self, PyObject *args, PyObject *kwargs) +{ + if (!_PyArg_NoPositional("__replace__", args)) { + return NULL; + } + + PyObject *result = PyObject_CallNoArgs((PyObject *)Py_TYPE(self)); + if (!result) { + return NULL; + } + if (PyDict_Update(((_PyNamespaceObject*)result)->ns_dict, + ((_PyNamespaceObject*)self)->ns_dict) < 0) + { + Py_DECREF(result); + return NULL; + } + if (kwargs) { + if (PyDict_Update(((_PyNamespaceObject*)result)->ns_dict, kwargs) < 0) { + Py_DECREF(result); + return NULL; + } + } + return result; +} + + static PyMethodDef namespace_methods[] = { {"__reduce__", (PyCFunction)namespace_reduce, METH_NOARGS, namespace_reduce__doc__}, + {"__replace__", _PyCFunction_CAST(namespace_replace), METH_VARARGS|METH_KEYWORDS, NULL}, {NULL, NULL} // sentinel }; From 85a5d3dbe19249ba8d414d63328ae84241a35528 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sun, 10 Sep 2023 09:06:08 +0200 Subject: [PATCH 44/66] gh-93627: Align Python implementation of pickle with C implementation of pickle (GH-103035) If a method like __reduce_ex_ or __reduce__ is set to None, a TypeError is raised. --- Lib/pickle.py | 22 +++---- Lib/test/pickletester.py | 59 +++++++++++++++++++ ...3-03-26-19-11-10.gh-issue-93627.0UgwBL.rst | 1 + 3 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-26-19-11-10.gh-issue-93627.0UgwBL.rst diff --git a/Lib/pickle.py b/Lib/pickle.py index fe86f80f51d3b9..4f5ad5b71e8899 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -396,6 +396,8 @@ def decode_long(data): return int.from_bytes(data, byteorder='little', signed=True) +_NoValue = object() + # Pickling machinery class _Pickler: @@ -542,8 +544,8 @@ def save(self, obj, save_persistent_id=True): return rv = NotImplemented - reduce = getattr(self, "reducer_override", None) - if reduce is not None: + reduce = getattr(self, "reducer_override", _NoValue) + if reduce is not _NoValue: rv = reduce(obj) if rv is NotImplemented: @@ -556,8 +558,8 @@ def save(self, obj, save_persistent_id=True): # Check private dispatch table if any, or else # copyreg.dispatch_table - reduce = getattr(self, 'dispatch_table', dispatch_table).get(t) - if reduce is not None: + reduce = getattr(self, 'dispatch_table', dispatch_table).get(t, _NoValue) + if reduce is not _NoValue: rv = reduce(obj) else: # Check for a class with a custom metaclass; treat as regular @@ -567,12 +569,12 @@ def save(self, obj, save_persistent_id=True): return # Check for a __reduce_ex__ method, fall back to __reduce__ - reduce = getattr(obj, "__reduce_ex__", None) - if reduce is not None: + reduce = getattr(obj, "__reduce_ex__", _NoValue) + if reduce is not _NoValue: rv = reduce(self.proto) else: - reduce = getattr(obj, "__reduce__", None) - if reduce is not None: + reduce = getattr(obj, "__reduce__", _NoValue) + if reduce is not _NoValue: rv = reduce() else: raise PicklingError("Can't pickle %r object: %r" % @@ -1705,8 +1707,8 @@ def load_build(self): stack = self.stack state = stack.pop() inst = stack[-1] - setstate = getattr(inst, "__setstate__", None) - if setstate is not None: + setstate = getattr(inst, "__setstate__", _NoValue) + if setstate is not _NoValue: setstate(state) return slotstate = None diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index a687fe0629080a..ddb180ef5ef825 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2408,6 +2408,22 @@ def test_reduce_calls_base(self): y = self.loads(s) self.assertEqual(y._reduce_called, 1) + def test_reduce_ex_None(self): + c = REX_None() + with self.assertRaises(TypeError): + self.dumps(c) + + def test_reduce_None(self): + c = R_None() + with self.assertRaises(TypeError): + self.dumps(c) + + def test_pickle_setstate_None(self): + c = C_None_setstate() + p = self.dumps(c) + with self.assertRaises(TypeError): + self.loads(p) + @no_tracing def test_bad_getattr(self): # Issue #3514: crash when there is an infinite loop in __getattr__ @@ -3349,6 +3365,21 @@ def __setstate__(self, state): def __reduce__(self): return type(self), (), self.state +class REX_None: + """ Setting __reduce_ex__ to None should fail """ + __reduce_ex__ = None + +class R_None: + """ Setting __reduce__ to None should fail """ + __reduce__ = None + +class C_None_setstate: + """ Setting __setstate__ to None should fail """ + def __getstate__(self): + return 1 + + __setstate__ = None + # Test classes for newobj @@ -3752,6 +3783,25 @@ def test_unpickling_buffering_readline(self): unpickler = self.unpickler_class(f) self.assertEqual(unpickler.load(), data) + def test_pickle_invalid_reducer_override(self): + # gh-103035 + obj = object() + + f = io.BytesIO() + class MyPickler(self.pickler_class): + pass + pickler = MyPickler(f) + pickler.dump(obj) + + pickler.clear_memo() + pickler.reducer_override = None + with self.assertRaises(TypeError): + pickler.dump(obj) + + pickler.clear_memo() + pickler.reducer_override = 10 + with self.assertRaises(TypeError): + pickler.dump(obj) # Tests for dispatch_table attribute @@ -3914,6 +3964,15 @@ def dumps(obj, protocol=None): self._test_dispatch_table(dumps, dt) + def test_dispatch_table_None_item(self): + # gh-93627 + obj = object() + f = io.BytesIO() + pickler = self.pickler_class(f) + pickler.dispatch_table = {type(obj): None} + with self.assertRaises(TypeError): + pickler.dump(obj) + def _test_dispatch_table(self, dumps, dispatch_table): def custom_load_dump(obj): return pickle.loads(dumps(obj, 0)) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-26-19-11-10.gh-issue-93627.0UgwBL.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-26-19-11-10.gh-issue-93627.0UgwBL.rst new file mode 100644 index 00000000000000..854da44b560b21 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-26-19-11-10.gh-issue-93627.0UgwBL.rst @@ -0,0 +1 @@ +Update the Python pickle module implementation to match the C implementation of the pickle module. For objects setting reduction methods like :meth:`~object.__reduce_ex__` or :meth:`~object.__reduce__` to ``None``, pickling will result in a :exc:`TypeError`. From 2dd6a86c4ee604b331ed739c2508b0d0114993c6 Mon Sep 17 00:00:00 2001 From: Delgan <4193924+Delgan@users.noreply.github.com> Date: Sun, 10 Sep 2023 12:55:56 +0200 Subject: [PATCH 45/66] Fix "FSTRING_MIDDLE" typo in py312 "What's New" (#109222) --- Doc/whatsnew/3.12.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index bdccbe1012c9e2..a46c913c4997ba 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1599,7 +1599,7 @@ Changes in the Python API functions is now changed due to the changes introduced in :pep:`701`. This means that ``STRING`` tokens are not emitted any more for f-strings and the tokens described in :pep:`701` are now produced instead: ``FSTRING_START``, - ``FSRING_MIDDLE`` and ``FSTRING_END`` are now emitted for f-string "string" + ``FSTRING_MIDDLE`` and ``FSTRING_END`` are now emitted for f-string "string" parts in addition to the appropriate tokens for the tokenization in the expression components. For example for the f-string ``f"start {1+1} end"`` the old version of the tokenizer emitted:: From 429749969621b149c1a7c3c004bd44f52bec8f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=91line?= <31395137+yunline@users.noreply.github.com> Date: Sun, 10 Sep 2023 20:04:24 +0800 Subject: [PATCH 46/66] gh-109207: Fix SystemError when printing symtable entry object. (GH-109225) --- Lib/test/test_symtable.py | 4 ++++ .../2023-09-10-18-53-55.gh-issue-109207.Fei8bY.rst | 1 + Python/symtable.c | 5 ++--- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-10-18-53-55.gh-issue-109207.Fei8bY.rst diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 25714aecda3a15..36cb7b3f242e4c 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -251,6 +251,10 @@ def test_symtable_repr(self): self.assertEqual(str(self.top), "") self.assertEqual(str(self.spam), "") + def test_symtable_entry_repr(self): + expected = f"" + self.assertEqual(repr(self.top._table), expected) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-10-18-53-55.gh-issue-109207.Fei8bY.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-10-18-53-55.gh-issue-109207.Fei8bY.rst new file mode 100644 index 00000000000000..f9da3ac4d1abbd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-10-18-53-55.gh-issue-109207.Fei8bY.rst @@ -0,0 +1 @@ +Fix a SystemError in ``__repr__`` of symtable entry object. diff --git a/Python/symtable.c b/Python/symtable.c index 7eef6f7231c035..6c28b49a0655c6 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -150,9 +150,8 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, static PyObject * ste_repr(PySTEntryObject *ste) { - return PyUnicode_FromFormat("", - ste->ste_name, - PyLong_AS_LONG(ste->ste_id), ste->ste_lineno); + return PyUnicode_FromFormat("", + ste->ste_name, ste->ste_id, ste->ste_lineno); } static void From 71b6e2602c0b5b06d51855f179cdb30094a67968 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 18:21:13 +0200 Subject: [PATCH 47/66] gh-109054: Don't use libatomic on cross-compilation (#109211) configure no longer uses libatomic by default when Python is cross-compiled. The LIBATOMIC variable can be set manually in this case: ./configure LIBATOMIC="-latomic" (...) --- configure | 7 ++----- configure.ac | 7 ++++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/configure b/configure index c78c45d11260f6..8326a1db06c2da 100755 --- a/configure +++ b/configure @@ -27774,7 +27774,7 @@ then : else $as_nop if test "$cross_compiling" = yes then : - ac_cv_libatomic_needed=yes + ac_cv_libatomic_needed=no else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -27811,17 +27811,14 @@ int main() _ACEOF if ac_fn_c_try_run "$LINENO" then : - ac_cv_libatomic_needed=no - else $as_nop - ac_cv_libatomic_needed=yes + ac_cv_libatomic_needed=yes fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext fi - fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_libatomic_needed" >&5 printf "%s\n" "$ac_cv_libatomic_needed" >&6; } diff --git a/configure.ac b/configure.ac index f833755a466012..843f2b267a5253 100644 --- a/configure.ac +++ b/configure.ac @@ -7007,9 +7007,10 @@ int main() } return 0; // all good } -]])],[ - ac_cv_libatomic_needed=no -],[ac_cv_libatomic_needed=yes],[ac_cv_libatomic_needed=yes]) +]])], + [ac_cv_libatomic_needed=no], dnl build succeeded + [ac_cv_libatomic_needed=yes], dnl build failed + [ac_cv_libatomic_needed=no]) dnl cross compilation ]) AS_VAR_IF([ac_cv_libatomic_needed], [yes], From d6892c2b9263b39ea1c7905667942914b6a24b2c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 10 Sep 2023 20:06:09 +0300 Subject: [PATCH 48/66] gh-50644: Forbid pickling of codecs streams (GH-109180) Attempts to pickle or create a shallow or deep copy of codecs streams now raise a TypeError. Previously, copying failed with a RecursionError, while pickling produced wrong results that eventually caused unpickling to fail with a RecursionError. --- Lib/codecs.py | 12 +++ Lib/test/test_codecs.py | 79 +++++++++++++++++++ ...3-09-09-15-08-37.gh-issue-50644.JUAZOh.rst | 4 + 3 files changed, 95 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-09-09-15-08-37.gh-issue-50644.JUAZOh.rst diff --git a/Lib/codecs.py b/Lib/codecs.py index c1c55d8afef389..82f23983e719c2 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -414,6 +414,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### class StreamReader(Codec): @@ -663,6 +666,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### class StreamReaderWriter: @@ -750,6 +756,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### class StreamRecoder: @@ -866,6 +875,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### Shortcuts def open(filename, mode='r', encoding=None, errors='strict', buffering=-1): diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 91d7eaf997ae20..b5e9271ac0c3cd 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1,7 +1,9 @@ import codecs import contextlib +import copy import io import locale +import pickle import sys import unittest import encodings @@ -1771,6 +1773,61 @@ def test_readlines(self): f = self.reader(self.stream) self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00']) + def test_copy(self): + f = self.reader(Queue(b'\xed\x95\x9c\n\xea\xb8\x80')) + with self.assertRaisesRegex(TypeError, 'StreamReader'): + copy.copy(f) + with self.assertRaisesRegex(TypeError, 'StreamReader'): + copy.deepcopy(f) + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + f = self.reader(Queue(b'\xed\x95\x9c\n\xea\xb8\x80')) + with self.assertRaisesRegex(TypeError, 'StreamReader'): + pickle.dumps(f, proto) + + +class StreamWriterTest(unittest.TestCase): + + def setUp(self): + self.writer = codecs.getwriter('utf-8') + + def test_copy(self): + f = self.writer(Queue(b'')) + with self.assertRaisesRegex(TypeError, 'StreamWriter'): + copy.copy(f) + with self.assertRaisesRegex(TypeError, 'StreamWriter'): + copy.deepcopy(f) + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + f = self.writer(Queue(b'')) + with self.assertRaisesRegex(TypeError, 'StreamWriter'): + pickle.dumps(f, proto) + + +class StreamReaderWriterTest(unittest.TestCase): + + def setUp(self): + self.reader = codecs.getreader('latin1') + self.writer = codecs.getwriter('utf-8') + + def test_copy(self): + f = codecs.StreamReaderWriter(Queue(b''), self.reader, self.writer) + with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'): + copy.copy(f) + with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'): + copy.deepcopy(f) + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + f = codecs.StreamReaderWriter(Queue(b''), self.reader, self.writer) + with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'): + pickle.dumps(f, proto) + class EncodedFileTest(unittest.TestCase): @@ -3346,6 +3403,28 @@ def test_seeking_write(self): self.assertEqual(sr.readline(), b'abc\n') self.assertEqual(sr.readline(), b'789\n') + def test_copy(self): + bio = io.BytesIO() + codec = codecs.lookup('ascii') + sr = codecs.StreamRecoder(bio, codec.encode, codec.decode, + encodings.ascii.StreamReader, encodings.ascii.StreamWriter) + + with self.assertRaisesRegex(TypeError, 'StreamRecoder'): + copy.copy(sr) + with self.assertRaisesRegex(TypeError, 'StreamRecoder'): + copy.deepcopy(sr) + + def test_pickle(self): + q = Queue(b'') + codec = codecs.lookup('ascii') + sr = codecs.StreamRecoder(q, codec.encode, codec.decode, + encodings.ascii.StreamReader, encodings.ascii.StreamWriter) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + with self.assertRaisesRegex(TypeError, 'StreamRecoder'): + pickle.dumps(sr, proto) + @unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module') class LocaleCodecTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2023-09-09-15-08-37.gh-issue-50644.JUAZOh.rst b/Misc/NEWS.d/next/Library/2023-09-09-15-08-37.gh-issue-50644.JUAZOh.rst new file mode 100644 index 00000000000000..a7a442e35289d3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-09-15-08-37.gh-issue-50644.JUAZOh.rst @@ -0,0 +1,4 @@ +Attempts to pickle or create a shallow or deep copy of :mod:`codecs` streams +now raise a TypeError. Previously, copying failed with a RecursionError, +while pickling produced wrong results that eventually caused unpickling +to fail with a RecursionError. From cbb3a6f8ada3d133c3ab9f9465b65067fce5bb42 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Sep 2023 00:04:35 +0200 Subject: [PATCH 49/66] gh-109237: Fix test_site for non-ASCII working directory (#109238) Fix test_site.test_underpth_basic() when the working directory contains at least one non-ASCII character: encode the "._pth" file to UTF-8 and enable the UTF-8 Mode to use UTF-8 for the child process stdout. --- Lib/test/test_site.py | 4 ++-- .../next/Tests/2023-09-10-22-32-20.gh-issue-109237.SvgKwD.rst | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-10-22-32-20.gh-issue-109237.SvgKwD.rst diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 20a96169e8be4e..e8ec3b35881fec 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -576,7 +576,7 @@ def _create_underpth_exe(self, lines, exe_pth=True): _pth_file = os.path.splitext(exe_file)[0] + '._pth' else: _pth_file = os.path.splitext(dll_file)[0] + '._pth' - with open(_pth_file, 'w') as f: + with open(_pth_file, 'w', encoding='utf8') as f: for line in lines: print(line, file=f) return exe_file @@ -613,7 +613,7 @@ def test_underpth_basic(self): os.path.dirname(exe_file), pth_lines) - output = subprocess.check_output([exe_file, '-c', + output = subprocess.check_output([exe_file, '-X', 'utf8', '-c', 'import sys; print("\\n".join(sys.path) if sys.flags.no_site else "")' ], encoding='utf-8', errors='surrogateescape') actual_sys_path = output.rstrip().split('\n') diff --git a/Misc/NEWS.d/next/Tests/2023-09-10-22-32-20.gh-issue-109237.SvgKwD.rst b/Misc/NEWS.d/next/Tests/2023-09-10-22-32-20.gh-issue-109237.SvgKwD.rst new file mode 100644 index 00000000000000..1d762bbe1d2592 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-10-22-32-20.gh-issue-109237.SvgKwD.rst @@ -0,0 +1,4 @@ +Fix ``test_site.test_underpth_basic()`` when the working directory contains +at least one non-ASCII character: encode the ``._pth`` file to UTF-8 and +enable the UTF-8 Mode to use UTF-8 for the child process stdout. Patch by +Victor Stinner. From e55aab95786e0e9fb36a9a1122d2d0fb3d2403cd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Sep 2023 00:16:59 +0200 Subject: [PATCH 50/66] gh-109230: test_pyexpat no longer depends on the current directory (#109233) Fix test_pyexpat.test_exception(): it can now be run from a directory different than Python source code directory. Before, the test failed in this case. Skip the test if Modules/pyexpat.c source is not available. Skip also the test on Python implementations other than CPython. --- Lib/test/test_pyexpat.py | 72 ++++++++++++------- ...-09-10-19-59-57.gh-issue-109230.SRNLFQ.rst | 5 ++ 2 files changed, 53 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-10-19-59-57.gh-issue-109230.SRNLFQ.rst diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index abe1ad517d2246..41f4d172f8d057 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -1,13 +1,15 @@ # XXX TypeErrors on calling handlers, or on bad return values from a # handler, are obscure and unhelpful. -from io import BytesIO import os import platform import sys import sysconfig import unittest import traceback +from io import BytesIO +from test import support +from test.support import os_helper from xml.parsers import expat from xml.parsers.expat import errors @@ -439,37 +441,59 @@ def test7(self): # Test handling of exception from callback: class HandlerExceptionTest(unittest.TestCase): def StartElementHandler(self, name, attrs): - raise RuntimeError(name) + raise RuntimeError(f'StartElementHandler: <{name}>') def check_traceback_entry(self, entry, filename, funcname): - self.assertEqual(os.path.basename(entry[0]), filename) - self.assertEqual(entry[2], funcname) + self.assertEqual(os.path.basename(entry.filename), filename) + self.assertEqual(entry.name, funcname) + @support.cpython_only def test_exception(self): + # gh-66652: test _PyTraceback_Add() used by pyexpat.c to inject frames + + # Change the current directory to the Python source code directory + # if it is available. + src_dir = sysconfig.get_config_var('abs_builddir') + if src_dir: + have_source = os.path.isdir(src_dir) + else: + have_source = False + if have_source: + with os_helper.change_cwd(src_dir): + self._test_exception(have_source) + else: + self._test_exception(have_source) + + def _test_exception(self, have_source): + # Use path relative to the current directory which should be the Python + # source code directory (if it is available). + PYEXPAT_C = os.path.join('Modules', 'pyexpat.c') + parser = expat.ParserCreate() parser.StartElementHandler = self.StartElementHandler try: parser.Parse(b"", True) - self.fail() - except RuntimeError as e: - self.assertEqual(e.args[0], 'a', - "Expected RuntimeError for element 'a', but" + \ - " found %r" % e.args[0]) - # Check that the traceback contains the relevant line in pyexpat.c - entries = traceback.extract_tb(e.__traceback__) - self.assertEqual(len(entries), 3) - self.check_traceback_entry(entries[0], - "test_pyexpat.py", "test_exception") - self.check_traceback_entry(entries[1], - "pyexpat.c", "StartElement") - self.check_traceback_entry(entries[2], - "test_pyexpat.py", "StartElementHandler") - if (sysconfig.is_python_build() - and not (sys.platform == 'win32' and platform.machine() == 'ARM') - and not is_emscripten - and not is_wasi - ): - self.assertIn('call_with_frame("StartElement"', entries[1][3]) + + self.fail("the parser did not raise RuntimeError") + except RuntimeError as exc: + self.assertEqual(exc.args[0], 'StartElementHandler: ', exc) + entries = traceback.extract_tb(exc.__traceback__) + + self.assertEqual(len(entries), 3, entries) + self.check_traceback_entry(entries[0], + "test_pyexpat.py", "_test_exception") + self.check_traceback_entry(entries[1], + os.path.basename(PYEXPAT_C), + "StartElement") + self.check_traceback_entry(entries[2], + "test_pyexpat.py", "StartElementHandler") + + # Check that the traceback contains the relevant line in + # Modules/pyexpat.c. Skip the test if Modules/pyexpat.c is not + # available. + if have_source and os.path.exists(PYEXPAT_C): + self.assertIn('call_with_frame("StartElement"', + entries[1].line) # Test Current* members: diff --git a/Misc/NEWS.d/next/Tests/2023-09-10-19-59-57.gh-issue-109230.SRNLFQ.rst b/Misc/NEWS.d/next/Tests/2023-09-10-19-59-57.gh-issue-109230.SRNLFQ.rst new file mode 100644 index 00000000000000..18e1e85242005a --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-10-19-59-57.gh-issue-109230.SRNLFQ.rst @@ -0,0 +1,5 @@ +Fix ``test_pyexpat.test_exception()``: it can now be run from a directory +different than Python source code directory. Before, the test failed in this +case. Skip the test if Modules/pyexpat.c source is not available. Skip also +the test on Python implementations other than CPython. Patch by Victor +Stinner. From a939b65aa63e2cae1eec7d27e1dc56324aee01d7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Sep 2023 01:11:22 +0200 Subject: [PATCH 51/66] gh-109162: libregrtest: add worker.py (#109229) Add new worker.py file: * Move create_worker_process() and worker_process() to this file. * Add main() function to worker.py. create_worker_process() now runs the command: "python -m test.libregrtest.worker JSON". * create_worker_process() now starts the worker process in the current working directory. Regrtest now gets the absolute path of the reflog.txt filename: -R command line option filename. * Remove --worker-json command line option. Remove test_regrtest.test_worker_json(). Related changes: * Add write_json() and from_json() methods to TestResult. * Rename select_temp_dir() to get_temp_dir() and move it to utils. * Rename make_temp_dir() to get_work_dir() and move it to utils. It no longer calls os.makedirs(): Regrtest.main() now calls it. * Move fix_umask() to utils. The function is now called by setup_tests(). * Move StrPath to utils. * Add exit_timeout() context manager to utils. * RunTests: Replace junit_filename (StrPath) with use_junit (bool). --- Lib/test/libregrtest/cmdline.py | 1 - Lib/test/libregrtest/main.py | 108 +++++----------------------- Lib/test/libregrtest/refleak.py | 1 - Lib/test/libregrtest/results.py | 5 +- Lib/test/libregrtest/runtest.py | 39 ++++++++-- Lib/test/libregrtest/runtest_mp.py | 110 ++--------------------------- Lib/test/libregrtest/setup.py | 8 ++- Lib/test/libregrtest/utils.py | 81 +++++++++++++++++++++ Lib/test/libregrtest/worker.py | 93 ++++++++++++++++++++++++ Lib/test/test_regrtest.py | 5 -- 10 files changed, 238 insertions(+), 213 deletions(-) create mode 100644 Lib/test/libregrtest/worker.py diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 41d969625d04d2..ab8efb427a14a5 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -216,7 +216,6 @@ def _create_parser(): group.add_argument('--wait', action='store_true', help='wait for user input, e.g., allow a debugger ' 'to be attached') - group.add_argument('--worker-json', metavar='ARGS') group.add_argument('-S', '--start', metavar='START', help='the name of the test at which to start.' + more_details) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 74ef69b7c65307..ed0813d6f30c10 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,34 +1,27 @@ -import faulthandler import locale import os import platform import random import re import sys -import sysconfig -import tempfile import time import unittest + +from test import support +from test.support import os_helper + from test.libregrtest.cmdline import _parse_args, Namespace from test.libregrtest.logger import Logger from test.libregrtest.runtest import ( findtests, split_test_packages, run_single_test, abs_module_name, PROGRESS_MIN_TIME, State, RunTests, HuntRefleak, - FilterTuple, TestList, StrPath, StrJSON, TestName) + FilterTuple, TestList, StrJSON, TestName) from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.results import TestResults -from test.libregrtest.utils import (strip_py_suffix, count, format_duration, - printlist, get_build_info) -from test import support -from test.support import os_helper -from test.support import threading_helper - - -# bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()). -# Used to protect against threading._shutdown() hang. -# Must be smaller than buildbot "1200 seconds without output" limit. -EXIT_TIMEOUT = 120.0 +from test.libregrtest.utils import ( + strip_py_suffix, count, format_duration, StrPath, + printlist, get_build_info, get_temp_dir, get_work_dir, exit_timeout) class Regrtest: @@ -104,7 +97,9 @@ def __init__(self, ns: Namespace): self.verbose: bool = ns.verbose self.quiet: bool = ns.quiet if ns.huntrleaks: - self.hunt_refleak: HuntRefleak = HuntRefleak(*ns.huntrleaks) + warmups, runs, filename = ns.huntrleaks + filename = os.path.abspath(filename) + self.hunt_refleak: HuntRefleak = HuntRefleak(warmups, runs, filename) else: self.hunt_refleak = None self.test_dir: StrPath | None = ns.testdir @@ -454,64 +449,6 @@ def display_summary(self): state = self.get_state() print(f"Result: {state}") - @staticmethod - def fix_umask(): - if support.is_emscripten: - # Emscripten has default umask 0o777, which breaks some tests. - # see https://github.com/emscripten-core/emscripten/issues/17269 - old_mask = os.umask(0) - if old_mask == 0o777: - os.umask(0o027) - else: - os.umask(old_mask) - - @staticmethod - def select_temp_dir(tmp_dir): - if tmp_dir: - tmp_dir = os.path.expanduser(tmp_dir) - else: - # When tests are run from the Python build directory, it is best practice - # to keep the test files in a subfolder. This eases the cleanup of leftover - # files using the "make distclean" command. - if sysconfig.is_python_build(): - tmp_dir = sysconfig.get_config_var('abs_builddir') - if tmp_dir is None: - # bpo-30284: On Windows, only srcdir is available. Using - # abs_builddir mostly matters on UNIX when building Python - # out of the source tree, especially when the source tree - # is read only. - tmp_dir = sysconfig.get_config_var('srcdir') - tmp_dir = os.path.join(tmp_dir, 'build') - else: - tmp_dir = tempfile.gettempdir() - - return os.path.abspath(tmp_dir) - - def is_worker(self): - return (self.worker_json is not None) - - @staticmethod - def make_temp_dir(tmp_dir: StrPath, is_worker: bool): - os.makedirs(tmp_dir, exist_ok=True) - - # Define a writable temp dir that will be used as cwd while running - # the tests. The name of the dir includes the pid to allow parallel - # testing (see the -j option). - # Emscripten and WASI have stubbed getpid(), Emscripten has only - # milisecond clock resolution. Use randint() instead. - if sys.platform in {"emscripten", "wasi"}: - nounce = random.randint(0, 1_000_000) - else: - nounce = os.getpid() - - if is_worker: - work_dir = 'test_python_worker_{}'.format(nounce) - else: - work_dir = 'test_python_{}'.format(nounce) - work_dir += os_helper.FS_NONASCII - work_dir = os.path.join(tmp_dir, work_dir) - return work_dir - @staticmethod def cleanup_temp_dir(tmp_dir: StrPath): import glob @@ -534,17 +471,16 @@ def main(self, tests: TestList | None = None): strip_py_suffix(self.cmdline_args) - self.tmp_dir = self.select_temp_dir(self.tmp_dir) - - self.fix_umask() + self.tmp_dir = get_temp_dir(self.tmp_dir) if self.want_cleanup: self.cleanup_temp_dir(self.tmp_dir) sys.exit(0) - work_dir = self.make_temp_dir(self.tmp_dir, self.is_worker()) + os.makedirs(self.tmp_dir, exist_ok=True) + work_dir = get_work_dir(parent_dir=self.tmp_dir) - try: + with exit_timeout(): # Run the tests in a context manager that temporarily changes the # CWD to a temporary and writable directory. If it's not possible # to create or change the CWD, the original CWD will be used. @@ -556,13 +492,6 @@ def main(self, tests: TestList | None = None): # processes. self._main() - except SystemExit as exc: - # bpo-38203: Python can hang at exit in Py_Finalize(), especially - # on threading._shutdown() call: put a timeout - if threading_helper.can_start_thread: - faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True) - - sys.exit(exc.code) def create_run_tests(self): return RunTests( @@ -579,7 +508,7 @@ def create_run_tests(self): quiet=self.quiet, hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, - junit_filename=self.junit_filename, + use_junit=(self.junit_filename is not None), memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, use_resources=self.use_resources, @@ -634,11 +563,6 @@ def run_tests(self) -> int: self.fail_rerun) def _main(self): - if self.is_worker(): - from test.libregrtest.runtest_mp import worker_process - worker_process(self.worker_json) - return - if self.want_wait: input("Press any key to continue...") diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 2e9f17e1c1eee6..81f163c47e5665 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -68,7 +68,6 @@ def get_pooled_int(value): warmups = hunt_refleak.warmups runs = hunt_refleak.runs filename = hunt_refleak.filename - filename = os.path.join(os_helper.SAVEDCWD, filename) repcount = warmups + runs # Pre-allocate to ensure that the loop doesn't allocate anything new diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 1df15c23770cc1..e44301938c6527 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -2,9 +2,10 @@ from test.support import TestStats from test.libregrtest.runtest import ( - TestName, TestTuple, TestList, FilterDict, StrPath, State, + TestName, TestTuple, TestList, FilterDict, State, TestResult, RunTests) -from test.libregrtest.utils import printlist, count, format_duration +from test.libregrtest.utils import ( + printlist, count, format_duration, StrPath) EXITCODE_BAD_TEST = 2 diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 6607e912330b52..a12c7fcaee8bc6 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -17,11 +17,11 @@ from test.support import os_helper from test.support import threading_helper from test.libregrtest.save_env import saved_test_environment -from test.libregrtest.utils import clear_caches, format_duration, print_warning +from test.libregrtest.utils import ( + clear_caches, format_duration, print_warning, StrPath) StrJSON = str -StrPath = str TestName = str TestTuple = tuple[TestName, ...] TestList = list[TestName] @@ -215,6 +215,33 @@ def get_rerun_match_tests(self) -> FilterTuple | None: return None return tuple(match_tests) + def write_json(self, file) -> None: + json.dump(self, file, cls=_EncodeTestResult) + + @staticmethod + def from_json(worker_json) -> 'TestResult': + return json.loads(worker_json, object_hook=_decode_test_result) + + +class _EncodeTestResult(json.JSONEncoder): + def default(self, o: Any) -> dict[str, Any]: + if isinstance(o, TestResult): + result = dataclasses.asdict(o) + result["__test_result__"] = o.__class__.__name__ + return result + else: + return super().default(o) + + +def _decode_test_result(data: dict[str, Any]) -> TestResult | dict[str, Any]: + if "__test_result__" in data: + data.pop('__test_result__') + if data['stats'] is not None: + data['stats'] = TestStats(**data['stats']) + return TestResult(**data) + else: + return data + @dataclasses.dataclass(slots=True, frozen=True) class RunTests: @@ -234,7 +261,7 @@ class RunTests: quiet: bool = False hunt_refleak: HuntRefleak | None = None test_dir: StrPath | None = None - junit_filename: StrPath | None = None + use_junit: bool = False memory_limit: str | None = None gc_threshold: int | None = None use_resources: list[str] = None @@ -358,7 +385,7 @@ def setup_support(runtests: RunTests): support.set_match_tests(runtests.match_tests, runtests.ignore_tests) support.failfast = runtests.fail_fast support.verbose = runtests.verbose - if runtests.junit_filename: + if runtests.use_junit: support.junit_xml_list = [] else: support.junit_xml_list = None @@ -434,8 +461,8 @@ def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: Returns a TestResult. - If runtests.junit_filename is not None, xml_data is a list containing each - generated testsuite element. + If runtests.use_junit, xml_data is a list containing each generated + testsuite element. """ start_time = time.perf_counter() result = TestResult(test_name) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index c4cffff57b14c4..c1bd911c43e2c4 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -1,6 +1,5 @@ import dataclasses import faulthandler -import json import os.path import queue import signal @@ -10,19 +9,19 @@ import threading import time import traceback -from typing import NoReturn, Literal, Any, TextIO +from typing import Literal, TextIO from test import support from test.support import os_helper -from test.support import TestStats from test.libregrtest.main import Regrtest from test.libregrtest.runtest import ( - run_single_test, TestResult, State, PROGRESS_MIN_TIME, - FilterTuple, RunTests, StrPath, StrJSON, TestName) -from test.libregrtest.setup import setup_tests, setup_test_dir + TestResult, State, PROGRESS_MIN_TIME, + RunTests, TestName) from test.libregrtest.results import TestResults -from test.libregrtest.utils import format_duration, print_warning +from test.libregrtest.utils import ( + format_duration, print_warning, StrPath) +from test.libregrtest.worker import create_worker_process, USE_PROCESS_GROUP if sys.platform == 'win32': import locale @@ -41,75 +40,6 @@ # Time to wait until a worker completes: should be immediate JOIN_TIMEOUT = 30.0 # seconds -USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) - - -@dataclasses.dataclass(slots=True) -class WorkerJob: - runtests: RunTests - - -def create_worker_process(runtests: RunTests, - output_file: TextIO, - tmp_dir: StrPath | None = None) -> subprocess.Popen: - python_cmd = runtests.python_cmd - worker_json = runtests.as_json() - - if python_cmd is not None: - executable = python_cmd - else: - executable = [sys.executable] - cmd = [*executable, *support.args_from_interpreter_flags(), - '-u', # Unbuffered stdout and stderr - '-m', 'test.regrtest', - '--worker-json', worker_json] - - env = dict(os.environ) - if tmp_dir is not None: - env['TMPDIR'] = tmp_dir - env['TEMP'] = tmp_dir - env['TMP'] = tmp_dir - - # Running the child from the same working directory as regrtest's original - # invocation ensures that TEMPDIR for the child is the same when - # sysconfig.is_python_build() is true. See issue 15300. - kw = dict( - env=env, - stdout=output_file, - # bpo-45410: Write stderr into stdout to keep messages order - stderr=output_file, - text=True, - close_fds=(os.name != 'nt'), - cwd=os_helper.SAVEDCWD, - ) - if USE_PROCESS_GROUP: - kw['start_new_session'] = True - return subprocess.Popen(cmd, **kw) - - -def worker_process(worker_json: StrJSON) -> NoReturn: - runtests = RunTests.from_json(worker_json) - test_name = runtests.tests[0] - match_tests: FilterTuple | None = runtests.match_tests - - setup_test_dir(runtests.test_dir) - setup_tests(runtests) - - if runtests.rerun: - if match_tests: - matching = "matching: " + ", ".join(match_tests) - print(f"Re-running {test_name} in verbose mode ({matching})", flush=True) - else: - print(f"Re-running {test_name} in verbose mode", flush=True) - - result = run_single_test(test_name, runtests) - print() # Force a newline (just in case) - - # Serialize TestResult as dict in JSON - json.dump(result, sys.stdout, cls=EncodeTestResult) - sys.stdout.flush() - sys.exit(0) - # We do not use a generator so multiple threads can call next(). class MultiprocessIterator: @@ -340,9 +270,7 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: err_msg = "Failed to parse worker stdout" else: try: - # deserialize run_tests_worker() output - result = json.loads(worker_json, - object_hook=decode_test_result) + result = TestResult.from_json(worker_json) except Exception as exc: err_msg = "Failed to parse worker JSON: %s" % exc @@ -562,27 +490,3 @@ def run(self) -> None: # worker when we exit this function self.pending.stop() self.stop_workers() - - -class EncodeTestResult(json.JSONEncoder): - """Encode a TestResult (sub)class object into a JSON dict.""" - - def default(self, o: Any) -> dict[str, Any]: - if isinstance(o, TestResult): - result = dataclasses.asdict(o) - result["__test_result__"] = o.__class__.__name__ - return result - - return super().default(o) - - -def decode_test_result(d: dict[str, Any]) -> TestResult | dict[str, Any]: - """Decode a TestResult (sub)class object from a JSON dict.""" - - if "__test_result__" not in d: - return d - - d.pop('__test_result__') - if d['stats'] is not None: - d['stats'] = TestStats(**d['stats']) - return TestResult(**d) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 594498333fe792..48eb8b6800af1e 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -11,8 +11,8 @@ except ImportError: gc = None -from test.libregrtest.utils import (setup_unraisable_hook, - setup_threading_excepthook) +from test.libregrtest.utils import ( + setup_unraisable_hook, setup_threading_excepthook, fix_umask) UNICODE_GUARD_ENV = "PYTHONREGRTEST_UNICODE_GUARD" @@ -26,6 +26,8 @@ def setup_test_dir(testdir: str | None) -> None: def setup_tests(runtests): + fix_umask() + try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): @@ -102,7 +104,7 @@ def _test_audit_hook(name, args): support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, timeout) support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, timeout) - if runtests.junit_filename: + if runtests.use_junit: from test.support.testresult import RegressionTestResult RegressionTestResult.USE_XML = True diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 57d85432bbfc95..e77772cc2577fe 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,13 +1,28 @@ +import contextlib +import faulthandler import math import os.path +import random import sys import sysconfig +import tempfile import textwrap + from test import support +from test.support import os_helper +from test.support import threading_helper MS_WINDOWS = (sys.platform == 'win32') +# bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()). +# Used to protect against threading._shutdown() hang. +# Must be smaller than buildbot "1200 seconds without output" limit. +EXIT_TIMEOUT = 120.0 + + +StrPath = str + def format_duration(seconds): ms = math.ceil(seconds * 1e3) @@ -308,3 +323,69 @@ def get_build_info(): build.append("dtrace") return build + + +def get_temp_dir(tmp_dir): + if tmp_dir: + tmp_dir = os.path.expanduser(tmp_dir) + else: + # When tests are run from the Python build directory, it is best practice + # to keep the test files in a subfolder. This eases the cleanup of leftover + # files using the "make distclean" command. + if sysconfig.is_python_build(): + tmp_dir = sysconfig.get_config_var('abs_builddir') + if tmp_dir is None: + # bpo-30284: On Windows, only srcdir is available. Using + # abs_builddir mostly matters on UNIX when building Python + # out of the source tree, especially when the source tree + # is read only. + tmp_dir = sysconfig.get_config_var('srcdir') + tmp_dir = os.path.join(tmp_dir, 'build') + else: + tmp_dir = tempfile.gettempdir() + + return os.path.abspath(tmp_dir) + + +def fix_umask(): + if support.is_emscripten: + # Emscripten has default umask 0o777, which breaks some tests. + # see https://github.com/emscripten-core/emscripten/issues/17269 + old_mask = os.umask(0) + if old_mask == 0o777: + os.umask(0o027) + else: + os.umask(old_mask) + + +def get_work_dir(*, parent_dir: StrPath = '', worker: bool = False): + # Define a writable temp dir that will be used as cwd while running + # the tests. The name of the dir includes the pid to allow parallel + # testing (see the -j option). + # Emscripten and WASI have stubbed getpid(), Emscripten has only + # milisecond clock resolution. Use randint() instead. + if sys.platform in {"emscripten", "wasi"}: + nounce = random.randint(0, 1_000_000) + else: + nounce = os.getpid() + + if worker: + work_dir = 'test_python_worker_{}'.format(nounce) + else: + work_dir = 'test_python_{}'.format(nounce) + work_dir += os_helper.FS_NONASCII + if parent_dir: + work_dir = os.path.join(parent_dir, work_dir) + return work_dir + + +@contextlib.contextmanager +def exit_timeout(): + try: + yield + except SystemExit as exc: + # bpo-38203: Python can hang at exit in Py_Finalize(), especially + # on threading._shutdown() call: put a timeout + if threading_helper.can_start_thread: + faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True) + sys.exit(exc.code) diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py new file mode 100644 index 00000000000000..033a0a3ff62260 --- /dev/null +++ b/Lib/test/libregrtest/worker.py @@ -0,0 +1,93 @@ +import subprocess +import sys +import os +from typing import TextIO, NoReturn + +from test import support +from test.support import os_helper + +from test.libregrtest.setup import setup_tests, setup_test_dir +from test.libregrtest.runtest import ( + run_single_test, StrJSON, FilterTuple, RunTests) +from test.libregrtest.utils import get_work_dir, exit_timeout, StrPath + + +USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) + + +def create_worker_process(runtests: RunTests, + output_file: TextIO, + tmp_dir: StrPath | None = None) -> subprocess.Popen: + python_cmd = runtests.python_cmd + worker_json = runtests.as_json() + + if python_cmd is not None: + executable = python_cmd + else: + executable = [sys.executable] + cmd = [*executable, *support.args_from_interpreter_flags(), + '-u', # Unbuffered stdout and stderr + '-m', 'test.libregrtest.worker', + worker_json] + + env = dict(os.environ) + if tmp_dir is not None: + env['TMPDIR'] = tmp_dir + env['TEMP'] = tmp_dir + env['TMP'] = tmp_dir + + # Running the child from the same working directory as regrtest's original + # invocation ensures that TEMPDIR for the child is the same when + # sysconfig.is_python_build() is true. See issue 15300. + kw = dict( + env=env, + stdout=output_file, + # bpo-45410: Write stderr into stdout to keep messages order + stderr=output_file, + text=True, + close_fds=(os.name != 'nt'), + ) + if USE_PROCESS_GROUP: + kw['start_new_session'] = True + return subprocess.Popen(cmd, **kw) + + +def worker_process(worker_json: StrJSON) -> NoReturn: + runtests = RunTests.from_json(worker_json) + test_name = runtests.tests[0] + match_tests: FilterTuple | None = runtests.match_tests + + setup_test_dir(runtests.test_dir) + setup_tests(runtests) + + if runtests.rerun: + if match_tests: + matching = "matching: " + ", ".join(match_tests) + print(f"Re-running {test_name} in verbose mode ({matching})", flush=True) + else: + print(f"Re-running {test_name} in verbose mode", flush=True) + + result = run_single_test(test_name, runtests) + print() # Force a newline (just in case) + + # Serialize TestResult as dict in JSON + result.write_json(sys.stdout) + sys.stdout.flush() + sys.exit(0) + + +def main(): + if len(sys.argv) != 2: + print("usage: python -m test.libregrtest.worker JSON") + sys.exit(1) + worker_json = sys.argv[1] + + work_dir = get_work_dir(worker=True) + + with exit_timeout(): + with os_helper.temp_cwd(work_dir, quiet=True): + worker_process(worker_json) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 23896fdedaccac..a5ee4c2155536e 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -75,11 +75,6 @@ def test_wait(self): ns = libregrtest._parse_args(['--wait']) self.assertTrue(ns.wait) - def test_worker_json(self): - ns = libregrtest._parse_args(['--worker-json', '[[], {}]']) - self.assertEqual(ns.worker_json, '[[], {}]') - self.checkError(['--worker-json'], 'expected one argument') - def test_start(self): for opt in '-S', '--start': with self.subTest(opt=opt): From 1ec45378e97a2d3812ed8bc18e1bd1eb8ddcc9a0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Sep 2023 02:07:18 +0200 Subject: [PATCH 52/66] gh-109162: libregrtest: add single.py and result.py (#109243) * Add single.py and result.py files. * Rename runtest.py to runtests.py. * Move run_single_test() function and its helper functions to single.py. * Move remove_testfn(), abs_module_name() and normalize_test_name() to utils.py. * Move setup_support() to setup.py. * Move type hints like TestName to utils.py. * Rename runtest.py to runtests.py. --- Lib/test/libregrtest/findtests.py | 58 +++ Lib/test/libregrtest/logger.py | 2 +- Lib/test/libregrtest/main.py | 14 +- Lib/test/libregrtest/refleak.py | 5 +- Lib/test/libregrtest/result.py | 184 ++++++++ Lib/test/libregrtest/results.py | 8 +- Lib/test/libregrtest/runtest.py | 676 ----------------------------- Lib/test/libregrtest/runtest_mp.py | 9 +- Lib/test/libregrtest/runtests.py | 83 ++++ Lib/test/libregrtest/setup.py | 13 + Lib/test/libregrtest/single.py | 275 ++++++++++++ Lib/test/libregrtest/utils.py | 82 ++++ Lib/test/libregrtest/worker.py | 8 +- Lib/test/test_regrtest.py | 2 +- 14 files changed, 722 insertions(+), 697 deletions(-) create mode 100644 Lib/test/libregrtest/findtests.py create mode 100644 Lib/test/libregrtest/result.py delete mode 100644 Lib/test/libregrtest/runtest.py create mode 100644 Lib/test/libregrtest/runtests.py create mode 100644 Lib/test/libregrtest/single.py diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py new file mode 100644 index 00000000000000..88570be7dccdfd --- /dev/null +++ b/Lib/test/libregrtest/findtests.py @@ -0,0 +1,58 @@ +import os + +from test.libregrtest.utils import StrPath, TestName, TestList + + +# If these test directories are encountered recurse into them and treat each +# "test_*.py" file or each sub-directory as a separate test module. This can +# increase parallelism. +# +# Beware this can't generally be done for any directory with sub-tests as the +# __init__.py may do things which alter what tests are to be run. +SPLITTESTDIRS: set[TestName] = { + "test_asyncio", + "test_concurrent_futures", + "test_multiprocessing_fork", + "test_multiprocessing_forkserver", + "test_multiprocessing_spawn", +} + + +def findtestdir(path=None): + return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir + + +def findtests(*, testdir: StrPath | None = None, exclude=(), + split_test_dirs: set[TestName] = SPLITTESTDIRS, + base_mod: str = "") -> TestList: + """Return a list of all applicable test modules.""" + testdir = findtestdir(testdir) + tests = [] + for name in os.listdir(testdir): + mod, ext = os.path.splitext(name) + if (not mod.startswith("test_")) or (mod in exclude): + continue + if mod in split_test_dirs: + subdir = os.path.join(testdir, mod) + mod = f"{base_mod or 'test'}.{mod}" + tests.extend(findtests(testdir=subdir, exclude=exclude, + split_test_dirs=split_test_dirs, + base_mod=mod)) + elif ext in (".py", ""): + tests.append(f"{base_mod}.{mod}" if base_mod else mod) + return sorted(tests) + + +def split_test_packages(tests, *, testdir: StrPath | None = None, exclude=(), + split_test_dirs=SPLITTESTDIRS): + testdir = findtestdir(testdir) + splitted = [] + for name in tests: + if name in split_test_dirs: + subdir = os.path.join(testdir, name) + splitted.extend(findtests(testdir=subdir, exclude=exclude, + split_test_dirs=split_test_dirs, + base_mod=name)) + else: + splitted.append(name) + return splitted diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py index c4498a43545545..05b9307c97ded1 100644 --- a/Lib/test/libregrtest/logger.py +++ b/Lib/test/libregrtest/logger.py @@ -1,7 +1,7 @@ import os import time -from test.libregrtest.runtest import RunTests +from test.libregrtest.runtests import RunTests from test.libregrtest.utils import print_warning, MS_WINDOWS if MS_WINDOWS: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index ed0813d6f30c10..0227a2d1fc612e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -11,17 +11,19 @@ from test.support import os_helper from test.libregrtest.cmdline import _parse_args, Namespace +from test.libregrtest.findtests import findtests, split_test_packages from test.libregrtest.logger import Logger -from test.libregrtest.runtest import ( - findtests, split_test_packages, run_single_test, abs_module_name, - PROGRESS_MIN_TIME, State, RunTests, HuntRefleak, - FilterTuple, TestList, StrJSON, TestName) +from test.libregrtest.result import State +from test.libregrtest.runtests import RunTests, HuntRefleak from test.libregrtest.setup import setup_tests, setup_test_dir +from test.libregrtest.single import run_single_test, PROGRESS_MIN_TIME from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.results import TestResults from test.libregrtest.utils import ( - strip_py_suffix, count, format_duration, StrPath, - printlist, get_build_info, get_temp_dir, get_work_dir, exit_timeout) + StrPath, StrJSON, TestName, TestList, FilterTuple, + strip_py_suffix, count, format_duration, + printlist, get_build_info, get_temp_dir, get_work_dir, exit_timeout, + abs_module_name) class Regrtest: diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 81f163c47e5665..6b1d7082ba46ba 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -1,10 +1,11 @@ -import os import sys import warnings from inspect import isabstract + from test import support from test.support import os_helper -from test.libregrtest.runtest import HuntRefleak + +from test.libregrtest.runtests import HuntRefleak from test.libregrtest.utils import clear_caches try: diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py new file mode 100644 index 00000000000000..4a68872369c9e7 --- /dev/null +++ b/Lib/test/libregrtest/result.py @@ -0,0 +1,184 @@ +import dataclasses +import json +from typing import Any + +from test.support import TestStats + +from test.libregrtest.utils import ( + TestName, FilterTuple, + format_duration, normalize_test_name, print_warning) + + +# Avoid enum.Enum to reduce the number of imports when tests are run +class State: + PASSED = "PASSED" + FAILED = "FAILED" + SKIPPED = "SKIPPED" + UNCAUGHT_EXC = "UNCAUGHT_EXC" + REFLEAK = "REFLEAK" + ENV_CHANGED = "ENV_CHANGED" + RESOURCE_DENIED = "RESOURCE_DENIED" + INTERRUPTED = "INTERRUPTED" + MULTIPROCESSING_ERROR = "MULTIPROCESSING_ERROR" + DID_NOT_RUN = "DID_NOT_RUN" + TIMEOUT = "TIMEOUT" + + @staticmethod + def is_failed(state): + return state in { + State.FAILED, + State.UNCAUGHT_EXC, + State.REFLEAK, + State.MULTIPROCESSING_ERROR, + State.TIMEOUT} + + @staticmethod + def has_meaningful_duration(state): + # Consider that the duration is meaningless for these cases. + # For example, if a whole test file is skipped, its duration + # is unlikely to be the duration of executing its tests, + # but just the duration to execute code which skips the test. + return state not in { + State.SKIPPED, + State.RESOURCE_DENIED, + State.INTERRUPTED, + State.MULTIPROCESSING_ERROR, + State.DID_NOT_RUN} + + @staticmethod + def must_stop(state): + return state in { + State.INTERRUPTED, + State.MULTIPROCESSING_ERROR} + + +@dataclasses.dataclass(slots=True) +class TestResult: + test_name: TestName + state: str | None = None + # Test duration in seconds + duration: float | None = None + xml_data: list[str] | None = None + stats: TestStats | None = None + + # errors and failures copied from support.TestFailedWithDetails + errors: list[tuple[str, str]] | None = None + failures: list[tuple[str, str]] | None = None + + def is_failed(self, fail_env_changed: bool) -> bool: + if self.state == State.ENV_CHANGED: + return fail_env_changed + return State.is_failed(self.state) + + def _format_failed(self): + if self.errors and self.failures: + le = len(self.errors) + lf = len(self.failures) + error_s = "error" + ("s" if le > 1 else "") + failure_s = "failure" + ("s" if lf > 1 else "") + return f"{self.test_name} failed ({le} {error_s}, {lf} {failure_s})" + + if self.errors: + le = len(self.errors) + error_s = "error" + ("s" if le > 1 else "") + return f"{self.test_name} failed ({le} {error_s})" + + if self.failures: + lf = len(self.failures) + failure_s = "failure" + ("s" if lf > 1 else "") + return f"{self.test_name} failed ({lf} {failure_s})" + + return f"{self.test_name} failed" + + def __str__(self) -> str: + match self.state: + case State.PASSED: + return f"{self.test_name} passed" + case State.FAILED: + return self._format_failed() + case State.SKIPPED: + return f"{self.test_name} skipped" + case State.UNCAUGHT_EXC: + return f"{self.test_name} failed (uncaught exception)" + case State.REFLEAK: + return f"{self.test_name} failed (reference leak)" + case State.ENV_CHANGED: + return f"{self.test_name} failed (env changed)" + case State.RESOURCE_DENIED: + return f"{self.test_name} skipped (resource denied)" + case State.INTERRUPTED: + return f"{self.test_name} interrupted" + case State.MULTIPROCESSING_ERROR: + return f"{self.test_name} process crashed" + case State.DID_NOT_RUN: + return f"{self.test_name} ran no tests" + case State.TIMEOUT: + return f"{self.test_name} timed out ({format_duration(self.duration)})" + case _: + raise ValueError("unknown result state: {state!r}") + + def has_meaningful_duration(self): + return State.has_meaningful_duration(self.state) + + def set_env_changed(self): + if self.state is None or self.state == State.PASSED: + self.state = State.ENV_CHANGED + + def must_stop(self, fail_fast: bool, fail_env_changed: bool) -> bool: + if State.must_stop(self.state): + return True + if fail_fast and self.is_failed(fail_env_changed): + return True + return False + + def get_rerun_match_tests(self) -> FilterTuple | None: + match_tests = [] + + errors = self.errors or [] + failures = self.failures or [] + for error_list, is_error in ( + (errors, True), + (failures, False), + ): + for full_name, *_ in error_list: + match_name = normalize_test_name(full_name, is_error=is_error) + if match_name is None: + # 'setUpModule (test.test_sys)': don't filter tests + return None + if not match_name: + error_type = "ERROR" if is_error else "FAIL" + print_warning(f"rerun failed to parse {error_type} test name: " + f"{full_name!r}: don't filter tests") + return None + match_tests.append(match_name) + + if not match_tests: + return None + return tuple(match_tests) + + def write_json(self, file) -> None: + json.dump(self, file, cls=_EncodeTestResult) + + @staticmethod + def from_json(worker_json) -> 'TestResult': + return json.loads(worker_json, object_hook=_decode_test_result) + + +class _EncodeTestResult(json.JSONEncoder): + def default(self, o: Any) -> dict[str, Any]: + if isinstance(o, TestResult): + result = dataclasses.asdict(o) + result["__test_result__"] = o.__class__.__name__ + return result + else: + return super().default(o) + + +def _decode_test_result(data: dict[str, Any]) -> TestResult | dict[str, Any]: + if "__test_result__" in data: + data.pop('__test_result__') + if data['stats'] is not None: + data['stats'] = TestStats(**data['stats']) + return TestResult(**data) + else: + return data diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index e44301938c6527..b7a044eae25aae 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,11 +1,11 @@ import sys from test.support import TestStats -from test.libregrtest.runtest import ( - TestName, TestTuple, TestList, FilterDict, State, - TestResult, RunTests) +from test.libregrtest.runtests import RunTests +from test.libregrtest.result import State, TestResult from test.libregrtest.utils import ( - printlist, count, format_duration, StrPath) + StrPath, TestName, TestTuple, TestList, FilterDict, + printlist, count, format_duration) EXITCODE_BAD_TEST = 2 diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py deleted file mode 100644 index a12c7fcaee8bc6..00000000000000 --- a/Lib/test/libregrtest/runtest.py +++ /dev/null @@ -1,676 +0,0 @@ -import dataclasses -import doctest -import faulthandler -import gc -import importlib -import io -import json -import os -import sys -import time -import traceback -import unittest -from typing import Any - -from test import support -from test.support import TestStats -from test.support import os_helper -from test.support import threading_helper -from test.libregrtest.save_env import saved_test_environment -from test.libregrtest.utils import ( - clear_caches, format_duration, print_warning, StrPath) - - -StrJSON = str -TestName = str -TestTuple = tuple[TestName, ...] -TestList = list[TestName] - -# --match and --ignore options: list of patterns -# ('*' joker character can be used) -FilterTuple = tuple[TestName, ...] -FilterDict = dict[TestName, FilterTuple] - - -@dataclasses.dataclass(slots=True, frozen=True) -class HuntRefleak: - warmups: int - runs: int - filename: StrPath - - -# Avoid enum.Enum to reduce the number of imports when tests are run -class State: - PASSED = "PASSED" - FAILED = "FAILED" - SKIPPED = "SKIPPED" - UNCAUGHT_EXC = "UNCAUGHT_EXC" - REFLEAK = "REFLEAK" - ENV_CHANGED = "ENV_CHANGED" - RESOURCE_DENIED = "RESOURCE_DENIED" - INTERRUPTED = "INTERRUPTED" - MULTIPROCESSING_ERROR = "MULTIPROCESSING_ERROR" - DID_NOT_RUN = "DID_NOT_RUN" - TIMEOUT = "TIMEOUT" - - @staticmethod - def is_failed(state): - return state in { - State.FAILED, - State.UNCAUGHT_EXC, - State.REFLEAK, - State.MULTIPROCESSING_ERROR, - State.TIMEOUT} - - @staticmethod - def has_meaningful_duration(state): - # Consider that the duration is meaningless for these cases. - # For example, if a whole test file is skipped, its duration - # is unlikely to be the duration of executing its tests, - # but just the duration to execute code which skips the test. - return state not in { - State.SKIPPED, - State.RESOURCE_DENIED, - State.INTERRUPTED, - State.MULTIPROCESSING_ERROR, - State.DID_NOT_RUN} - - @staticmethod - def must_stop(state): - return state in { - State.INTERRUPTED, - State.MULTIPROCESSING_ERROR} - - -# gh-90681: When rerunning tests, we might need to rerun the whole -# class or module suite if some its life-cycle hooks fail. -# Test level hooks are not affected. -_TEST_LIFECYCLE_HOOKS = frozenset(( - 'setUpClass', 'tearDownClass', - 'setUpModule', 'tearDownModule', -)) - -def normalize_test_name(test_full_name, *, is_error=False): - short_name = test_full_name.split(" ")[0] - if is_error and short_name in _TEST_LIFECYCLE_HOOKS: - if test_full_name.startswith(('setUpModule (', 'tearDownModule (')): - # if setUpModule() or tearDownModule() failed, don't filter - # tests with the test file name, don't use use filters. - return None - - # This means that we have a failure in a life-cycle hook, - # we need to rerun the whole module or class suite. - # Basically the error looks like this: - # ERROR: setUpClass (test.test_reg_ex.RegTest) - # or - # ERROR: setUpModule (test.test_reg_ex) - # So, we need to parse the class / module name. - lpar = test_full_name.index('(') - rpar = test_full_name.index(')') - return test_full_name[lpar + 1: rpar].split('.')[-1] - return short_name - - -@dataclasses.dataclass(slots=True) -class TestResult: - test_name: TestName - state: str | None = None - # Test duration in seconds - duration: float | None = None - xml_data: list[str] | None = None - stats: TestStats | None = None - - # errors and failures copied from support.TestFailedWithDetails - errors: list[tuple[str, str]] | None = None - failures: list[tuple[str, str]] | None = None - - def is_failed(self, fail_env_changed: bool) -> bool: - if self.state == State.ENV_CHANGED: - return fail_env_changed - return State.is_failed(self.state) - - def _format_failed(self): - if self.errors and self.failures: - le = len(self.errors) - lf = len(self.failures) - error_s = "error" + ("s" if le > 1 else "") - failure_s = "failure" + ("s" if lf > 1 else "") - return f"{self.test_name} failed ({le} {error_s}, {lf} {failure_s})" - - if self.errors: - le = len(self.errors) - error_s = "error" + ("s" if le > 1 else "") - return f"{self.test_name} failed ({le} {error_s})" - - if self.failures: - lf = len(self.failures) - failure_s = "failure" + ("s" if lf > 1 else "") - return f"{self.test_name} failed ({lf} {failure_s})" - - return f"{self.test_name} failed" - - def __str__(self) -> str: - match self.state: - case State.PASSED: - return f"{self.test_name} passed" - case State.FAILED: - return self._format_failed() - case State.SKIPPED: - return f"{self.test_name} skipped" - case State.UNCAUGHT_EXC: - return f"{self.test_name} failed (uncaught exception)" - case State.REFLEAK: - return f"{self.test_name} failed (reference leak)" - case State.ENV_CHANGED: - return f"{self.test_name} failed (env changed)" - case State.RESOURCE_DENIED: - return f"{self.test_name} skipped (resource denied)" - case State.INTERRUPTED: - return f"{self.test_name} interrupted" - case State.MULTIPROCESSING_ERROR: - return f"{self.test_name} process crashed" - case State.DID_NOT_RUN: - return f"{self.test_name} ran no tests" - case State.TIMEOUT: - return f"{self.test_name} timed out ({format_duration(self.duration)})" - case _: - raise ValueError("unknown result state: {state!r}") - - def has_meaningful_duration(self): - return State.has_meaningful_duration(self.state) - - def set_env_changed(self): - if self.state is None or self.state == State.PASSED: - self.state = State.ENV_CHANGED - - def must_stop(self, fail_fast: bool, fail_env_changed: bool) -> bool: - if State.must_stop(self.state): - return True - if fail_fast and self.is_failed(fail_env_changed): - return True - return False - - def get_rerun_match_tests(self) -> FilterTuple | None: - match_tests = [] - - errors = self.errors or [] - failures = self.failures or [] - for error_list, is_error in ( - (errors, True), - (failures, False), - ): - for full_name, *_ in error_list: - match_name = normalize_test_name(full_name, is_error=is_error) - if match_name is None: - # 'setUpModule (test.test_sys)': don't filter tests - return None - if not match_name: - error_type = "ERROR" if is_error else "FAIL" - print_warning(f"rerun failed to parse {error_type} test name: " - f"{full_name!r}: don't filter tests") - return None - match_tests.append(match_name) - - if not match_tests: - return None - return tuple(match_tests) - - def write_json(self, file) -> None: - json.dump(self, file, cls=_EncodeTestResult) - - @staticmethod - def from_json(worker_json) -> 'TestResult': - return json.loads(worker_json, object_hook=_decode_test_result) - - -class _EncodeTestResult(json.JSONEncoder): - def default(self, o: Any) -> dict[str, Any]: - if isinstance(o, TestResult): - result = dataclasses.asdict(o) - result["__test_result__"] = o.__class__.__name__ - return result - else: - return super().default(o) - - -def _decode_test_result(data: dict[str, Any]) -> TestResult | dict[str, Any]: - if "__test_result__" in data: - data.pop('__test_result__') - if data['stats'] is not None: - data['stats'] = TestStats(**data['stats']) - return TestResult(**data) - else: - return data - - -@dataclasses.dataclass(slots=True, frozen=True) -class RunTests: - tests: TestTuple - fail_fast: bool = False - fail_env_changed: bool = False - match_tests: FilterTuple | None = None - ignore_tests: FilterTuple | None = None - match_tests_dict: FilterDict | None = None - rerun: bool = False - forever: bool = False - pgo: bool = False - pgo_extended: bool = False - output_on_failure: bool = False - timeout: float | None = None - verbose: bool = False - quiet: bool = False - hunt_refleak: HuntRefleak | None = None - test_dir: StrPath | None = None - use_junit: bool = False - memory_limit: str | None = None - gc_threshold: int | None = None - use_resources: list[str] = None - python_cmd: list[str] | None = None - - def copy(self, **override): - state = dataclasses.asdict(self) - state.update(override) - return RunTests(**state) - - def get_match_tests(self, test_name) -> FilterTuple | None: - if self.match_tests_dict is not None: - return self.match_tests_dict.get(test_name, None) - else: - return None - - def iter_tests(self): - if self.forever: - while True: - yield from self.tests - else: - yield from self.tests - - def as_json(self) -> StrJSON: - return json.dumps(self, cls=_EncodeRunTests) - - @staticmethod - def from_json(worker_json: StrJSON) -> 'RunTests': - return json.loads(worker_json, object_hook=_decode_runtests) - - -class _EncodeRunTests(json.JSONEncoder): - def default(self, o: Any) -> dict[str, Any]: - if isinstance(o, RunTests): - result = dataclasses.asdict(o) - result["__runtests__"] = True - return result - else: - return super().default(o) - - -def _decode_runtests(data: dict[str, Any]) -> RunTests | dict[str, Any]: - if "__runtests__" in data: - data.pop('__runtests__') - if data['hunt_refleak']: - data['hunt_refleak'] = HuntRefleak(**data['hunt_refleak']) - return RunTests(**data) - else: - return data - - -# Minimum duration of a test to display its duration or to mention that -# the test is running in background -PROGRESS_MIN_TIME = 30.0 # seconds - -#If these test directories are encountered recurse into them and treat each -# test_ .py or dir as a separate test module. This can increase parallelism. -# Beware this can't generally be done for any directory with sub-tests as the -# __init__.py may do things which alter what tests are to be run. - -SPLITTESTDIRS: set[TestName] = { - "test_asyncio", - "test_concurrent_futures", - "test_multiprocessing_fork", - "test_multiprocessing_forkserver", - "test_multiprocessing_spawn", -} - - -def findtestdir(path=None): - return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir - - -def findtests(*, testdir: StrPath | None = None, exclude=(), - split_test_dirs: set[TestName] = SPLITTESTDIRS, - base_mod: str = "") -> TestList: - """Return a list of all applicable test modules.""" - testdir = findtestdir(testdir) - tests = [] - for name in os.listdir(testdir): - mod, ext = os.path.splitext(name) - if (not mod.startswith("test_")) or (mod in exclude): - continue - if mod in split_test_dirs: - subdir = os.path.join(testdir, mod) - mod = f"{base_mod or 'test'}.{mod}" - tests.extend(findtests(testdir=subdir, exclude=exclude, - split_test_dirs=split_test_dirs, - base_mod=mod)) - elif ext in (".py", ""): - tests.append(f"{base_mod}.{mod}" if base_mod else mod) - return sorted(tests) - - -def split_test_packages(tests, *, testdir: StrPath | None = None, exclude=(), - split_test_dirs=SPLITTESTDIRS): - testdir = findtestdir(testdir) - splitted = [] - for name in tests: - if name in split_test_dirs: - subdir = os.path.join(testdir, name) - splitted.extend(findtests(testdir=subdir, exclude=exclude, - split_test_dirs=split_test_dirs, - base_mod=name)) - else: - splitted.append(name) - return splitted - - -def abs_module_name(test_name: TestName, test_dir: StrPath | None) -> TestName: - if test_name.startswith('test.') or test_dir: - return test_name - else: - # Import it from the test package - return 'test.' + test_name - - -def setup_support(runtests: RunTests): - support.PGO = runtests.pgo - support.PGO_EXTENDED = runtests.pgo_extended - support.set_match_tests(runtests.match_tests, runtests.ignore_tests) - support.failfast = runtests.fail_fast - support.verbose = runtests.verbose - if runtests.use_junit: - support.junit_xml_list = [] - else: - support.junit_xml_list = None - - -def _runtest(result: TestResult, runtests: RunTests) -> None: - # Capture stdout and stderr, set faulthandler timeout, - # and create JUnit XML report. - verbose = runtests.verbose - output_on_failure = runtests.output_on_failure - timeout = runtests.timeout - - use_timeout = ( - timeout is not None and threading_helper.can_start_thread - ) - if use_timeout: - faulthandler.dump_traceback_later(timeout, exit=True) - - try: - setup_support(runtests) - - if output_on_failure: - support.verbose = True - - stream = io.StringIO() - orig_stdout = sys.stdout - orig_stderr = sys.stderr - print_warning = support.print_warning - orig_print_warnings_stderr = print_warning.orig_stderr - - output = None - try: - sys.stdout = stream - sys.stderr = stream - # print_warning() writes into the temporary stream to preserve - # messages order. If support.environment_altered becomes true, - # warnings will be written to sys.stderr below. - print_warning.orig_stderr = stream - - _runtest_env_changed_exc(result, runtests, display_failure=False) - # Ignore output if the test passed successfully - if result.state != State.PASSED: - output = stream.getvalue() - finally: - sys.stdout = orig_stdout - sys.stderr = orig_stderr - print_warning.orig_stderr = orig_print_warnings_stderr - - if output is not None: - sys.stderr.write(output) - sys.stderr.flush() - else: - # Tell tests to be moderately quiet - support.verbose = verbose - _runtest_env_changed_exc(result, runtests, - display_failure=not verbose) - - xml_list = support.junit_xml_list - if xml_list: - import xml.etree.ElementTree as ET - result.xml_data = [ET.tostring(x).decode('us-ascii') - for x in xml_list] - finally: - if use_timeout: - faulthandler.cancel_dump_traceback_later() - support.junit_xml_list = None - - -def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: - """Run a single test. - - test_name -- the name of the test - - Returns a TestResult. - - If runtests.use_junit, xml_data is a list containing each generated - testsuite element. - """ - start_time = time.perf_counter() - result = TestResult(test_name) - pgo = runtests.pgo - try: - _runtest(result, runtests) - except: - if not pgo: - msg = traceback.format_exc() - print(f"test {test_name} crashed -- {msg}", - file=sys.stderr, flush=True) - result.state = State.UNCAUGHT_EXC - result.duration = time.perf_counter() - start_time - return result - - -def run_unittest(test_mod): - loader = unittest.TestLoader() - tests = loader.loadTestsFromModule(test_mod) - for error in loader.errors: - print(error, file=sys.stderr) - if loader.errors: - raise Exception("errors while loading tests") - return support.run_unittest(tests) - - -def save_env(test_name: TestName, runtests: RunTests): - return saved_test_environment(test_name, runtests.verbose, runtests.quiet, - pgo=runtests.pgo) - - -def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: - # Run test_func(), collect statistics, and detect reference and memory - # leaks. - if runtests.hunt_refleak: - from test.libregrtest.refleak import runtest_refleak - refleak, test_result = runtest_refleak(result.test_name, test_func, - runtests.hunt_refleak, - runtests.quiet) - else: - test_result = test_func() - refleak = False - - if refleak: - result.state = State.REFLEAK - - match test_result: - case TestStats(): - stats = test_result - case unittest.TestResult(): - stats = TestStats.from_unittest(test_result) - case doctest.TestResults(): - stats = TestStats.from_doctest(test_result) - case None: - print_warning(f"{result.test_name} test runner returned None: {test_func}") - stats = None - case _: - print_warning(f"Unknown test result type: {type(test_result)}") - stats = None - - result.stats = stats - - -# Storage of uncollectable objects -FOUND_GARBAGE = [] - - -def _load_run_test(result: TestResult, runtests: RunTests) -> None: - # Load the test function, run the test function. - module_name = abs_module_name(result.test_name, runtests.test_dir) - - # Remove the module from sys.module to reload it if it was already imported - sys.modules.pop(module_name, None) - - test_mod = importlib.import_module(module_name) - - if hasattr(test_mod, "test_main"): - # https://github.com/python/cpython/issues/89392 - raise Exception(f"Module {result.test_name} defines test_main() which is no longer supported by regrtest") - def test_func(): - return run_unittest(test_mod) - - try: - with save_env(result.test_name, runtests): - regrtest_runner(result, test_func, runtests) - finally: - # First kill any dangling references to open files etc. - # This can also issue some ResourceWarnings which would otherwise get - # triggered during the following test run, and possibly produce - # failures. - support.gc_collect() - - remove_testfn(result.test_name, runtests.verbose) - - if gc.garbage: - support.environment_altered = True - print_warning(f"{result.test_name} created {len(gc.garbage)} " - f"uncollectable object(s)") - - # move the uncollectable objects somewhere, - # so we don't see them again - FOUND_GARBAGE.extend(gc.garbage) - gc.garbage.clear() - - support.reap_children() - - -def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, - display_failure: bool = True) -> None: - # Detect environment changes, handle exceptions. - - # Reset the environment_altered flag to detect if a test altered - # the environment - support.environment_altered = False - - pgo = runtests.pgo - if pgo: - display_failure = False - quiet = runtests.quiet - - test_name = result.test_name - try: - clear_caches() - support.gc_collect() - - with save_env(test_name, runtests): - _load_run_test(result, runtests) - except support.ResourceDenied as msg: - if not quiet and not pgo: - print(f"{test_name} skipped -- {msg}", flush=True) - result.state = State.RESOURCE_DENIED - return - except unittest.SkipTest as msg: - if not quiet and not pgo: - print(f"{test_name} skipped -- {msg}", flush=True) - result.state = State.SKIPPED - return - except support.TestFailedWithDetails as exc: - msg = f"test {test_name} failed" - if display_failure: - msg = f"{msg} -- {exc}" - print(msg, file=sys.stderr, flush=True) - result.state = State.FAILED - result.errors = exc.errors - result.failures = exc.failures - result.stats = exc.stats - return - except support.TestFailed as exc: - msg = f"test {test_name} failed" - if display_failure: - msg = f"{msg} -- {exc}" - print(msg, file=sys.stderr, flush=True) - result.state = State.FAILED - result.stats = exc.stats - return - except support.TestDidNotRun: - result.state = State.DID_NOT_RUN - return - except KeyboardInterrupt: - print() - result.state = State.INTERRUPTED - return - except: - if not pgo: - msg = traceback.format_exc() - print(f"test {test_name} crashed -- {msg}", - file=sys.stderr, flush=True) - result.state = State.UNCAUGHT_EXC - return - - if support.environment_altered: - result.set_env_changed() - # Don't override the state if it was already set (REFLEAK or ENV_CHANGED) - if result.state is None: - result.state = State.PASSED - - -def remove_testfn(test_name: TestName, verbose: int) -> None: - # Try to clean up os_helper.TESTFN if left behind. - # - # While tests shouldn't leave any files or directories behind, when a test - # fails that can be tedious for it to arrange. The consequences can be - # especially nasty on Windows, since if a test leaves a file open, it - # cannot be deleted by name (while there's nothing we can do about that - # here either, we can display the name of the offending test, which is a - # real help). - name = os_helper.TESTFN - if not os.path.exists(name): - return - - if os.path.isdir(name): - import shutil - kind, nuker = "directory", shutil.rmtree - elif os.path.isfile(name): - kind, nuker = "file", os.unlink - else: - raise RuntimeError(f"os.path says {name!r} exists but is neither " - f"directory nor file") - - if verbose: - print_warning(f"{test_name} left behind {kind} {name!r}") - support.environment_altered = True - - try: - import stat - # fix possible permissions problems that might prevent cleanup - os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - nuker(name) - except Exception as exc: - print_warning(f"{test_name} left behind {kind} {name!r} " - f"and it couldn't be removed: {exc}") diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index c1bd911c43e2c4..f576d49e85db93 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -15,12 +15,13 @@ from test.support import os_helper from test.libregrtest.main import Regrtest -from test.libregrtest.runtest import ( - TestResult, State, PROGRESS_MIN_TIME, - RunTests, TestName) +from test.libregrtest.result import TestResult, State from test.libregrtest.results import TestResults +from test.libregrtest.runtests import RunTests +from test.libregrtest.single import PROGRESS_MIN_TIME from test.libregrtest.utils import ( - format_duration, print_warning, StrPath) + StrPath, TestName, + format_duration, print_warning) from test.libregrtest.worker import create_worker_process, USE_PROCESS_GROUP if sys.platform == 'win32': diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py new file mode 100644 index 00000000000000..366c6f1e7a1046 --- /dev/null +++ b/Lib/test/libregrtest/runtests.py @@ -0,0 +1,83 @@ +import dataclasses +import json +from typing import Any + +from test.libregrtest.utils import ( + StrPath, StrJSON, TestTuple, FilterTuple, FilterDict) + + +@dataclasses.dataclass(slots=True, frozen=True) +class HuntRefleak: + warmups: int + runs: int + filename: StrPath + + +@dataclasses.dataclass(slots=True, frozen=True) +class RunTests: + tests: TestTuple + fail_fast: bool = False + fail_env_changed: bool = False + match_tests: FilterTuple | None = None + ignore_tests: FilterTuple | None = None + match_tests_dict: FilterDict | None = None + rerun: bool = False + forever: bool = False + pgo: bool = False + pgo_extended: bool = False + output_on_failure: bool = False + timeout: float | None = None + verbose: bool = False + quiet: bool = False + hunt_refleak: HuntRefleak | None = None + test_dir: StrPath | None = None + use_junit: bool = False + memory_limit: str | None = None + gc_threshold: int | None = None + use_resources: list[str] = None + python_cmd: list[str] | None = None + + def copy(self, **override): + state = dataclasses.asdict(self) + state.update(override) + return RunTests(**state) + + def get_match_tests(self, test_name) -> FilterTuple | None: + if self.match_tests_dict is not None: + return self.match_tests_dict.get(test_name, None) + else: + return None + + def iter_tests(self): + if self.forever: + while True: + yield from self.tests + else: + yield from self.tests + + def as_json(self) -> StrJSON: + return json.dumps(self, cls=_EncodeRunTests) + + @staticmethod + def from_json(worker_json: StrJSON) -> 'RunTests': + return json.loads(worker_json, object_hook=_decode_runtests) + + +class _EncodeRunTests(json.JSONEncoder): + def default(self, o: Any) -> dict[str, Any]: + if isinstance(o, RunTests): + result = dataclasses.asdict(o) + result["__runtests__"] = True + return result + else: + return super().default(o) + + +def _decode_runtests(data: dict[str, Any]) -> RunTests | dict[str, Any]: + if "__runtests__" in data: + data.pop('__runtests__') + if data['hunt_refleak']: + data['hunt_refleak'] = HuntRefleak(**data['hunt_refleak']) + return RunTests(**data) + else: + return data diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 48eb8b6800af1e..20ef3dc38cbd04 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -11,6 +11,7 @@ except ImportError: gc = None +from test.libregrtest.runtests import RunTests from test.libregrtest.utils import ( setup_unraisable_hook, setup_threading_excepthook, fix_umask) @@ -25,6 +26,18 @@ def setup_test_dir(testdir: str | None) -> None: sys.path.insert(0, os.path.abspath(testdir)) +def setup_support(runtests: RunTests): + support.PGO = runtests.pgo + support.PGO_EXTENDED = runtests.pgo_extended + support.set_match_tests(runtests.match_tests, runtests.ignore_tests) + support.failfast = runtests.fail_fast + support.verbose = runtests.verbose + if runtests.use_junit: + support.junit_xml_list = [] + else: + support.junit_xml_list = None + + def setup_tests(runtests): fix_umask() diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py new file mode 100644 index 00000000000000..bb33387fee0d35 --- /dev/null +++ b/Lib/test/libregrtest/single.py @@ -0,0 +1,275 @@ +import doctest +import faulthandler +import gc +import importlib +import io +import sys +import time +import traceback +import unittest + +from test import support +from test.support import TestStats +from test.support import threading_helper + +from test.libregrtest.result import State, TestResult +from test.libregrtest.runtests import RunTests +from test.libregrtest.save_env import saved_test_environment +from test.libregrtest.setup import setup_support +from test.libregrtest.utils import ( + TestName, + clear_caches, remove_testfn, abs_module_name, print_warning) + + +# Minimum duration of a test to display its duration or to mention that +# the test is running in background +PROGRESS_MIN_TIME = 30.0 # seconds + + +def run_unittest(test_mod): + loader = unittest.TestLoader() + tests = loader.loadTestsFromModule(test_mod) + for error in loader.errors: + print(error, file=sys.stderr) + if loader.errors: + raise Exception("errors while loading tests") + return support.run_unittest(tests) + + +def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: + # Run test_func(), collect statistics, and detect reference and memory + # leaks. + if runtests.hunt_refleak: + from test.libregrtest.refleak import runtest_refleak + refleak, test_result = runtest_refleak(result.test_name, test_func, + runtests.hunt_refleak, + runtests.quiet) + else: + test_result = test_func() + refleak = False + + if refleak: + result.state = State.REFLEAK + + match test_result: + case TestStats(): + stats = test_result + case unittest.TestResult(): + stats = TestStats.from_unittest(test_result) + case doctest.TestResults(): + stats = TestStats.from_doctest(test_result) + case None: + print_warning(f"{result.test_name} test runner returned None: {test_func}") + stats = None + case _: + print_warning(f"Unknown test result type: {type(test_result)}") + stats = None + + result.stats = stats + + +def save_env(test_name: TestName, runtests: RunTests): + return saved_test_environment(test_name, runtests.verbose, runtests.quiet, + pgo=runtests.pgo) + + +# Storage of uncollectable GC objects (gc.garbage) +GC_GARBAGE = [] + + +def _load_run_test(result: TestResult, runtests: RunTests) -> None: + # Load the test function, run the test function. + module_name = abs_module_name(result.test_name, runtests.test_dir) + + # Remove the module from sys.module to reload it if it was already imported + sys.modules.pop(module_name, None) + + test_mod = importlib.import_module(module_name) + + if hasattr(test_mod, "test_main"): + # https://github.com/python/cpython/issues/89392 + raise Exception(f"Module {result.test_name} defines test_main() which is no longer supported by regrtest") + def test_func(): + return run_unittest(test_mod) + + try: + with save_env(result.test_name, runtests): + regrtest_runner(result, test_func, runtests) + finally: + # First kill any dangling references to open files etc. + # This can also issue some ResourceWarnings which would otherwise get + # triggered during the following test run, and possibly produce + # failures. + support.gc_collect() + + remove_testfn(result.test_name, runtests.verbose) + + if gc.garbage: + support.environment_altered = True + print_warning(f"{result.test_name} created {len(gc.garbage)} " + f"uncollectable object(s)") + + # move the uncollectable objects somewhere, + # so we don't see them again + GC_GARBAGE.extend(gc.garbage) + gc.garbage.clear() + + support.reap_children() + + +def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, + display_failure: bool = True) -> None: + # Detect environment changes, handle exceptions. + + # Reset the environment_altered flag to detect if a test altered + # the environment + support.environment_altered = False + + pgo = runtests.pgo + if pgo: + display_failure = False + quiet = runtests.quiet + + test_name = result.test_name + try: + clear_caches() + support.gc_collect() + + with save_env(test_name, runtests): + _load_run_test(result, runtests) + except support.ResourceDenied as msg: + if not quiet and not pgo: + print(f"{test_name} skipped -- {msg}", flush=True) + result.state = State.RESOURCE_DENIED + return + except unittest.SkipTest as msg: + if not quiet and not pgo: + print(f"{test_name} skipped -- {msg}", flush=True) + result.state = State.SKIPPED + return + except support.TestFailedWithDetails as exc: + msg = f"test {test_name} failed" + if display_failure: + msg = f"{msg} -- {exc}" + print(msg, file=sys.stderr, flush=True) + result.state = State.FAILED + result.errors = exc.errors + result.failures = exc.failures + result.stats = exc.stats + return + except support.TestFailed as exc: + msg = f"test {test_name} failed" + if display_failure: + msg = f"{msg} -- {exc}" + print(msg, file=sys.stderr, flush=True) + result.state = State.FAILED + result.stats = exc.stats + return + except support.TestDidNotRun: + result.state = State.DID_NOT_RUN + return + except KeyboardInterrupt: + print() + result.state = State.INTERRUPTED + return + except: + if not pgo: + msg = traceback.format_exc() + print(f"test {test_name} crashed -- {msg}", + file=sys.stderr, flush=True) + result.state = State.UNCAUGHT_EXC + return + + if support.environment_altered: + result.set_env_changed() + # Don't override the state if it was already set (REFLEAK or ENV_CHANGED) + if result.state is None: + result.state = State.PASSED + + +def _runtest(result: TestResult, runtests: RunTests) -> None: + # Capture stdout and stderr, set faulthandler timeout, + # and create JUnit XML report. + verbose = runtests.verbose + output_on_failure = runtests.output_on_failure + timeout = runtests.timeout + + use_timeout = ( + timeout is not None and threading_helper.can_start_thread + ) + if use_timeout: + faulthandler.dump_traceback_later(timeout, exit=True) + + try: + setup_support(runtests) + + if output_on_failure: + support.verbose = True + + stream = io.StringIO() + orig_stdout = sys.stdout + orig_stderr = sys.stderr + print_warning = support.print_warning + orig_print_warnings_stderr = print_warning.orig_stderr + + output = None + try: + sys.stdout = stream + sys.stderr = stream + # print_warning() writes into the temporary stream to preserve + # messages order. If support.environment_altered becomes true, + # warnings will be written to sys.stderr below. + print_warning.orig_stderr = stream + + _runtest_env_changed_exc(result, runtests, display_failure=False) + # Ignore output if the test passed successfully + if result.state != State.PASSED: + output = stream.getvalue() + finally: + sys.stdout = orig_stdout + sys.stderr = orig_stderr + print_warning.orig_stderr = orig_print_warnings_stderr + + if output is not None: + sys.stderr.write(output) + sys.stderr.flush() + else: + # Tell tests to be moderately quiet + support.verbose = verbose + _runtest_env_changed_exc(result, runtests, + display_failure=not verbose) + + xml_list = support.junit_xml_list + if xml_list: + import xml.etree.ElementTree as ET + result.xml_data = [ET.tostring(x).decode('us-ascii') + for x in xml_list] + finally: + if use_timeout: + faulthandler.cancel_dump_traceback_later() + support.junit_xml_list = None + + +def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: + """Run a single test. + + test_name -- the name of the test + + Returns a TestResult. + + If runtests.use_junit, xml_data is a list containing each generated + testsuite element. + """ + start_time = time.perf_counter() + result = TestResult(test_name) + pgo = runtests.pgo + try: + _runtest(result, runtests) + except: + if not pgo: + msg = traceback.format_exc() + print(f"test {test_name} crashed -- {msg}", + file=sys.stderr, flush=True) + result.state = State.UNCAUGHT_EXC + result.duration = time.perf_counter() - start_time + return result diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index e77772cc2577fe..011d287e1674cd 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -21,7 +21,16 @@ EXIT_TIMEOUT = 120.0 +# Types for types hints StrPath = str +TestName = str +StrJSON = str +TestTuple = tuple[TestName, ...] +TestList = list[TestName] +# --match and --ignore options: list of patterns +# ('*' joker character can be used) +FilterTuple = tuple[TestName, ...] +FilterDict = dict[TestName, FilterTuple] def format_duration(seconds): @@ -389,3 +398,76 @@ def exit_timeout(): if threading_helper.can_start_thread: faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True) sys.exit(exc.code) + + +def remove_testfn(test_name: TestName, verbose: int) -> None: + # Try to clean up os_helper.TESTFN if left behind. + # + # While tests shouldn't leave any files or directories behind, when a test + # fails that can be tedious for it to arrange. The consequences can be + # especially nasty on Windows, since if a test leaves a file open, it + # cannot be deleted by name (while there's nothing we can do about that + # here either, we can display the name of the offending test, which is a + # real help). + name = os_helper.TESTFN + if not os.path.exists(name): + return + + if os.path.isdir(name): + import shutil + kind, nuker = "directory", shutil.rmtree + elif os.path.isfile(name): + kind, nuker = "file", os.unlink + else: + raise RuntimeError(f"os.path says {name!r} exists but is neither " + f"directory nor file") + + if verbose: + print_warning(f"{test_name} left behind {kind} {name!r}") + support.environment_altered = True + + try: + import stat + # fix possible permissions problems that might prevent cleanup + os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + nuker(name) + except Exception as exc: + print_warning(f"{test_name} left behind {kind} {name!r} " + f"and it couldn't be removed: {exc}") + + +def abs_module_name(test_name: TestName, test_dir: StrPath | None) -> TestName: + if test_name.startswith('test.') or test_dir: + return test_name + else: + # Import it from the test package + return 'test.' + test_name + + +# gh-90681: When rerunning tests, we might need to rerun the whole +# class or module suite if some its life-cycle hooks fail. +# Test level hooks are not affected. +_TEST_LIFECYCLE_HOOKS = frozenset(( + 'setUpClass', 'tearDownClass', + 'setUpModule', 'tearDownModule', +)) + +def normalize_test_name(test_full_name, *, is_error=False): + short_name = test_full_name.split(" ")[0] + if is_error and short_name in _TEST_LIFECYCLE_HOOKS: + if test_full_name.startswith(('setUpModule (', 'tearDownModule (')): + # if setUpModule() or tearDownModule() failed, don't filter + # tests with the test file name, don't use use filters. + return None + + # This means that we have a failure in a life-cycle hook, + # we need to rerun the whole module or class suite. + # Basically the error looks like this: + # ERROR: setUpClass (test.test_reg_ex.RegTest) + # or + # ERROR: setUpModule (test.test_reg_ex) + # So, we need to parse the class / module name. + lpar = test_full_name.index('(') + rpar = test_full_name.index(')') + return test_full_name[lpar + 1: rpar].split('.')[-1] + return short_name diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 033a0a3ff62260..24251c35cdd2f1 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -7,9 +7,11 @@ from test.support import os_helper from test.libregrtest.setup import setup_tests, setup_test_dir -from test.libregrtest.runtest import ( - run_single_test, StrJSON, FilterTuple, RunTests) -from test.libregrtest.utils import get_work_dir, exit_timeout, StrPath +from test.libregrtest.runtests import RunTests +from test.libregrtest.single import run_single_test +from test.libregrtest.utils import ( + StrPath, StrJSON, FilterTuple, + get_work_dir, exit_timeout) USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a5ee4c2155536e..21babb5185548f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -22,7 +22,7 @@ from test import support from test.support import os_helper, TestStats from test.libregrtest import utils, setup -from test.libregrtest.runtest import normalize_test_name +from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") From 0b6b05391bcc5893d2b71032cb704995696ecd0b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Sep 2023 03:46:26 +0200 Subject: [PATCH 53/66] gh-109162: libregrtest: fix Logger (#109246) * Pass results, quiet and pgo to Logger constructor. * Move display_progress() method from Regrtest to Logger. * No longer pass Regrtest to RunWorkers, but logger and results. --- Lib/test/libregrtest/logger.py | 18 +++++++++++++- Lib/test/libregrtest/main.py | 39 +++++++++++------------------- Lib/test/libregrtest/runtest_mp.py | 11 ++++++--- 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py index 05b9307c97ded1..6195b5ddd4ebc0 100644 --- a/Lib/test/libregrtest/logger.py +++ b/Lib/test/libregrtest/logger.py @@ -1,6 +1,7 @@ import os import time +from test.libregrtest.results import TestResults from test.libregrtest.runtests import RunTests from test.libregrtest.utils import print_warning, MS_WINDOWS @@ -9,11 +10,14 @@ class Logger: - def __init__(self): + def __init__(self, results: TestResults, quiet: bool, pgo: bool): self.start_time = time.perf_counter() self.test_count_text = '' self.test_count_width = 3 self.win_load_tracker = None + self._results: TestResults = results + self._quiet: bool = quiet + self._pgo: bool = pgo def log(self, line: str = '') -> None: empty = not line @@ -43,6 +47,18 @@ def get_load_avg(self) -> float | None: return self.win_load_tracker.getloadavg() return None + def display_progress(self, test_index: int, text: str) -> None: + if self._quiet: + return + results = self._results + + # "[ 51/405/1] test_tcl passed" + line = f"{test_index:{self.test_count_width}}{self.test_count_text}" + fails = len(results.bad) + len(results.env_changed) + if fails and not self._pgo: + line = f"{line}/{fails}" + self.log(f"[{line}] {text}") + def set_tests(self, runtests: RunTests) -> None: if runtests.forever: self.test_count_text = '' diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 0227a2d1fc612e..0864cbb3276d2b 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -50,7 +50,18 @@ class Regrtest: on the command line. """ def __init__(self, ns: Namespace): - self.logger = Logger() + # Log verbosity + self.verbose: bool = ns.verbose + self.quiet: bool = ns.quiet + self.pgo: bool = ns.pgo + self.pgo_extended: bool = ns.pgo_extended + + # Test results + self.results: TestResults = TestResults() + self.first_state: str | None = None + + # Logger + self.logger = Logger(self.results, self.quiet, self.pgo) # Actions self.want_header: bool = ns.header @@ -92,12 +103,8 @@ def __init__(self, ns: Namespace): self.forever: bool = ns.forever self.randomize: bool = ns.randomize self.random_seed: int | None = ns.random_seed - self.pgo: bool = ns.pgo - self.pgo_extended: bool = ns.pgo_extended self.output_on_failure: bool = ns.verbose3 self.timeout: float | None = ns.timeout - self.verbose: bool = ns.verbose - self.quiet: bool = ns.quiet if ns.huntrleaks: warmups, runs, filename = ns.huntrleaks filename = os.path.abspath(filename) @@ -119,18 +126,11 @@ def __init__(self, ns: Namespace): self.selected: TestList = [] self.first_runtests: RunTests | None = None - # test results - self.results: TestResults = TestResults() - - self.first_state: str | None = None - # used by --slowest self.print_slowest: bool = ns.print_slow # used to display the progress bar "[ 3/100]" self.start_time = time.perf_counter() - self.test_count_text = '' - self.test_count_width = 1 # used by --single self.single_test_run: bool = ns.single @@ -140,17 +140,6 @@ def __init__(self, ns: Namespace): def log(self, line=''): self.logger.log(line) - def display_progress(self, test_index, text): - if self.quiet: - return - - # "[ 51/405/1] test_tcl passed" - line = f"{test_index:{self.test_count_width}}{self.test_count_text}" - fails = len(self.results.bad) + len(self.results.env_changed) - if fails and not self.pgo: - line = f"{line}/{fails}" - self.log(f"[{line}] {text}") - def find_tests(self): if self.single_test_run: self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') @@ -344,7 +333,7 @@ def run_tests_sequentially(self, runtests): text = test_name if previous_test: text = '%s -- %s' % (text, previous_test) - self.display_progress(test_index, text) + self.logger.display_progress(test_index, text) result = self.run_test(test_name, runtests, tracer) @@ -416,7 +405,7 @@ def get_state(self): def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from test.libregrtest.runtest_mp import RunWorkers - RunWorkers(self, runtests, num_workers).run() + RunWorkers(num_workers, runtests, self.logger, self.results).run() def finalize_tests(self, tracer): if self.next_single_filename: diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index f576d49e85db93..96b2ac521b9d82 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -14,6 +14,7 @@ from test import support from test.support import os_helper +from test.libregrtest.logger import Logger from test.libregrtest.main import Regrtest from test.libregrtest.result import TestResult, State from test.libregrtest.results import TestResults @@ -360,12 +361,14 @@ def get_running(workers: list[WorkerThread]) -> list[str]: class RunWorkers: - def __init__(self, regrtest: Regrtest, runtests: RunTests, num_workers: int) -> None: - self.results: TestResults = regrtest.results - self.log = regrtest.logger.log - self.display_progress = regrtest.display_progress + def __init__(self, num_workers: int, runtests: RunTests, + logger: Logger, results: TestResult) -> None: self.num_workers = num_workers self.runtests = runtests + self.log = logger.log + self.display_progress = logger.display_progress + self.results: TestResults = results + self.output: queue.Queue[QueueOutput] = queue.Queue() tests_iter = runtests.iter_tests() self.pending = MultiprocessIterator(tests_iter) From 0c139b5f2fb9c8a9e6df58e5da9d4a992d17926d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Sep 2023 05:27:37 +0200 Subject: [PATCH 54/66] gh-109162: libregrtest: rename runtest_mp.py to run_workers.py (#109248) * Rename runtest_mp.py to run_workers.py * Move exit_timeout() and temp_cwd() context managers from Regrtest.main() to Regrtest.run_tests(). Actions like --list-tests or --list-cases don't need these protections. * Regrtest: remove selected and tests attributes. Pass 'selected' to list_tests(), list_cases() and run_tests(). display_result() now expects a TestTuple, instead of TestList. * Rename setup_tests() to setup_process() and rename setup_support() to setup_tests(). * Move _adjust_resource_limits() to utils and rename it to adjust_rlimit_nofile(). * Move replace_stdout() to utils. * Fix RunTests.verbose type: it's an int. --- Lib/test/libregrtest/main.py | 133 +++++++++--------- Lib/test/libregrtest/result.py | 4 +- Lib/test/libregrtest/results.py | 4 +- .../{runtest_mp.py => run_workers.py} | 5 +- Lib/test/libregrtest/runtests.py | 4 +- Lib/test/libregrtest/setup.py | 124 +++++----------- Lib/test/libregrtest/single.py | 4 +- Lib/test/libregrtest/utils.py | 53 +++++++ Lib/test/libregrtest/worker.py | 4 +- 9 files changed, 171 insertions(+), 164 deletions(-) rename Lib/test/libregrtest/{runtest_mp.py => run_workers.py} (98%) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 0864cbb3276d2b..31eab99ca76351 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -15,12 +15,12 @@ from test.libregrtest.logger import Logger from test.libregrtest.result import State from test.libregrtest.runtests import RunTests, HuntRefleak -from test.libregrtest.setup import setup_tests, setup_test_dir +from test.libregrtest.setup import setup_process, setup_test_dir from test.libregrtest.single import run_single_test, PROGRESS_MIN_TIME from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.results import TestResults from test.libregrtest.utils import ( - StrPath, StrJSON, TestName, TestList, FilterTuple, + StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple, strip_py_suffix, count, format_duration, printlist, get_build_info, get_temp_dir, get_work_dir, exit_timeout, abs_module_name) @@ -51,7 +51,7 @@ class Regrtest: """ def __init__(self, ns: Namespace): # Log verbosity - self.verbose: bool = ns.verbose + self.verbose: int = int(ns.verbose) self.quiet: bool = ns.quiet self.pgo: bool = ns.pgo self.pgo_extended: bool = ns.pgo_extended @@ -122,8 +122,6 @@ def __init__(self, ns: Namespace): self.tmp_dir: StrPath | None = ns.tempdir # tests - self.tests = [] - self.selected: TestList = [] self.first_runtests: RunTests | None = None # used by --slowest @@ -140,18 +138,18 @@ def __init__(self, ns: Namespace): def log(self, line=''): self.logger.log(line) - def find_tests(self): + def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList | None]: if self.single_test_run: self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') try: with open(self.next_single_filename, 'r') as fp: next_test = fp.read().strip() - self.tests = [next_test] + tests = [next_test] except OSError: pass if self.fromfile: - self.tests = [] + tests = [] # regex to match 'test_builtin' in line: # '0:00:00 [ 4/400] test_builtin -- test_dict took 1 sec' regex = re.compile(r'\btest_[a-zA-Z0-9_]+\b') @@ -161,9 +159,9 @@ def find_tests(self): line = line.strip() match = regex.search(line) if match is not None: - self.tests.append(match.group()) + tests.append(match.group()) - strip_py_suffix(self.tests) + strip_py_suffix(tests) if self.pgo: # add default PGO tests if no tests are specified @@ -179,18 +177,18 @@ def find_tests(self): exclude=exclude_tests) if not self.fromfile: - self.selected = self.tests or self.cmdline_args - if self.selected: - self.selected = split_test_packages(self.selected) + selected = tests or self.cmdline_args + if selected: + selected = split_test_packages(selected) else: - self.selected = alltests + selected = alltests else: - self.selected = self.tests + selected = tests if self.single_test_run: - self.selected = self.selected[:1] + selected = selected[:1] try: - pos = alltests.index(self.selected[0]) + pos = alltests.index(selected[0]) self.next_single_test = alltests[pos + 1] except IndexError: pass @@ -198,7 +196,7 @@ def find_tests(self): # Remove all the selected tests that precede start if it's set. if self.starting_test: try: - del self.selected[:self.selected.index(self.starting_test)] + del selected[:selected.index(self.starting_test)] except ValueError: print(f"Cannot find starting test: {self.starting_test}") sys.exit(1) @@ -207,10 +205,12 @@ def find_tests(self): if self.random_seed is None: self.random_seed = random.randrange(100_000_000) random.seed(self.random_seed) - random.shuffle(self.selected) + random.shuffle(selected) + + return (tuple(selected), tests) @staticmethod - def list_tests(tests: TestList): + def list_tests(tests: TestTuple): for name in tests: print(name) @@ -224,12 +224,12 @@ def _list_cases(self, suite): if support.match_test(test): print(test.id()) - def list_cases(self): + def list_cases(self, tests: TestTuple): support.verbose = False support.set_match_tests(self.match_tests, self.ignore_tests) skipped = [] - for test_name in self.selected: + for test_name in tests: module_name = abs_module_name(test_name, self.test_dir) try: suite = unittest.defaultTestLoader.loadTestsFromName(module_name) @@ -247,6 +247,10 @@ def list_cases(self): def _rerun_failed_tests(self, runtests: RunTests): # Configure the runner to re-run tests if self.num_workers == 0: + # Always run tests in fresh processes to have more deterministic + # initial state. Don't re-run tests in parallel but limit to a + # single worker process to have side effects (on the system load + # and timings) between tests. self.num_workers = 1 tests, match_tests_dict = self.results.prepare_rerun() @@ -294,7 +298,8 @@ def display_result(self, runtests): print() print(f"== Tests result: {state} ==") - self.results.display_result(self.selected, self.quiet, self.print_slowest) + self.results.display_result(runtests.tests, + self.quiet, self.print_slowest) def run_test(self, test_name: TestName, runtests: RunTests, tracer): if tracer is not None: @@ -404,7 +409,7 @@ def get_state(self): return state def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: - from test.libregrtest.runtest_mp import RunWorkers + from test.libregrtest.run_workers import RunWorkers RunWorkers(num_workers, runtests, self.logger, self.results).run() def finalize_tests(self, tracer): @@ -454,39 +459,9 @@ def cleanup_temp_dir(tmp_dir: StrPath): print("Remove file: %s" % name) os_helper.unlink(name) - def main(self, tests: TestList | None = None): - if self.junit_filename and not os.path.isabs(self.junit_filename): - self.junit_filename = os.path.abspath(self.junit_filename) - - self.tests = tests - - strip_py_suffix(self.cmdline_args) - - self.tmp_dir = get_temp_dir(self.tmp_dir) - - if self.want_cleanup: - self.cleanup_temp_dir(self.tmp_dir) - sys.exit(0) - - os.makedirs(self.tmp_dir, exist_ok=True) - work_dir = get_work_dir(parent_dir=self.tmp_dir) - - with exit_timeout(): - # Run the tests in a context manager that temporarily changes the - # CWD to a temporary and writable directory. If it's not possible - # to create or change the CWD, the original CWD will be used. - # The original CWD is available from os_helper.SAVEDCWD. - with os_helper.temp_cwd(work_dir, quiet=True): - # When using multiprocessing, worker processes will use - # work_dir as their parent temporary directory. So when the - # main process exit, it removes also subdirectories of worker - # processes. - - self._main() - - def create_run_tests(self): + def create_run_tests(self, tests: TestTuple): return RunTests( - tuple(self.selected), + tests, fail_fast=self.fail_fast, match_tests=self.match_tests, ignore_tests=self.ignore_tests, @@ -506,7 +481,7 @@ def create_run_tests(self): python_cmd=self.python_cmd, ) - def run_tests(self) -> int: + def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if self.hunt_refleak and self.hunt_refleak.warmups < 3: msg = ("WARNING: Running tests with --huntrleaks/-R and " "less than 3 warmup repetitions can give false positives!") @@ -520,17 +495,17 @@ def run_tests(self) -> int: # For a partial run, we do not need to clutter the output. if (self.want_header or not(self.pgo or self.quiet or self.single_test_run - or self.tests or self.cmdline_args)): + or tests or self.cmdline_args)): self.display_header() if self.randomize: print("Using random seed", self.random_seed) - runtests = self.create_run_tests() + runtests = self.create_run_tests(selected) self.first_runtests = runtests self.logger.set_tests(runtests) - setup_tests(runtests) + setup_process() self.logger.start_load_tracker() try: @@ -553,20 +528,48 @@ def run_tests(self) -> int: return self.results.get_exitcode(self.fail_env_changed, self.fail_rerun) - def _main(self): + def run_tests(self, selected: TestTuple, tests: TestList | None) -> int: + os.makedirs(self.tmp_dir, exist_ok=True) + work_dir = get_work_dir(parent_dir=self.tmp_dir) + + # Put a timeout on Python exit + with exit_timeout(): + # Run the tests in a context manager that temporarily changes the + # CWD to a temporary and writable directory. If it's not possible + # to create or change the CWD, the original CWD will be used. + # The original CWD is available from os_helper.SAVEDCWD. + with os_helper.temp_cwd(work_dir, quiet=True): + # When using multiprocessing, worker processes will use + # work_dir as their parent temporary directory. So when the + # main process exit, it removes also subdirectories of worker + # processes. + return self._run_tests(selected, tests) + + def main(self, tests: TestList | None = None): + if self.junit_filename and not os.path.isabs(self.junit_filename): + self.junit_filename = os.path.abspath(self.junit_filename) + + strip_py_suffix(self.cmdline_args) + + self.tmp_dir = get_temp_dir(self.tmp_dir) + + if self.want_cleanup: + self.cleanup_temp_dir(self.tmp_dir) + sys.exit(0) + if self.want_wait: input("Press any key to continue...") setup_test_dir(self.test_dir) - self.find_tests() + selected, tests = self.find_tests(tests) exitcode = 0 if self.want_list_tests: - self.list_tests(self.selected) + self.list_tests(selected) elif self.want_list_cases: - self.list_cases() + self.list_cases(selected) else: - exitcode = self.run_tests() + exitcode = self.run_tests(selected, tests) sys.exit(exitcode) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 4a68872369c9e7..b73494d07583fc 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -5,7 +5,7 @@ from test.support import TestStats from test.libregrtest.utils import ( - TestName, FilterTuple, + StrJSON, TestName, FilterTuple, format_duration, normalize_test_name, print_warning) @@ -160,7 +160,7 @@ def write_json(self, file) -> None: json.dump(self, file, cls=_EncodeTestResult) @staticmethod - def from_json(worker_json) -> 'TestResult': + def from_json(worker_json: StrJSON) -> 'TestResult': return json.loads(worker_json, object_hook=_decode_test_result) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index b7a044eae25aae..6a07c2fcf3092c 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -106,7 +106,7 @@ def accumulate_result(self, result: TestResult, runtests: RunTests): xml_data = result.xml_data if xml_data: - self.add_junit(result.xml_data) + self.add_junit(xml_data) def need_rerun(self): return bool(self.bad_results) @@ -163,7 +163,7 @@ def write_junit(self, filename: StrPath): for s in ET.tostringlist(root): f.write(s) - def display_result(self, tests: TestList, quiet: bool, print_slowest: bool): + def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool): if self.interrupted: print("Test suite interrupted by signal SIGINT.") diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/run_workers.py similarity index 98% rename from Lib/test/libregrtest/runtest_mp.py rename to Lib/test/libregrtest/run_workers.py index 96b2ac521b9d82..6267fe5a924d9f 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/run_workers.py @@ -15,7 +15,6 @@ from test.support import os_helper from test.libregrtest.logger import Logger -from test.libregrtest.main import Regrtest from test.libregrtest.result import TestResult, State from test.libregrtest.results import TestResults from test.libregrtest.runtests import RunTests @@ -154,10 +153,10 @@ def mp_result_error( ) -> MultiprocessResult: return MultiprocessResult(test_result, stdout, err_msg) - def _run_process(self, worker_job, output_file: TextIO, + def _run_process(self, runtests: RunTests, output_file: TextIO, tmp_dir: StrPath | None = None) -> int: try: - popen = create_worker_process(worker_job, output_file, tmp_dir) + popen = create_worker_process(runtests, output_file, tmp_dir) self._killed = False self._popen = popen diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 366c6f1e7a1046..e16e79e990c8f1 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -27,14 +27,14 @@ class RunTests: pgo_extended: bool = False output_on_failure: bool = False timeout: float | None = None - verbose: bool = False + verbose: int = 0 quiet: bool = False hunt_refleak: HuntRefleak | None = None test_dir: StrPath | None = None use_junit: bool = False memory_limit: str | None = None gc_threshold: int | None = None - use_resources: list[str] = None + use_resources: list[str] = dataclasses.field(default_factory=list) python_cmd: list[str] | None = None def copy(self, **override): diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 20ef3dc38cbd04..c3d81273163860 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -1,4 +1,3 @@ -import atexit import faulthandler import os import signal @@ -13,7 +12,8 @@ from test.libregrtest.runtests import RunTests from test.libregrtest.utils import ( - setup_unraisable_hook, setup_threading_excepthook, fix_umask) + setup_unraisable_hook, setup_threading_excepthook, fix_umask, + replace_stdout, adjust_rlimit_nofile) UNICODE_GUARD_ENV = "PYTHONREGRTEST_UNICODE_GUARD" @@ -26,19 +26,7 @@ def setup_test_dir(testdir: str | None) -> None: sys.path.insert(0, os.path.abspath(testdir)) -def setup_support(runtests: RunTests): - support.PGO = runtests.pgo - support.PGO_EXTENDED = runtests.pgo_extended - support.set_match_tests(runtests.match_tests, runtests.ignore_tests) - support.failfast = runtests.fail_fast - support.verbose = runtests.verbose - if runtests.use_junit: - support.junit_xml_list = [] - else: - support.junit_xml_list = None - - -def setup_tests(runtests): +def setup_process(): fix_umask() try: @@ -62,7 +50,7 @@ def setup_tests(runtests): for signum in signals: faulthandler.register(signum, chain=True, file=stderr_fd) - _adjust_resource_limits() + adjust_rlimit_nofile() replace_stdout() support.record_original_stdout(sys.stdout) @@ -83,19 +71,6 @@ def setup_tests(runtests): if getattr(module, '__file__', None): module.__file__ = os.path.abspath(module.__file__) - if runtests.hunt_refleak: - unittest.BaseTestSuite._cleanup = False - - if runtests.memory_limit is not None: - support.set_memlimit(runtests.memory_limit) - - if runtests.gc_threshold is not None: - gc.set_threshold(runtests.gc_threshold) - - support.suppress_msvcrt_asserts(runtests.verbose and runtests.verbose >= 2) - - support.use_resources = runtests.use_resources - if hasattr(sys, 'addaudithook'): # Add an auditing hook for all tests to ensure PySys_Audit is tested def _test_audit_hook(name, args): @@ -105,6 +80,36 @@ def _test_audit_hook(name, args): setup_unraisable_hook() setup_threading_excepthook() + # Ensure there's a non-ASCII character in env vars at all times to force + # tests consider this case. See BPO-44647 for details. + if TESTFN_UNDECODABLE and os.supports_bytes_environ: + os.environb.setdefault(UNICODE_GUARD_ENV.encode(), TESTFN_UNDECODABLE) + elif FS_NONASCII: + os.environ.setdefault(UNICODE_GUARD_ENV, FS_NONASCII) + + +def setup_tests(runtests: RunTests): + support.verbose = runtests.verbose + support.failfast = runtests.fail_fast + support.PGO = runtests.pgo + support.PGO_EXTENDED = runtests.pgo_extended + + support.set_match_tests(runtests.match_tests, runtests.ignore_tests) + + if runtests.use_junit: + support.junit_xml_list = [] + from test.support.testresult import RegressionTestResult + RegressionTestResult.USE_XML = True + else: + support.junit_xml_list = None + + if runtests.memory_limit is not None: + support.set_memlimit(runtests.memory_limit) + + support.suppress_msvcrt_asserts(runtests.verbose >= 2) + + support.use_resources = runtests.use_resources + timeout = runtests.timeout if timeout is not None: # For a slow buildbot worker, increase SHORT_TIMEOUT and LONG_TIMEOUT @@ -117,61 +122,8 @@ def _test_audit_hook(name, args): support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, timeout) support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, timeout) - if runtests.use_junit: - from test.support.testresult import RegressionTestResult - RegressionTestResult.USE_XML = True - - # Ensure there's a non-ASCII character in env vars at all times to force - # tests consider this case. See BPO-44647 for details. - if TESTFN_UNDECODABLE and os.supports_bytes_environ: - os.environb.setdefault(UNICODE_GUARD_ENV.encode(), TESTFN_UNDECODABLE) - elif FS_NONASCII: - os.environ.setdefault(UNICODE_GUARD_ENV, FS_NONASCII) - - -def replace_stdout(): - """Set stdout encoder error handler to backslashreplace (as stderr error - handler) to avoid UnicodeEncodeError when printing a traceback""" - stdout = sys.stdout - try: - fd = stdout.fileno() - except ValueError: - # On IDLE, sys.stdout has no file descriptor and is not a TextIOWrapper - # object. Leaving sys.stdout unchanged. - # - # Catch ValueError to catch io.UnsupportedOperation on TextIOBase - # and ValueError on a closed stream. - return - - sys.stdout = open(fd, 'w', - encoding=stdout.encoding, - errors="backslashreplace", - closefd=False, - newline='\n') - - def restore_stdout(): - sys.stdout.close() - sys.stdout = stdout - atexit.register(restore_stdout) - + if runtests.hunt_refleak: + unittest.BaseTestSuite._cleanup = False -def _adjust_resource_limits(): - """Adjust the system resource limits (ulimit) if needed.""" - try: - import resource - from resource import RLIMIT_NOFILE - except ImportError: - return - fd_limit, max_fds = resource.getrlimit(RLIMIT_NOFILE) - # On macOS the default fd limit is sometimes too low (256) for our - # test suite to succeed. Raise it to something more reasonable. - # 1024 is a common Linux default. - desired_fds = 1024 - if fd_limit < desired_fds and fd_limit < max_fds: - new_fd_limit = min(desired_fds, max_fds) - try: - resource.setrlimit(RLIMIT_NOFILE, (new_fd_limit, max_fds)) - print(f"Raised RLIMIT_NOFILE: {fd_limit} -> {new_fd_limit}") - except (ValueError, OSError) as err: - print(f"Unable to raise RLIMIT_NOFILE from {fd_limit} to " - f"{new_fd_limit}: {err}.") + if runtests.gc_threshold is not None: + gc.set_threshold(runtests.gc_threshold) diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index bb33387fee0d35..0cb31925787893 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -15,7 +15,7 @@ from test.libregrtest.result import State, TestResult from test.libregrtest.runtests import RunTests from test.libregrtest.save_env import saved_test_environment -from test.libregrtest.setup import setup_support +from test.libregrtest.setup import setup_tests from test.libregrtest.utils import ( TestName, clear_caches, remove_testfn, abs_module_name, print_warning) @@ -201,7 +201,7 @@ def _runtest(result: TestResult, runtests: RunTests) -> None: faulthandler.dump_traceback_later(timeout, exit=True) try: - setup_support(runtests) + setup_tests(runtests) if output_on_failure: support.verbose = True diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 011d287e1674cd..f97e3fd4bb7106 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,3 +1,4 @@ +import atexit import contextlib import faulthandler import math @@ -471,3 +472,55 @@ def normalize_test_name(test_full_name, *, is_error=False): rpar = test_full_name.index(')') return test_full_name[lpar + 1: rpar].split('.')[-1] return short_name + + +def replace_stdout(): + """Set stdout encoder error handler to backslashreplace (as stderr error + handler) to avoid UnicodeEncodeError when printing a traceback""" + stdout = sys.stdout + try: + fd = stdout.fileno() + except ValueError: + # On IDLE, sys.stdout has no file descriptor and is not a TextIOWrapper + # object. Leaving sys.stdout unchanged. + # + # Catch ValueError to catch io.UnsupportedOperation on TextIOBase + # and ValueError on a closed stream. + return + + sys.stdout = open(fd, 'w', + encoding=stdout.encoding, + errors="backslashreplace", + closefd=False, + newline='\n') + + def restore_stdout(): + sys.stdout.close() + sys.stdout = stdout + atexit.register(restore_stdout) + + +def adjust_rlimit_nofile(): + """ + On macOS the default fd limit (RLIMIT_NOFILE) is sometimes too low (256) + for our test suite to succeed. Raise it to something more reasonable. 1024 + is a common Linux default. + """ + try: + import resource + except ImportError: + return + + fd_limit, max_fds = resource.getrlimit(resource.RLIMIT_NOFILE) + + desired_fds = 1024 + + if fd_limit < desired_fds and fd_limit < max_fds: + new_fd_limit = min(desired_fds, max_fds) + try: + resource.setrlimit(resource.RLIMIT_NOFILE, + (new_fd_limit, max_fds)) + print(f"Raised RLIMIT_NOFILE: {fd_limit} -> {new_fd_limit}") + except (ValueError, OSError) as err: + print_warning(f"Unable to raise RLIMIT_NOFILE from {fd_limit} to " + f"{new_fd_limit}: {err}.") diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 24251c35cdd2f1..b9fb031764349a 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -6,7 +6,7 @@ from test import support from test.support import os_helper -from test.libregrtest.setup import setup_tests, setup_test_dir +from test.libregrtest.setup import setup_process, setup_test_dir from test.libregrtest.runtests import RunTests from test.libregrtest.single import run_single_test from test.libregrtest.utils import ( @@ -60,7 +60,7 @@ def worker_process(worker_json: StrJSON) -> NoReturn: match_tests: FilterTuple | None = runtests.match_tests setup_test_dir(runtests.test_dir) - setup_tests(runtests) + setup_process() if runtests.rerun: if match_tests: From 7aa8fcc8eb700bd4ae4bc19e086ce7ff5916f7bd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Sep 2023 09:02:35 +0200 Subject: [PATCH 55/66] gh-109162: libregrtest: use relative imports (#109250) libregrtest.__init__ no longer exposes any symbol, so "python -m test.libregrtest.worker" imports less modules. --- Lib/test/__main__.py | 2 +- Lib/test/autotest.py | 2 +- Lib/test/libregrtest/__init__.py | 2 - Lib/test/libregrtest/findtests.py | 2 +- Lib/test/libregrtest/logger.py | 8 +- Lib/test/libregrtest/main.py | 22 ++--- Lib/test/libregrtest/refleak.py | 4 +- Lib/test/libregrtest/result.py | 2 +- Lib/test/libregrtest/results.py | 6 +- Lib/test/libregrtest/run_workers.py | 14 ++-- Lib/test/libregrtest/runtests.py | 2 +- Lib/test/libregrtest/save_env.py | 4 +- Lib/test/libregrtest/setup.py | 4 +- Lib/test/libregrtest/single.py | 12 +-- Lib/test/libregrtest/worker.py | 8 +- Lib/test/regrtest.py | 2 +- Lib/test/test_regrtest.py | 119 +++++++++++++++------------- 17 files changed, 110 insertions(+), 105 deletions(-) diff --git a/Lib/test/__main__.py b/Lib/test/__main__.py index 19a6b2b8904526..e5780b784b4b05 100644 --- a/Lib/test/__main__.py +++ b/Lib/test/__main__.py @@ -1,2 +1,2 @@ -from test.libregrtest import main +from test.libregrtest.main import main main() diff --git a/Lib/test/autotest.py b/Lib/test/autotest.py index fa85cc153a133a..b5a1fab404c72d 100644 --- a/Lib/test/autotest.py +++ b/Lib/test/autotest.py @@ -1,5 +1,5 @@ # This should be equivalent to running regrtest.py from the cmdline. # It can be especially handy if you're in an interactive shell, e.g., # from test import autotest. -from test.libregrtest import main +from test.libregrtest.main import main main() diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index 5e8dba5dbde71a..e69de29bb2d1d6 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1,2 +0,0 @@ -from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES, ALL_RESOURCES -from test.libregrtest.main import main diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index 88570be7dccdfd..0d38b8e60eb722 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -1,6 +1,6 @@ import os -from test.libregrtest.utils import StrPath, TestName, TestList +from .utils import StrPath, TestName, TestList # If these test directories are encountered recurse into them and treat each diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py index 6195b5ddd4ebc0..f74bdff6322f13 100644 --- a/Lib/test/libregrtest/logger.py +++ b/Lib/test/libregrtest/logger.py @@ -1,12 +1,12 @@ import os import time -from test.libregrtest.results import TestResults -from test.libregrtest.runtests import RunTests -from test.libregrtest.utils import print_warning, MS_WINDOWS +from .results import TestResults +from .runtests import RunTests +from .utils import print_warning, MS_WINDOWS if MS_WINDOWS: - from test.libregrtest.win_utils import WindowsLoadTracker + from .win_utils import WindowsLoadTracker class Logger: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 31eab99ca76351..0f289c06325bd0 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -10,16 +10,16 @@ from test import support from test.support import os_helper -from test.libregrtest.cmdline import _parse_args, Namespace -from test.libregrtest.findtests import findtests, split_test_packages -from test.libregrtest.logger import Logger -from test.libregrtest.result import State -from test.libregrtest.runtests import RunTests, HuntRefleak -from test.libregrtest.setup import setup_process, setup_test_dir -from test.libregrtest.single import run_single_test, PROGRESS_MIN_TIME -from test.libregrtest.pgo import setup_pgo_tests -from test.libregrtest.results import TestResults -from test.libregrtest.utils import ( +from .cmdline import _parse_args, Namespace +from .findtests import findtests, split_test_packages +from .logger import Logger +from .result import State +from .runtests import RunTests, HuntRefleak +from .setup import setup_process, setup_test_dir +from .single import run_single_test, PROGRESS_MIN_TIME +from .pgo import setup_pgo_tests +from .results import TestResults +from .utils import ( StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple, strip_py_suffix, count, format_duration, printlist, get_build_info, get_temp_dir, get_work_dir, exit_timeout, @@ -409,7 +409,7 @@ def get_state(self): return state def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: - from test.libregrtest.run_workers import RunWorkers + from .run_workers import RunWorkers RunWorkers(num_workers, runtests, self.logger, self.results).run() def finalize_tests(self, tracer): diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 6b1d7082ba46ba..edf8569a8f95fd 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -5,8 +5,8 @@ from test import support from test.support import os_helper -from test.libregrtest.runtests import HuntRefleak -from test.libregrtest.utils import clear_caches +from .runtests import HuntRefleak +from .utils import clear_caches try: from _abc import _get_dump diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index b73494d07583fc..cfd08ecc41b7d4 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -4,7 +4,7 @@ from test.support import TestStats -from test.libregrtest.utils import ( +from .utils import ( StrJSON, TestName, FilterTuple, format_duration, normalize_test_name, print_warning) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 6a07c2fcf3092c..94654fd6ab23f9 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,9 +1,9 @@ import sys from test.support import TestStats -from test.libregrtest.runtests import RunTests -from test.libregrtest.result import State, TestResult -from test.libregrtest.utils import ( +from .runtests import RunTests +from .result import State, TestResult +from .utils import ( StrPath, TestName, TestTuple, TestList, FilterDict, printlist, count, format_duration) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 6267fe5a924d9f..768625cb507f9a 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -14,15 +14,15 @@ from test import support from test.support import os_helper -from test.libregrtest.logger import Logger -from test.libregrtest.result import TestResult, State -from test.libregrtest.results import TestResults -from test.libregrtest.runtests import RunTests -from test.libregrtest.single import PROGRESS_MIN_TIME -from test.libregrtest.utils import ( +from .logger import Logger +from .result import TestResult, State +from .results import TestResults +from .runtests import RunTests +from .single import PROGRESS_MIN_TIME +from .utils import ( StrPath, TestName, format_duration, print_warning) -from test.libregrtest.worker import create_worker_process, USE_PROCESS_GROUP +from .worker import create_worker_process, USE_PROCESS_GROUP if sys.platform == 'win32': import locale diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index e16e79e990c8f1..e843cc2dadf734 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -2,7 +2,7 @@ import json from typing import Any -from test.libregrtest.utils import ( +from .utils import ( StrPath, StrJSON, TestTuple, FilterTuple, FilterDict) diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 164fe9806b5f0d..55c1f7801489b0 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -3,9 +3,11 @@ import os import sys import threading + from test import support from test.support import os_helper -from test.libregrtest.utils import print_warning + +from .utils import print_warning class SkipTestEnvironment(Exception): diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index c3d81273163860..353a0f70b94ab2 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -10,8 +10,8 @@ except ImportError: gc = None -from test.libregrtest.runtests import RunTests -from test.libregrtest.utils import ( +from .runtests import RunTests +from .utils import ( setup_unraisable_hook, setup_threading_excepthook, fix_umask, replace_stdout, adjust_rlimit_nofile) diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 0cb31925787893..635b4f93702b04 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -12,11 +12,11 @@ from test.support import TestStats from test.support import threading_helper -from test.libregrtest.result import State, TestResult -from test.libregrtest.runtests import RunTests -from test.libregrtest.save_env import saved_test_environment -from test.libregrtest.setup import setup_tests -from test.libregrtest.utils import ( +from .result import State, TestResult +from .runtests import RunTests +from .save_env import saved_test_environment +from .setup import setup_tests +from .utils import ( TestName, clear_caches, remove_testfn, abs_module_name, print_warning) @@ -40,7 +40,7 @@ def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: # Run test_func(), collect statistics, and detect reference and memory # leaks. if runtests.hunt_refleak: - from test.libregrtest.refleak import runtest_refleak + from .refleak import runtest_refleak refleak, test_result = runtest_refleak(result.test_name, test_func, runtests.hunt_refleak, runtests.quiet) diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index b9fb031764349a..ed1286e5700679 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -6,10 +6,10 @@ from test import support from test.support import os_helper -from test.libregrtest.setup import setup_process, setup_test_dir -from test.libregrtest.runtests import RunTests -from test.libregrtest.single import run_single_test -from test.libregrtest.utils import ( +from .setup import setup_process, setup_test_dir +from .runtests import RunTests +from .single import run_single_test +from .utils import ( StrPath, StrJSON, FilterTuple, get_work_dir, exit_timeout) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 0ffb3ed454eda0..46a74fe276f553 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -8,7 +8,7 @@ import os import sys -from test.libregrtest import main +from test.libregrtest.main import main # Alias for backward compatibility (just in case) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 21babb5185548f..f587a8f919dcde 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -18,10 +18,11 @@ import tempfile import textwrap import unittest -from test import libregrtest from test import support from test.support import os_helper, TestStats -from test.libregrtest import utils, setup +from test.libregrtest import cmdline +from test.libregrtest import utils +from test.libregrtest import setup from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -52,9 +53,13 @@ class ParseArgsTestCase(unittest.TestCase): Test regrtest's argument parsing, function _parse_args(). """ + @staticmethod + def parse_args(args): + return cmdline._parse_args(args) + def checkError(self, args, msg): with support.captured_stderr() as err, self.assertRaises(SystemExit): - libregrtest._parse_args(args) + self.parse_args(args) self.assertIn(msg, err.getvalue()) def test_help(self): @@ -62,78 +67,78 @@ def test_help(self): with self.subTest(opt=opt): with support.captured_stdout() as out, \ self.assertRaises(SystemExit): - libregrtest._parse_args([opt]) + self.parse_args([opt]) self.assertIn('Run Python regression tests.', out.getvalue()) def test_timeout(self): - ns = libregrtest._parse_args(['--timeout', '4.2']) + ns = self.parse_args(['--timeout', '4.2']) self.assertEqual(ns.timeout, 4.2) self.checkError(['--timeout'], 'expected one argument') self.checkError(['--timeout', 'foo'], 'invalid float value') def test_wait(self): - ns = libregrtest._parse_args(['--wait']) + ns = self.parse_args(['--wait']) self.assertTrue(ns.wait) def test_start(self): for opt in '-S', '--start': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'foo']) + ns = self.parse_args([opt, 'foo']) self.assertEqual(ns.start, 'foo') self.checkError([opt], 'expected one argument') def test_verbose(self): - ns = libregrtest._parse_args(['-v']) + ns = self.parse_args(['-v']) self.assertEqual(ns.verbose, 1) - ns = libregrtest._parse_args(['-vvv']) + ns = self.parse_args(['-vvv']) self.assertEqual(ns.verbose, 3) - ns = libregrtest._parse_args(['--verbose']) + ns = self.parse_args(['--verbose']) self.assertEqual(ns.verbose, 1) - ns = libregrtest._parse_args(['--verbose'] * 3) + ns = self.parse_args(['--verbose'] * 3) self.assertEqual(ns.verbose, 3) - ns = libregrtest._parse_args([]) + ns = self.parse_args([]) self.assertEqual(ns.verbose, 0) def test_rerun(self): for opt in '-w', '--rerun', '--verbose2': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.rerun) def test_verbose3(self): for opt in '-W', '--verbose3': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.verbose3) def test_quiet(self): for opt in '-q', '--quiet': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_slowest(self): for opt in '-o', '--slowest': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.print_slow) def test_header(self): - ns = libregrtest._parse_args(['--header']) + ns = self.parse_args(['--header']) self.assertTrue(ns.header) - ns = libregrtest._parse_args(['--verbose']) + ns = self.parse_args(['--verbose']) self.assertTrue(ns.header) def test_randomize(self): for opt in '-r', '--randomize': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.randomize) def test_randseed(self): - ns = libregrtest._parse_args(['--randseed', '12345']) + ns = self.parse_args(['--randseed', '12345']) self.assertEqual(ns.random_seed, 12345) self.assertTrue(ns.randomize) self.checkError(['--randseed'], 'expected one argument') @@ -142,7 +147,7 @@ def test_randseed(self): def test_fromfile(self): for opt in '-f', '--fromfile': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'foo']) + ns = self.parse_args([opt, 'foo']) self.assertEqual(ns.fromfile, 'foo') self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo', '-s'], "don't go together") @@ -150,20 +155,20 @@ def test_fromfile(self): def test_exclude(self): for opt in '-x', '--exclude': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.exclude) def test_single(self): for opt in '-s', '--single': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") def test_ignore(self): for opt in '-i', '--ignore': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'pattern']) + ns = self.parse_args([opt, 'pattern']) self.assertEqual(ns.ignore_tests, ['pattern']) self.checkError([opt], 'expected one argument') @@ -173,7 +178,7 @@ def test_ignore(self): print('matchfile2', file=fp) filename = os.path.abspath(os_helper.TESTFN) - ns = libregrtest._parse_args(['-m', 'match', + ns = self.parse_args(['-m', 'match', '--ignorefile', filename]) self.assertEqual(ns.ignore_tests, ['matchfile1', 'matchfile2']) @@ -181,11 +186,11 @@ def test_ignore(self): def test_match(self): for opt in '-m', '--match': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'pattern']) + ns = self.parse_args([opt, 'pattern']) self.assertEqual(ns.match_tests, ['pattern']) self.checkError([opt], 'expected one argument') - ns = libregrtest._parse_args(['-m', 'pattern1', + ns = self.parse_args(['-m', 'pattern1', '-m', 'pattern2']) self.assertEqual(ns.match_tests, ['pattern1', 'pattern2']) @@ -195,7 +200,7 @@ def test_match(self): print('matchfile2', file=fp) filename = os.path.abspath(os_helper.TESTFN) - ns = libregrtest._parse_args(['-m', 'match', + ns = self.parse_args(['-m', 'match', '--matchfile', filename]) self.assertEqual(ns.match_tests, ['match', 'matchfile1', 'matchfile2']) @@ -203,65 +208,65 @@ def test_match(self): def test_failfast(self): for opt in '-G', '--failfast': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, '-v']) + ns = self.parse_args([opt, '-v']) self.assertTrue(ns.failfast) - ns = libregrtest._parse_args([opt, '-W']) + ns = self.parse_args([opt, '-W']) self.assertTrue(ns.failfast) self.checkError([opt], '-G/--failfast needs either -v or -W') def test_use(self): for opt in '-u', '--use': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'gui,network']) + ns = self.parse_args([opt, 'gui,network']) self.assertEqual(ns.use_resources, ['gui', 'network']) - ns = libregrtest._parse_args([opt, 'gui,none,network']) + ns = self.parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(libregrtest.ALL_RESOURCES) + expected = list(cmdline.ALL_RESOURCES) expected.remove('gui') - ns = libregrtest._parse_args([opt, 'all,-gui']) + ns = self.parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid resource') # all + a resource not part of "all" - ns = libregrtest._parse_args([opt, 'all,tzdata']) + ns = self.parse_args([opt, 'all,tzdata']) self.assertEqual(ns.use_resources, - list(libregrtest.ALL_RESOURCES) + ['tzdata']) + list(cmdline.ALL_RESOURCES) + ['tzdata']) # test another resource which is not part of "all" - ns = libregrtest._parse_args([opt, 'extralargefile']) + ns = self.parse_args([opt, 'extralargefile']) self.assertEqual(ns.use_resources, ['extralargefile']) def test_memlimit(self): for opt in '-M', '--memlimit': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, '4G']) + ns = self.parse_args([opt, '4G']) self.assertEqual(ns.memlimit, '4G') self.checkError([opt], 'expected one argument') def test_testdir(self): - ns = libregrtest._parse_args(['--testdir', 'foo']) + ns = self.parse_args(['--testdir', 'foo']) self.assertEqual(ns.testdir, os.path.join(os_helper.SAVEDCWD, 'foo')) self.checkError(['--testdir'], 'expected one argument') def test_runleaks(self): for opt in '-L', '--runleaks': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.runleaks) def test_huntrleaks(self): for opt in '-R', '--huntrleaks': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, ':']) + ns = self.parse_args([opt, ':']) self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt')) - ns = libregrtest._parse_args([opt, '6:']) + ns = self.parse_args([opt, '6:']) self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt')) - ns = libregrtest._parse_args([opt, ':3']) + ns = self.parse_args([opt, ':3']) self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt')) - ns = libregrtest._parse_args([opt, '6:3:leaks.log']) + ns = self.parse_args([opt, '6:3:leaks.log']) self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log')) self.checkError([opt], 'expected one argument') self.checkError([opt, '6'], @@ -272,7 +277,7 @@ def test_huntrleaks(self): def test_multiprocess(self): for opt in '-j', '--multiprocess': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, '2']) + ns = self.parse_args([opt, '2']) self.assertEqual(ns.use_mp, 2) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') @@ -282,13 +287,13 @@ def test_multiprocess(self): def test_coverage(self): for opt in '-T', '--coverage': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.trace) def test_coverdir(self): for opt in '-D', '--coverdir': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'foo']) + ns = self.parse_args([opt, 'foo']) self.assertEqual(ns.coverdir, os.path.join(os_helper.SAVEDCWD, 'foo')) self.checkError([opt], 'expected one argument') @@ -296,13 +301,13 @@ def test_coverdir(self): def test_nocoverdir(self): for opt in '-N', '--nocoverdir': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertIsNone(ns.coverdir) def test_threshold(self): for opt in '-t', '--threshold': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, '1000']) + ns = self.parse_args([opt, '1000']) self.assertEqual(ns.threshold, 1000) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') @@ -311,7 +316,7 @@ def test_nowindows(self): for opt in '-n', '--nowindows': with self.subTest(opt=opt): with contextlib.redirect_stderr(io.StringIO()) as stderr: - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.nowindows) err = stderr.getvalue() self.assertIn('the --nowindows (-n) option is deprecated', err) @@ -319,39 +324,39 @@ def test_nowindows(self): def test_forever(self): for opt in '-F', '--forever': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.forever) def test_unrecognized_argument(self): self.checkError(['--xxx'], 'usage:') def test_long_option__partial(self): - ns = libregrtest._parse_args(['--qui']) + ns = self.parse_args(['--qui']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_two_options(self): - ns = libregrtest._parse_args(['--quiet', '--exclude']) + ns = self.parse_args(['--quiet', '--exclude']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertTrue(ns.exclude) def test_option_with_empty_string_value(self): - ns = libregrtest._parse_args(['--start', '']) + ns = self.parse_args(['--start', '']) self.assertEqual(ns.start, '') def test_arg(self): - ns = libregrtest._parse_args(['foo']) + ns = self.parse_args(['foo']) self.assertEqual(ns.args, ['foo']) def test_option_and_arg(self): - ns = libregrtest._parse_args(['--quiet', 'foo']) + ns = self.parse_args(['--quiet', 'foo']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertEqual(ns.args, ['foo']) def test_arg_option_arg(self): - ns = libregrtest._parse_args(['test_unaryop', '-v', 'test_binop']) + ns = self.parse_args(['test_unaryop', '-v', 'test_binop']) self.assertEqual(ns.verbose, 1) self.assertEqual(ns.args, ['test_unaryop', 'test_binop']) From 3b2ecbc1275bd05534885cee9ac1389987238561 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Sep 2023 09:34:04 +0200 Subject: [PATCH 56/66] GH-108614: Increase importlib MAGIC for RESUME_CHECK instruction (#109247) --- Lib/importlib/_bootstrap_external.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 0717e202ecbcde..7a83a58d0874df 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -455,6 +455,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) # Python 3.13a1 3558 (Reorder the stack items for CALL) # Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c) +# Python 3.13a1 3560 (Add RESUME_CHECK instruction) # Python 3.14 will start with 3600 @@ -471,7 +472,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3559).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3560).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c From a9b1f84790e977fb09f75b148c4c4f5924a6ef99 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Sep 2023 10:11:31 +0200 Subject: [PATCH 57/66] gh-107219: Fix concurrent.futures terminate_broken() (#109244) Fix a race condition in concurrent.futures. When a process in the process pool was terminated abruptly (while the future was running or pending), close the connection write end. If the call queue is blocked on sending bytes to a worker process, closing the connection write end interrupts the send, so the queue can be closed. Changes: * _ExecutorManagerThread.terminate_broken() now closes call_queue._writer. * multiprocessing PipeConnection.close() now interrupts WaitForMultipleObjects() in _send_bytes() by cancelling the overlapped operation. --- Lib/concurrent/futures/process.py | 4 ++++ Lib/multiprocessing/connection.py | 18 ++++++++++++++++++ ...3-09-11-00-32-18.gh-issue-107219.3zqyFT.rst | 5 +++++ 3 files changed, 27 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-09-11-00-32-18.gh-issue-107219.3zqyFT.rst diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 9933d3d0e040ea..f4b5cd1d869067 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -510,6 +510,10 @@ def terminate_broken(self, cause): # https://github.com/python/cpython/issues/94777 self.call_queue._reader.close() + # gh-107219: Close the connection writer which can unblock + # Queue._feed() if it was stuck in send_bytes(). + self.call_queue._writer.close() + # clean up resources self.join_executor_internals() diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 04eaea811cfbbe..7c425a2d8e7034 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -9,6 +9,7 @@ __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ] +import errno import io import os import sys @@ -41,6 +42,7 @@ BUFSIZE = 8192 # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. +WSA_OPERATION_ABORTED = 995 _mmap_counter = itertools.count() @@ -271,12 +273,22 @@ class PipeConnection(_ConnectionBase): with FILE_FLAG_OVERLAPPED. """ _got_empty_message = False + _send_ov = None def _close(self, _CloseHandle=_winapi.CloseHandle): + ov = self._send_ov + if ov is not None: + # Interrupt WaitForMultipleObjects() in _send_bytes() + ov.cancel() _CloseHandle(self._handle) def _send_bytes(self, buf): + if self._send_ov is not None: + # A connection should only be used by a single thread + raise ValueError("concurrent send_bytes() calls " + "are not supported") ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True) + self._send_ov = ov try: if err == _winapi.ERROR_IO_PENDING: waitres = _winapi.WaitForMultipleObjects( @@ -286,7 +298,13 @@ def _send_bytes(self, buf): ov.cancel() raise finally: + self._send_ov = None nwritten, err = ov.GetOverlappedResult(True) + if err == WSA_OPERATION_ABORTED: + # close() was called by another thread while + # WaitForMultipleObjects() was waiting for the overlapped + # operation. + raise OSError(errno.EPIPE, "handle is closed") assert err == 0 assert nwritten == len(buf) diff --git a/Misc/NEWS.d/next/Library/2023-09-11-00-32-18.gh-issue-107219.3zqyFT.rst b/Misc/NEWS.d/next/Library/2023-09-11-00-32-18.gh-issue-107219.3zqyFT.rst new file mode 100644 index 00000000000000..10afbcf823386a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-11-00-32-18.gh-issue-107219.3zqyFT.rst @@ -0,0 +1,5 @@ +Fix a race condition in ``concurrent.futures``. When a process in the +process pool was terminated abruptly (while the future was running or +pending), close the connection write end. If the call queue is blocked on +sending bytes to a worker process, closing the connection write end interrupts +the send, so the queue can be closed. Patch by Victor Stinner. From c439f6a72d828a55aa373fd42c8a0ef771e303cd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Sep 2023 10:52:03 +0200 Subject: [PATCH 58/66] gh-109162: libregrtest: move code around (#109253) * Move Regrtest.display_header() to utils.py. * Move cleanup_temp_dir() to utils.py. * Move list_cases() to findtests.py. --- Lib/test/libregrtest/findtests.py | 42 ++++++++++++- Lib/test/libregrtest/main.py | 101 +++--------------------------- Lib/test/libregrtest/utils.py | 55 ++++++++++++++++ 3 files changed, 105 insertions(+), 93 deletions(-) diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index 0d38b8e60eb722..f4a8b9ae26ae65 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -1,6 +1,12 @@ import os +import sys +import unittest -from .utils import StrPath, TestName, TestList +from test import support + +from .utils import ( + StrPath, TestName, TestTuple, TestList, FilterTuple, + abs_module_name, count, printlist) # If these test directories are encountered recurse into them and treat each @@ -56,3 +62,37 @@ def split_test_packages(tests, *, testdir: StrPath | None = None, exclude=(), else: splitted.append(name) return splitted + + +def _list_cases(suite): + for test in suite: + if isinstance(test, unittest.loader._FailedTest): + continue + if isinstance(test, unittest.TestSuite): + _list_cases(test) + elif isinstance(test, unittest.TestCase): + if support.match_test(test): + print(test.id()) + +def list_cases(tests: TestTuple, *, + match_tests: FilterTuple | None = None, + ignore_tests: FilterTuple | None = None, + test_dir: StrPath | None = None): + support.verbose = False + support.set_match_tests(match_tests, ignore_tests) + + skipped = [] + for test_name in tests: + module_name = abs_module_name(test_name, test_dir) + try: + suite = unittest.defaultTestLoader.loadTestsFromName(module_name) + _list_cases(suite) + except unittest.SkipTest: + skipped.append(test_name) + + if skipped: + sys.stdout.flush() + stderr = sys.stderr + print(file=stderr) + print(count(len(skipped), "test"), "skipped:", file=stderr) + printlist(skipped, file=stderr) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 0f289c06325bd0..2c0a6c204373cc 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,17 +1,14 @@ -import locale import os -import platform import random import re import sys import time -import unittest from test import support from test.support import os_helper from .cmdline import _parse_args, Namespace -from .findtests import findtests, split_test_packages +from .findtests import findtests, split_test_packages, list_cases from .logger import Logger from .result import State from .runtests import RunTests, HuntRefleak @@ -22,8 +19,8 @@ from .utils import ( StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple, strip_py_suffix, count, format_duration, - printlist, get_build_info, get_temp_dir, get_work_dir, exit_timeout, - abs_module_name) + printlist, get_temp_dir, get_work_dir, exit_timeout, + display_header, cleanup_temp_dir) class Regrtest: @@ -214,36 +211,6 @@ def list_tests(tests: TestTuple): for name in tests: print(name) - def _list_cases(self, suite): - for test in suite: - if isinstance(test, unittest.loader._FailedTest): - continue - if isinstance(test, unittest.TestSuite): - self._list_cases(test) - elif isinstance(test, unittest.TestCase): - if support.match_test(test): - print(test.id()) - - def list_cases(self, tests: TestTuple): - support.verbose = False - support.set_match_tests(self.match_tests, self.ignore_tests) - - skipped = [] - for test_name in tests: - module_name = abs_module_name(test_name, self.test_dir) - try: - suite = unittest.defaultTestLoader.loadTestsFromName(module_name) - self._list_cases(suite) - except unittest.SkipTest: - skipped.append(test_name) - - if skipped: - sys.stdout.flush() - stderr = sys.stderr - print(file=stderr) - print(count(len(skipped), "test"), "skipped:", file=stderr) - printlist(skipped, file=stderr) - def _rerun_failed_tests(self, runtests: RunTests): # Configure the runner to re-run tests if self.num_workers == 0: @@ -363,45 +330,6 @@ def run_tests_sequentially(self, runtests): return tracer - @staticmethod - def display_header(): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("==", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== Python build:", ' '.join(get_build_info())) - print("== cwd:", os.getcwd()) - cpu_count = os.cpu_count() - if cpu_count: - print("== CPU count:", cpu_count) - print("== encodings: locale=%s, FS=%s" - % (locale.getencoding(), sys.getfilesystemencoding())) - - # This makes it easier to remember what to set in your local - # environment when trying to reproduce a sanitizer failure. - asan = support.check_sanitizer(address=True) - msan = support.check_sanitizer(memory=True) - ubsan = support.check_sanitizer(ub=True) - sanitizers = [] - if asan: - sanitizers.append("address") - if msan: - sanitizers.append("memory") - if ubsan: - sanitizers.append("undefined behavior") - if not sanitizers: - return - - print(f"== sanitizers: {', '.join(sanitizers)}") - for sanitizer, env_var in ( - (asan, "ASAN_OPTIONS"), - (msan, "MSAN_OPTIONS"), - (ubsan, "UBSAN_OPTIONS"), - ): - options= os.environ.get(env_var) - if sanitizer and options is not None: - print(f"== {env_var}={options!r}") - def get_state(self): state = self.results.get_state(self.fail_env_changed) if self.first_state: @@ -445,20 +373,6 @@ def display_summary(self): state = self.get_state() print(f"Result: {state}") - @staticmethod - def cleanup_temp_dir(tmp_dir: StrPath): - import glob - - path = os.path.join(glob.escape(tmp_dir), 'test_python_*') - print("Cleanup %s directory" % tmp_dir) - for name in glob.glob(path): - if os.path.isdir(name): - print("Remove directory: %s" % name) - os_helper.rmtree(name) - else: - print("Remove file: %s" % name) - os_helper.unlink(name) - def create_run_tests(self, tests: TestTuple): return RunTests( tests, @@ -496,7 +410,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if (self.want_header or not(self.pgo or self.quiet or self.single_test_run or tests or self.cmdline_args)): - self.display_header() + display_header() if self.randomize: print("Using random seed", self.random_seed) @@ -554,7 +468,7 @@ def main(self, tests: TestList | None = None): self.tmp_dir = get_temp_dir(self.tmp_dir) if self.want_cleanup: - self.cleanup_temp_dir(self.tmp_dir) + cleanup_temp_dir(self.tmp_dir) sys.exit(0) if self.want_wait: @@ -567,7 +481,10 @@ def main(self, tests: TestList | None = None): if self.want_list_tests: self.list_tests(selected) elif self.want_list_cases: - self.list_cases(selected) + list_cases(selected, + match_tests=self.match_tests, + ignore_tests=self.ignore_tests, + test_dir=self.test_dir) else: exitcode = self.run_tests(selected, tests) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index f97e3fd4bb7106..b46cec6f0eec70 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,8 +1,10 @@ import atexit import contextlib import faulthandler +import locale import math import os.path +import platform import random import sys import sysconfig @@ -524,3 +526,56 @@ def adjust_rlimit_nofile(): except (ValueError, OSError) as err: print_warning(f"Unable to raise RLIMIT_NOFILE from {fd_limit} to " f"{new_fd_limit}: {err}.") + + +def display_header(): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("==", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== Python build:", ' '.join(get_build_info())) + print("== cwd:", os.getcwd()) + cpu_count = os.cpu_count() + if cpu_count: + print("== CPU count:", cpu_count) + print("== encodings: locale=%s, FS=%s" + % (locale.getencoding(), sys.getfilesystemencoding())) + + # This makes it easier to remember what to set in your local + # environment when trying to reproduce a sanitizer failure. + asan = support.check_sanitizer(address=True) + msan = support.check_sanitizer(memory=True) + ubsan = support.check_sanitizer(ub=True) + sanitizers = [] + if asan: + sanitizers.append("address") + if msan: + sanitizers.append("memory") + if ubsan: + sanitizers.append("undefined behavior") + if not sanitizers: + return + + print(f"== sanitizers: {', '.join(sanitizers)}") + for sanitizer, env_var in ( + (asan, "ASAN_OPTIONS"), + (msan, "MSAN_OPTIONS"), + (ubsan, "UBSAN_OPTIONS"), + ): + options= os.environ.get(env_var) + if sanitizer and options is not None: + print(f"== {env_var}={options!r}") + + +def cleanup_temp_dir(tmp_dir: StrPath): + import glob + + path = os.path.join(glob.escape(tmp_dir), 'test_python_*') + print("Cleanup %s directory" % tmp_dir) + for name in glob.glob(path): + if os.path.isdir(name): + print("Remove directory: %s" % name) + os_helper.rmtree(name) + else: + print("Remove file: %s" % name) + os_helper.unlink(name) From 0abc935086931d4915ea3c45cffffecb31e7a45c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 11 Sep 2023 14:03:30 +0300 Subject: [PATCH 59/66] Test DocTestFinder directly instead of calling support.run_doctest() (GH-108917) --- Lib/test/test_doctest.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 9cc460c8b913f6..6e12e82a7a0084 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -784,15 +784,13 @@ class TestDocTestFinder(unittest.TestCase): def test_issue35753(self): # This import of `call` should trigger issue35753 when - # `support.run_doctest` is called due to unwrap failing, + # DocTestFinder.find() is called due to inspect.unwrap() failing, # however with a patched doctest this should succeed. from unittest.mock import call dummy_module = types.ModuleType("dummy") dummy_module.__dict__['inject_call'] = call - try: - support.run_doctest(dummy_module, verbosity=True) - except ValueError as e: - raise support.TestFailed("Doctest unwrap failed") from e + finder = doctest.DocTestFinder() + self.assertEqual(finder.find(dummy_module), []) def test_empty_namespace_package(self): pkg_name = 'doctest_empty_pkg' From 60b8341d07649194aa73108369a1e04ed1848794 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 11 Sep 2023 14:05:30 +0300 Subject: [PATCH 60/66] Better integration of doctest and unittest in test_ctypes.test_objects (GH-108922) Better integration of docrtest and unittest in test_ctypes.test_objects --- Lib/test/test_ctypes/test_objects.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_ctypes/test_objects.py b/Lib/test/test_ctypes/test_objects.py index 23c92b01a11107..fb01421b955951 100644 --- a/Lib/test/test_ctypes/test_objects.py +++ b/Lib/test/test_ctypes/test_objects.py @@ -55,14 +55,12 @@ import doctest import unittest -import test.test_ctypes.test_objects -class TestCase(unittest.TestCase): - def test(self): - failures, tests = doctest.testmod(test.test_ctypes.test_objects) - self.assertFalse(failures, 'doctests failed, see output above') +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite()) + return tests if __name__ == '__main__': - doctest.testmod(test.test_ctypes.test_objects) + unittest.main() From 4a69301ea4539da172a00a80e78c07e9b41c1f8e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 11 Sep 2023 14:37:09 +0100 Subject: [PATCH 61/66] GH-108976. Keep monitoring data structures valid during de-optimization during callback. (GH-109131) --- Lib/test/test_monitoring.py | 8 ++ Lib/test/test_pdb.py | 18 +++ ...-09-06-22-50-25.gh-issue-108976.MUKaIJ.rst | 2 + Python/instrumentation.c | 106 +++++++++--------- 4 files changed, 79 insertions(+), 55 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-06-22-50-25.gh-issue-108976.MUKaIJ.rst diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 8a7d300b734dcc..e575eca42fabf9 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1719,6 +1719,14 @@ def make_foo_optimized_then_set_event(): finally: sys.monitoring.set_events(TEST_TOOL, 0) + def test_gh108976(self): + sys.monitoring.use_tool_id(0, "test") + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.LINE, lambda *args: sys.monitoring.set_events(0, 0)) + sys.monitoring.register_callback(0, E.INSTRUCTION, lambda *args: 0) + sys.monitoring.set_events(0, E.LINE | E.INSTRUCTION) + sys.monitoring.set_events(0, 0) + class TestOptimizer(MonitoringTestBase, unittest.TestCase): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index f6bed84808ed7f..f337656121643c 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2297,6 +2297,24 @@ def test_pdb_issue_gh_101517(): (Pdb) continue """ +def test_pdb_issue_gh_108976(): + """See GH-108976 + Make sure setting f_trace_opcodes = True won't crash pdb + >>> def test_function(): + ... import sys + ... sys._getframe().f_trace_opcodes = True + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... a = 1 + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'continue' + ... ]): + ... test_function() + bdb.Bdb.dispatch: unknown debugging event: 'opcode' + > (5)test_function() + -> a = 1 + (Pdb) continue + """ + def test_pdb_ambiguous_statements(): """See GH-104301 diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-06-22-50-25.gh-issue-108976.MUKaIJ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-06-22-50-25.gh-issue-108976.MUKaIJ.rst new file mode 100644 index 00000000000000..4b89375f0f57ef --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-06-22-50-25.gh-issue-108976.MUKaIJ.rst @@ -0,0 +1,2 @@ +Fix crash that occurs after de-instrumenting a code object in a monitoring +callback. diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 74c3adf0c1f475..5f714602eafbb8 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -364,7 +364,7 @@ dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoMonitoringDat } static void -dump_monitors(const char *prefix, _Py_Monitors monitors, FILE*out) +dump_global_monitors(const char *prefix, _Py_GlobalMonitors monitors, FILE*out) { fprintf(out, "%s monitors:\n", prefix); for (int event = 0; event < _PY_MONITORING_UNGROUPED_EVENTS; event++) { @@ -372,37 +372,13 @@ dump_monitors(const char *prefix, _Py_Monitors monitors, FILE*out) } } -/* Like _Py_GetBaseOpcode but without asserts. - * Does its best to give the right answer, but won't abort - * if something is wrong */ -static int -get_base_opcode_best_attempt(PyCodeObject *code, int offset) +static void +dump_local_monitors(const char *prefix, _Py_LocalMonitors monitors, FILE*out) { - int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); - if (INSTRUMENTED_OPCODES[opcode] != opcode) { - /* Not instrumented */ - return _PyOpcode_Deopt[opcode] == 0 ? opcode : _PyOpcode_Deopt[opcode]; - } - if (opcode == INSTRUMENTED_INSTRUCTION) { - if (code->_co_monitoring->per_instruction_opcodes[offset] == 0) { - return opcode; - } - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - } - if (opcode == INSTRUMENTED_LINE) { - if (code->_co_monitoring->lines[offset].original_opcode == 0) { - return opcode; - } - opcode = code->_co_monitoring->lines[offset].original_opcode; - } - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - return deinstrumented; - } - if (_PyOpcode_Deopt[opcode] == 0) { - return opcode; + fprintf(out, "%s monitors:\n", prefix); + for (int event = 0; event < _PY_MONITORING_LOCAL_EVENTS; event++) { + fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]); } - return _PyOpcode_Deopt[opcode]; } /* No error checking -- Don't use this for anything but experimental debugging */ @@ -417,9 +393,9 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) fprintf(out, "NULL\n"); return; } - dump_monitors("Global", _PyInterpreterState_GET()->monitors, out); - dump_monitors("Code", data->local_monitors, out); - dump_monitors("Active", data->active_monitors, out); + dump_global_monitors("Global", _PyInterpreterState_GET()->monitors, out); + dump_local_monitors("Code", data->local_monitors, out); + dump_local_monitors("Active", data->active_monitors, out); int code_len = (int)Py_SIZE(code); bool starred = false; for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) { @@ -458,6 +434,9 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) static bool valid_opcode(int opcode) { + if (opcode == INSTRUMENTED_LINE) { + return true; + } if (IS_VALID_OPCODE(opcode) && opcode != CACHE && opcode != RESERVED && @@ -475,18 +454,23 @@ sanity_check_instrumentation(PyCodeObject *code) if (data == NULL) { return; } - _Py_GlobalMonitors active_monitors = _PyInterpreterState_GET()->monitors; + _Py_GlobalMonitors global_monitors = _PyInterpreterState_GET()->monitors; + _Py_LocalMonitors active_monitors; if (code->_co_monitoring) { - _Py_Monitors local_monitors = code->_co_monitoring->local_monitors; - active_monitors = local_union(active_monitors, local_monitors); + _Py_LocalMonitors local_monitors = code->_co_monitoring->local_monitors; + active_monitors = local_union(global_monitors, local_monitors); + } + else { + _Py_LocalMonitors empty = (_Py_LocalMonitors) { 0 }; + active_monitors = local_union(global_monitors, empty); } assert(monitors_equals( code->_co_monitoring->active_monitors, - active_monitors) - ); + active_monitors)); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len;) { - int opcode = _PyCode_CODE(code)[i].op.code; + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = instr->op.code; int base_opcode = _Py_GetBaseOpcode(code, i); CHECK(valid_opcode(opcode)); CHECK(valid_opcode(base_opcode)); @@ -506,23 +490,30 @@ sanity_check_instrumentation(PyCodeObject *code) opcode = data->lines[i].original_opcode; CHECK(opcode != END_FOR); CHECK(opcode != RESUME); + CHECK(opcode != RESUME_CHECK); CHECK(opcode != INSTRUMENTED_RESUME); if (!is_instrumented(opcode)) { CHECK(_PyOpcode_Deopt[opcode] == opcode); } CHECK(opcode != INSTRUMENTED_LINE); } - else if (data->lines && !is_instrumented(opcode)) { - CHECK(data->lines[i].original_opcode == 0 || - data->lines[i].original_opcode == base_opcode || - DE_INSTRUMENT[data->lines[i].original_opcode] == base_opcode); + else if (data->lines) { + /* If original_opcode is INSTRUMENTED_INSTRUCTION + * *and* we are executing a INSTRUMENTED_LINE instruction + * that has de-instrumented itself, then we will execute + * an invalid INSTRUMENTED_INSTRUCTION */ + CHECK(data->lines[i].original_opcode != INSTRUMENTED_INSTRUCTION); + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + CHECK(data->per_instruction_opcodes[i] != 0); + opcode = data->per_instruction_opcodes[i]; } if (is_instrumented(opcode)) { CHECK(DE_INSTRUMENT[opcode] == base_opcode); int event = EVENT_FOR_OPCODE[DE_INSTRUMENT[opcode]]; if (event < 0) { /* RESUME fixup */ - event = _PyCode_CODE(code)[i].op.arg; + event = instr->op.arg ? 1: 0; } CHECK(active_monitors.tools[event] != 0); } @@ -608,30 +599,30 @@ static void de_instrument_line(PyCodeObject *code, int i) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - uint8_t *opcode_ptr = &instr->op.code; - int opcode =*opcode_ptr; + int opcode = instr->op.code; if (opcode != INSTRUMENTED_LINE) { return; } _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; int original_opcode = lines->original_opcode; + if (original_opcode == INSTRUMENTED_INSTRUCTION) { + lines->original_opcode = code->_co_monitoring->per_instruction_opcodes[i]; + } CHECK(original_opcode != 0); CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); - *opcode_ptr = instr->op.code = original_opcode; + instr->op.code = original_opcode; if (_PyOpcode_Caches[original_opcode]) { instr[1].cache = adaptive_counter_warmup(); } - assert(*opcode_ptr != INSTRUMENTED_LINE); assert(instr->op.code != INSTRUMENTED_LINE); } - static void de_instrument_per_instruction(PyCodeObject *code, int i) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; uint8_t *opcode_ptr = &instr->op.code; - int opcode =*opcode_ptr; + int opcode = *opcode_ptr; if (opcode == INSTRUMENTED_LINE) { opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; opcode = *opcode_ptr; @@ -642,10 +633,11 @@ de_instrument_per_instruction(PyCodeObject *code, int i) int original_opcode = code->_co_monitoring->per_instruction_opcodes[i]; CHECK(original_opcode != 0); CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); - instr->op.code = original_opcode; + *opcode_ptr = original_opcode; if (_PyOpcode_Caches[original_opcode]) { instr[1].cache = adaptive_counter_warmup(); } + assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION); assert(instr->op.code != INSTRUMENTED_INSTRUCTION); /* Keep things clean for sanity check */ code->_co_monitoring->per_instruction_opcodes[i] = 0; @@ -685,7 +677,7 @@ static void instrument_line(PyCodeObject *code, int i) { uint8_t *opcode_ptr = &_PyCode_CODE(code)[i].op.code; - int opcode =*opcode_ptr; + int opcode = *opcode_ptr; if (opcode == INSTRUMENTED_LINE) { return; } @@ -700,13 +692,14 @@ instrument_per_instruction(PyCodeObject *code, int i) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; uint8_t *opcode_ptr = &instr->op.code; - int opcode =*opcode_ptr; + int opcode = *opcode_ptr; if (opcode == INSTRUMENTED_LINE) { _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; opcode_ptr = &lines->original_opcode; opcode = *opcode_ptr; } if (opcode == INSTRUMENTED_INSTRUCTION) { + assert(code->_co_monitoring->per_instruction_opcodes[i] > 0); return; } CHECK(opcode != 0); @@ -1129,7 +1122,6 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _PyCoMonitoringData *monitoring = code->_co_monitoring; _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; - uint8_t original_opcode = line_data->original_opcode; if (tstate->tracing) { goto done; } @@ -1212,7 +1204,9 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, } } while (tools); Py_DECREF(line_obj); + uint8_t original_opcode; done: + original_opcode = line_data->original_opcode; assert(original_opcode != 0); assert(original_opcode != INSTRUMENTED_LINE); assert(_PyOpcode_Deopt[original_opcode] == original_opcode); @@ -1655,7 +1649,9 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) i += _PyInstruction_GetLength(code, i); } } - +#ifdef INSTRUMENT_DEBUG + sanity_check_instrumentation(code); +#endif uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; From c0f488b88f2a54d76256818e2841d868fecfd396 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 11 Sep 2023 17:50:33 +0300 Subject: [PATCH 62/66] gh-109182: Fix and improve tests for gh-108654 (GH-109189) --- Lib/test/test_listcomps.py | 46 +++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index c1089574d71b02..12f7bbd123b30c 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -125,7 +125,7 @@ def get_output(moddict, name): self.assertIs(type(e), raises) else: for k, v in (outputs or {}).items(): - self.assertEqual(get_output(newns, k), v) + self.assertEqual(get_output(newns, k), v, k) def test_lambdas_with_iteration_var_as_default(self): code = """ @@ -563,28 +563,38 @@ def test_iter_var_available_in_locals(self): def test_comp_in_try_except(self): template = """ - value = ["a"] + value = ["ab"] + result = snapshot = None try: - [{func}(value) for value in value] + result = [{func}(value) for value in value] except: - pass + snapshot = value + raise """ - for func in ["str", "int"]: - code = template.format(func=func) - raises = func != "str" - with self.subTest(raises=raises): - self._check_in_scopes(code, {"value": ["a"]}) + # No exception. + code = template.format(func='len') + self._check_in_scopes(code, {"value": ["ab"], "result": [2], "snapshot": None}) + # Handles exception. + code = template.format(func='int') + self._check_in_scopes(code, {"value": ["ab"], "result": None, "snapshot": ["ab"]}, + raises=ValueError) def test_comp_in_try_finally(self): - code = """ - def f(value): - try: - [{func}(value) for value in value] - finally: - return value - ret = f(["a"]) - """ - self._check_in_scopes(code, {"ret": ["a"]}) + template = """ + value = ["ab"] + result = snapshot = None + try: + result = [{func}(value) for value in value] + finally: + snapshot = value + """ + # No exception. + code = template.format(func='len') + self._check_in_scopes(code, {"value": ["ab"], "result": [2], "snapshot": ["ab"]}) + # Handles exception. + code = template.format(func='int') + self._check_in_scopes(code, {"value": ["ab"], "result": None, "snapshot": ["ab"]}, + raises=ValueError) def test_exception_in_post_comp_call(self): code = """ From 517cd82ea7d01b344804413ef05610934a43a241 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Sep 2023 17:27:03 +0200 Subject: [PATCH 63/66] gh-108987: Fix _thread.start_new_thread() race condition (#109135) Fix _thread.start_new_thread() race condition. If a thread is created during Python finalization, the newly spawned thread now exits immediately instead of trying to access freed memory and lead to a crash. thread_run() calls PyEval_AcquireThread() which checks if the thread must exit. The problem was that tstate was dereferenced earlier in _PyThreadState_Bind() which leads to a crash most of the time. Move _PyThreadState_CheckConsistency() from thread_run() to _PyThreadState_Bind(). --- Include/internal/pycore_pystate.h | 2 + ...-09-08-12-09-55.gh-issue-108987.x5AIG8.rst | 4 ++ Modules/_threadmodule.c | 47 ++++++++++++------- Python/ceval_gil.c | 28 ++--------- Python/pystate.c | 29 ++++++++++++ 5 files changed, 69 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-08-12-09-55.gh-issue-108987.x5AIG8.rst diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 9c0e42e7bad06c..9fc8ae903b2ac0 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -71,6 +71,8 @@ extern _Py_thread_local PyThreadState *_Py_tss_tstate; extern int _PyThreadState_CheckConsistency(PyThreadState *tstate); #endif +int _PyThreadState_MustExit(PyThreadState *tstate); + // Export for most shared extensions, used via _PyThreadState_GET() static // inline function. PyAPI_FUNC(PyThreadState *) _PyThreadState_GetCurrent(void); diff --git a/Misc/NEWS.d/next/Library/2023-09-08-12-09-55.gh-issue-108987.x5AIG8.rst b/Misc/NEWS.d/next/Library/2023-09-08-12-09-55.gh-issue-108987.x5AIG8.rst new file mode 100644 index 00000000000000..16526ee748d869 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-08-12-09-55.gh-issue-108987.x5AIG8.rst @@ -0,0 +1,4 @@ +Fix :func:`_thread.start_new_thread` race condition. If a thread is created +during Python finalization, the newly spawned thread now exits immediately +instead of trying to access freed memory and lead to a crash. Patch by +Victor Stinner. diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 05bb49756c9303..7692bacccc4909 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1051,21 +1051,21 @@ _localdummy_destroyed(PyObject *localweakref, PyObject *dummyweakref) /* Module functions */ struct bootstate { - PyInterpreterState *interp; + PyThreadState *tstate; PyObject *func; PyObject *args; PyObject *kwargs; - PyThreadState *tstate; - _PyRuntimeState *runtime; }; static void -thread_bootstate_free(struct bootstate *boot) +thread_bootstate_free(struct bootstate *boot, int decref) { - Py_DECREF(boot->func); - Py_DECREF(boot->args); - Py_XDECREF(boot->kwargs); + if (decref) { + Py_DECREF(boot->func); + Py_DECREF(boot->args); + Py_XDECREF(boot->kwargs); + } PyMem_Free(boot); } @@ -1076,9 +1076,24 @@ thread_run(void *boot_raw) struct bootstate *boot = (struct bootstate *) boot_raw; PyThreadState *tstate = boot->tstate; - // gh-104690: If Python is being finalized and PyInterpreterState_Delete() - // was called, tstate becomes a dangling pointer. - assert(_PyThreadState_CheckConsistency(tstate)); + // gh-108987: If _thread.start_new_thread() is called before or while + // Python is being finalized, thread_run() can called *after*. + // _PyRuntimeState_SetFinalizing() is called. At this point, all Python + // threads must exit, except of the thread calling Py_Finalize() whch holds + // the GIL and must not exit. + // + // At this stage, tstate can be a dangling pointer (point to freed memory), + // it's ok to call _PyThreadState_MustExit() with a dangling pointer. + if (_PyThreadState_MustExit(tstate)) { + // Don't call PyThreadState_Clear() nor _PyThreadState_DeleteCurrent(). + // These functions are called on tstate indirectly by Py_Finalize() + // which calls _PyInterpreterState_Clear(). + // + // Py_DECREF() cannot be called because the GIL is not held: leak + // references on purpose. Python is being finalized anyway. + thread_bootstate_free(boot, 0); + goto exit; + } _PyThreadState_Bind(tstate); PyEval_AcquireThread(tstate); @@ -1097,14 +1112,17 @@ thread_run(void *boot_raw) Py_DECREF(res); } - thread_bootstate_free(boot); + thread_bootstate_free(boot, 1); + tstate->interp->threads.count--; PyThreadState_Clear(tstate); _PyThreadState_DeleteCurrent(tstate); +exit: // bpo-44434: Don't call explicitly PyThread_exit_thread(). On Linux with // the glibc, pthread_exit() can abort the whole process if dlopen() fails // to open the libgcc_s.so library (ex: EMFILE error). + return; } static PyObject * @@ -1128,7 +1146,6 @@ and False otherwise.\n"); static PyObject * thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs) { - _PyRuntimeState *runtime = &_PyRuntime; PyObject *func, *args, *kwargs = NULL; if (!PyArg_UnpackTuple(fargs, "start_new_thread", 2, 3, @@ -1171,8 +1188,7 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs) if (boot == NULL) { return PyErr_NoMemory(); } - boot->interp = _PyInterpreterState_GET(); - boot->tstate = _PyThreadState_New(boot->interp); + boot->tstate = _PyThreadState_New(interp); if (boot->tstate == NULL) { PyMem_Free(boot); if (!PyErr_Occurred()) { @@ -1180,7 +1196,6 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs) } return NULL; } - boot->runtime = runtime; boot->func = Py_NewRef(func); boot->args = Py_NewRef(args); boot->kwargs = Py_XNewRef(kwargs); @@ -1189,7 +1204,7 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs) if (ident == PYTHREAD_INVALID_THREAD_ID) { PyErr_SetString(ThreadError, "can't start new thread"); PyThreadState_Clear(boot->tstate); - thread_bootstate_free(boot); + thread_bootstate_free(boot, 1); return NULL; } return PyLong_FromUnsignedLong(ident); diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index cef5317b46bf8e..3b7e6cb1bda3ff 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -329,28 +329,6 @@ drop_gil(struct _ceval_state *ceval, PyThreadState *tstate) } -/* Check if a Python thread must exit immediately, rather than taking the GIL - if Py_Finalize() has been called. - - When this function is called by a daemon thread after Py_Finalize() has been - called, the GIL does no longer exist. - - tstate must be non-NULL. */ -static inline int -tstate_must_exit(PyThreadState *tstate) -{ - /* bpo-39877: Access _PyRuntime directly rather than using - tstate->interp->runtime to support calls from Python daemon threads. - After Py_Finalize() has been called, tstate can be a dangling pointer: - point to PyThreadState freed memory. */ - PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(&_PyRuntime); - if (finalizing == NULL) { - finalizing = _PyInterpreterState_GetFinalizing(tstate->interp); - } - return (finalizing != NULL && finalizing != tstate); -} - - /* Take the GIL. The function saves errno at entry and restores its value at exit. @@ -366,7 +344,7 @@ take_gil(PyThreadState *tstate) // XXX It may be more correct to check tstate->_status.finalizing. // XXX assert(!tstate->_status.cleared); - if (tstate_must_exit(tstate)) { + if (_PyThreadState_MustExit(tstate)) { /* bpo-39877: If Py_Finalize() has been called and tstate is not the thread which called Py_Finalize(), exit immediately the thread. @@ -404,7 +382,7 @@ take_gil(PyThreadState *tstate) _Py_atomic_load_relaxed(&gil->locked) && gil->switch_number == saved_switchnum) { - if (tstate_must_exit(tstate)) { + if (_PyThreadState_MustExit(tstate)) { MUTEX_UNLOCK(gil->mutex); // gh-96387: If the loop requested a drop request in a previous // iteration, reset the request. Otherwise, drop_gil() can @@ -444,7 +422,7 @@ take_gil(PyThreadState *tstate) MUTEX_UNLOCK(gil->switch_mutex); #endif - if (tstate_must_exit(tstate)) { + if (_PyThreadState_MustExit(tstate)) { /* bpo-36475: If Py_Finalize() has been called and tstate is not the thread which called Py_Finalize(), exit immediately the thread. diff --git a/Python/pystate.c b/Python/pystate.c index 09c3538ad7b872..b5c4fd7fb50616 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1907,6 +1907,10 @@ PyThreadState_Swap(PyThreadState *newts) void _PyThreadState_Bind(PyThreadState *tstate) { + // gh-104690: If Python is being finalized and PyInterpreterState_Delete() + // was called, tstate becomes a dangling pointer. + assert(_PyThreadState_CheckConsistency(tstate)); + bind_tstate(tstate); // This makes sure there's a gilstate tstate bound // as soon as possible. @@ -2908,6 +2912,31 @@ _PyThreadState_CheckConsistency(PyThreadState *tstate) #endif +// Check if a Python thread must exit immediately, rather than taking the GIL +// if Py_Finalize() has been called. +// +// When this function is called by a daemon thread after Py_Finalize() has been +// called, the GIL does no longer exist. +// +// tstate can be a dangling pointer (point to freed memory): only tstate value +// is used, the pointer is not deferenced. +// +// tstate must be non-NULL. +int +_PyThreadState_MustExit(PyThreadState *tstate) +{ + /* bpo-39877: Access _PyRuntime directly rather than using + tstate->interp->runtime to support calls from Python daemon threads. + After Py_Finalize() has been called, tstate can be a dangling pointer: + point to PyThreadState freed memory. */ + PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(&_PyRuntime); + if (finalizing == NULL) { + finalizing = _PyInterpreterState_GetFinalizing(tstate->interp); + } + return (finalizing != NULL && finalizing != tstate); +} + + #ifdef __cplusplus } #endif From 57b6205523d934d61b6308d63ef72c494c7d2b7e Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 11 Sep 2023 10:06:16 -0600 Subject: [PATCH 64/66] gh-109190: What's New in 3.12: Add subheadings to removals for easy linking (#109159) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/whatsnew/3.12.rst | 159 ++++++++++++++++++++++++++++-------------- 1 file changed, 107 insertions(+), 52 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index a46c913c4997ba..447e81855a4a66 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1274,13 +1274,19 @@ although there is currently no date scheduled for their removal. Removed ======= -* ``asynchat`` and ``asyncore``: These two modules have been removed +asynchat and asyncore +--------------------- + +* These two modules have been removed according to the schedule in :pep:`594`, having been deprecated in Python 3.6. Use :mod:`asyncio` instead. (Contributed by Nikita Sobolev in :gh:`96580`.) -* :mod:`configparser`: Several names deprecated in the :mod:`configparser` way back in 3.2 have +configparser +------------ + +* Several names deprecated in the :mod:`configparser` way back in 3.2 have been removed per :gh:`89336`: * :class:`configparser.ParsingError` no longer has a ``filename`` attribute @@ -1290,13 +1296,19 @@ Removed * :class:`configparser.ConfigParser` no longer has a ``readfp`` method. Use :meth:`~configparser.ConfigParser.read_file` instead. -* ``distutils``: Remove the ``distutils`` package. It was deprecated in Python 3.10 by +distutils +--------- + +* Remove the :py:mod:`!distutils` package. It was deprecated in Python 3.10 by :pep:`632` "Deprecate distutils module". For projects still using ``distutils`` and cannot be updated to something else, the ``setuptools`` project can be installed: it still provides ``distutils``. (Contributed by Victor Stinner in :gh:`92584`.) -* :mod:`ensurepip`: Remove the bundled setuptools wheel from :mod:`ensurepip`, +ensurepip +--------- + +* Remove the bundled setuptools wheel from :mod:`ensurepip`, and stop installing setuptools in environments created by :mod:`venv`. ``pip (>= 22.1)`` does not require setuptools to be installed in the @@ -1314,27 +1326,42 @@ Removed (Contributed by Pradyun Gedam in :gh:`95299`.) -* :mod:`enum`: Remove ``EnumMeta.__getattr__``, which is no longer needed for +enum +---- + +* Remove :mod:`enum`'s ``EnumMeta.__getattr__``, which is no longer needed for enum attribute access. (Contributed by Ethan Furman in :gh:`95083`.) -* :mod:`ftplib`: Remove the ``FTP_TLS.ssl_version`` class attribute: use the +ftplib +------ + +* Remove :mod:`ftplib`'s ``FTP_TLS.ssl_version`` class attribute: use the *context* parameter instead. (Contributed by Victor Stinner in :gh:`94172`.) -* :mod:`gzip`: Remove the ``filename`` attribute of :class:`gzip.GzipFile`, +gzip +---- + +* Remove the ``filename`` attribute of :mod:`gzip`'s :class:`gzip.GzipFile`, deprecated since Python 2.6, use the :attr:`~gzip.GzipFile.name` attribute instead. In write mode, the ``filename`` attribute added ``'.gz'`` file extension if it was not present. (Contributed by Victor Stinner in :gh:`94196`.) -* :mod:`hashlib`: Remove the pure Python implementation of +hashlib +------- + +* Remove the pure Python implementation of :mod:`hashlib`'s :func:`hashlib.pbkdf2_hmac()`, deprecated in Python 3.10. Python 3.10 and newer requires OpenSSL 1.1.1 (:pep:`644`): this OpenSSL version provides a C implementation of :func:`~hashlib.pbkdf2_hmac()` which is faster. (Contributed by Victor Stinner in :gh:`94199`.) -* :mod:`importlib`: Many previously deprecated cleanups in :mod:`importlib` have now been +importlib +--------- + +* Many previously deprecated cleanups in :mod:`importlib` have now been completed: * References to, and support for :meth:`!module_repr()` has been removed. @@ -1350,10 +1377,13 @@ Removed * ``importlib.abc.Finder``, ``pkgutil.ImpImporter``, and ``pkgutil.ImpLoader`` have been removed. (Contributed by Barry Warsaw in :gh:`98040`.) - * The :mod:`!imp` module has been removed. (Contributed by Barry Warsaw in - :gh:`98040`.) +imp +--- - * Replace removed :mod:`!imp` functions with :mod:`importlib` functions: +* The :mod:`!imp` module has been removed. (Contributed by Barry Warsaw in + :gh:`98040`.) + +* Replace removed :mod:`!imp` functions with :mod:`importlib` functions: ================================= ======================================= imp importlib @@ -1370,7 +1400,7 @@ Removed ``imp.source_from_cache()`` :func:`importlib.util.source_from_cache` ================================= ======================================= - * Replace ``imp.load_source()`` with:: +* Replace ``imp.load_source()`` with:: import importlib.util import importlib.machinery @@ -1385,28 +1415,34 @@ Removed loader.exec_module(module) return module - * Removed :mod:`!imp` functions and attributes with no replacements: +* Removed :mod:`!imp` functions and attributes with no replacements: + + * undocumented functions: - * undocumented functions: + * ``imp.init_builtin()`` + * ``imp.load_compiled()`` + * ``imp.load_dynamic()`` + * ``imp.load_package()`` - * ``imp.init_builtin()`` - * ``imp.load_compiled()`` - * ``imp.load_dynamic()`` - * ``imp.load_package()`` + * ``imp.lock_held()``, ``imp.acquire_lock()``, ``imp.release_lock()``: + the locking scheme has changed in Python 3.3 to per-module locks. + * ``imp.find_module()`` constants: ``SEARCH_ERROR``, ``PY_SOURCE``, + ``PY_COMPILED``, ``C_EXTENSION``, ``PY_RESOURCE``, ``PKG_DIRECTORY``, + ``C_BUILTIN``, ``PY_FROZEN``, ``PY_CODERESOURCE``, ``IMP_HOOK``. - * ``imp.lock_held()``, ``imp.acquire_lock()``, ``imp.release_lock()``: - the locking scheme has changed in Python 3.3 to per-module locks. - * ``imp.find_module()`` constants: ``SEARCH_ERROR``, ``PY_SOURCE``, - ``PY_COMPILED``, ``C_EXTENSION``, ``PY_RESOURCE``, ``PKG_DIRECTORY``, - ``C_BUILTIN``, ``PY_FROZEN``, ``PY_CODERESOURCE``, ``IMP_HOOK``. +io +-- -* :mod:`io`: Remove ``io.OpenWrapper`` and ``_pyio.OpenWrapper``, deprecated in Python +* Remove :mod:`io`'s ``io.OpenWrapper`` and ``_pyio.OpenWrapper``, deprecated in Python 3.10: just use :func:`open` instead. The :func:`open` (:func:`io.open`) function is a built-in function. Since Python 3.10, :func:`!_pyio.open` is also a static method. (Contributed by Victor Stinner in :gh:`94169`.) -* :mod:`locale`: Remove the :func:`!locale.format` function, deprecated in Python 3.7: +locale +------ + +* Remove :mod:`locale`'s :func:`!locale.format` function, deprecated in Python 3.7: use :func:`locale.format_string` instead. (Contributed by Victor Stinner in :gh:`94226`.) @@ -1418,7 +1454,10 @@ Removed .. _aiosmtpd: https://pypi.org/project/aiosmtpd/ -* :mod:`sqlite3`: The following undocumented :mod:`sqlite3` features, deprecated in Python +sqlite3 +------- + +* The following undocumented :mod:`sqlite3` features, deprecated in Python 3.10, are now removed: * ``sqlite3.enable_shared_cache()`` @@ -1434,30 +1473,34 @@ Removed (Contributed by Erlend E. Aasland in :gh:`92548`.) -* :mod:`ssl`: +ssl +--- - * Remove the :func:`!ssl.RAND_pseudo_bytes` function, deprecated in Python 3.6: - use :func:`os.urandom` or :func:`ssl.RAND_bytes` instead. - (Contributed by Victor Stinner in :gh:`94199`.) +* Remove :mod:`ssl`'s :func:`!ssl.RAND_pseudo_bytes` function, deprecated in Python 3.6: + use :func:`os.urandom` or :func:`ssl.RAND_bytes` instead. + (Contributed by Victor Stinner in :gh:`94199`.) + +* Remove the :func:`!ssl.match_hostname` function. + It was deprecated in Python 3.7. OpenSSL performs + hostname matching since Python 3.7, Python no longer uses the + :func:`!ssl.match_hostname` function. + (Contributed by Victor Stinner in :gh:`94199`.) - * Remove the :func:`!ssl.match_hostname` function. - It was deprecated in Python 3.7. OpenSSL performs - hostname matching since Python 3.7, Python no longer uses the - :func:`!ssl.match_hostname` function. - (Contributed by Victor Stinner in :gh:`94199`.) +* Remove the :func:`!ssl.wrap_socket` function, deprecated in Python 3.7: + instead, create a :class:`ssl.SSLContext` object and call its + :class:`ssl.SSLContext.wrap_socket` method. Any package that still uses + :func:`!ssl.wrap_socket` is broken and insecure. The function neither sends a + SNI TLS extension nor validates server hostname. Code is subject to `CWE-295 + `_: Improper Certificate + Validation. + (Contributed by Victor Stinner in :gh:`94199`.) - * Remove the :func:`!ssl.wrap_socket` function, deprecated in Python 3.7: - instead, create a :class:`ssl.SSLContext` object and call its - :class:`ssl.SSLContext.wrap_socket` method. Any package that still uses - :func:`!ssl.wrap_socket` is broken and insecure. The function neither sends a - SNI TLS extension nor validates server hostname. Code is subject to `CWE-295 - `_: Improper Certificate - Validation. - (Contributed by Victor Stinner in :gh:`94199`.) +unittest +-------- -* :mod:`unittest`: Removed many old deprecated :mod:`unittest` features: +* Removed many old deprecated :mod:`unittest` features: - - A number of :class:`~unittest.TestCase` method aliases: + * A number of :class:`~unittest.TestCase` method aliases: ============================ =============================== =============== Deprecated alias Method Name Deprecated in @@ -1482,33 +1525,45 @@ Removed You can use https://github.com/isidentical/teyit to automatically modernise your unit tests. - - Undocumented and broken :class:`~unittest.TestCase` method + * Undocumented and broken :class:`~unittest.TestCase` method ``assertDictContainsSubset`` (deprecated in Python 3.2). - - Undocumented :meth:`TestLoader.loadTestsFromModule + * Undocumented :meth:`TestLoader.loadTestsFromModule ` parameter *use_load_tests* (deprecated and ignored since Python 3.2). - - An alias of the :class:`~unittest.TextTestResult` class: + * An alias of the :class:`~unittest.TextTestResult` class: ``_TextTestResult`` (deprecated in Python 3.2). (Contributed by Serhiy Storchaka in :issue:`45162`.) -* :mod:`webbrowser`: Remove support for obsolete browsers from :mod:`webbrowser`. +webbrowser +---------- + +* Remove support for obsolete browsers from :mod:`webbrowser`. Removed browsers include: Grail, Mosaic, Netscape, Galeon, Skipstone, Iceape, Firebird, and Firefox versions 35 and below (:gh:`102871`). -* :mod:`xml.etree.ElementTree`: Remove the ``ElementTree.Element.copy()`` method of the +xml.etree.ElementTree +--------------------- + +* Remove the ``ElementTree.Element.copy()`` method of the pure Python implementation, deprecated in Python 3.10, use the :func:`copy.copy` function instead. The C implementation of :mod:`xml.etree.ElementTree` has no ``copy()`` method, only a ``__copy__()`` method. (Contributed by Victor Stinner in :gh:`94383`.) -* :mod:`zipimport`: Remove ``find_loader()`` and ``find_module()`` methods, +zipimport +--------- + +* Remove :mod:`zipimport`'s ``find_loader()`` and ``find_module()`` methods, deprecated in Python 3.10: use the ``find_spec()`` method instead. See :pep:`451` for the rationale. (Contributed by Victor Stinner in :gh:`94379`.) +Others +------ + * Removed the ``suspicious`` rule from the documentation Makefile, and removed ``Doc/tools/rstlint.py``, both in favor of `sphinx-lint `_. From baa6dc8e388e71b2a00347143ecefb2ad3a8e53b Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 11 Sep 2023 19:13:37 +0300 Subject: [PATCH 65/66] gh-90805: Make sure test_functools works with and without _functoolsmodule (GH-108644) --- Lib/test/test_functools.py | 58 ++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 5ba7f51c91f3b5..e4de2c5ede15f1 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -26,10 +26,16 @@ py_functools = import_helper.import_fresh_module('functools', blocked=['_functools']) -c_functools = import_helper.import_fresh_module('functools') +c_functools = import_helper.import_fresh_module('functools', + fresh=['_functools']) decimal = import_helper.import_fresh_module('decimal', fresh=['_decimal']) +_partial_types = [py_functools.partial] +if c_functools: + _partial_types.append(c_functools.partial) + + @contextlib.contextmanager def replaced_module(name, replacement): original_module = sys.modules[name] @@ -201,7 +207,7 @@ def test_repr(self): kwargs = {'a': object(), 'b': object()} kwargs_reprs = ['a={a!r}, b={b!r}'.format_map(kwargs), 'b={b!r}, a={a!r}'.format_map(kwargs)] - if self.partial in (c_functools.partial, py_functools.partial): + if self.partial in _partial_types: name = 'functools.partial' else: name = self.partial.__name__ @@ -223,7 +229,7 @@ def test_repr(self): for kwargs_repr in kwargs_reprs]) def test_recursive_repr(self): - if self.partial in (c_functools.partial, py_functools.partial): + if self.partial in _partial_types: name = 'functools.partial' else: name = self.partial.__name__ @@ -250,7 +256,7 @@ def test_recursive_repr(self): f.__setstate__((capture, (), {}, {})) def test_pickle(self): - with self.AllowPickle(): + with replaced_module('functools', self.module): f = self.partial(signature, ['asdf'], bar=[True]) f.attr = [] for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -333,7 +339,7 @@ def test_setstate_subclasses(self): self.assertIs(type(r[0]), tuple) def test_recursive_pickle(self): - with self.AllowPickle(): + with replaced_module('functools', self.module): f = self.partial(capture) f.__setstate__((f, (), {}, {})) try: @@ -387,14 +393,9 @@ def __getitem__(self, key): @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestPartialC(TestPartial, unittest.TestCase): if c_functools: + module = c_functools partial = c_functools.partial - class AllowPickle: - def __enter__(self): - return self - def __exit__(self, type, value, tb): - return False - def test_attributes_unwritable(self): # attributes should not be writable p = self.partial(capture, 1, 2, a=10, b=20) @@ -437,15 +438,9 @@ def __str__(self): class TestPartialPy(TestPartial, unittest.TestCase): + module = py_functools partial = py_functools.partial - class AllowPickle: - def __init__(self): - self._cm = replaced_module("functools", py_functools) - def __enter__(self): - return self._cm.__enter__() - def __exit__(self, type, value, tb): - return self._cm.__exit__(type, value, tb) if c_functools: class CPartialSubclass(c_functools.partial): @@ -1872,9 +1867,10 @@ def orig(): ... def py_cached_func(x, y): return 3 * x + y -@c_functools.lru_cache() -def c_cached_func(x, y): - return 3 * x + y +if c_functools: + @c_functools.lru_cache() + def c_cached_func(x, y): + return 3 * x + y class TestLRUPy(TestLRU, unittest.TestCase): @@ -1891,18 +1887,20 @@ def cached_staticmeth(x, y): return 3 * x + y +@unittest.skipUnless(c_functools, 'requires the C _functools module') class TestLRUC(TestLRU, unittest.TestCase): - module = c_functools - cached_func = c_cached_func, + if c_functools: + module = c_functools + cached_func = c_cached_func, - @module.lru_cache() - def cached_meth(self, x, y): - return 3 * x + y + @module.lru_cache() + def cached_meth(self, x, y): + return 3 * x + y - @staticmethod - @module.lru_cache() - def cached_staticmeth(x, y): - return 3 * x + y + @staticmethod + @module.lru_cache() + def cached_staticmeth(x, y): + return 3 * x + y class TestSingleDispatch(unittest.TestCase): From de5f8f7d13c0bbc723eaea83284dc78b37be54b4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Sep 2023 19:33:42 +0200 Subject: [PATCH 66/66] gh-109276: libregrtest: use separated file for JSON (#109277) libregrtest now uses a separated file descriptor to write test result as JSON. Previously, if a test wrote debug messages late around the JSON, the main test process failed to parse JSON. Rename TestResult.write_json() to TestResult.write_json_into(). worker_process() no longer writes an empty line at the end. There is no need to separate test process output from the JSON output anymore, since JSON is now written into a separated file descriptor. create_worker_process() now always spawn the process with close_fds=True. --- Lib/test/libregrtest/result.py | 2 +- Lib/test/libregrtest/run_workers.py | 66 ++++++++++++------- Lib/test/libregrtest/runtests.py | 3 + Lib/test/libregrtest/single.py | 4 ++ Lib/test/libregrtest/worker.py | 42 ++++++++---- ...-09-11-18-19-52.gh-issue-109276.btfFtT.rst | 3 + 6 files changed, 84 insertions(+), 36 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-11-18-19-52.gh-issue-109276.btfFtT.rst diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index cfd08ecc41b7d4..bf885264657d5c 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -156,7 +156,7 @@ def get_rerun_match_tests(self) -> FilterTuple | None: return None return tuple(match_tests) - def write_json(self, file) -> None: + def write_json_into(self, file) -> None: json.dump(self, file, cls=_EncodeTestResult) @staticmethod diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 768625cb507f9a..5c665abfeb57bd 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -20,12 +20,14 @@ from .runtests import RunTests from .single import PROGRESS_MIN_TIME from .utils import ( - StrPath, TestName, + StrPath, StrJSON, TestName, MS_WINDOWS, format_duration, print_warning) from .worker import create_worker_process, USE_PROCESS_GROUP -if sys.platform == 'win32': +if MS_WINDOWS: import locale + import msvcrt + # Display the running tests if nothing happened last N seconds @@ -153,10 +155,11 @@ def mp_result_error( ) -> MultiprocessResult: return MultiprocessResult(test_result, stdout, err_msg) - def _run_process(self, runtests: RunTests, output_file: TextIO, + def _run_process(self, runtests: RunTests, output_fd: int, json_fd: int, tmp_dir: StrPath | None = None) -> int: try: - popen = create_worker_process(runtests, output_file, tmp_dir) + popen = create_worker_process(runtests, output_fd, json_fd, + tmp_dir) self._killed = False self._popen = popen @@ -208,7 +211,7 @@ def _run_process(self, runtests: RunTests, output_file: TextIO, def _runtest(self, test_name: TestName) -> MultiprocessResult: self.current_test_name = test_name - if sys.platform == 'win32': + if MS_WINDOWS: # gh-95027: When stdout is not a TTY, Python uses the ANSI code # page for the sys.stdout encoding. If the main process runs in a # terminal, sys.stdout uses WindowsConsoleIO with UTF-8 encoding. @@ -221,14 +224,25 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: match_tests = self.runtests.get_match_tests(test_name) else: match_tests = None - kwargs = {} - if match_tests: - kwargs['match_tests'] = match_tests - worker_runtests = self.runtests.copy(tests=tests, **kwargs) + err_msg = None # gh-94026: Write stdout+stderr to a tempfile as workaround for # non-blocking pipes on Emscripten with NodeJS. - with tempfile.TemporaryFile('w+', encoding=encoding) as stdout_file: + with (tempfile.TemporaryFile('w+', encoding=encoding) as stdout_file, + tempfile.TemporaryFile('w+', encoding='utf8') as json_file): + stdout_fd = stdout_file.fileno() + json_fd = json_file.fileno() + if MS_WINDOWS: + json_fd = msvcrt.get_osfhandle(json_fd) + + kwargs = {} + if match_tests: + kwargs['match_tests'] = match_tests + worker_runtests = self.runtests.copy( + tests=tests, + json_fd=json_fd, + **kwargs) + # gh-93353: Check for leaked temporary files in the parent process, # since the deletion of temporary files can happen late during # Python finalization: too late for libregrtest. @@ -239,12 +253,14 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: tmp_dir = tempfile.mkdtemp(prefix="test_python_") tmp_dir = os.path.abspath(tmp_dir) try: - retcode = self._run_process(worker_runtests, stdout_file, tmp_dir) + retcode = self._run_process(worker_runtests, + stdout_fd, json_fd, tmp_dir) finally: tmp_files = os.listdir(tmp_dir) os_helper.rmtree(tmp_dir) else: - retcode = self._run_process(worker_runtests, stdout_file) + retcode = self._run_process(worker_runtests, + stdout_fd, json_fd) tmp_files = () stdout_file.seek(0) @@ -257,23 +273,27 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: result = TestResult(test_name, state=State.MULTIPROCESSING_ERROR) return self.mp_result_error(result, err_msg=err_msg) + try: + # deserialize run_tests_worker() output + json_file.seek(0) + worker_json: StrJSON = json_file.read() + if worker_json: + result = TestResult.from_json(worker_json) + else: + err_msg = f"empty JSON" + except Exception as exc: + # gh-101634: Catch UnicodeDecodeError if stdout cannot be + # decoded from encoding + err_msg = f"Fail to read or parser worker process JSON: {exc}" + result = TestResult(test_name, state=State.MULTIPROCESSING_ERROR) + return self.mp_result_error(result, stdout, err_msg=err_msg) + if retcode is None: result = TestResult(test_name, state=State.TIMEOUT) return self.mp_result_error(result, stdout) - err_msg = None if retcode != 0: err_msg = "Exit code %s" % retcode - else: - stdout, _, worker_json = stdout.rpartition("\n") - stdout = stdout.rstrip() - if not worker_json: - err_msg = "Failed to parse worker stdout" - else: - try: - result = TestResult.from_json(worker_json) - except Exception as exc: - err_msg = "Failed to parse worker JSON: %s" % exc if err_msg: result = TestResult(test_name, state=State.MULTIPROCESSING_ERROR) diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index e843cc2dadf734..64f8f6ab0ff305 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -36,6 +36,9 @@ class RunTests: gc_threshold: int | None = None use_resources: list[str] = dataclasses.field(default_factory=list) python_cmd: list[str] | None = None + # On Unix, it's a file descriptor. + # On Windows, it's a handle. + json_fd: int | None = None def copy(self, **override): state = dataclasses.asdict(self) diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 635b4f93702b04..c26e542cf15ef5 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -271,5 +271,9 @@ def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: print(f"test {test_name} crashed -- {msg}", file=sys.stderr, flush=True) result.state = State.UNCAUGHT_EXC + + sys.stdout.flush() + sys.stderr.flush() + result.duration = time.perf_counter() - start_time return result diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index ed1286e5700679..b3b204f65f92ec 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -10,7 +10,7 @@ from .runtests import RunTests from .single import run_single_test from .utils import ( - StrPath, StrJSON, FilterTuple, + StrPath, StrJSON, FilterTuple, MS_WINDOWS, get_work_dir, exit_timeout) @@ -18,7 +18,7 @@ def create_worker_process(runtests: RunTests, - output_file: TextIO, + output_fd: int, json_fd: int, tmp_dir: StrPath | None = None) -> subprocess.Popen: python_cmd = runtests.python_cmd worker_json = runtests.as_json() @@ -41,23 +41,42 @@ def create_worker_process(runtests: RunTests, # Running the child from the same working directory as regrtest's original # invocation ensures that TEMPDIR for the child is the same when # sysconfig.is_python_build() is true. See issue 15300. - kw = dict( + kwargs = dict( env=env, - stdout=output_file, + stdout=output_fd, # bpo-45410: Write stderr into stdout to keep messages order - stderr=output_file, + stderr=output_fd, text=True, - close_fds=(os.name != 'nt'), + close_fds=True, ) + if not MS_WINDOWS: + kwargs['pass_fds'] = [json_fd] + else: + startupinfo = subprocess.STARTUPINFO() + startupinfo.lpAttributeList = {"handle_list": [json_fd]} + kwargs['startupinfo'] = startupinfo if USE_PROCESS_GROUP: - kw['start_new_session'] = True - return subprocess.Popen(cmd, **kw) + kwargs['start_new_session'] = True + + if MS_WINDOWS: + os.set_handle_inheritable(json_fd, True) + try: + return subprocess.Popen(cmd, **kwargs) + finally: + if MS_WINDOWS: + os.set_handle_inheritable(json_fd, False) def worker_process(worker_json: StrJSON) -> NoReturn: runtests = RunTests.from_json(worker_json) test_name = runtests.tests[0] match_tests: FilterTuple | None = runtests.match_tests + json_fd: int = runtests.json_fd + + if MS_WINDOWS: + import msvcrt + json_fd = msvcrt.open_osfhandle(json_fd, os.O_WRONLY) + setup_test_dir(runtests.test_dir) setup_process() @@ -70,11 +89,10 @@ def worker_process(worker_json: StrJSON) -> NoReturn: print(f"Re-running {test_name} in verbose mode", flush=True) result = run_single_test(test_name, runtests) - print() # Force a newline (just in case) - # Serialize TestResult as dict in JSON - result.write_json(sys.stdout) - sys.stdout.flush() + with open(json_fd, 'w', encoding='utf-8') as json_file: + result.write_json_into(json_file) + sys.exit(0) diff --git a/Misc/NEWS.d/next/Tests/2023-09-11-18-19-52.gh-issue-109276.btfFtT.rst b/Misc/NEWS.d/next/Tests/2023-09-11-18-19-52.gh-issue-109276.btfFtT.rst new file mode 100644 index 00000000000000..5fcf6624f2e84d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-11-18-19-52.gh-issue-109276.btfFtT.rst @@ -0,0 +1,3 @@ +libregrtest now uses a separated file descriptor to write test result as JSON. +Previously, if a test wrote debug messages late around the JSON, the main test +process failed to parse JSON. Patch by Victor Stinner.