From 48f024b6d72daa9e9481f67b4c50f86795333f4c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 2 Sep 2024 02:08:45 +0800 Subject: [PATCH 01/46] skeleton --- Include/internal/pycore_optimizer.h | 1 + Makefile.pre.in | 11 +- Python/optimizer_analysis.c | 100 + Python/optimizer_symbols.c | 3 + Python/partial_evaluator_bytecodes.c | 283 +++ Python/partial_evaluator_cases.c.h | 1895 +++++++++++++++++ .../partial_evaluator_generator.py | 234 ++ 7 files changed, 2526 insertions(+), 1 deletion(-) create mode 100644 Python/partial_evaluator_bytecodes.c create mode 100644 Python/partial_evaluator_cases.c.h create mode 100644 Tools/cases_generator/partial_evaluator_generator.py diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 19e54bf122a8bb..2d648d79655954 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -152,6 +152,7 @@ struct _Py_UopsSymbol { PyTypeObject *typ; // Borrowed reference PyObject *const_val; // Owned reference (!) unsigned int type_version; // currently stores type version + bool is_static; // used for binding-time analysis }; #define UOP_FORMAT_TARGET 0 diff --git a/Makefile.pre.in b/Makefile.pre.in index 46733d0cb44f72..4f214eed60059e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1951,7 +1951,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c regen-cases: \ regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ regen-generated-cases regen-executor-cases regen-optimizer-cases \ - regen-opcode-metadata regen-uop-metadata + regen-partial-evaluator-cases regen-opcode-metadata regen-uop-metadata .PHONY: regen-opcode-ids regen-opcode-ids: @@ -1997,6 +1997,15 @@ regen-optimizer-cases: $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new +.PHONY: regen-partial-evaluator-cases +regen-partial-evaluator-cases: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/partial_evaluator_generator.py \ + -o $(srcdir)/Python/partial_evaluator_cases.c.h.new \ + $(srcdir)/Python/partial_evaluator_bytecodes.c \ + $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/partial_evaluator_cases.c.h $(srcdir)/Python/partial_evaluator_cases.c.h.new + + .PHONY: regen-opcode-metadata regen-opcode-metadata: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index f7adb44c9e09ef..caca5c397143c6 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -486,6 +486,106 @@ optimize_uops( } +/* 1 for success, 0 for not ready, cannot error at the moment. */ +static int +partial_evaluate_uops( + PyCodeObject *co, + _PyUOpInstruction *trace, + int trace_len, + int curr_stacklen, + _PyBloomFilter *dependencies +) +{ + + _Py_UOpsContext context; + _Py_UOpsContext *ctx = &context; + uint32_t opcode = UINT16_MAX; + int curr_space = 0; + int max_space = 0; + _PyUOpInstruction *first_valid_check_stack = NULL; + _PyUOpInstruction *corresponding_check_stack = NULL; + + _Py_uop_abstractcontext_init(ctx); + _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, curr_stacklen, NULL, 0); + if (frame == NULL) { + return -1; + } + ctx->curr_frame_depth++; + ctx->frame = frame; + ctx->done = false; + ctx->out_of_space = false; + ctx->contradiction = false; + + _PyUOpInstruction *this_instr = NULL; + for (int i = 0; !ctx->done; i++) { + assert(i < trace_len); + this_instr = &trace[i]; + + int oparg = this_instr->oparg; + opcode = this_instr->opcode; + _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; + +#ifdef Py_DEBUG + if (get_lltrace() >= 3) { + printf("%4d pe: ", (int)(this_instr - trace)); + _PyUOpPrint(this_instr); + printf(" "); + } +#endif + + switch (opcode) { + +#include "partial_evaluator_cases.c.h" + + default: + DPRINTF(1, "\nUnknown opcode in pe's abstract interpreter\n"); + Py_UNREACHABLE(); + } + assert(ctx->frame != NULL); + DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); + ctx->frame->stack_pointer = stack_pointer; + assert(STACK_LEVEL() >= 0); + } + if (ctx->out_of_space) { + DPRINTF(3, "\n"); + DPRINTF(1, "Out of space in pe's abstract interpreter\n"); + } + if (ctx->contradiction) { + // Attempted to push a "bottom" (contradiction) symbol onto the stack. + // This means that the abstract interpreter has hit unreachable code. + // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting + // bottom indicates type instability, so we are probably better off + // retrying later. + DPRINTF(3, "\n"); + DPRINTF(1, "Hit bottom in pe's abstract interpreter\n"); + _Py_uop_abstractcontext_fini(ctx); + return 0; + } + + /* Either reached the end or cannot optimize further, but there + * would be no benefit in retrying later */ + _Py_uop_abstractcontext_fini(ctx); + if (first_valid_check_stack != NULL) { + assert(first_valid_check_stack->opcode == _CHECK_STACK_SPACE); + assert(max_space > 0); + assert(max_space <= INT_MAX); + assert(max_space <= INT32_MAX); + first_valid_check_stack->opcode = _CHECK_STACK_SPACE_OPERAND; + first_valid_check_stack->operand = max_space; + } + return trace_len; + + error: + DPRINTF(3, "\n"); + DPRINTF(1, "Encountered error in pe's abstract interpreter\n"); + if (opcode <= MAX_UOP_ID) { + OPT_ERROR_IN_OPCODE(opcode); + } + _Py_uop_abstractcontext_fini(ctx); + return -1; + +} + static int remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) { diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 40cbf95e3d6d39..9b0be091c5666d 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -77,6 +77,7 @@ sym_new(_Py_UOpsContext *ctx) self->typ = NULL; self->const_val = NULL; self->type_version = 0; + self->is_static = false; return self; } @@ -187,6 +188,7 @@ _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const sym->typ = typ; sym->const_val = Py_NewRef(const_val); } + sym->is_static = true; } void @@ -196,6 +198,7 @@ _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) sym_set_bottom(ctx, sym); } sym_set_flag(sym, IS_NULL); + sym->is_static = true; } void diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c new file mode 100644 index 00000000000000..30c20537afcdce --- /dev/null +++ b/Python/partial_evaluator_bytecodes.c @@ -0,0 +1,283 @@ +#include "Python.h" +#include "pycore_optimizer.h" +#include "pycore_uops.h" +#include "pycore_uop_ids.h" +#include "internal/pycore_moduleobject.h" + +#define op(name, ...) /* NAME is ignored */ + +typedef struct _Py_UopsSymbol _Py_UopsSymbol; +typedef struct _Py_UOpsContext _Py_UOpsContext; +typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; + +/* Shortened forms for convenience */ +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_matches_type_version _Py_uop_sym_matches_type_version +#define sym_get_type _Py_uop_sym_get_type +#define sym_has_type _Py_uop_sym_has_type +#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) +#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) +#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_is_bottom _Py_uop_sym_is_bottom +#define frame_new _Py_uop_frame_new +#define frame_pop _Py_uop_frame_pop + +extern int +optimize_to_bool( + _PyUOpInstruction *this_instr, + _Py_UOpsContext *ctx, + _Py_UopsSymbol *value, + _Py_UopsSymbol **result_ptr); + +extern void +eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit); + +extern PyCodeObject *get_code(_PyUOpInstruction *op); + +static int +dummy_func(void) { + + PyCodeObject *co; + int oparg; + _Py_UopsSymbol *flag; + _Py_UopsSymbol *left; + _Py_UopsSymbol *right; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + _Py_UopsSymbol *iter; + _Py_UopsSymbol *top; + _Py_UopsSymbol *bottom; + _Py_UOpsAbstractFrame *frame; + _Py_UOpsAbstractFrame *new_frame; + _Py_UOpsContext *ctx; + _PyUOpInstruction *this_instr; + +// BEGIN BYTECODES // + + op(_LOAD_FAST_CHECK, (-- value)) { + value = GETLOCAL(oparg); + // We guarantee this will error - just bail and don't optimize it. + if (sym_is_null(value)) { + ctx->done = true; + } + } + + op(_LOAD_FAST, (-- value)) { + value = GETLOCAL(oparg); + } + + op(_LOAD_FAST_AND_CLEAR, (-- value)) { + value = GETLOCAL(oparg); + _Py_UopsSymbol *temp = sym_new_null(ctx); + GETLOCAL(oparg) = temp; + } + + op(_STORE_FAST, (value --)) { + GETLOCAL(oparg) = value; + } + + op(_PUSH_NULL, (-- res)) { + res = sym_new_null(ctx); + } + + op(_LOAD_CONST, (-- value)) { + // Should never happen. This should be run after the specializer pass. + Py_UNREACHABLE(); + } + + op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + value = sym_new_const(ctx, ptr); + } + + op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + value = sym_new_const(ctx, ptr); + } + + op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); + } + + op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); + } + + op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { + assert(oparg > 0); + top = bottom; + } + + op(_SWAP, (bottom, unused[oparg-2], top -- + top, unused[oparg-2], bottom)) { + } + + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + int argcount = oparg; + + (void)callable; + + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + uint64_t push_operand = (this_instr + 2)->operand; + if (push_operand & 1) { + co = (PyCodeObject *)(push_operand & ~1); + DPRINTF(3, "code=%p ", co); + assert(PyCode_Check(co)); + } + else { + PyFunctionObject *func = (PyFunctionObject *)push_operand; + DPRINTF(3, "func=%p ", func); + if (func == NULL) { + DPRINTF(3, "\n"); + DPRINTF(1, "Missing function\n"); + ctx->done = true; + break; + } + co = (PyCodeObject *)func->func_code; + DPRINTF(3, "code=%p ", co); + } + + assert(self_or_null != NULL); + assert(args != NULL); + new_frame = frame_new(ctx, co, 0, NULL, 0); + } + + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ + (void)callable; + (void)self_or_null; + (void)args; + new_frame = NULL; + ctx->done = true; + } + + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { + (void)callable; + (void)self_or_null; + (void)args; + (void)kwnames; + new_frame = NULL; + ctx->done = true; + } + + op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { + (void)self; + (void)init; + (void)args; + init_frame = NULL; + ctx->done = true; + } + + op(_RETURN_VALUE, (retval -- res)) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = retval; + + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + } + + op(_RETURN_GENERATOR, ( -- res)) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = sym_new_unknown(ctx); + + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + } + + op(_YIELD_VALUE, (unused -- res)) { + res = sym_new_unknown(ctx); + } + + op(_FOR_ITER_GEN_FRAME, ( -- )) { + /* We are about to hit the end of the trace */ + ctx->done = true; + } + + op(_SEND_GEN_FRAME, ( -- )) { + // We are about to hit the end of the trace: + ctx->done = true; + } + + op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + ctx->frame = new_frame; + ctx->curr_frame_depth++; + stack_pointer = new_frame->stack_pointer; + co = get_code(this_instr); + if (co == NULL) { + // should be about to _EXIT_TRACE anyway + ctx->done = true; + break; + } + } + + op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { + /* This has to be done manually */ + (void)seq; + for (int i = 0; i < oparg; i++) { + values[i] = sym_new_unknown(ctx); + } + } + + op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8])) { + /* This has to be done manually */ + (void)seq; + int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; + for (int i = 0; i < totalargs; i++) { + values[i] = sym_new_unknown(ctx); + } + } + + op(_JUMP_TO_TOP, (--)) { + ctx->done = true; + } + + op(_EXIT_TRACE, (exit_p/4 --)) { + (void)exit_p; + ctx->done = true; + } + +// END BYTECODES // + +} diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h new file mode 100644 index 00000000000000..3b05eadd290111 --- /dev/null +++ b/Python/partial_evaluator_cases.c.h @@ -0,0 +1,1895 @@ +// This file is generated by Tools/cases_generator/partial_evaluator_generator.py +// from: +// Python/partial_evaluator_bytecodes.c +// Do not edit! + + case _NOP: { + break; + } + + case _CHECK_PERIODIC: { + break; + } + + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { + break; + } + + /* _QUICKEN_RESUME is not a viable micro-op for tier 2 */ + + case _RESUME_CHECK: { + break; + } + + /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ + + case _LOAD_FAST_CHECK: { + _Py_UopsSymbol *value; + value = GETLOCAL(oparg); + // We guarantee this will error - just bail and don't optimize it. + if (sym_is_null(value)) { + ctx->done = true; + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST: { + _Py_UopsSymbol *value; + value = GETLOCAL(oparg); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST_AND_CLEAR: { + _Py_UopsSymbol *value; + value = GETLOCAL(oparg); + _Py_UopsSymbol *temp = sym_new_null(ctx); + GETLOCAL(oparg) = temp; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST: { + _Py_UopsSymbol *value; + // Should never happen. This should be run after the specializer pass. + Py_UNREACHABLE(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_FAST: { + _Py_UopsSymbol *value; + value = stack_pointer[-1]; + GETLOCAL(oparg) = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _POP_TOP: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _PUSH_NULL: { + _Py_UopsSymbol *res; + res = sym_new_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _END_SEND: { + _Py_UopsSymbol *value; + value = sym_new_not_null(ctx); + stack_pointer[-2] = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNARY_NEGATIVE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _UNARY_NOT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_BOOL: { + break; + } + + case _TO_BOOL_INT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_LIST: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_NONE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_STR: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _REPLACE_WITH_TRUE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _UNARY_INVERT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _GUARD_BOTH_INT: { + break; + } + + case _GUARD_NOS_INT: { + break; + } + + case _GUARD_TOS_INT: { + break; + } + + case _BINARY_OP_MULTIPLY_INT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_INT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_INT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_BOTH_FLOAT: { + break; + } + + case _GUARD_NOS_FLOAT: { + break; + } + + case _GUARD_TOS_FLOAT: { + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_FLOAT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_BOTH_UNICODE: { + break; + } + + case _BINARY_OP_ADD_UNICODE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_INPLACE_ADD_UNICODE: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SLICE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SLICE: { + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_LIST_INT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_STR_INT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_TUPLE_INT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_DICT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_CHECK_FUNC: { + break; + } + + case _BINARY_SUBSCR_INIT_CALL: { + _PyInterpreterFrame *new_frame; + new_frame = sym_new_not_null(ctx); + stack_pointer[-2] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LIST_APPEND: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _SET_ADD: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SUBSCR: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SUBSCR_LIST_INT: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SUBSCR_DICT: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_SUBSCR: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_INTRINSIC_1: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _CALL_INTRINSIC_2: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _RETURN_VALUE: { + _Py_UopsSymbol *retval; + _Py_UopsSymbol *res; + retval = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = retval; + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GET_AITER: { + _Py_UopsSymbol *iter; + iter = sym_new_not_null(ctx); + stack_pointer[-1] = iter; + break; + } + + case _GET_ANEXT: { + _Py_UopsSymbol *awaitable; + awaitable = sym_new_not_null(ctx); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GET_AWAITABLE: { + _Py_UopsSymbol *iter; + iter = sym_new_not_null(ctx); + stack_pointer[-1] = iter; + break; + } + + /* _SEND is not a viable micro-op for tier 2 */ + + case _SEND_GEN_FRAME: { + // We are about to hit the end of the trace: + ctx->done = true; + break; + } + + case _YIELD_VALUE: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + stack_pointer[-1] = res; + break; + } + + case _POP_EXCEPT: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_COMMON_CONSTANT: { + _Py_UopsSymbol *value; + value = sym_new_not_null(ctx); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_BUILD_CLASS: { + _Py_UopsSymbol *bc; + bc = sym_new_not_null(ctx); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_NAME: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_NAME: { + break; + } + + case _UNPACK_SEQUENCE: { + _Py_UopsSymbol *seq; + _Py_UopsSymbol **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + /* This has to be done manually */ + (void)seq; + for (int i = 0; i < oparg; i++) { + values[i] = sym_new_unknown(ctx); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNPACK_SEQUENCE_TWO_TUPLE: { + _Py_UopsSymbol *val1; + _Py_UopsSymbol *val0; + val1 = sym_new_not_null(ctx); + val0 = sym_new_not_null(ctx); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNPACK_SEQUENCE_TUPLE: { + _Py_UopsSymbol **values; + values = &stack_pointer[-1]; + for (int _i = oparg; --_i >= 0;) { + values[_i] = sym_new_not_null(ctx); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNPACK_SEQUENCE_LIST: { + _Py_UopsSymbol **values; + values = &stack_pointer[-1]; + for (int _i = oparg; --_i >= 0;) { + values[_i] = sym_new_not_null(ctx); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNPACK_EX: { + _Py_UopsSymbol *seq; + _Py_UopsSymbol **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + /* This has to be done manually */ + (void)seq; + int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; + for (int i = 0; i < totalargs; i++) { + values[i] = sym_new_unknown(ctx); + } + stack_pointer += (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_ATTR: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_ATTR: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_GLOBAL: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_GLOBAL: { + break; + } + + case _LOAD_LOCALS: { + _Py_UopsSymbol *locals; + locals = sym_new_not_null(ctx); + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ + + case _LOAD_NAME: { + _Py_UopsSymbol *v; + v = sym_new_not_null(ctx); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_GLOBAL: { + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_GLOBALS_VERSION: { + break; + } + + case _GUARD_BUILTINS_VERSION: { + break; + } + + case _LOAD_GLOBAL_MODULE: { + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_GLOBAL_BUILTINS: { + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_FAST: { + break; + } + + case _MAKE_CELL: { + break; + } + + case _DELETE_DEREF: { + break; + } + + case _LOAD_FROM_DICT_OR_DEREF: { + _Py_UopsSymbol *value; + value = sym_new_not_null(ctx); + stack_pointer[-1] = value; + break; + } + + case _LOAD_DEREF: { + _Py_UopsSymbol *value; + value = sym_new_not_null(ctx); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_DEREF: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COPY_FREE_VARS: { + break; + } + + case _BUILD_STRING: { + _Py_UopsSymbol *str; + str = sym_new_not_null(ctx); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_TUPLE: { + _Py_UopsSymbol *tup; + tup = sym_new_not_null(ctx); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_LIST: { + _Py_UopsSymbol *list; + list = sym_new_not_null(ctx); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LIST_EXTEND: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _SET_UPDATE: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_SET: { + _Py_UopsSymbol *set; + set = sym_new_not_null(ctx); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_MAP: { + _Py_UopsSymbol *map; + map = sym_new_not_null(ctx); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _SETUP_ANNOTATIONS: { + break; + } + + case _DICT_UPDATE: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DICT_MERGE: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MAP_ADD: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ + + case _LOAD_SUPER_ATTR_ATTR: { + _Py_UopsSymbol *attr_st; + attr_st = sym_new_not_null(ctx); + stack_pointer[-3] = attr_st; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_SUPER_ATTR_METHOD: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null; + attr = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null = NULL; + attr = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_TYPE_VERSION: { + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { + break; + } + + case _LOAD_ATTR_INSTANCE_VALUE: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_ATTR_MODULE: { + break; + } + + case _LOAD_ATTR_MODULE: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_ATTR_WITH_HINT: { + break; + } + + case _LOAD_ATTR_WITH_HINT: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR_SLOT: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_ATTR_CLASS: { + break; + } + + case _LOAD_ATTR_CLASS: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR_PROPERTY_FRAME: { + _PyInterpreterFrame *new_frame; + new_frame = sym_new_not_null(ctx); + stack_pointer[-1] = (_Py_UopsSymbol *)new_frame; + break; + } + + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ + + case _GUARD_DORV_NO_DICT: { + break; + } + + case _STORE_ATTR_INSTANCE_VALUE: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_ATTR_WITH_HINT: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_ATTR_SLOT: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COMPARE_OP: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COMPARE_OP_FLOAT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COMPARE_OP_INT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COMPARE_OP_STR: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _IS_OP: { + _Py_UopsSymbol *b; + b = sym_new_not_null(ctx); + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP: { + _Py_UopsSymbol *b; + b = sym_new_not_null(ctx); + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP_SET: { + _Py_UopsSymbol *b; + b = sym_new_not_null(ctx); + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP_DICT: { + _Py_UopsSymbol *b; + b = sym_new_not_null(ctx); + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_EG_MATCH: { + _Py_UopsSymbol *rest; + _Py_UopsSymbol *match; + rest = sym_new_not_null(ctx); + match = sym_new_not_null(ctx); + stack_pointer[-2] = rest; + stack_pointer[-1] = match; + break; + } + + case _CHECK_EXC_MATCH: { + _Py_UopsSymbol *b; + b = sym_new_not_null(ctx); + stack_pointer[-1] = b; + break; + } + + case _IMPORT_NAME: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _IMPORT_FROM: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + + case _IS_NONE: { + _Py_UopsSymbol *b; + b = sym_new_not_null(ctx); + stack_pointer[-1] = b; + break; + } + + case _GET_LEN: { + _Py_UopsSymbol *len; + len = sym_new_not_null(ctx); + stack_pointer[0] = len; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MATCH_CLASS: { + _Py_UopsSymbol *attrs; + attrs = sym_new_not_null(ctx); + stack_pointer[-3] = attrs; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MATCH_MAPPING: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MATCH_SEQUENCE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MATCH_KEYS: { + _Py_UopsSymbol *values_or_none; + values_or_none = sym_new_not_null(ctx); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GET_ITER: { + _Py_UopsSymbol *iter; + iter = sym_new_not_null(ctx); + stack_pointer[-1] = iter; + break; + } + + case _GET_YIELD_FROM_ITER: { + _Py_UopsSymbol *iter; + iter = sym_new_not_null(ctx); + stack_pointer[-1] = iter; + break; + } + + /* _FOR_ITER is not a viable micro-op for tier 2 */ + + case _FOR_ITER_TIER_TWO: { + _Py_UopsSymbol *next; + next = sym_new_not_null(ctx); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 */ + + case _ITER_CHECK_LIST: { + break; + } + + /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_EXHAUSTED_LIST: { + break; + } + + case _ITER_NEXT_LIST: { + _Py_UopsSymbol *next; + next = sym_new_not_null(ctx); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _ITER_CHECK_TUPLE: { + break; + } + + /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_EXHAUSTED_TUPLE: { + break; + } + + case _ITER_NEXT_TUPLE: { + _Py_UopsSymbol *next; + next = sym_new_not_null(ctx); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _ITER_CHECK_RANGE: { + break; + } + + /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_EXHAUSTED_RANGE: { + break; + } + + case _ITER_NEXT_RANGE: { + _Py_UopsSymbol *next; + next = sym_new_not_null(ctx); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _FOR_ITER_GEN_FRAME: { + /* We are about to hit the end of the trace */ + ctx->done = true; + break; + } + + case _LOAD_SPECIAL: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null; + attr = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); + stack_pointer[-1] = attr; + stack_pointer[0] = self_or_null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _WITH_EXCEPT_START: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _PUSH_EXC_INFO: { + _Py_UopsSymbol *prev_exc; + _Py_UopsSymbol *new_exc; + prev_exc = sym_new_not_null(ctx); + new_exc = sym_new_not_null(ctx); + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { + break; + } + + case _GUARD_KEYS_VERSION: { + break; + } + + case _LOAD_ATTR_METHOD_WITH_VALUES: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; + attr = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; + attr = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { + _Py_UopsSymbol *attr; + attr = sym_new_not_null(ctx); + stack_pointer[-1] = attr; + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { + _Py_UopsSymbol *attr; + attr = sym_new_not_null(ctx); + stack_pointer[-1] = attr; + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT: { + break; + } + + case _LOAD_ATTR_METHOD_LAZY_DICT: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; + attr = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MAYBE_EXPAND_METHOD: { + _Py_UopsSymbol *func; + _Py_UopsSymbol *maybe_self; + _Py_UopsSymbol **args; + args = &stack_pointer[-oparg]; + func = sym_new_not_null(ctx); + maybe_self = sym_new_not_null(ctx); + for (int _i = oparg; --_i >= 0;) { + args[_i] = sym_new_not_null(ctx); + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + break; + } + + /* _DO_CALL is not a viable micro-op for tier 2 */ + + /* _MONITOR_CALL is not a viable micro-op for tier 2 */ + + case _PY_FRAME_GENERAL: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UOpsAbstractFrame *new_frame; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ + (void)callable; + (void)self_or_null; + (void)args; + new_frame = NULL; + ctx->done = true; + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION: { + break; + } + + case _CHECK_METHOD_VERSION: { + break; + } + + case _EXPAND_METHOD: { + _Py_UopsSymbol *method; + _Py_UopsSymbol *self; + method = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = method; + stack_pointer[-1 - oparg] = self; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE: { + break; + } + + case _CALL_NON_PY_GENERAL: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { + break; + } + + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { + _Py_UopsSymbol *func; + _Py_UopsSymbol *self; + func = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = self; + break; + } + + case _CHECK_PEP_523: { + break; + } + + case _CHECK_FUNCTION_EXACT_ARGS: { + break; + } + + case _CHECK_STACK_SPACE: { + break; + } + + case _INIT_CALL_PY_EXACT_ARGS: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UOpsAbstractFrame *new_frame; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + (void)callable; + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + uint64_t push_operand = (this_instr + 2)->operand; + if (push_operand & 1) { + co = (PyCodeObject *)(push_operand & ~1); + DPRINTF(3, "code=%p ", co); + assert(PyCode_Check(co)); + } + else { + PyFunctionObject *func = (PyFunctionObject *)push_operand; + DPRINTF(3, "func=%p ", func); + if (func == NULL) { + DPRINTF(3, "\n"); + DPRINTF(1, "Missing function\n"); + ctx->done = true; + break; + } + co = (PyCodeObject *)func->func_code; + DPRINTF(3, "code=%p ", co); + } + assert(self_or_null != NULL); + assert(args != NULL); + new_frame = frame_new(ctx, co, 0, NULL, 0); + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _PUSH_FRAME: { + _Py_UOpsAbstractFrame *new_frame; + new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + ctx->frame->stack_pointer = stack_pointer; + ctx->frame = new_frame; + ctx->curr_frame_depth++; + stack_pointer = new_frame->stack_pointer; + co = get_code(this_instr); + if (co == NULL) { + // should be about to _EXIT_TRACE anyway + ctx->done = true; + break; + } + break; + } + + case _CALL_TYPE_1: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_STR_1: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_TUPLE_1: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_AND_ALLOCATE_OBJECT: { + _Py_UopsSymbol *self; + _Py_UopsSymbol *init; + _Py_UopsSymbol **args; + self = sym_new_not_null(ctx); + init = sym_new_not_null(ctx); + for (int _i = oparg; --_i >= 0;) { + args[_i] = sym_new_not_null(ctx); + } + stack_pointer[-2 - oparg] = self; + stack_pointer[-1 - oparg] = init; + break; + } + + case _CREATE_INIT_FRAME: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *init; + _Py_UopsSymbol *self; + _Py_UOpsAbstractFrame *init_frame; + args = &stack_pointer[-oparg]; + init = stack_pointer[-1 - oparg]; + self = stack_pointer[-2 - oparg]; + (void)self; + (void)init; + (void)args; + init_frame = NULL; + ctx->done = true; + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)init_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _EXIT_INIT_CHECK: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_CLASS: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_O: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_FAST: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_LEN: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_ISINSTANCE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_LIST_APPEND: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_O: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_NOARGS: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ + + /* _DO_CALL_KW is not a viable micro-op for tier 2 */ + + case _PY_FRAME_KW: { + _Py_UopsSymbol *kwnames; + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UOpsAbstractFrame *new_frame; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + (void)callable; + (void)self_or_null; + (void)args; + (void)kwnames; + new_frame = NULL; + ctx->done = true; + stack_pointer[-3 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION_KW: { + break; + } + + case _CHECK_METHOD_VERSION_KW: { + break; + } + + case _EXPAND_METHOD_KW: { + _Py_UopsSymbol *method; + _Py_UopsSymbol *self; + _Py_UopsSymbol *kwnames; + method = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); + kwnames = sym_new_not_null(ctx); + stack_pointer[-3 - oparg] = method; + stack_pointer[-2 - oparg] = self; + stack_pointer[-1] = kwnames; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_KW: { + break; + } + + case _CALL_KW_NON_PY: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + case _MAKE_FUNCTION: { + _Py_UopsSymbol *func; + func = sym_new_not_null(ctx); + stack_pointer[-1] = func; + break; + } + + case _SET_FUNCTION_ATTRIBUTE: { + _Py_UopsSymbol *func_st; + func_st = sym_new_not_null(ctx); + stack_pointer[-2] = func_st; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _RETURN_GENERATOR: { + _Py_UopsSymbol *res; + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = sym_new_unknown(ctx); + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_SLICE: { + _Py_UopsSymbol *slice; + slice = sym_new_not_null(ctx); + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONVERT_VALUE: { + _Py_UopsSymbol *result; + result = sym_new_not_null(ctx); + stack_pointer[-1] = result; + break; + } + + case _FORMAT_SIMPLE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _FORMAT_WITH_SPEC: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COPY: { + _Py_UopsSymbol *bottom; + _Py_UopsSymbol *top; + bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); + top = bottom; + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _SWAP: { + _Py_UopsSymbol *top; + _Py_UopsSymbol *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + stack_pointer[-2 - (oparg-2)] = top; + stack_pointer[-1] = bottom; + break; + } + + /* _INSTRUMENTED_LINE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ + + /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ + + case _GUARD_IS_TRUE_POP: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_IS_FALSE_POP: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_IS_NONE_POP: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_IS_NOT_NONE_POP: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _JUMP_TO_TOP: { + ctx->done = true; + break; + } + + case _SET_IP: { + break; + } + + case _CHECK_STACK_SPACE_OPERAND: { + break; + } + + case _SAVE_RETURN_OFFSET: { + break; + } + + case _EXIT_TRACE: { + PyObject *exit_p = (PyObject *)this_instr->operand; + (void)exit_p; + ctx->done = true; + break; + } + + case _CHECK_VALIDITY: { + break; + } + + case _LOAD_CONST_INLINE: { + _Py_UopsSymbol *value; + PyObject *ptr = (PyObject *)this_instr->operand; + value = sym_new_const(ctx, ptr); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST_INLINE_BORROW: { + _Py_UopsSymbol *value; + PyObject *ptr = (PyObject *)this_instr->operand; + value = sym_new_const(ctx, ptr); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _POP_TOP_LOAD_CONST_INLINE_BORROW: { + _Py_UopsSymbol *value; + value = sym_new_not_null(ctx); + stack_pointer[-1] = value; + break; + } + + case _LOAD_CONST_INLINE_WITH_NULL: { + _Py_UopsSymbol *value; + _Py_UopsSymbol *null; + PyObject *ptr = (PyObject *)this_instr->operand; + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { + _Py_UopsSymbol *value; + _Py_UopsSymbol *null; + PyObject *ptr = (PyObject *)this_instr->operand; + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION: { + break; + } + + case _INTERNAL_INCREMENT_OPT_COUNTER: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DYNAMIC_EXIT: { + break; + } + + case _START_EXECUTOR: { + break; + } + + case _FATAL_ERROR: { + break; + } + + case _CHECK_VALIDITY_AND_SET_IP: { + break; + } + + case _DEOPT: { + break; + } + + case _ERROR_POP_N: { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _TIER2_RESUME_CHECK: { + break; + } + diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py new file mode 100644 index 00000000000000..67a96cc899fdf1 --- /dev/null +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -0,0 +1,234 @@ +"""Generate the cases for the tier 2 optimizer. +Reads the instruction definitions from bytecodes.c and optimizer_bytecodes.c +Writes the cases to optimizer_cases.c.h, which is #included in Python/optimizer_analysis.c. +""" + +import argparse + +from analyzer import ( + Analysis, + Instruction, + Uop, + analyze_files, + StackItem, + analysis_error, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + Emitter, +) +from cwriter import CWriter +from typing import TextIO, Iterator +from lexer import Token +from stack import Local, Stack, StackError + +DEFAULT_OUTPUT = ROOT / "Python/partial_evaluator_cases.c.h" +DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/partial_evaluator_bytecodes.c").absolute().as_posix() + + +def validate_uop(override: Uop, uop: Uop) -> None: + # To do + pass + + +def type_name(var: StackItem) -> str: + if var.is_array(): + return f"_Py_UopsSymbol **" + if var.type: + return var.type + return f"_Py_UopsSymbol *" + + +def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: + variables = {"unused"} + if not skip_inputs: + for var in reversed(uop.stack.inputs): + if var.name not in variables: + variables.add(var.name) + if var.condition: + out.emit(f"{type_name(var)}{var.name} = NULL;\n") + else: + out.emit(f"{type_name(var)}{var.name};\n") + for var in uop.stack.outputs: + if var.peek: + continue + if var.name not in variables: + variables.add(var.name) + if var.condition: + out.emit(f"{type_name(var)}{var.name} = NULL;\n") + else: + out.emit(f"{type_name(var)}{var.name};\n") + + +def decref_inputs( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + out.emit_at("", tkn) + + +def emit_default(out: CWriter, uop: Uop) -> None: + for i, var in enumerate(uop.stack.outputs): + if var.name != "unused" and not var.peek: + if var.is_array(): + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") + out.emit("}\n") + elif var.name == "null": + out.emit(f"{var.name} = sym_new_null(ctx);\n") + else: + out.emit(f"{var.name} = sym_new_not_null(ctx);\n") + + +class OptimizerEmitter(Emitter): + pass + + +def write_uop( + override: Uop | None, + uop: Uop, + out: CWriter, + stack: Stack, + debug: bool, + skip_inputs: bool, +) -> None: + locals: dict[str, Local] = {} + try: + prototype = override if override else uop + is_override = override is not None + out.start_line() + for var in reversed(prototype.stack.inputs): + code, local = stack.pop(var, extract_bits=True) + if not skip_inputs: + out.emit(code) + if local.defined: + locals[local.name] = local + out.emit(stack.define_output_arrays(prototype.stack.outputs)) + if debug: + args = [] + for var in prototype.stack.inputs: + if not var.peek or is_override: + args.append(var.name) + out.emit(f'DEBUG_PRINTF({", ".join(args)});\n') + if override: + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = cast = "PyObject *" + else: + type = f"uint{cache.size*16}_t " + cast = f"uint{cache.size*16}_t" + out.emit(f"{type}{cache.name} = ({cast})this_instr->operand;\n") + if override: + emitter = OptimizerEmitter(out) + emitter.emit_tokens(override, stack, None) + else: + emit_default(out, uop) + + for var in prototype.stack.outputs: + if var.name in locals: + local = locals[var.name] + else: + local = Local.local(var) + stack.push(local) + out.start_line() + stack.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=True) + except StackError as ex: + raise analysis_error(ex.args[0], uop.body[0]) + + +SKIPS = ("_EXTENDED_ARG",) + + +def generate_abstract_interpreter( + filenames: list[str], + abstract: Analysis, + base: Analysis, + outfile: TextIO, + debug: bool, +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 2, False) + out.emit("\n") + base_uop_names = set([uop.name for uop in base.uops.values()]) + for abstract_uop_name in abstract.uops: + assert ( + abstract_uop_name in base_uop_names + ), f"All abstract uops should override base uops, but {abstract_uop_name} is not." + + for uop in base.uops.values(): + override: Uop | None = None + if uop.name in abstract.uops: + override = abstract.uops[uop.name] + validate_uop(override, uop) + if uop.properties.tier == 1: + continue + if uop.replicates: + continue + if uop.is_super(): + continue + if not uop.is_viable(): + out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n") + continue + out.emit(f"case {uop.name}: {{\n") + if override: + declare_variables(override, out, skip_inputs=False) + else: + declare_variables(uop, out, skip_inputs=True) + stack = Stack() + write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) + out.start_line() + out.emit("break;\n") + out.emit("}") + out.emit("\n\n") + + +def generate_tier2_abstract_from_files( + filenames: list[str], outfilename: str, debug: bool = False +) -> None: + assert len(filenames) == 2, "Need a base file and an abstract cases file." + base = analyze_files([filenames[0]]) + abstract = analyze_files([filenames[1]]) + with open(outfilename, "w") as outfile: + generate_abstract_interpreter(filenames, abstract, base, outfile, debug) + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the tier 2 interpreter.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + + +arg_parser.add_argument("input", nargs="*", help="Abstract interpreter definition file") + +arg_parser.add_argument( + "base", nargs="*", help="The base instruction definition file(s)" +) + +arg_parser.add_argument("-d", "--debug", help="Insert debug calls", action="store_true") + +if __name__ == "__main__": + args = arg_parser.parse_args() + if not args.input: + args.base.append(DEFAULT_INPUT) + args.input.append(DEFAULT_ABSTRACT_INPUT) + else: + args.base.append(args.input[-1]) + args.input.pop() + abstract = analyze_files(args.input) + base = analyze_files(args.base) + with open(args.output, "w") as outfile: + generate_abstract_interpreter(args.input, abstract, base, outfile, args.debug) From dd5cfe79f145907f64559ec1640491ebfc8b86e6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 3 Sep 2024 02:23:10 +0800 Subject: [PATCH 02/46] generate the pe from the base optimizer --- Include/internal/pycore_optimizer.h | 2 + Makefile.pre.in | 1 + Python/optimizer_analysis.c | 19 +- Python/optimizer_symbols.c | 11 + Python/partial_evaluator_bytecodes.c | 196 +----- Python/partial_evaluator_cases.c.h | 589 ++++++++++++++++-- .../partial_evaluator_generator.py | 5 +- 7 files changed, 580 insertions(+), 243 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 2d648d79655954..54a08de252b4d0 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -153,6 +153,7 @@ struct _Py_UopsSymbol { PyObject *const_val; // Owned reference (!) unsigned int type_version; // currently stores type version bool is_static; // used for binding-time analysis + int locals_idx; }; #define UOP_FORMAT_TARGET 0 @@ -242,6 +243,7 @@ extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); extern bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym); extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); extern bool _Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version); +extern void _Py_uop_sym_set_locals_idx(_Py_UopsSymbol *sym, int locals_idx); extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ); diff --git a/Makefile.pre.in b/Makefile.pre.in index 4f214eed60059e..4c72011a07b358 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2001,6 +2001,7 @@ regen-optimizer-cases: regen-partial-evaluator-cases: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/partial_evaluator_generator.py \ -o $(srcdir)/Python/partial_evaluator_cases.c.h.new \ + $(srcdir)/Python/optimizer_bytecodes.c \ $(srcdir)/Python/partial_evaluator_bytecodes.c \ $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/partial_evaluator_cases.c.h $(srcdir)/Python/partial_evaluator_cases.c.h.new diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index caca5c397143c6..b871010ee6a141 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -324,6 +324,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) #define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_set_locals_idx _Py_uop_sym_set_locals_idx #define sym_is_bottom _Py_uop_sym_is_bottom #define sym_truthiness _Py_uop_sym_truthiness #define frame_new _Py_uop_frame_new @@ -565,17 +566,9 @@ partial_evaluate_uops( /* Either reached the end or cannot optimize further, but there * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); - if (first_valid_check_stack != NULL) { - assert(first_valid_check_stack->opcode == _CHECK_STACK_SPACE); - assert(max_space > 0); - assert(max_space <= INT_MAX); - assert(max_space <= INT32_MAX); - first_valid_check_stack->opcode = _CHECK_STACK_SPACE_OPERAND; - first_valid_check_stack->operand = max_space; - } return trace_len; - error: +error: DPRINTF(3, "\n"); DPRINTF(1, "Encountered error in pe's abstract interpreter\n"); if (opcode <= MAX_UOP_ID) { @@ -698,6 +691,14 @@ _Py_uop_analyze_and_optimize( return length; } + length = partial_evaluate_uops( + _PyFrame_GetCode(frame), buffer, + length, curr_stacklen, dependencies); + + if (length <= 0) { + return length; + } + length = remove_unneeded_uops(buffer, length); assert(length > 0); diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 9b0be091c5666d..3962ced2dbecd2 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -78,6 +78,7 @@ sym_new(_Py_UOpsContext *ctx) self->const_val = NULL; self->type_version = 0; self->is_static = false; + self->locals_idx = -1; return self; } @@ -300,6 +301,12 @@ _Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version) return _Py_uop_sym_get_type_version(sym) == version; } +void +_Py_uop_sym_set_locals_idx(_Py_UopsSymbol *sym, int locals_idx) +{ + assert(locals_idx >= 0); + sym->locals_idx = locals_idx; +} int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym) @@ -370,6 +377,10 @@ _Py_uop_frame_new( frame->locals[i] = local; } + for (int i = 0; i < co->co_nlocalsplus; i++) { + frame->locals[i]->locals_idx = i; + } + // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index 30c20537afcdce..2364db07f342fa 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -65,7 +65,7 @@ dummy_func(void) { // BEGIN BYTECODES // - op(_LOAD_FAST_CHECK, (-- value)) { + override op(_LOAD_FAST_CHECK, (-- value)) { value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. if (sym_is_null(value)) { @@ -73,209 +73,31 @@ dummy_func(void) { } } - op(_LOAD_FAST, (-- value)) { + override op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); } - op(_LOAD_FAST_AND_CLEAR, (-- value)) { + override op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); _Py_UopsSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; } - op(_STORE_FAST, (value --)) { + override op(_STORE_FAST, (value --)) { GETLOCAL(oparg) = value; + sym_set_locals_idx(value, oparg); } - op(_PUSH_NULL, (-- res)) { + override op(_PUSH_NULL, (-- res)) { res = sym_new_null(ctx); } - op(_LOAD_CONST, (-- value)) { - // Should never happen. This should be run after the specializer pass. + override op(_LOAD_CONST, (-- value)) { + // Should've all been converted by specializer. Py_UNREACHABLE(); } - op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); - } - - op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); - } - - op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { - value = sym_new_const(ctx, ptr); - null = sym_new_null(ctx); - } - - op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { - value = sym_new_const(ctx, ptr); - null = sym_new_null(ctx); - } - - op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { - assert(oparg > 0); - top = bottom; - } - - op(_SWAP, (bottom, unused[oparg-2], top -- - top, unused[oparg-2], bottom)) { - } - - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { - int argcount = oparg; - - (void)callable; - - PyCodeObject *co = NULL; - assert((this_instr + 2)->opcode == _PUSH_FRAME); - uint64_t push_operand = (this_instr + 2)->operand; - if (push_operand & 1) { - co = (PyCodeObject *)(push_operand & ~1); - DPRINTF(3, "code=%p ", co); - assert(PyCode_Check(co)); - } - else { - PyFunctionObject *func = (PyFunctionObject *)push_operand; - DPRINTF(3, "func=%p ", func); - if (func == NULL) { - DPRINTF(3, "\n"); - DPRINTF(1, "Missing function\n"); - ctx->done = true; - break; - } - co = (PyCodeObject *)func->func_code; - DPRINTF(3, "code=%p ", co); - } - - assert(self_or_null != NULL); - assert(args != NULL); - new_frame = frame_new(ctx, co, 0, NULL, 0); - } - - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { - /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ - (void)callable; - (void)self_or_null; - (void)args; - new_frame = NULL; - ctx->done = true; - } - - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { - (void)callable; - (void)self_or_null; - (void)args; - (void)kwnames; - new_frame = NULL; - ctx->done = true; - } - - op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { - (void)self; - (void)init; - (void)args; - init_frame = NULL; - ctx->done = true; - } - - op(_RETURN_VALUE, (retval -- res)) { - SYNC_SP(); - ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); - stack_pointer = ctx->frame->stack_pointer; - res = retval; - - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; - - co = get_code(this_instr); - if (co == NULL) { - // might be impossible, but bailing is still safe - ctx->done = true; - } - } - - op(_RETURN_GENERATOR, ( -- res)) { - SYNC_SP(); - ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); - stack_pointer = ctx->frame->stack_pointer; - res = sym_new_unknown(ctx); - - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; - - co = get_code(this_instr); - if (co == NULL) { - // might be impossible, but bailing is still safe - ctx->done = true; - } - } - - op(_YIELD_VALUE, (unused -- res)) { - res = sym_new_unknown(ctx); - } - - op(_FOR_ITER_GEN_FRAME, ( -- )) { - /* We are about to hit the end of the trace */ - ctx->done = true; - } - - op(_SEND_GEN_FRAME, ( -- )) { - // We are about to hit the end of the trace: - ctx->done = true; - } - - op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { - SYNC_SP(); - ctx->frame->stack_pointer = stack_pointer; - ctx->frame = new_frame; - ctx->curr_frame_depth++; - stack_pointer = new_frame->stack_pointer; - co = get_code(this_instr); - if (co == NULL) { - // should be about to _EXIT_TRACE anyway - ctx->done = true; - break; - } - } - - op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { - /* This has to be done manually */ - (void)seq; - for (int i = 0; i < oparg; i++) { - values[i] = sym_new_unknown(ctx); - } - } - - op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8])) { - /* This has to be done manually */ - (void)seq; - int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; - for (int i = 0; i < totalargs; i++) { - values[i] = sym_new_unknown(ctx); - } - } - - op(_JUMP_TO_TOP, (--)) { - ctx->done = true; - } - - op(_EXIT_TRACE, (exit_p/4 --)) { - (void)exit_p; - ctx->done = true; + override op (_CHECK_STACK_SPACE_OPERAND, ( -- )) { } // END BYTECODES // diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index 3b05eadd290111..ddf5f18ee054fa 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -1,6 +1,6 @@ // This file is generated by Tools/cases_generator/partial_evaluator_generator.py // from: -// Python/partial_evaluator_bytecodes.c +// Python/optimizer_bytecodes.c, Python/partial_evaluator_bytecodes.c // Do not edit! case _NOP: { @@ -58,7 +58,7 @@ case _LOAD_CONST: { _Py_UopsSymbol *value; - // Should never happen. This should be run after the specializer pass. + // Should've all been converted by specializer. Py_UNREACHABLE(); stack_pointer[0] = value; stack_pointer += 1; @@ -70,6 +70,7 @@ _Py_UopsSymbol *value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; + sym_set_locals_idx(value, oparg); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -114,40 +115,72 @@ } case _TO_BOOL: { + _Py_UopsSymbol *value; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); + } stack_pointer[-1] = res; break; } case _TO_BOOL_BOOL: { + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyBool_Type); + res = value; + } + stack_pointer[-1] = res; break; } case _TO_BOOL_INT: { + _Py_UopsSymbol *value; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyLong_Type); + res = sym_new_type(ctx, &PyBool_Type); + } stack_pointer[-1] = res; break; } case _TO_BOOL_LIST: { + _Py_UopsSymbol *value; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyList_Type); + res = sym_new_type(ctx, &PyBool_Type); + } stack_pointer[-1] = res; break; } case _TO_BOOL_NONE: { + _Py_UopsSymbol *value; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_const(value, Py_None); + res = sym_new_const(ctx, Py_False); + } stack_pointer[-1] = res; break; } case _TO_BOOL_STR: { + _Py_UopsSymbol *value; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); + sym_set_type(value, &PyUnicode_Type); + } stack_pointer[-1] = res; break; } @@ -167,6 +200,25 @@ } case _GUARD_BOTH_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyLong_Type)) { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_INT, 0, 0); + } + } + else { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0); + } + } + sym_set_type(left, &PyLong_Type); + sym_set_type(right, &PyLong_Type); break; } @@ -179,8 +231,29 @@ } case _BINARY_OP_MULTIPLY_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + res = sym_new_type(ctx, &PyLong_Type); + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -188,8 +261,29 @@ } case _BINARY_OP_ADD_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + res = sym_new_type(ctx, &PyLong_Type); + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -197,8 +291,29 @@ } case _BINARY_OP_SUBTRACT_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + res = sym_new_type(ctx, &PyLong_Type); + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -206,6 +321,25 @@ } case _GUARD_BOTH_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyFloat_Type)) { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_FLOAT, 0, 0); + } + } + else { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0); + } + } + sym_set_type(left, &PyFloat_Type); + sym_set_type(right, &PyFloat_Type); break; } @@ -218,8 +352,30 @@ } case _BINARY_OP_MULTIPLY_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) * + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -227,8 +383,30 @@ } case _BINARY_OP_ADD_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) + + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -236,8 +414,30 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) - + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -245,12 +445,37 @@ } case _GUARD_BOTH_UNICODE: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyUnicode_Type) && + sym_matches_type(right, &PyUnicode_Type)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + sym_set_type(left, &PyUnicode_Type); + sym_set_type(left, &PyUnicode_Type); break; } case _BINARY_OP_ADD_UNICODE: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { + PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + } + else { + res = sym_new_type(ctx, &PyUnicode_Type); + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -328,8 +553,15 @@ } case _BINARY_SUBSCR_INIT_CALL: { - _PyInterpreterFrame *new_frame; - new_frame = sym_new_not_null(ctx); + _Py_UopsSymbol *sub; + _Py_UopsSymbol *container; + _Py_UOpsAbstractFrame *new_frame; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + (void)container; + (void)sub; + new_frame = NULL; + ctx->done = true; stack_pointer[-2] = (_Py_UopsSymbol *)new_frame; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -780,10 +1012,15 @@ } case _LOAD_ATTR: { + _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *self_or_null = NULL; + owner = stack_pointer[-1]; + (void)owner; attr = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + if (oparg & 1) { + self_or_null = sym_new_unknown(ctx); + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; stack_pointer += (oparg & 1); @@ -792,6 +1029,28 @@ } case _GUARD_TYPE_VERSION: { + _Py_UopsSymbol *owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)this_instr->operand; + assert(type_version); + if (sym_matches_type_version(owner, type_version)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } else { + // add watcher so that whenever the type changes we invalidate this + PyTypeObject *type = _PyType_LookupByVersion(type_version); + // if the type is null, it was not found in the cache (there was a conflict) + // with the key, in which case we can't trust the version + if (type) { + // if the type version was set properly, then add a watcher + // if it wasn't this means that the type version was previously set to something else + // and we set the owner to bottom, so we don't need to add a watcher because we must have + // already added one earlier. + if (sym_set_type_version(owner, type_version)) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); + _Py_BloomFilter_Add(dependencies, type); + } + } + } break; } @@ -800,10 +1059,15 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { + _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + uint16_t offset = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); + (void)offset; + (void)owner; stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -812,14 +1076,51 @@ } case _CHECK_ATTR_MODULE: { + _Py_UopsSymbol *owner; + owner = stack_pointer[-1]; + uint32_t dict_version = (uint32_t)this_instr->operand; + (void)dict_version; + if (sym_is_const(owner)) { + PyObject *cnst = sym_get_const(owner); + if (PyModule_CheckExact(cnst)) { + PyModuleObject *mod = (PyModuleObject *)cnst; + PyObject *dict = mod->md_dict; + uint64_t watched_mutations = get_mutations(dict); + if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + PyDict_Watch(GLOBALS_WATCHER_ID, dict); + _Py_BloomFilter_Add(dependencies, dict); + this_instr->opcode = _NOP; + } + } + } break; } case _LOAD_ATTR_MODULE: { + _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; - attr = sym_new_not_null(ctx); + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)this_instr->operand; + (void)index; null = sym_new_null(ctx); + attr = NULL; + if (this_instr[-1].opcode == _NOP) { + // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. + assert(sym_is_const(owner)); + PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + assert(PyModule_CheckExact(mod)); + PyObject *dict = mod->md_dict; + PyObject *res = convert_global_to_const(this_instr, dict); + if (res != NULL) { + this_instr[-1].opcode = _POP_TOP; + attr = sym_new_const(ctx, res); + } + } + if (attr == NULL) { + /* No conversion made. We don't know what `attr` is. */ + attr = sym_new_not_null(ctx); + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -832,10 +1133,15 @@ } case _LOAD_ATTR_WITH_HINT: { + _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + uint16_t hint = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); + (void)hint; + (void)owner; stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -844,10 +1150,15 @@ } case _LOAD_ATTR_SLOT: { + _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); + (void)index; + (void)owner; stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -860,10 +1171,15 @@ } case _LOAD_ATTR_CLASS: { + _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); + (void)descr; + (void)owner; stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -872,8 +1188,14 @@ } case _LOAD_ATTR_PROPERTY_FRAME: { - _PyInterpreterFrame *new_frame; - new_frame = sym_new_not_null(ctx); + _Py_UopsSymbol *owner; + _Py_UOpsAbstractFrame *new_frame; + owner = stack_pointer[-1]; + PyObject *fget = (PyObject *)this_instr->operand; + (void)fget; + (void)owner; + new_frame = NULL; + ctx->done = true; stack_pointer[-1] = (_Py_UopsSymbol *)new_frame; break; } @@ -903,8 +1225,19 @@ } case _COMPARE_OP: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + if (oparg & 16) { + res = sym_new_type(ctx, &PyBool_Type); + } + else { + res = _Py_uop_sym_new_not_null(ctx); + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -912,8 +1245,14 @@ } case _COMPARE_OP_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -921,8 +1260,14 @@ } case _COMPARE_OP_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -930,8 +1275,14 @@ } case _COMPARE_OP_STR: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -939,18 +1290,30 @@ } case _IS_OP: { - _Py_UopsSymbol *b; - b = sym_new_not_null(ctx); - stack_pointer[-2] = b; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _CONTAINS_OP: { - _Py_UopsSymbol *b; - b = sym_new_not_null(ctx); - stack_pointer[-2] = b; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1141,8 +1504,11 @@ } case _ITER_NEXT_RANGE: { + _Py_UopsSymbol *iter; _Py_UopsSymbol *next; - next = sym_new_not_null(ctx); + iter = stack_pointer[-1]; + next = sym_new_type(ctx, &PyLong_Type); + (void)iter; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1156,10 +1522,13 @@ } case _LOAD_SPECIAL: { + _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *self_or_null; + owner = stack_pointer[-1]; + (void)owner; attr = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + self_or_null = sym_new_unknown(ctx); stack_pointer[-1] = attr; stack_pointer[0] = self_or_null; stack_pointer += 1; @@ -1197,10 +1566,14 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { + _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *self = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; attr = sym_new_not_null(ctx); - self = sym_new_not_null(ctx); + self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; @@ -1209,10 +1582,14 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { + _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *self = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; attr = sym_new_not_null(ctx); - self = sym_new_not_null(ctx); + self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; @@ -1239,10 +1616,14 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { + _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *self = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; attr = sym_new_not_null(ctx); - self = sym_new_not_null(ctx); + self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; @@ -1251,15 +1632,20 @@ } case _MAYBE_EXPAND_METHOD: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; _Py_UopsSymbol *func; _Py_UopsSymbol *maybe_self; - _Py_UopsSymbol **args; args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-oparg]; + (void)callable; + (void)self_or_null; + (void)args; func = sym_new_not_null(ctx); maybe_self = sym_new_not_null(ctx); - for (int _i = oparg; --_i >= 0;) { - args[_i] = sym_new_not_null(ctx); - } stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = maybe_self; break; @@ -1321,12 +1707,21 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { + _Py_UopsSymbol *null; + _Py_UopsSymbol *callable; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); break; } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { + _Py_UopsSymbol *callable; _Py_UopsSymbol *func; _Py_UopsSymbol *self; + callable = stack_pointer[-2 - oparg]; + (void)callable; func = sym_new_not_null(ctx); self = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = func; @@ -1335,14 +1730,27 @@ } case _CHECK_PEP_523: { + /* Setting the eval frame function invalidates + * all executors, so no need to check dynamically */ + if (_PyInterpreterState_GET()->eval_frame == NULL) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } break; } case _CHECK_FUNCTION_EXACT_ARGS: { + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + sym_set_type(callable, &PyFunction_Type); + (void)self_or_null; break; } case _CHECK_STACK_SPACE: { + assert(corresponding_check_stack == NULL); + corresponding_check_stack = this_instr; break; } @@ -1378,7 +1786,16 @@ } assert(self_or_null != NULL); assert(args != NULL); - new_frame = frame_new(ctx, co, 0, NULL, 0); + if (sym_is_not_null(self_or_null)) { + // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM + args--; + argcount++; + } + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + new_frame = frame_new(ctx, co, 0, args, argcount); + } else { + new_frame = frame_new(ctx, co, 0, NULL, 0); + } stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1400,6 +1817,24 @@ ctx->done = true; break; } + /* Stack space handling */ + int framesize = co->co_framesize; + assert(framesize > 0); + curr_space += framesize; + if (curr_space < 0 || curr_space > INT32_MAX) { + // won't fit in signed 32-bit int + ctx->done = true; + break; + } + max_space = curr_space > max_space ? curr_space : max_space; + if (first_valid_check_stack == NULL) { + first_valid_check_stack = corresponding_check_stack; + } + else if (corresponding_check_stack) { + // delete all but the first valid _CHECK_STACK_SPACE + corresponding_check_stack->opcode = _NOP; + } + corresponding_check_stack = NULL; break; } @@ -1431,14 +1866,22 @@ } case _CHECK_AND_ALLOCATE_OBJECT: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *null; + _Py_UopsSymbol *callable; _Py_UopsSymbol *self; _Py_UopsSymbol *init; - _Py_UopsSymbol **args; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-oparg]; + uint32_t type_version = (uint32_t)this_instr->operand; + (void)type_version; + (void)callable; + (void)null; + (void)args; self = sym_new_not_null(ctx); init = sym_new_not_null(ctx); - for (int _i = oparg; --_i >= 0;) { - args[_i] = sym_new_not_null(ctx); - } stack_pointer[-2 - oparg] = self; stack_pointer[-1 - oparg] = init; break; @@ -1714,8 +2157,27 @@ } case _BINARY_OP: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyTypeObject *ltype = sym_get_type(left); + PyTypeObject *rtype = sym_get_type(right); + if (ltype != NULL && (ltype == &PyLong_Type || ltype == &PyFloat_Type) && + rtype != NULL && (rtype == &PyLong_Type || rtype == &PyFloat_Type)) + { + if (oparg != NB_TRUE_DIVIDE && oparg != NB_INPLACE_TRUE_DIVIDE && + ltype == &PyLong_Type && rtype == &PyLong_Type) { + /* If both inputs are ints and the op is not division the result is an int */ + res = sym_new_type(ctx, &PyLong_Type); + } + else { + /* For any other op combining ints/floats the result is a float */ + res = sym_new_type(ctx, &PyFloat_Type); + } + } + res = sym_new_unknown(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1749,24 +2211,60 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { + _Py_UopsSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_True); + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_FALSE_POP: { + _Py_UopsSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_False); + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NONE_POP: { + _Py_UopsSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, !Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, true); + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NOT_NONE_POP: { + _Py_UopsSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, false); + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1782,6 +2280,7 @@ } case _CHECK_STACK_SPACE_OPERAND: { + uint32_t framesize = (uint32_t)this_instr->operand; break; } diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py index 67a96cc899fdf1..4e278b86136833 100644 --- a/Tools/cases_generator/partial_evaluator_generator.py +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -25,6 +25,7 @@ from stack import Local, Stack, StackError DEFAULT_OUTPUT = ROOT / "Python/partial_evaluator_cases.c.h" +DEFAULT_SPECIALIZER_INPUT = (ROOT / "Python/optimizer_bytecodes.c").absolute().as_posix() DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/partial_evaluator_bytecodes.c").absolute().as_posix() @@ -211,8 +212,7 @@ def generate_tier2_abstract_from_files( "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT ) - -arg_parser.add_argument("input", nargs="*", help="Abstract interpreter definition file") +arg_parser.add_argument("input", nargs="*", help="Partial evaluator definition file") arg_parser.add_argument( "base", nargs="*", help="The base instruction definition file(s)" @@ -224,6 +224,7 @@ def generate_tier2_abstract_from_files( args = arg_parser.parse_args() if not args.input: args.base.append(DEFAULT_INPUT) + args.input.append(DEFAULT_SPECIALIZER_INPUT) args.input.append(DEFAULT_ABSTRACT_INPUT) else: args.base.append(args.input[-1]) From 2d6884d2bb2c5246f2769668fc0c01778441e8f7 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 3 Sep 2024 02:40:05 +0800 Subject: [PATCH 03/46] cleanup --- Makefile.pre.in | 2 +- Python/partial_evaluator_cases.c.h | 2 +- .../partial_evaluator_generator.py | 235 ------------------ 3 files changed, 2 insertions(+), 237 deletions(-) delete mode 100644 Tools/cases_generator/partial_evaluator_generator.py diff --git a/Makefile.pre.in b/Makefile.pre.in index 4c72011a07b358..d53866d8fb3ae2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1999,7 +1999,7 @@ regen-optimizer-cases: .PHONY: regen-partial-evaluator-cases regen-partial-evaluator-cases: - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/partial_evaluator_generator.py \ + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/optimizer_generator.py \ -o $(srcdir)/Python/partial_evaluator_cases.c.h.new \ $(srcdir)/Python/optimizer_bytecodes.c \ $(srcdir)/Python/partial_evaluator_bytecodes.c \ diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index ddf5f18ee054fa..eb73d2d68b8cb2 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -1,4 +1,4 @@ -// This file is generated by Tools/cases_generator/partial_evaluator_generator.py +// This file is generated by Tools/cases_generator/optimizer_generator.py // from: // Python/optimizer_bytecodes.c, Python/partial_evaluator_bytecodes.c // Do not edit! diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py deleted file mode 100644 index 4e278b86136833..00000000000000 --- a/Tools/cases_generator/partial_evaluator_generator.py +++ /dev/null @@ -1,235 +0,0 @@ -"""Generate the cases for the tier 2 optimizer. -Reads the instruction definitions from bytecodes.c and optimizer_bytecodes.c -Writes the cases to optimizer_cases.c.h, which is #included in Python/optimizer_analysis.c. -""" - -import argparse - -from analyzer import ( - Analysis, - Instruction, - Uop, - analyze_files, - StackItem, - analysis_error, -) -from generators_common import ( - DEFAULT_INPUT, - ROOT, - write_header, - Emitter, -) -from cwriter import CWriter -from typing import TextIO, Iterator -from lexer import Token -from stack import Local, Stack, StackError - -DEFAULT_OUTPUT = ROOT / "Python/partial_evaluator_cases.c.h" -DEFAULT_SPECIALIZER_INPUT = (ROOT / "Python/optimizer_bytecodes.c").absolute().as_posix() -DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/partial_evaluator_bytecodes.c").absolute().as_posix() - - -def validate_uop(override: Uop, uop: Uop) -> None: - # To do - pass - - -def type_name(var: StackItem) -> str: - if var.is_array(): - return f"_Py_UopsSymbol **" - if var.type: - return var.type - return f"_Py_UopsSymbol *" - - -def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: - variables = {"unused"} - if not skip_inputs: - for var in reversed(uop.stack.inputs): - if var.name not in variables: - variables.add(var.name) - if var.condition: - out.emit(f"{type_name(var)}{var.name} = NULL;\n") - else: - out.emit(f"{type_name(var)}{var.name};\n") - for var in uop.stack.outputs: - if var.peek: - continue - if var.name not in variables: - variables.add(var.name) - if var.condition: - out.emit(f"{type_name(var)}{var.name} = NULL;\n") - else: - out.emit(f"{type_name(var)}{var.name};\n") - - -def decref_inputs( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - out.emit_at("", tkn) - - -def emit_default(out: CWriter, uop: Uop) -> None: - for i, var in enumerate(uop.stack.outputs): - if var.name != "unused" and not var.peek: - if var.is_array(): - out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") - out.emit("}\n") - elif var.name == "null": - out.emit(f"{var.name} = sym_new_null(ctx);\n") - else: - out.emit(f"{var.name} = sym_new_not_null(ctx);\n") - - -class OptimizerEmitter(Emitter): - pass - - -def write_uop( - override: Uop | None, - uop: Uop, - out: CWriter, - stack: Stack, - debug: bool, - skip_inputs: bool, -) -> None: - locals: dict[str, Local] = {} - try: - prototype = override if override else uop - is_override = override is not None - out.start_line() - for var in reversed(prototype.stack.inputs): - code, local = stack.pop(var, extract_bits=True) - if not skip_inputs: - out.emit(code) - if local.defined: - locals[local.name] = local - out.emit(stack.define_output_arrays(prototype.stack.outputs)) - if debug: - args = [] - for var in prototype.stack.inputs: - if not var.peek or is_override: - args.append(var.name) - out.emit(f'DEBUG_PRINTF({", ".join(args)});\n') - if override: - for cache in uop.caches: - if cache.name != "unused": - if cache.size == 4: - type = cast = "PyObject *" - else: - type = f"uint{cache.size*16}_t " - cast = f"uint{cache.size*16}_t" - out.emit(f"{type}{cache.name} = ({cast})this_instr->operand;\n") - if override: - emitter = OptimizerEmitter(out) - emitter.emit_tokens(override, stack, None) - else: - emit_default(out, uop) - - for var in prototype.stack.outputs: - if var.name in locals: - local = locals[var.name] - else: - local = Local.local(var) - stack.push(local) - out.start_line() - stack.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=True) - except StackError as ex: - raise analysis_error(ex.args[0], uop.body[0]) - - -SKIPS = ("_EXTENDED_ARG",) - - -def generate_abstract_interpreter( - filenames: list[str], - abstract: Analysis, - base: Analysis, - outfile: TextIO, - debug: bool, -) -> None: - write_header(__file__, filenames, outfile) - out = CWriter(outfile, 2, False) - out.emit("\n") - base_uop_names = set([uop.name for uop in base.uops.values()]) - for abstract_uop_name in abstract.uops: - assert ( - abstract_uop_name in base_uop_names - ), f"All abstract uops should override base uops, but {abstract_uop_name} is not." - - for uop in base.uops.values(): - override: Uop | None = None - if uop.name in abstract.uops: - override = abstract.uops[uop.name] - validate_uop(override, uop) - if uop.properties.tier == 1: - continue - if uop.replicates: - continue - if uop.is_super(): - continue - if not uop.is_viable(): - out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n") - continue - out.emit(f"case {uop.name}: {{\n") - if override: - declare_variables(override, out, skip_inputs=False) - else: - declare_variables(uop, out, skip_inputs=True) - stack = Stack() - write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) - out.start_line() - out.emit("break;\n") - out.emit("}") - out.emit("\n\n") - - -def generate_tier2_abstract_from_files( - filenames: list[str], outfilename: str, debug: bool = False -) -> None: - assert len(filenames) == 2, "Need a base file and an abstract cases file." - base = analyze_files([filenames[0]]) - abstract = analyze_files([filenames[1]]) - with open(outfilename, "w") as outfile: - generate_abstract_interpreter(filenames, abstract, base, outfile, debug) - - -arg_parser = argparse.ArgumentParser( - description="Generate the code for the tier 2 interpreter.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, -) - -arg_parser.add_argument( - "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT -) - -arg_parser.add_argument("input", nargs="*", help="Partial evaluator definition file") - -arg_parser.add_argument( - "base", nargs="*", help="The base instruction definition file(s)" -) - -arg_parser.add_argument("-d", "--debug", help="Insert debug calls", action="store_true") - -if __name__ == "__main__": - args = arg_parser.parse_args() - if not args.input: - args.base.append(DEFAULT_INPUT) - args.input.append(DEFAULT_SPECIALIZER_INPUT) - args.input.append(DEFAULT_ABSTRACT_INPUT) - else: - args.base.append(args.input[-1]) - args.input.pop() - abstract = analyze_files(args.input) - base = analyze_files(args.base) - with open(args.output, "w") as outfile: - generate_abstract_interpreter(args.input, abstract, base, outfile, args.debug) From e5e736422d8196c9f6352081778908c6662c9242 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 3 Sep 2024 04:10:24 +0800 Subject: [PATCH 04/46] basic copying setup --- Include/internal/pycore_optimizer.h | 3 +++ Python/optimizer_analysis.c | 28 ++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 54a08de252b4d0..8bd4d3201b2e9a 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -226,6 +226,9 @@ struct _Py_UOpsContext { _Py_UopsSymbol **n_consumed; _Py_UopsSymbol **limit; _Py_UopsSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; + + _PyUOpInstruction *trace_dest; + int n_trace_dest; }; typedef struct _Py_UOpsContext _Py_UOpsContext; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index b871010ee6a141..945cce3ea06988 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -486,6 +486,10 @@ optimize_uops( } +#define WRITE_OP(INST, OP, ARG, OPERAND) \ + (INST)->opcode = OP; \ + (INST)->oparg = ARG; \ + (INST)->operand = OPERAND; /* 1 for success, 0 for not ready, cannot error at the moment. */ static int @@ -498,7 +502,10 @@ partial_evaluate_uops( ) { + _PyUOpInstruction trace_dest[UOP_MAX_TRACE_LENGTH]; _Py_UOpsContext context; + context.trace_dest = trace_dest; + context.n_trace_dest = 0; _Py_UOpsContext *ctx = &context; uint32_t opcode = UINT16_MAX; int curr_space = 0; @@ -521,9 +528,11 @@ partial_evaluate_uops( for (int i = 0; !ctx->done; i++) { assert(i < trace_len); this_instr = &trace[i]; + trace_dest[ctx->n_trace_dest] = *this_instr; int oparg = this_instr->oparg; opcode = this_instr->opcode; + uint64_t operand = this_instr->operand; _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; #ifdef Py_DEBUG @@ -546,6 +555,8 @@ partial_evaluate_uops( DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); ctx->frame->stack_pointer = stack_pointer; assert(STACK_LEVEL() >= 0); + WRITE_OP(&trace_dest[ctx->n_trace_dest], opcode, oparg, operand); + ctx->n_trace_dest++; } if (ctx->out_of_space) { DPRINTF(3, "\n"); @@ -563,10 +574,19 @@ partial_evaluate_uops( return 0; } - /* Either reached the end or cannot optimize further, but there - * would be no benefit in retrying later */ - _Py_uop_abstractcontext_fini(ctx); - return trace_len; + if (ctx->out_of_space || !is_terminator(this_instr)) { + _Py_uop_abstractcontext_fini(ctx); + return trace_len; + } + else { + // We MUST not have bailed early here. + // That's the only time the PE's residual is valid. + assert(ctx->n_trace_dest < UOP_MAX_TRACE_LENGTH); + assert(is_terminator(this_instr)); + memcpy(trace, trace_dest, ctx->n_trace_dest * sizeof(_PyUOpInstruction)); + _Py_uop_abstractcontext_fini(ctx); + return trace_len; + } error: DPRINTF(3, "\n"); From 9722b40db3d00803fbb12a1e735733e797dfa8fd Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 3 Sep 2024 15:39:15 +0800 Subject: [PATCH 05/46] fix compilation --- Include/internal/pycore_optimizer.h | 64 ++- Lib/test/test_capi/test_opt.py | 13 + Python/optimizer_analysis.c | 94 +++- Python/optimizer_bytecodes.c | 40 +- Python/optimizer_cases.c.h | 544 +++++++++--------- Python/optimizer_symbols.c | 157 +++--- Python/partial_evaluator_bytecodes.c | 16 +- Python/partial_evaluator_cases.c.h | 550 ++++++++++--------- Tools/cases_generator/optimizer_generator.py | 10 +- Tools/cases_generator/stack.py | 2 +- 10 files changed, 800 insertions(+), 690 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 8bd4d3201b2e9a..68b59d1c877785 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -152,8 +152,8 @@ struct _Py_UopsSymbol { PyTypeObject *typ; // Borrowed reference PyObject *const_val; // Owned reference (!) unsigned int type_version; // currently stores type version - bool is_static; // used for binding-time analysis int locals_idx; + char is_static; // used for binding-time analysis }; #define UOP_FORMAT_TARGET 0 @@ -191,16 +191,22 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) // handle before rejoining the rest of the program. #define MAX_CHAIN_DEPTH 4 + typedef struct _Py_UopsSymbol _Py_UopsSymbol; +typedef struct _Py_UopsLocalsPlusSlot { + _Py_UopsSymbol *sym; + char is_virtual; +} _Py_UopsLocalsPlusSlot; + struct _Py_UOpsAbstractFrame { // Max stacklen int stack_len; int locals_len; - _Py_UopsSymbol **stack_pointer; - _Py_UopsSymbol **stack; - _Py_UopsSymbol **locals; + _Py_UopsLocalsPlusSlot *stack_pointer; + _Py_UopsLocalsPlusSlot *stack; + _Py_UopsLocalsPlusSlot *locals; }; typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; @@ -223,9 +229,9 @@ struct _Py_UOpsContext { // Arena for the symbolic types. ty_arena t_arena; - _Py_UopsSymbol **n_consumed; - _Py_UopsSymbol **limit; - _Py_UopsSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; + _Py_UopsLocalsPlusSlot *n_consumed; + _Py_UopsLocalsPlusSlot *limit; + _Py_UopsLocalsPlusSlot locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; _PyUOpInstruction *trace_dest; int n_trace_dest; @@ -233,28 +239,28 @@ struct _Py_UOpsContext { typedef struct _Py_UOpsContext _Py_UOpsContext; -extern bool _Py_uop_sym_is_null(_Py_UopsSymbol *sym); -extern bool _Py_uop_sym_is_not_null(_Py_UopsSymbol *sym); -extern bool _Py_uop_sym_is_const(_Py_UopsSymbol *sym); -extern PyObject *_Py_uop_sym_get_const(_Py_UopsSymbol *sym); -extern _Py_UopsSymbol *_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx); -extern _Py_UopsSymbol *_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx); -extern _Py_UopsSymbol *_Py_uop_sym_new_type( +extern bool _Py_uop_sym_is_null(_Py_UopsLocalsPlusSlot sym); +extern bool _Py_uop_sym_is_not_null(_Py_UopsLocalsPlusSlot sym); +extern bool _Py_uop_sym_is_const(_Py_UopsLocalsPlusSlot sym); +extern PyObject *_Py_uop_sym_get_const(_Py_UopsLocalsPlusSlot sym); +extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_unknown(_Py_UOpsContext *ctx); +extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_not_null(_Py_UOpsContext *ctx); +extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_type( _Py_UOpsContext *ctx, PyTypeObject *typ); -extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); -extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); -extern bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym); -extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version); -extern void _Py_uop_sym_set_locals_idx(_Py_UopsSymbol *sym, int locals_idx); -extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); -extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); -extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version); -extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val); -extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); -extern int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym); -extern PyTypeObject *_Py_uop_sym_get_type(_Py_UopsSymbol *sym); +extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); +extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_null(_Py_UOpsContext *ctx); +extern bool _Py_uop_sym_has_type(_Py_UopsLocalsPlusSlot sym); +extern bool _Py_uop_sym_matches_type(_Py_UopsLocalsPlusSlot sym, PyTypeObject *typ); +extern bool _Py_uop_sym_matches_type_version(_Py_UopsLocalsPlusSlot sym, unsigned int version); +extern void _Py_uop_sym_set_locals_idx(_Py_UopsLocalsPlusSlot sym, int locals_idx); +extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym); +extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym); +extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, unsigned int version); +extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyObject *const_val); +extern bool _Py_uop_sym_is_bottom(_Py_UopsLocalsPlusSlot sym); +extern int _Py_uop_sym_truthiness(_Py_UopsLocalsPlusSlot sym); +extern PyTypeObject *_Py_uop_sym_get_type(_Py_UopsLocalsPlusSlot sym); extern void _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); @@ -264,7 +270,7 @@ extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( _Py_UOpsContext *ctx, PyCodeObject *co, int curr_stackentries, - _Py_UopsSymbol **args, + _Py_UopsLocalsPlusSlot *args, int arg_len); extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index f1ab72180d714d..449d589b984de8 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1481,6 +1481,19 @@ def fn(a): fn(A()) + def test_pe_load_fast_pop_top(self): + def thing(a): + x = 0 + for i in range(20): + i + return i + + + res, ex = self._run_with_optimizer(thing, 1) + self.assertEqual(res, 19) + self.assertIsNotNone(ex) + self.assertEqual(list(iter_opnames(ex)).count("_POP_TOP"), 0) + self.assertTrue(ex.is_valid()) if __name__ == "__main__": unittest.main() diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 945cce3ea06988..5758d404553565 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -334,8 +334,8 @@ static int optimize_to_bool( _PyUOpInstruction *this_instr, _Py_UOpsContext *ctx, - _Py_UopsSymbol *value, - _Py_UopsSymbol **result_ptr) + _Py_UopsLocalsPlusSlot value, + _Py_UopsLocalsPlusSlot *result_ptr) { if (sym_matches_type(value, &PyBool_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -386,6 +386,12 @@ get_code(_PyUOpInstruction *op) return co; } +static inline _Py_UopsLocalsPlusSlot +sym_to_slot(_Py_UopsSymbol *sym) +{ + return (_Py_UopsLocalsPlusSlot){sym, 0}; +} + /* 1 for success, 0 for not ready, cannot error at the moment. */ static int optimize_uops( @@ -423,7 +429,7 @@ optimize_uops( int oparg = this_instr->oparg; opcode = this_instr->opcode; - _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; + _Py_UopsLocalsPlusSlot *stack_pointer = ctx->frame->stack_pointer; #ifdef Py_DEBUG if (get_lltrace() >= 3) { @@ -491,6 +497,44 @@ optimize_uops( (INST)->oparg = ARG; \ (INST)->operand = OPERAND; +#define SET_STATIC_INST() instr_is_truly_static = true; + +static void +reify_shadow_stack(_Py_UOpsContext *ctx) +{ + _PyUOpInstruction *trace_dest = ctx->trace_dest; + for (_Py_UopsLocalsPlusSlot *sp = ctx->frame->stack; sp < ctx->frame->stack_pointer; sp++) { + _Py_UopsSymbol *sym = sp->sym; + assert(sym != NULL); + // Need reifying. +// if (sym->is_virtual) { +// if (sym->const_val) { +// WRITE_OP(&trace_dest[ctx->n_trace_dest], _Py_IsImmortal(sym->const_val) ? +// _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, sym->locals_idx, (uint64_t)sym->const_val); +// } +// else if (sym->locals_idx >= 0) { +// printf("pe reified LOAD_FAST %d\n", sym->locals_idx); +// WRITE_OP(&trace_dest[ctx->n_trace_dest], _LOAD_FAST, sym->locals_idx, 0); +// } +// else if (sym_is_null(sym)) { +// WRITE_OP(&trace_dest[ctx->n_trace_dest], _PUSH_NULL, sym->locals_idx, 0); +// } +// else { +// // Is static but not a constant value of locals or NULL. +// // How is that possible? +// Py_UNREACHABLE(); +// } +// ctx->n_trace_dest++; +// sym->is_virtual = false; +// } +// if (ctx->n_trace_dest >= UOP_MAX_TRACE_LENGTH) { +// ctx->out_of_space = true; +// ctx->done = true; +// return; +// } + } +} + /* 1 for success, 0 for not ready, cannot error at the moment. */ static int partial_evaluate_uops( @@ -525,7 +569,9 @@ partial_evaluate_uops( ctx->contradiction = false; _PyUOpInstruction *this_instr = NULL; - for (int i = 0; !ctx->done; i++) { + int i = 0; + bool prev_instr_is_truly_static = false; + for (; !ctx->done; i++) { assert(i < trace_len); this_instr = &trace[i]; trace_dest[ctx->n_trace_dest] = *this_instr; @@ -533,7 +579,12 @@ partial_evaluate_uops( int oparg = this_instr->oparg; opcode = this_instr->opcode; uint64_t operand = this_instr->operand; - _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; + _Py_UopsLocalsPlusSlot *stack_pointer = ctx->frame->stack_pointer; + + // An instruction is candidate static if it has no escapes, and all its inputs + // are static. + // If so, whether it can be eliminated is up to whether it has an implementation. + bool instr_is_truly_static = false; #ifdef Py_DEBUG if (get_lltrace() >= 3) { @@ -555,8 +606,32 @@ partial_evaluate_uops( DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); ctx->frame->stack_pointer = stack_pointer; assert(STACK_LEVEL() >= 0); - WRITE_OP(&trace_dest[ctx->n_trace_dest], opcode, oparg, operand); - ctx->n_trace_dest++; + if (ctx->done) { + break; + } + // Always write these instructions for bookkeeping. + if (opcode == _CHECK_VALIDITY_AND_SET_IP || opcode == _SET_IP || opcode == _CHECK_VALIDITY) { + WRITE_OP(&trace_dest[ctx->n_trace_dest], opcode, oparg, operand); + ctx->n_trace_dest++; + } + // If the instruction is not static, + // reify the shadow stack, and write the op. + else if (!instr_is_truly_static) { + reify_shadow_stack(ctx); + WRITE_OP(&trace_dest[ctx->n_trace_dest], opcode, oparg, operand); + ctx->n_trace_dest++; + } + else { +//#ifdef Py_DEBUG +// if (get_lltrace() >= 3) { + printf("%4d pe STATIC: ", (int) (this_instr - trace)); + _PyUOpPrint(this_instr); + printf("\n"); +// } +//#endif + // Inst is static. Nothing written :)! + } + prev_instr_is_truly_static = instr_is_truly_static; } if (ctx->out_of_space) { DPRINTF(3, "\n"); @@ -583,7 +658,8 @@ partial_evaluate_uops( // That's the only time the PE's residual is valid. assert(ctx->n_trace_dest < UOP_MAX_TRACE_LENGTH); assert(is_terminator(this_instr)); - memcpy(trace, trace_dest, ctx->n_trace_dest * sizeof(_PyUOpInstruction)); + // Copy rest of trace to dest + memcpy(trace, trace_dest, ctx->n_trace_dest); _Py_uop_abstractcontext_fini(ctx); return trace_len; } @@ -644,7 +720,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } if (last->opcode == _LOAD_CONST_INLINE || last->opcode == _LOAD_CONST_INLINE_BORROW || - last->opcode == _LOAD_FAST || +// last->opcode == _LOAD_FAST || last->opcode == _COPY ) { last->opcode = _NOP; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 9a1b9da52f4bb5..77514cfd0627ff 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -85,7 +85,7 @@ dummy_func(void) { op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - _Py_UopsSymbol *temp = sym_new_null(ctx); + _Py_UopsLocalsPlusSlot temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; } @@ -329,10 +329,10 @@ dummy_func(void) { } } - op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame)) { (void)container; (void)sub; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; } @@ -487,7 +487,7 @@ dummy_func(void) { op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { (void)index; null = sym_new_null(ctx); - attr = NULL; + attr = (_Py_UopsLocalsPlusSlot){NULL, 0}; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. assert(sym_is_const(owner)); @@ -500,7 +500,7 @@ dummy_func(void) { attr = sym_new_const(ctx, res); } } - if (attr == NULL) { + if (attr.sym == NULL) { /* No conversion made. We don't know what `attr` is. */ attr = sym_new_not_null(ctx); } @@ -545,10 +545,10 @@ dummy_func(void) { self = owner; } - op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { (void)fget; (void)owner; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; } @@ -568,7 +568,7 @@ dummy_func(void) { sym_set_type(callable, &PyMethod_Type); } - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { int argcount = oparg; (void)callable; @@ -594,7 +594,7 @@ dummy_func(void) { DPRINTF(3, "code=%p ", co); } - assert(self_or_null != NULL); + assert(self_or_null.sym != NULL); assert(args != NULL); if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM @@ -603,9 +603,9 @@ dummy_func(void) { } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = frame_new(ctx, co, 0, args, argcount); + new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, args, argcount); } else { - new_frame = frame_new(ctx, co, 0, NULL, 0); + new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, NULL, 0); } } @@ -618,21 +618,21 @@ dummy_func(void) { maybe_self = sym_new_not_null(ctx); } - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ (void)callable; (void)self_or_null; (void)args; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { (void)callable; (void)self_or_null; (void)args; (void)kwnames; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; } @@ -645,11 +645,11 @@ dummy_func(void) { init = sym_new_not_null(ctx); } - op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { + op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame)) { (void)self; (void)init; (void)args; - init_frame = NULL; + init_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; } @@ -723,12 +723,12 @@ dummy_func(void) { Py_UNREACHABLE(); } - op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { + op(_PUSH_FRAME, (new_frame -- unused if (0))) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)new_frame.sym; ctx->curr_frame_depth++; - stack_pointer = new_frame->stack_pointer; + stack_pointer = ((_Py_UOpsAbstractFrame *)new_frame.sym)->stack_pointer; co = get_code(this_instr); if (co == NULL) { // should be about to _EXIT_TRACE anyway diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 672fec3946f2fb..86ea241d0cb048 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -24,7 +24,7 @@ /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. if (sym_is_null(value)) { @@ -37,7 +37,7 @@ } case _LOAD_FAST: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); stack_pointer[0] = value; stack_pointer += 1; @@ -46,9 +46,9 @@ } case _LOAD_FAST_AND_CLEAR: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); - _Py_UopsSymbol *temp = sym_new_null(ctx); + _Py_UopsLocalsPlusSlot temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; @@ -57,7 +57,7 @@ } case _LOAD_CONST: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); @@ -69,7 +69,7 @@ } case _STORE_FAST: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; stack_pointer += -1; @@ -84,7 +84,7 @@ } case _PUSH_NULL: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -93,7 +93,7 @@ } case _END_SEND: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; @@ -102,22 +102,22 @@ } case _UNARY_NEGATIVE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_NOT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { res = sym_new_type(ctx, &PyBool_Type); @@ -127,8 +127,8 @@ } case _TO_BOOL_BOOL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyBool_Type); @@ -139,8 +139,8 @@ } case _TO_BOOL_INT: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyLong_Type); @@ -151,8 +151,8 @@ } case _TO_BOOL_LIST: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyList_Type); @@ -163,8 +163,8 @@ } case _TO_BOOL_NONE: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_const(value, Py_None); @@ -175,8 +175,8 @@ } case _TO_BOOL_STR: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { res = sym_new_type(ctx, &PyBool_Type); @@ -187,22 +187,22 @@ } case _REPLACE_WITH_TRUE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_INVERT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _GUARD_BOTH_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyLong_Type)) { @@ -232,9 +232,9 @@ } case _BINARY_OP_MULTIPLY_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -262,9 +262,9 @@ } case _BINARY_OP_ADD_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -292,9 +292,9 @@ } case _BINARY_OP_SUBTRACT_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -322,8 +322,8 @@ } case _GUARD_BOTH_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyFloat_Type)) { @@ -353,9 +353,9 @@ } case _BINARY_OP_MULTIPLY_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -384,9 +384,9 @@ } case _BINARY_OP_ADD_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -415,9 +415,9 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -446,8 +446,8 @@ } case _GUARD_BOTH_UNICODE: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyUnicode_Type) && @@ -460,9 +460,9 @@ } case _BINARY_OP_ADD_UNICODE: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -490,7 +490,7 @@ } case _BINARY_SUBSCR: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -499,7 +499,7 @@ } case _BINARY_SLICE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -514,7 +514,7 @@ } case _BINARY_SUBSCR_LIST_INT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -523,7 +523,7 @@ } case _BINARY_SUBSCR_STR_INT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -532,7 +532,7 @@ } case _BINARY_SUBSCR_TUPLE_INT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -541,7 +541,7 @@ } case _BINARY_SUBSCR_DICT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -554,16 +554,16 @@ } case _BINARY_SUBSCR_INIT_CALL: { - _Py_UopsSymbol *sub; - _Py_UopsSymbol *container; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot sub; + _Py_UopsLocalsPlusSlot container; + _Py_UopsLocalsPlusSlot new_frame; sub = stack_pointer[-1]; container = stack_pointer[-2]; (void)container; (void)sub; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-2] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-2] = new_frame; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -606,14 +606,14 @@ } case _CALL_INTRINSIC_1: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -622,8 +622,8 @@ } case _RETURN_VALUE: { - _Py_UopsSymbol *retval; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot retval; + _Py_UopsLocalsPlusSlot res; retval = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -650,14 +650,14 @@ } case _GET_AITER: { - _Py_UopsSymbol *iter; + _Py_UopsLocalsPlusSlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - _Py_UopsSymbol *awaitable; + _Py_UopsLocalsPlusSlot awaitable; awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; @@ -666,7 +666,7 @@ } case _GET_AWAITABLE: { - _Py_UopsSymbol *iter; + _Py_UopsLocalsPlusSlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -681,7 +681,7 @@ } case _YIELD_VALUE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_unknown(ctx); stack_pointer[-1] = res; break; @@ -694,7 +694,7 @@ } case _LOAD_COMMON_CONSTANT: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -703,7 +703,7 @@ } case _LOAD_BUILD_CLASS: { - _Py_UopsSymbol *bc; + _Py_UopsLocalsPlusSlot bc; bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; @@ -722,8 +722,8 @@ } case _UNPACK_SEQUENCE: { - _Py_UopsSymbol *seq; - _Py_UopsSymbol **values; + _Py_UopsLocalsPlusSlot seq; + _Py_UopsLocalsPlusSlot *values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ @@ -737,8 +737,8 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { - _Py_UopsSymbol *val1; - _Py_UopsSymbol *val0; + _Py_UopsLocalsPlusSlot val1; + _Py_UopsLocalsPlusSlot val0; val1 = sym_new_not_null(ctx); val0 = sym_new_not_null(ctx); stack_pointer[-1] = val1; @@ -749,7 +749,7 @@ } case _UNPACK_SEQUENCE_TUPLE: { - _Py_UopsSymbol **values; + _Py_UopsLocalsPlusSlot *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); @@ -760,7 +760,7 @@ } case _UNPACK_SEQUENCE_LIST: { - _Py_UopsSymbol **values; + _Py_UopsLocalsPlusSlot *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); @@ -771,8 +771,8 @@ } case _UNPACK_EX: { - _Py_UopsSymbol *seq; - _Py_UopsSymbol **values; + _Py_UopsLocalsPlusSlot seq; + _Py_UopsLocalsPlusSlot *values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ @@ -809,7 +809,7 @@ } case _LOAD_LOCALS: { - _Py_UopsSymbol *locals; + _Py_UopsLocalsPlusSlot locals; locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; @@ -820,7 +820,7 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ case _LOAD_NAME: { - _Py_UopsSymbol *v; + _Py_UopsLocalsPlusSlot v; v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; @@ -829,8 +829,8 @@ } case _LOAD_GLOBAL: { - _Py_UopsSymbol *res; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot res; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -849,8 +849,8 @@ } case _LOAD_GLOBAL_MODULE: { - _Py_UopsSymbol *res; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot res; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -861,8 +861,8 @@ } case _LOAD_GLOBAL_BUILTINS: { - _Py_UopsSymbol *res; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot res; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -885,14 +885,14 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -911,7 +911,7 @@ } case _BUILD_STRING: { - _Py_UopsSymbol *str; + _Py_UopsLocalsPlusSlot str; str = sym_new_not_null(ctx); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -920,7 +920,7 @@ } case _BUILD_TUPLE: { - _Py_UopsSymbol *tup; + _Py_UopsLocalsPlusSlot tup; tup = sym_new_not_null(ctx); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; @@ -929,7 +929,7 @@ } case _BUILD_LIST: { - _Py_UopsSymbol *list; + _Py_UopsLocalsPlusSlot list; list = sym_new_not_null(ctx); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -950,7 +950,7 @@ } case _BUILD_SET: { - _Py_UopsSymbol *set; + _Py_UopsLocalsPlusSlot set; set = sym_new_not_null(ctx); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -959,7 +959,7 @@ } case _BUILD_MAP: { - _Py_UopsSymbol *map; + _Py_UopsLocalsPlusSlot map; map = sym_new_not_null(ctx); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -992,7 +992,7 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ case _LOAD_SUPER_ATTR_ATTR: { - _Py_UopsSymbol *attr_st; + _Py_UopsLocalsPlusSlot attr_st; attr_st = sym_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -1001,8 +1001,8 @@ } case _LOAD_SUPER_ATTR_METHOD: { - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self_or_null; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); stack_pointer[-3] = attr; @@ -1013,9 +1013,9 @@ } case _LOAD_ATTR: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self_or_null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self_or_null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; (void)owner; attr = sym_new_not_null(ctx); @@ -1030,7 +1030,7 @@ } case _GUARD_TYPE_VERSION: { - _Py_UopsSymbol *owner; + _Py_UopsLocalsPlusSlot owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand; assert(type_version); @@ -1060,9 +1060,9 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; uint16_t offset = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1077,7 +1077,7 @@ } case _CHECK_ATTR_MODULE: { - _Py_UopsSymbol *owner; + _Py_UopsLocalsPlusSlot owner; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand; (void)dict_version; @@ -1098,14 +1098,14 @@ } case _LOAD_ATTR_MODULE: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; (void)index; null = sym_new_null(ctx); - attr = NULL; + attr = (_Py_UopsLocalsPlusSlot){NULL, 0}; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. assert(sym_is_const(owner)); @@ -1118,7 +1118,7 @@ attr = sym_new_const(ctx, res); } } - if (attr == NULL) { + if (attr.sym == NULL) { /* No conversion made. We don't know what `attr` is. */ attr = sym_new_not_null(ctx); } @@ -1134,9 +1134,9 @@ } case _LOAD_ATTR_WITH_HINT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; uint16_t hint = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1151,9 +1151,9 @@ } case _LOAD_ATTR_SLOT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1172,9 +1172,9 @@ } case _LOAD_ATTR_CLASS: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1189,15 +1189,15 @@ } case _LOAD_ATTR_PROPERTY_FRAME: { - _Py_UopsSymbol *owner; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot new_frame; owner = stack_pointer[-1]; PyObject *fget = (PyObject *)this_instr->operand; (void)fget; (void)owner; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-1] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-1] = new_frame; break; } @@ -1226,9 +1226,9 @@ } case _COMPARE_OP: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1246,9 +1246,9 @@ } case _COMPARE_OP_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1261,9 +1261,9 @@ } case _COMPARE_OP_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1276,9 +1276,9 @@ } case _COMPARE_OP_STR: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1291,9 +1291,9 @@ } case _IS_OP: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1306,9 +1306,9 @@ } case _CONTAINS_OP: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1321,7 +1321,7 @@ } case _CONTAINS_OP_SET: { - _Py_UopsSymbol *b; + _Py_UopsLocalsPlusSlot b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -1330,7 +1330,7 @@ } case _CONTAINS_OP_DICT: { - _Py_UopsSymbol *b; + _Py_UopsLocalsPlusSlot b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -1339,8 +1339,8 @@ } case _CHECK_EG_MATCH: { - _Py_UopsSymbol *rest; - _Py_UopsSymbol *match; + _Py_UopsLocalsPlusSlot rest; + _Py_UopsLocalsPlusSlot match; rest = sym_new_not_null(ctx); match = sym_new_not_null(ctx); stack_pointer[-2] = rest; @@ -1349,14 +1349,14 @@ } case _CHECK_EXC_MATCH: { - _Py_UopsSymbol *b; + _Py_UopsLocalsPlusSlot b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -1365,7 +1365,7 @@ } case _IMPORT_FROM: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1378,14 +1378,14 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - _Py_UopsSymbol *b; + _Py_UopsLocalsPlusSlot b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _GET_LEN: { - _Py_UopsSymbol *len; + _Py_UopsLocalsPlusSlot len; len = sym_new_not_null(ctx); stack_pointer[0] = len; stack_pointer += 1; @@ -1394,7 +1394,7 @@ } case _MATCH_CLASS: { - _Py_UopsSymbol *attrs; + _Py_UopsLocalsPlusSlot attrs; attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1403,7 +1403,7 @@ } case _MATCH_MAPPING: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1412,7 +1412,7 @@ } case _MATCH_SEQUENCE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1421,7 +1421,7 @@ } case _MATCH_KEYS: { - _Py_UopsSymbol *values_or_none; + _Py_UopsLocalsPlusSlot values_or_none; values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1430,14 +1430,14 @@ } case _GET_ITER: { - _Py_UopsSymbol *iter; + _Py_UopsLocalsPlusSlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_YIELD_FROM_ITER: { - _Py_UopsSymbol *iter; + _Py_UopsLocalsPlusSlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -1446,7 +1446,7 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - _Py_UopsSymbol *next; + _Py_UopsLocalsPlusSlot next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1467,7 +1467,7 @@ } case _ITER_NEXT_LIST: { - _Py_UopsSymbol *next; + _Py_UopsLocalsPlusSlot next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1486,7 +1486,7 @@ } case _ITER_NEXT_TUPLE: { - _Py_UopsSymbol *next; + _Py_UopsLocalsPlusSlot next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1505,8 +1505,8 @@ } case _ITER_NEXT_RANGE: { - _Py_UopsSymbol *iter; - _Py_UopsSymbol *next; + _Py_UopsLocalsPlusSlot iter; + _Py_UopsLocalsPlusSlot next; iter = stack_pointer[-1]; next = sym_new_type(ctx, &PyLong_Type); (void)iter; @@ -1523,9 +1523,9 @@ } case _LOAD_SPECIAL: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self_or_null; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self_or_null; owner = stack_pointer[-1]; (void)owner; attr = sym_new_not_null(ctx); @@ -1538,7 +1538,7 @@ } case _WITH_EXCEPT_START: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1547,8 +1547,8 @@ } case _PUSH_EXC_INFO: { - _Py_UopsSymbol *prev_exc; - _Py_UopsSymbol *new_exc; + _Py_UopsLocalsPlusSlot prev_exc; + _Py_UopsLocalsPlusSlot new_exc; prev_exc = sym_new_not_null(ctx); new_exc = sym_new_not_null(ctx); stack_pointer[-1] = prev_exc; @@ -1567,9 +1567,9 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; @@ -1583,9 +1583,9 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; @@ -1599,14 +1599,14 @@ } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - _Py_UopsSymbol *attr; + _Py_UopsLocalsPlusSlot attr; attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - _Py_UopsSymbol *attr; + _Py_UopsLocalsPlusSlot attr; attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; @@ -1617,9 +1617,9 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; @@ -1633,11 +1633,11 @@ } case _MAYBE_EXPAND_METHOD: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; - _Py_UopsSymbol *func; - _Py_UopsSymbol *maybe_self; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot func; + _Py_UopsLocalsPlusSlot maybe_self; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1657,10 +1657,10 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1668,9 +1668,9 @@ (void)callable; (void)self_or_null; (void)args; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -1685,8 +1685,8 @@ } case _EXPAND_METHOD: { - _Py_UopsSymbol *method; - _Py_UopsSymbol *self; + _Py_UopsLocalsPlusSlot method; + _Py_UopsLocalsPlusSlot self; method = sym_new_not_null(ctx); self = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = method; @@ -1699,7 +1699,7 @@ } case _CALL_NON_PY_GENERAL: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1708,8 +1708,8 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UopsSymbol *null; - _Py_UopsSymbol *callable; + _Py_UopsLocalsPlusSlot null; + _Py_UopsLocalsPlusSlot callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; sym_set_null(null); @@ -1718,9 +1718,9 @@ } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UopsSymbol *callable; - _Py_UopsSymbol *func; - _Py_UopsSymbol *self; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot func; + _Py_UopsLocalsPlusSlot self; callable = stack_pointer[-2 - oparg]; (void)callable; func = sym_new_not_null(ctx); @@ -1740,8 +1740,8 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; sym_set_type(callable, &PyFunction_Type); @@ -1756,10 +1756,10 @@ } case _INIT_CALL_PY_EXACT_ARGS: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1785,7 +1785,7 @@ co = (PyCodeObject *)func->func_code; DPRINTF(3, "code=%p ", co); } - assert(self_or_null != NULL); + assert(self_or_null.sym != NULL); assert(args != NULL); if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM @@ -1793,25 +1793,25 @@ argcount++; } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = frame_new(ctx, co, 0, args, argcount); + new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, args, argcount); } else { - new_frame = frame_new(ctx, co, 0, NULL, 0); + new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, NULL, 0); } - stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_FRAME: { - _Py_UOpsAbstractFrame *new_frame; - new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; + _Py_UopsLocalsPlusSlot new_frame; + new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)new_frame.sym; ctx->curr_frame_depth++; - stack_pointer = new_frame->stack_pointer; + stack_pointer = ((_Py_UOpsAbstractFrame *)new_frame.sym)->stack_pointer; co = get_code(this_instr); if (co == NULL) { // should be about to _EXIT_TRACE anyway @@ -1840,7 +1840,7 @@ } case _CALL_TYPE_1: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1849,7 +1849,7 @@ } case _CALL_STR_1: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1858,7 +1858,7 @@ } case _CALL_TUPLE_1: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1867,11 +1867,11 @@ } case _CHECK_AND_ALLOCATE_OBJECT: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *null; - _Py_UopsSymbol *callable; - _Py_UopsSymbol *self; - _Py_UopsSymbol *init; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot self; + _Py_UopsLocalsPlusSlot init; args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1889,19 +1889,19 @@ } case _CREATE_INIT_FRAME: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *init; - _Py_UopsSymbol *self; - _Py_UOpsAbstractFrame *init_frame; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot init; + _Py_UopsLocalsPlusSlot self; + _Py_UopsLocalsPlusSlot init_frame; args = &stack_pointer[-oparg]; init = stack_pointer[-1 - oparg]; self = stack_pointer[-2 - oparg]; (void)self; (void)init; (void)args; - init_frame = NULL; + init_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)init_frame; + stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -1914,7 +1914,7 @@ } case _CALL_BUILTIN_CLASS: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1923,7 +1923,7 @@ } case _CALL_BUILTIN_O: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1932,7 +1932,7 @@ } case _CALL_BUILTIN_FAST: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1941,7 +1941,7 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1950,7 +1950,7 @@ } case _CALL_LEN: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1959,7 +1959,7 @@ } case _CALL_ISINSTANCE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1974,7 +1974,7 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1983,7 +1983,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1992,7 +1992,7 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2001,7 +2001,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2014,11 +2014,11 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { - _Py_UopsSymbol *kwnames; - _Py_UopsSymbol **args; - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot kwnames; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot new_frame; kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; @@ -2027,9 +2027,9 @@ (void)self_or_null; (void)args; (void)kwnames; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-3 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -2044,9 +2044,9 @@ } case _EXPAND_METHOD_KW: { - _Py_UopsSymbol *method; - _Py_UopsSymbol *self; - _Py_UopsSymbol *kwnames; + _Py_UopsLocalsPlusSlot method; + _Py_UopsLocalsPlusSlot self; + _Py_UopsLocalsPlusSlot kwnames; method = sym_new_not_null(ctx); self = sym_new_not_null(ctx); kwnames = sym_new_not_null(ctx); @@ -2061,7 +2061,7 @@ } case _CALL_KW_NON_PY: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; @@ -2074,14 +2074,14 @@ /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - _Py_UopsSymbol *func; + _Py_UopsLocalsPlusSlot func; func = sym_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - _Py_UopsSymbol *func_st; + _Py_UopsLocalsPlusSlot func_st; func_st = sym_new_not_null(ctx); stack_pointer[-2] = func_st; stack_pointer += -1; @@ -2090,7 +2090,7 @@ } case _RETURN_GENERATOR: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -2114,7 +2114,7 @@ } case _BUILD_SLICE: { - _Py_UopsSymbol *slice; + _Py_UopsLocalsPlusSlot slice; slice = sym_new_not_null(ctx); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -2123,21 +2123,21 @@ } case _CONVERT_VALUE: { - _Py_UopsSymbol *result; + _Py_UopsLocalsPlusSlot result; result = sym_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -2146,8 +2146,8 @@ } case _COPY: { - _Py_UopsSymbol *bottom; - _Py_UopsSymbol *top; + _Py_UopsLocalsPlusSlot bottom; + _Py_UopsLocalsPlusSlot top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; @@ -2158,9 +2158,9 @@ } case _BINARY_OP: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; PyTypeObject *ltype = sym_get_type(left); @@ -2186,8 +2186,8 @@ } case _SWAP: { - _Py_UopsSymbol *top; - _Py_UopsSymbol *bottom; + _Py_UopsLocalsPlusSlot top; + _Py_UopsLocalsPlusSlot bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; stack_pointer[-2 - (oparg-2)] = top; @@ -2212,7 +2212,7 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { - _Py_UopsSymbol *flag; + _Py_UopsLocalsPlusSlot flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2225,7 +2225,7 @@ } case _GUARD_IS_FALSE_POP: { - _Py_UopsSymbol *flag; + _Py_UopsLocalsPlusSlot flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2238,7 +2238,7 @@ } case _GUARD_IS_NONE_POP: { - _Py_UopsSymbol *flag; + _Py_UopsLocalsPlusSlot flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2255,7 +2255,7 @@ } case _GUARD_IS_NOT_NONE_POP: { - _Py_UopsSymbol *flag; + _Py_UopsLocalsPlusSlot flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2305,7 +2305,7 @@ } case _LOAD_CONST_INLINE: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); stack_pointer[0] = value; @@ -2315,7 +2315,7 @@ } case _LOAD_CONST_INLINE_BORROW: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); stack_pointer[0] = value; @@ -2325,15 +2325,15 @@ } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_CONST_INLINE_WITH_NULL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *null; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot null; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); null = sym_new_null(ctx); @@ -2345,8 +2345,8 @@ } case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *null; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot null; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); null = sym_new_null(ctx); diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 3962ced2dbecd2..2be4477cc11e34 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -55,12 +55,17 @@ static _Py_UopsSymbol NO_SPACE_SYMBOL = { .type_version = 0, }; -_Py_UopsSymbol * +static _Py_UopsLocalsPlusSlot NO_SPACE_SLOT = { + .sym = &NO_SPACE_SYMBOL, + .is_virtual = 0, +}; + +_Py_UopsLocalsPlusSlot out_of_space(_Py_UOpsContext *ctx) { ctx->done = true; ctx->out_of_space = true; - return &NO_SPACE_SYMBOL; + return NO_SPACE_SLOT; } static _Py_UopsSymbol * @@ -84,24 +89,25 @@ sym_new(_Py_UOpsContext *ctx) } static inline void -sym_set_flag(_Py_UopsSymbol *sym, int flag) +sym_set_flag(_Py_UopsLocalsPlusSlot sym, int flag) { - sym->flags |= flag; + sym.sym->flags |= flag; } static inline void -sym_set_bottom(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +sym_set_bottom(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym) { sym_set_flag(sym, IS_NULL | NOT_NULL); - sym->typ = NULL; - Py_CLEAR(sym->const_val); + sym.sym->typ = NULL; + Py_CLEAR(sym.sym->const_val); ctx->done = true; ctx->contradiction = true; } bool -_Py_uop_sym_is_bottom(_Py_UopsSymbol *sym) +_Py_uop_sym_is_bottom(_Py_UopsLocalsPlusSlot sym_l) { + _Py_UopsSymbol *sym = sym_l.sym; if ((sym->flags & IS_NULL) && (sym->flags & NOT_NULL)) { assert(sym->flags == (IS_NULL | NOT_NULL)); assert(sym->typ == NULL); @@ -112,98 +118,98 @@ _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym) } bool -_Py_uop_sym_is_not_null(_Py_UopsSymbol *sym) +_Py_uop_sym_is_not_null(_Py_UopsLocalsPlusSlot sym) { - return sym->flags == NOT_NULL; + return sym.sym->flags == NOT_NULL; } bool -_Py_uop_sym_is_null(_Py_UopsSymbol *sym) +_Py_uop_sym_is_null(_Py_UopsLocalsPlusSlot sym) { - return sym->flags == IS_NULL; + return sym.sym->flags == IS_NULL; } bool -_Py_uop_sym_is_const(_Py_UopsSymbol *sym) +_Py_uop_sym_is_const(_Py_UopsLocalsPlusSlot sym) { - return sym->const_val != NULL; + return sym.sym->const_val != NULL; } PyObject * -_Py_uop_sym_get_const(_Py_UopsSymbol *sym) +_Py_uop_sym_get_const(_Py_UopsLocalsPlusSlot sym) { - return sym->const_val; + return sym.sym->const_val; } void -_Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ) +_Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); - if (sym->flags & IS_NULL) { + if (sym.sym->flags & IS_NULL) { sym_set_bottom(ctx, sym); return; } - if (sym->typ != NULL) { - if (sym->typ != typ) { + if (sym.sym->typ != NULL) { + if (sym.sym->typ != typ) { sym_set_bottom(ctx, sym); return; } } else { sym_set_flag(sym, NOT_NULL); - sym->typ = typ; + sym.sym->typ = typ; } } bool -_Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version) +_Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, unsigned int version) { // if the type version was already set, then it must be different and we should set it to bottom - if (sym->type_version) { + if (sym.sym->type_version) { sym_set_bottom(ctx, sym); return false; } - sym->type_version = version; + sym.sym->type_version = version; return true; } void -_Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val) +_Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyObject *const_val) { assert(const_val != NULL); - if (sym->flags & IS_NULL) { + if (sym.sym->flags & IS_NULL) { sym_set_bottom(ctx, sym); } PyTypeObject *typ = Py_TYPE(const_val); - if (sym->typ != NULL && sym->typ != typ) { + if (sym.sym->typ != NULL && sym.sym->typ != typ) { sym_set_bottom(ctx, sym); } - if (sym->const_val != NULL) { - if (sym->const_val != const_val) { + if (sym.sym->const_val != NULL) { + if (sym.sym->const_val != const_val) { // TODO: What if they're equal? sym_set_bottom(ctx, sym); } } else { sym_set_flag(sym, NOT_NULL); - sym->typ = typ; - sym->const_val = Py_NewRef(const_val); + sym.sym->typ = typ; + sym.sym->const_val = Py_NewRef(const_val); } - sym->is_static = true; + sym.sym->is_static = true; } void -_Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +_Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym) { if (_Py_uop_sym_is_not_null(sym)) { sym_set_bottom(ctx, sym); } sym_set_flag(sym, IS_NULL); - sym->is_static = true; + sym.sym->is_static = true; } void -_Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +_Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym) { if (_Py_uop_sym_is_null(sym)) { sym_set_bottom(ctx, sym); @@ -212,36 +218,37 @@ _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) } -_Py_UopsSymbol * +_Py_UopsLocalsPlusSlot _Py_uop_sym_new_unknown(_Py_UOpsContext *ctx) { - return sym_new(ctx); + return (_Py_UopsLocalsPlusSlot){sym_new(ctx), 0}; } -_Py_UopsSymbol * +_Py_UopsLocalsPlusSlot _Py_uop_sym_new_not_null(_Py_UOpsContext *ctx) { - _Py_UopsSymbol *res = _Py_uop_sym_new_unknown(ctx); - if (res == NULL) { + _Py_UopsLocalsPlusSlot res = _Py_uop_sym_new_unknown(ctx); + if (res.sym == NULL) { return out_of_space(ctx); } sym_set_flag(res, NOT_NULL); return res; } -_Py_UopsSymbol * +_Py_UopsLocalsPlusSlot _Py_uop_sym_new_type(_Py_UOpsContext *ctx, PyTypeObject *typ) { _Py_UopsSymbol *res = sym_new(ctx); if (res == NULL) { return out_of_space(ctx); } - _Py_uop_sym_set_type(ctx, res, typ); - return res; + _Py_UopsLocalsPlusSlot sym = {res, 0}; + _Py_uop_sym_set_type(ctx, sym, typ); + return sym; } // Adds a new reference to const_val, owned by the symbol. -_Py_UopsSymbol * +_Py_UopsLocalsPlusSlot _Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) { assert(const_val != NULL); @@ -249,15 +256,16 @@ _Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) if (res == NULL) { return out_of_space(ctx); } - _Py_uop_sym_set_const(ctx, res, const_val); - return res; + _Py_UopsLocalsPlusSlot sym = {res, 0}; + _Py_uop_sym_set_const(ctx, sym, const_val); + return sym; } -_Py_UopsSymbol * +_Py_UopsLocalsPlusSlot _Py_uop_sym_new_null(_Py_UOpsContext *ctx) { - _Py_UopsSymbol *null_sym = _Py_uop_sym_new_unknown(ctx); - if (null_sym == NULL) { + _Py_UopsLocalsPlusSlot null_sym = _Py_uop_sym_new_unknown(ctx); + if (null_sym.sym == NULL) { return out_of_space(ctx); } _Py_uop_sym_set_null(ctx, null_sym); @@ -265,51 +273,51 @@ _Py_uop_sym_new_null(_Py_UOpsContext *ctx) } PyTypeObject * -_Py_uop_sym_get_type(_Py_UopsSymbol *sym) +_Py_uop_sym_get_type(_Py_UopsLocalsPlusSlot sym) { if (_Py_uop_sym_is_bottom(sym)) { return NULL; } - return sym->typ; + return sym.sym->typ; } unsigned int -_Py_uop_sym_get_type_version(_Py_UopsSymbol *sym) +_Py_uop_sym_get_type_version(_Py_UopsLocalsPlusSlot sym) { - return sym->type_version; + return sym.sym->type_version; } bool -_Py_uop_sym_has_type(_Py_UopsSymbol *sym) +_Py_uop_sym_has_type(_Py_UopsLocalsPlusSlot sym) { if (_Py_uop_sym_is_bottom(sym)) { return false; } - return sym->typ != NULL; + return sym.sym->typ != NULL; } bool -_Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) +_Py_uop_sym_matches_type(_Py_UopsLocalsPlusSlot sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); return _Py_uop_sym_get_type(sym) == typ; } bool -_Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version) +_Py_uop_sym_matches_type_version(_Py_UopsLocalsPlusSlot sym, unsigned int version) { return _Py_uop_sym_get_type_version(sym) == version; } void -_Py_uop_sym_set_locals_idx(_Py_UopsSymbol *sym, int locals_idx) +_Py_uop_sym_set_locals_idx(_Py_UopsLocalsPlusSlot sym, int locals_idx) { assert(locals_idx >= 0); - sym->locals_idx = locals_idx; + sym.sym->locals_idx = locals_idx; } int -_Py_uop_sym_truthiness(_Py_UopsSymbol *sym) +_Py_uop_sym_truthiness(_Py_UopsLocalsPlusSlot sym) { /* There are some non-constant values for * which `bool(val)` always evaluates to @@ -348,7 +356,7 @@ _Py_uop_frame_new( _Py_UOpsContext *ctx, PyCodeObject *co, int curr_stackentries, - _Py_UopsSymbol **args, + _Py_UopsLocalsPlusSlot *args, int arg_len) { assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); @@ -373,19 +381,17 @@ _Py_uop_frame_new( } for (int i = arg_len; i < co->co_nlocalsplus; i++) { - _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx); - frame->locals[i] = local; + frame->locals[i] = _Py_uop_sym_new_unknown(ctx);; } for (int i = 0; i < co->co_nlocalsplus; i++) { - frame->locals[i]->locals_idx = i; + frame->locals[i].sym->locals_idx = i; } // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { - _Py_UopsSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); - frame->stack[i] = stackvar; + frame->stack[i] = _Py_uop_sym_new_unknown(ctx); } return frame; @@ -411,7 +417,8 @@ _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) ctx->n_consumed = ctx->locals_and_stack; #ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { - ctx->locals_and_stack[i] = NULL; + _Py_UopsLocalsPlusSlot slot = {NULL, 0}; + ctx->locals_and_stack[i] = slot; } #endif @@ -445,10 +452,10 @@ do { \ } \ } while (0) -static _Py_UopsSymbol * +static _Py_UopsLocalsPlusSlot make_bottom(_Py_UOpsContext *ctx) { - _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); + _Py_UopsLocalsPlusSlot sym = _Py_uop_sym_new_unknown(ctx); _Py_uop_sym_set_null(ctx, sym); _Py_uop_sym_set_non_null(ctx, sym); return sym; @@ -464,8 +471,8 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) PyObject *val_43 = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. - _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); - if (sym == NULL) { + _Py_UopsLocalsPlusSlot sym = _Py_uop_sym_new_unknown(ctx); + if (sym.sym == NULL) { goto fail; } TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); @@ -476,7 +483,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); sym = make_bottom(ctx); - if (sym == NULL) { + if (sym.sym == NULL) { goto fail; } TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); @@ -487,7 +494,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + if (sym.sym == NULL) { goto fail; } TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); @@ -512,7 +519,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) assert(_Py_IsImmortal(val_43)); sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + if (sym.sym == NULL) { goto fail; } _Py_uop_sym_set_const(ctx, sym, val_42); @@ -533,7 +540,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + if (sym.sym == NULL) { goto fail; } _Py_uop_sym_set_const(ctx, sym, val_42); diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index 2364db07f342fa..a5aff06f442e71 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -75,12 +75,12 @@ dummy_func(void) { override op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); + SET_STATIC_INST(); } override op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - _Py_UopsSymbol *temp = sym_new_null(ctx); - GETLOCAL(oparg) = temp; + GETLOCAL(oparg) = sym_new_null(ctx); } override op(_STORE_FAST, (value --)) { @@ -88,15 +88,19 @@ dummy_func(void) { sym_set_locals_idx(value, oparg); } - override op(_PUSH_NULL, (-- res)) { - res = sym_new_null(ctx); - } - override op(_LOAD_CONST, (-- value)) { // Should've all been converted by specializer. Py_UNREACHABLE(); } + + override op(_POP_TOP, (pop --)) { +// if (sym_is_virtual(pop)) { +// SET_STATIC_INST(); +// } + } + + override op (_CHECK_STACK_SPACE_OPERAND, ( -- )) { } diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index eb73d2d68b8cb2..a9f9ea59ebece4 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -24,7 +24,7 @@ /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. if (sym_is_null(value)) { @@ -37,8 +37,9 @@ } case _LOAD_FAST: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); + SET_STATIC_INST(); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -46,10 +47,9 @@ } case _LOAD_FAST_AND_CLEAR: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); - _Py_UopsSymbol *temp = sym_new_null(ctx); - GETLOCAL(oparg) = temp; + GETLOCAL(oparg) = sym_new_null(ctx); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -57,7 +57,7 @@ } case _LOAD_CONST: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; // Should've all been converted by specializer. Py_UNREACHABLE(); stack_pointer[0] = value; @@ -67,7 +67,7 @@ } case _STORE_FAST: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; sym_set_locals_idx(value, oparg); @@ -77,13 +77,17 @@ } case _POP_TOP: { + _Py_UopsLocalsPlusSlot pop; + // if (sym_is_virtual(pop)) { + // SET_STATIC_INST(); + // } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_NULL: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -92,7 +96,7 @@ } case _END_SEND: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; @@ -101,22 +105,22 @@ } case _UNARY_NEGATIVE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_NOT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { res = sym_new_type(ctx, &PyBool_Type); @@ -126,8 +130,8 @@ } case _TO_BOOL_BOOL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyBool_Type); @@ -138,8 +142,8 @@ } case _TO_BOOL_INT: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyLong_Type); @@ -150,8 +154,8 @@ } case _TO_BOOL_LIST: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyList_Type); @@ -162,8 +166,8 @@ } case _TO_BOOL_NONE: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_const(value, Py_None); @@ -174,8 +178,8 @@ } case _TO_BOOL_STR: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { res = sym_new_type(ctx, &PyBool_Type); @@ -186,22 +190,22 @@ } case _REPLACE_WITH_TRUE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_INVERT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _GUARD_BOTH_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyLong_Type)) { @@ -231,9 +235,9 @@ } case _BINARY_OP_MULTIPLY_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -261,9 +265,9 @@ } case _BINARY_OP_ADD_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -291,9 +295,9 @@ } case _BINARY_OP_SUBTRACT_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -321,8 +325,8 @@ } case _GUARD_BOTH_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyFloat_Type)) { @@ -352,9 +356,9 @@ } case _BINARY_OP_MULTIPLY_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -383,9 +387,9 @@ } case _BINARY_OP_ADD_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -414,9 +418,9 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -445,8 +449,8 @@ } case _GUARD_BOTH_UNICODE: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyUnicode_Type) && @@ -459,9 +463,9 @@ } case _BINARY_OP_ADD_UNICODE: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -489,7 +493,7 @@ } case _BINARY_SUBSCR: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -498,7 +502,7 @@ } case _BINARY_SLICE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -513,7 +517,7 @@ } case _BINARY_SUBSCR_LIST_INT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -522,7 +526,7 @@ } case _BINARY_SUBSCR_STR_INT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -531,7 +535,7 @@ } case _BINARY_SUBSCR_TUPLE_INT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -540,7 +544,7 @@ } case _BINARY_SUBSCR_DICT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -553,16 +557,16 @@ } case _BINARY_SUBSCR_INIT_CALL: { - _Py_UopsSymbol *sub; - _Py_UopsSymbol *container; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot sub; + _Py_UopsLocalsPlusSlot container; + _Py_UopsLocalsPlusSlot new_frame; sub = stack_pointer[-1]; container = stack_pointer[-2]; (void)container; (void)sub; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-2] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-2] = new_frame; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -605,14 +609,14 @@ } case _CALL_INTRINSIC_1: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -621,8 +625,8 @@ } case _RETURN_VALUE: { - _Py_UopsSymbol *retval; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot retval; + _Py_UopsLocalsPlusSlot res; retval = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -649,14 +653,14 @@ } case _GET_AITER: { - _Py_UopsSymbol *iter; + _Py_UopsLocalsPlusSlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - _Py_UopsSymbol *awaitable; + _Py_UopsLocalsPlusSlot awaitable; awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; @@ -665,7 +669,7 @@ } case _GET_AWAITABLE: { - _Py_UopsSymbol *iter; + _Py_UopsLocalsPlusSlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -680,7 +684,7 @@ } case _YIELD_VALUE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_unknown(ctx); stack_pointer[-1] = res; break; @@ -693,7 +697,7 @@ } case _LOAD_COMMON_CONSTANT: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -702,7 +706,7 @@ } case _LOAD_BUILD_CLASS: { - _Py_UopsSymbol *bc; + _Py_UopsLocalsPlusSlot bc; bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; @@ -721,8 +725,8 @@ } case _UNPACK_SEQUENCE: { - _Py_UopsSymbol *seq; - _Py_UopsSymbol **values; + _Py_UopsLocalsPlusSlot seq; + _Py_UopsLocalsPlusSlot *values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ @@ -736,8 +740,8 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { - _Py_UopsSymbol *val1; - _Py_UopsSymbol *val0; + _Py_UopsLocalsPlusSlot val1; + _Py_UopsLocalsPlusSlot val0; val1 = sym_new_not_null(ctx); val0 = sym_new_not_null(ctx); stack_pointer[-1] = val1; @@ -748,7 +752,7 @@ } case _UNPACK_SEQUENCE_TUPLE: { - _Py_UopsSymbol **values; + _Py_UopsLocalsPlusSlot *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); @@ -759,7 +763,7 @@ } case _UNPACK_SEQUENCE_LIST: { - _Py_UopsSymbol **values; + _Py_UopsLocalsPlusSlot *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); @@ -770,8 +774,8 @@ } case _UNPACK_EX: { - _Py_UopsSymbol *seq; - _Py_UopsSymbol **values; + _Py_UopsLocalsPlusSlot seq; + _Py_UopsLocalsPlusSlot *values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ @@ -808,7 +812,7 @@ } case _LOAD_LOCALS: { - _Py_UopsSymbol *locals; + _Py_UopsLocalsPlusSlot locals; locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; @@ -819,7 +823,7 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ case _LOAD_NAME: { - _Py_UopsSymbol *v; + _Py_UopsLocalsPlusSlot v; v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; @@ -828,8 +832,8 @@ } case _LOAD_GLOBAL: { - _Py_UopsSymbol *res; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot res; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -848,8 +852,8 @@ } case _LOAD_GLOBAL_MODULE: { - _Py_UopsSymbol *res; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot res; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -860,8 +864,8 @@ } case _LOAD_GLOBAL_BUILTINS: { - _Py_UopsSymbol *res; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot res; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -884,14 +888,14 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -910,7 +914,7 @@ } case _BUILD_STRING: { - _Py_UopsSymbol *str; + _Py_UopsLocalsPlusSlot str; str = sym_new_not_null(ctx); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -919,7 +923,7 @@ } case _BUILD_TUPLE: { - _Py_UopsSymbol *tup; + _Py_UopsLocalsPlusSlot tup; tup = sym_new_not_null(ctx); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; @@ -928,7 +932,7 @@ } case _BUILD_LIST: { - _Py_UopsSymbol *list; + _Py_UopsLocalsPlusSlot list; list = sym_new_not_null(ctx); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -949,7 +953,7 @@ } case _BUILD_SET: { - _Py_UopsSymbol *set; + _Py_UopsLocalsPlusSlot set; set = sym_new_not_null(ctx); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -958,7 +962,7 @@ } case _BUILD_MAP: { - _Py_UopsSymbol *map; + _Py_UopsLocalsPlusSlot map; map = sym_new_not_null(ctx); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -991,7 +995,7 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ case _LOAD_SUPER_ATTR_ATTR: { - _Py_UopsSymbol *attr_st; + _Py_UopsLocalsPlusSlot attr_st; attr_st = sym_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -1000,8 +1004,8 @@ } case _LOAD_SUPER_ATTR_METHOD: { - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self_or_null; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); stack_pointer[-3] = attr; @@ -1012,9 +1016,9 @@ } case _LOAD_ATTR: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self_or_null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self_or_null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; (void)owner; attr = sym_new_not_null(ctx); @@ -1029,7 +1033,7 @@ } case _GUARD_TYPE_VERSION: { - _Py_UopsSymbol *owner; + _Py_UopsLocalsPlusSlot owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand; assert(type_version); @@ -1059,9 +1063,9 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; uint16_t offset = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1076,7 +1080,7 @@ } case _CHECK_ATTR_MODULE: { - _Py_UopsSymbol *owner; + _Py_UopsLocalsPlusSlot owner; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand; (void)dict_version; @@ -1097,14 +1101,14 @@ } case _LOAD_ATTR_MODULE: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; (void)index; null = sym_new_null(ctx); - attr = NULL; + attr = (_Py_UopsLocalsPlusSlot){NULL, 0}; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. assert(sym_is_const(owner)); @@ -1117,7 +1121,7 @@ attr = sym_new_const(ctx, res); } } - if (attr == NULL) { + if (attr.sym == NULL) { /* No conversion made. We don't know what `attr` is. */ attr = sym_new_not_null(ctx); } @@ -1133,9 +1137,9 @@ } case _LOAD_ATTR_WITH_HINT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; uint16_t hint = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1150,9 +1154,9 @@ } case _LOAD_ATTR_SLOT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1171,9 +1175,9 @@ } case _LOAD_ATTR_CLASS: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1188,15 +1192,15 @@ } case _LOAD_ATTR_PROPERTY_FRAME: { - _Py_UopsSymbol *owner; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot new_frame; owner = stack_pointer[-1]; PyObject *fget = (PyObject *)this_instr->operand; (void)fget; (void)owner; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-1] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-1] = new_frame; break; } @@ -1225,9 +1229,9 @@ } case _COMPARE_OP: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1245,9 +1249,9 @@ } case _COMPARE_OP_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1260,9 +1264,9 @@ } case _COMPARE_OP_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1275,9 +1279,9 @@ } case _COMPARE_OP_STR: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1290,9 +1294,9 @@ } case _IS_OP: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1305,9 +1309,9 @@ } case _CONTAINS_OP: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1320,7 +1324,7 @@ } case _CONTAINS_OP_SET: { - _Py_UopsSymbol *b; + _Py_UopsLocalsPlusSlot b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -1329,7 +1333,7 @@ } case _CONTAINS_OP_DICT: { - _Py_UopsSymbol *b; + _Py_UopsLocalsPlusSlot b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -1338,8 +1342,8 @@ } case _CHECK_EG_MATCH: { - _Py_UopsSymbol *rest; - _Py_UopsSymbol *match; + _Py_UopsLocalsPlusSlot rest; + _Py_UopsLocalsPlusSlot match; rest = sym_new_not_null(ctx); match = sym_new_not_null(ctx); stack_pointer[-2] = rest; @@ -1348,14 +1352,14 @@ } case _CHECK_EXC_MATCH: { - _Py_UopsSymbol *b; + _Py_UopsLocalsPlusSlot b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -1364,7 +1368,7 @@ } case _IMPORT_FROM: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1377,14 +1381,14 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - _Py_UopsSymbol *b; + _Py_UopsLocalsPlusSlot b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _GET_LEN: { - _Py_UopsSymbol *len; + _Py_UopsLocalsPlusSlot len; len = sym_new_not_null(ctx); stack_pointer[0] = len; stack_pointer += 1; @@ -1393,7 +1397,7 @@ } case _MATCH_CLASS: { - _Py_UopsSymbol *attrs; + _Py_UopsLocalsPlusSlot attrs; attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1402,7 +1406,7 @@ } case _MATCH_MAPPING: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1411,7 +1415,7 @@ } case _MATCH_SEQUENCE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1420,7 +1424,7 @@ } case _MATCH_KEYS: { - _Py_UopsSymbol *values_or_none; + _Py_UopsLocalsPlusSlot values_or_none; values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1429,14 +1433,14 @@ } case _GET_ITER: { - _Py_UopsSymbol *iter; + _Py_UopsLocalsPlusSlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_YIELD_FROM_ITER: { - _Py_UopsSymbol *iter; + _Py_UopsLocalsPlusSlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -1445,7 +1449,7 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - _Py_UopsSymbol *next; + _Py_UopsLocalsPlusSlot next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1466,7 +1470,7 @@ } case _ITER_NEXT_LIST: { - _Py_UopsSymbol *next; + _Py_UopsLocalsPlusSlot next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1485,7 +1489,7 @@ } case _ITER_NEXT_TUPLE: { - _Py_UopsSymbol *next; + _Py_UopsLocalsPlusSlot next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1504,8 +1508,8 @@ } case _ITER_NEXT_RANGE: { - _Py_UopsSymbol *iter; - _Py_UopsSymbol *next; + _Py_UopsLocalsPlusSlot iter; + _Py_UopsLocalsPlusSlot next; iter = stack_pointer[-1]; next = sym_new_type(ctx, &PyLong_Type); (void)iter; @@ -1522,9 +1526,9 @@ } case _LOAD_SPECIAL: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self_or_null; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self_or_null; owner = stack_pointer[-1]; (void)owner; attr = sym_new_not_null(ctx); @@ -1537,7 +1541,7 @@ } case _WITH_EXCEPT_START: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1546,8 +1550,8 @@ } case _PUSH_EXC_INFO: { - _Py_UopsSymbol *prev_exc; - _Py_UopsSymbol *new_exc; + _Py_UopsLocalsPlusSlot prev_exc; + _Py_UopsLocalsPlusSlot new_exc; prev_exc = sym_new_not_null(ctx); new_exc = sym_new_not_null(ctx); stack_pointer[-1] = prev_exc; @@ -1566,9 +1570,9 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; @@ -1582,9 +1586,9 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; @@ -1598,14 +1602,14 @@ } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - _Py_UopsSymbol *attr; + _Py_UopsLocalsPlusSlot attr; attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - _Py_UopsSymbol *attr; + _Py_UopsLocalsPlusSlot attr; attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; @@ -1616,9 +1620,9 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; @@ -1632,11 +1636,11 @@ } case _MAYBE_EXPAND_METHOD: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; - _Py_UopsSymbol *func; - _Py_UopsSymbol *maybe_self; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot func; + _Py_UopsLocalsPlusSlot maybe_self; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1656,10 +1660,10 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1667,9 +1671,9 @@ (void)callable; (void)self_or_null; (void)args; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -1684,8 +1688,8 @@ } case _EXPAND_METHOD: { - _Py_UopsSymbol *method; - _Py_UopsSymbol *self; + _Py_UopsLocalsPlusSlot method; + _Py_UopsLocalsPlusSlot self; method = sym_new_not_null(ctx); self = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = method; @@ -1698,7 +1702,7 @@ } case _CALL_NON_PY_GENERAL: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1707,8 +1711,8 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UopsSymbol *null; - _Py_UopsSymbol *callable; + _Py_UopsLocalsPlusSlot null; + _Py_UopsLocalsPlusSlot callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; sym_set_null(null); @@ -1717,9 +1721,9 @@ } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UopsSymbol *callable; - _Py_UopsSymbol *func; - _Py_UopsSymbol *self; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot func; + _Py_UopsLocalsPlusSlot self; callable = stack_pointer[-2 - oparg]; (void)callable; func = sym_new_not_null(ctx); @@ -1739,8 +1743,8 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; sym_set_type(callable, &PyFunction_Type); @@ -1755,10 +1759,10 @@ } case _INIT_CALL_PY_EXACT_ARGS: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1784,7 +1788,7 @@ co = (PyCodeObject *)func->func_code; DPRINTF(3, "code=%p ", co); } - assert(self_or_null != NULL); + assert(self_or_null.sym != NULL); assert(args != NULL); if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM @@ -1792,25 +1796,25 @@ argcount++; } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = frame_new(ctx, co, 0, args, argcount); + new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, args, argcount); } else { - new_frame = frame_new(ctx, co, 0, NULL, 0); + new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, NULL, 0); } - stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_FRAME: { - _Py_UOpsAbstractFrame *new_frame; - new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; + _Py_UopsLocalsPlusSlot new_frame; + new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)new_frame.sym; ctx->curr_frame_depth++; - stack_pointer = new_frame->stack_pointer; + stack_pointer = ((_Py_UOpsAbstractFrame *)new_frame.sym)->stack_pointer; co = get_code(this_instr); if (co == NULL) { // should be about to _EXIT_TRACE anyway @@ -1839,7 +1843,7 @@ } case _CALL_TYPE_1: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1848,7 +1852,7 @@ } case _CALL_STR_1: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1857,7 +1861,7 @@ } case _CALL_TUPLE_1: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1866,11 +1870,11 @@ } case _CHECK_AND_ALLOCATE_OBJECT: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *null; - _Py_UopsSymbol *callable; - _Py_UopsSymbol *self; - _Py_UopsSymbol *init; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot self; + _Py_UopsLocalsPlusSlot init; args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1888,19 +1892,19 @@ } case _CREATE_INIT_FRAME: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *init; - _Py_UopsSymbol *self; - _Py_UOpsAbstractFrame *init_frame; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot init; + _Py_UopsLocalsPlusSlot self; + _Py_UopsLocalsPlusSlot init_frame; args = &stack_pointer[-oparg]; init = stack_pointer[-1 - oparg]; self = stack_pointer[-2 - oparg]; (void)self; (void)init; (void)args; - init_frame = NULL; + init_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)init_frame; + stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -1913,7 +1917,7 @@ } case _CALL_BUILTIN_CLASS: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1922,7 +1926,7 @@ } case _CALL_BUILTIN_O: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1931,7 +1935,7 @@ } case _CALL_BUILTIN_FAST: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1940,7 +1944,7 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1949,7 +1953,7 @@ } case _CALL_LEN: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1958,7 +1962,7 @@ } case _CALL_ISINSTANCE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1973,7 +1977,7 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1982,7 +1986,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1991,7 +1995,7 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2000,7 +2004,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2013,11 +2017,11 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { - _Py_UopsSymbol *kwnames; - _Py_UopsSymbol **args; - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot kwnames; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot new_frame; kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; @@ -2026,9 +2030,9 @@ (void)self_or_null; (void)args; (void)kwnames; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-3 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -2043,9 +2047,9 @@ } case _EXPAND_METHOD_KW: { - _Py_UopsSymbol *method; - _Py_UopsSymbol *self; - _Py_UopsSymbol *kwnames; + _Py_UopsLocalsPlusSlot method; + _Py_UopsLocalsPlusSlot self; + _Py_UopsLocalsPlusSlot kwnames; method = sym_new_not_null(ctx); self = sym_new_not_null(ctx); kwnames = sym_new_not_null(ctx); @@ -2060,7 +2064,7 @@ } case _CALL_KW_NON_PY: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; @@ -2073,14 +2077,14 @@ /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - _Py_UopsSymbol *func; + _Py_UopsLocalsPlusSlot func; func = sym_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - _Py_UopsSymbol *func_st; + _Py_UopsLocalsPlusSlot func_st; func_st = sym_new_not_null(ctx); stack_pointer[-2] = func_st; stack_pointer += -1; @@ -2089,7 +2093,7 @@ } case _RETURN_GENERATOR: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -2113,7 +2117,7 @@ } case _BUILD_SLICE: { - _Py_UopsSymbol *slice; + _Py_UopsLocalsPlusSlot slice; slice = sym_new_not_null(ctx); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -2122,21 +2126,21 @@ } case _CONVERT_VALUE: { - _Py_UopsSymbol *result; + _Py_UopsLocalsPlusSlot result; result = sym_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -2145,8 +2149,8 @@ } case _COPY: { - _Py_UopsSymbol *bottom; - _Py_UopsSymbol *top; + _Py_UopsLocalsPlusSlot bottom; + _Py_UopsLocalsPlusSlot top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; @@ -2157,9 +2161,9 @@ } case _BINARY_OP: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; PyTypeObject *ltype = sym_get_type(left); @@ -2185,8 +2189,8 @@ } case _SWAP: { - _Py_UopsSymbol *top; - _Py_UopsSymbol *bottom; + _Py_UopsLocalsPlusSlot top; + _Py_UopsLocalsPlusSlot bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; stack_pointer[-2 - (oparg-2)] = top; @@ -2211,7 +2215,7 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { - _Py_UopsSymbol *flag; + _Py_UopsLocalsPlusSlot flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2224,7 +2228,7 @@ } case _GUARD_IS_FALSE_POP: { - _Py_UopsSymbol *flag; + _Py_UopsLocalsPlusSlot flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2237,7 +2241,7 @@ } case _GUARD_IS_NONE_POP: { - _Py_UopsSymbol *flag; + _Py_UopsLocalsPlusSlot flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2254,7 +2258,7 @@ } case _GUARD_IS_NOT_NONE_POP: { - _Py_UopsSymbol *flag; + _Py_UopsLocalsPlusSlot flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2300,7 +2304,7 @@ } case _LOAD_CONST_INLINE: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); stack_pointer[0] = value; @@ -2310,7 +2314,7 @@ } case _LOAD_CONST_INLINE_BORROW: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); stack_pointer[0] = value; @@ -2320,15 +2324,15 @@ } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_CONST_INLINE_WITH_NULL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *null; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot null; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); null = sym_new_null(ctx); @@ -2340,8 +2344,8 @@ } case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *null; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot null; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); null = sym_new_null(ctx); diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index b74f627235ad84..8e3e6e2d6c6022 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -35,10 +35,10 @@ def validate_uop(override: Uop, uop: Uop) -> None: def type_name(var: StackItem) -> str: if var.is_array(): - return f"_Py_UopsSymbol **" + return f"_Py_UopsLocalsPlusSlot *" if var.type: return var.type - return f"_Py_UopsSymbol *" + return f"_Py_UopsLocalsPlusSlot " def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: @@ -48,7 +48,7 @@ def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: if var.name not in variables: variables.add(var.name) if var.condition: - out.emit(f"{type_name(var)}{var.name} = NULL;\n") + out.emit(f"{type_name(var)}{var.name} = (_Py_UopsLocalsPlusSlot){{NULL, 0}};\n") else: out.emit(f"{type_name(var)}{var.name};\n") for var in uop.stack.outputs: @@ -57,7 +57,7 @@ def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: if var.name not in variables: variables.add(var.name) if var.condition: - out.emit(f"{type_name(var)}{var.name} = NULL;\n") + out.emit(f"{type_name(var)}{var.name} = (_Py_UopsLocalsPlusSlot){{NULL, 0}};\n") else: out.emit(f"{type_name(var)}{var.name};\n") @@ -141,7 +141,7 @@ def write_uop( local = Local.local(var) stack.push(local) out.start_line() - stack.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=True) + stack.flush(out, cast_type="", extract_bits=True) except StackError as ex: raise analysis_error(ex.args[0], uop.body[0]) diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 34bf597f2f552d..91ef47c65c46a6 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -249,7 +249,7 @@ def _do_emit( cast_type: str = "uintptr_t", extract_bits: bool = False, ) -> None: - cast = f"({cast_type})" if var.type else "" + cast = f"({cast_type})" if var.type and cast_type else "" bits = ".bits" if cast and not extract_bits else "" if var.condition == "0": return From d8732fc19b8bb4473da7cccde7938ac92b164279 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 3 Sep 2024 20:43:36 +0800 Subject: [PATCH 06/46] baby pe --- Include/internal/pycore_opcode_metadata.h | 6 +- Include/internal/pycore_optimizer.h | 3 + Include/internal/pycore_uop_metadata.h | 22 +-- Python/bytecodes.c | 6 +- Python/optimizer_analysis.c | 139 ++++++++++-------- Python/partial_evaluator_bytecodes.c | 13 +- Python/partial_evaluator_cases.c.h | 12 +- Tools/cases_generator/analyzer.py | 2 + Tools/cases_generator/generators_common.py | 2 + Tools/cases_generator/lexer.py | 1 + .../opcode_metadata_generator.py | 1 + 11 files changed, 127 insertions(+), 80 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 97a8e4a00a9d55..b0fa37381e9b69 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -974,6 +974,7 @@ enum InstructionFormat { #define HAS_PASSTHROUGH_FLAG (4096) #define HAS_OPARG_AND_1_FLAG (8192) #define HAS_ERROR_NO_POP_FLAG (16384) +#define HAS_STATIC_FLAG (32768) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -989,6 +990,7 @@ enum InstructionFormat { #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) #define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) +#define OPCODE_HAS_STATIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_STATIC_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -1145,7 +1147,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1225,7 +1227,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [_DO_CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 68b59d1c877785..906931dc2abffb 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -158,6 +158,7 @@ struct _Py_UopsSymbol { #define UOP_FORMAT_TARGET 0 #define UOP_FORMAT_JUMP 1 +void _PyUOpPrint(const _PyUOpInstruction*); static inline uint32_t uop_get_target(const _PyUOpInstruction *inst) { @@ -207,6 +208,8 @@ struct _Py_UOpsAbstractFrame { _Py_UopsLocalsPlusSlot *stack_pointer; _Py_UopsLocalsPlusSlot *stack; _Py_UopsLocalsPlusSlot *locals; + + void *instr_ptr; }; typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index e2cba4dc0dfc81..9300d48b025a65 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -19,20 +19,20 @@ extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { - [_NOP] = HAS_PURE_FLAG, + [_NOP] = HAS_PURE_FLAG | HAS_STATIC_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_STATIC_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, @@ -47,7 +47,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_POP_TOP] = HAS_PURE_FLAG, + [_POP_TOP] = HAS_PURE_FLAG | HAS_STATIC_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_SEND] = HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 01e88a34d10b6a..a07fcf055332e7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -140,7 +140,7 @@ dummy_func( switch (opcode) { // BEGIN BYTECODES // - pure inst(NOP, (--)) { + pure _static inst(NOP, (--)) { } family(RESUME, 0) = { @@ -239,7 +239,7 @@ dummy_func( value = PyStackRef_DUP(value_s); } - replicate(8) pure inst(LOAD_FAST, (-- value)) { + replicate(8) _static inst(LOAD_FAST, (-- value)) { assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); } @@ -283,7 +283,7 @@ dummy_func( SETLOCAL(oparg2, value2); } - pure inst(POP_TOP, (value --)) { + pure _static inst(POP_TOP, (value --)) { DECREF_INPUTS(); } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 5758d404553565..c9061eabe96bed 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -39,12 +39,13 @@ extern void _PyUOpPrint(const _PyUOpInstruction *uop); static const char *const DEBUG_ENV = "PYTHON_OPT_DEBUG"; static inline int get_lltrace(void) { - char *uop_debug = Py_GETENV(DEBUG_ENV); - int lltrace = 0; - if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = *uop_debug - '0'; // TODO: Parse an int and all that - } - return lltrace; + return 5; +// char *uop_debug = Py_GETENV(DEBUG_ENV); +// int lltrace = 0; +// if (uop_debug != NULL && *uop_debug >= '0') { +// lltrace = *uop_debug - '0'; // TODO: Parse an int and all that +// } +// return lltrace; } #define DPRINTF(level, ...) \ if (get_lltrace() >= (level)) { printf(__VA_ARGS__); } @@ -501,37 +502,41 @@ optimize_uops( static void reify_shadow_stack(_Py_UOpsContext *ctx) -{ +{; + bool wrote_inst = false; _PyUOpInstruction *trace_dest = ctx->trace_dest; for (_Py_UopsLocalsPlusSlot *sp = ctx->frame->stack; sp < ctx->frame->stack_pointer; sp++) { - _Py_UopsSymbol *sym = sp->sym; - assert(sym != NULL); + _Py_UopsLocalsPlusSlot slot = *sp; + assert(slot.sym != NULL); // Need reifying. -// if (sym->is_virtual) { -// if (sym->const_val) { -// WRITE_OP(&trace_dest[ctx->n_trace_dest], _Py_IsImmortal(sym->const_val) ? -// _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, sym->locals_idx, (uint64_t)sym->const_val); -// } -// else if (sym->locals_idx >= 0) { -// printf("pe reified LOAD_FAST %d\n", sym->locals_idx); -// WRITE_OP(&trace_dest[ctx->n_trace_dest], _LOAD_FAST, sym->locals_idx, 0); -// } -// else if (sym_is_null(sym)) { -// WRITE_OP(&trace_dest[ctx->n_trace_dest], _PUSH_NULL, sym->locals_idx, 0); -// } -// else { -// // Is static but not a constant value of locals or NULL. -// // How is that possible? -// Py_UNREACHABLE(); -// } -// ctx->n_trace_dest++; -// sym->is_virtual = false; -// } -// if (ctx->n_trace_dest >= UOP_MAX_TRACE_LENGTH) { -// ctx->out_of_space = true; -// ctx->done = true; -// return; -// } + if (slot.is_virtual) { + wrote_inst = true; + if (slot.sym->const_val) { + WRITE_OP(&trace_dest[ctx->n_trace_dest], _Py_IsImmortal(slot.sym->const_val) ? + _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uint64_t)slot.sym->const_val); + } + else if (slot.sym->locals_idx >= 0) { + DPRINTF(3, "reifying LOAD_FAST %d\n", slot.sym->locals_idx); + WRITE_OP(&trace_dest[ctx->n_trace_dest], _LOAD_FAST, slot.sym->locals_idx, 0); + trace_dest[ctx->n_trace_dest].format = UOP_FORMAT_TARGET; + trace_dest[ctx->n_trace_dest].jump_target = 0; + } + else if (sym_is_null(slot)) { + WRITE_OP(&trace_dest[ctx->n_trace_dest], _PUSH_NULL, 0, 0); + } + else { + // Is static but not a constant value of locals or NULL. + // How is that possible? + Py_UNREACHABLE(); + } + ctx->n_trace_dest++; + if (ctx->n_trace_dest >= UOP_MAX_TRACE_LENGTH) { + ctx->out_of_space = true; + ctx->done = true; + return; + } + sp->is_virtual = false; + } } } @@ -570,21 +575,23 @@ partial_evaluate_uops( _PyUOpInstruction *this_instr = NULL; int i = 0; - bool prev_instr_is_truly_static = false; for (; !ctx->done; i++) { assert(i < trace_len); this_instr = &trace[i]; - trace_dest[ctx->n_trace_dest] = *this_instr; int oparg = this_instr->oparg; opcode = this_instr->opcode; uint64_t operand = this_instr->operand; _Py_UopsLocalsPlusSlot *stack_pointer = ctx->frame->stack_pointer; + _Py_UopsLocalsPlusSlot *old_sp = stack_pointer; // An instruction is candidate static if it has no escapes, and all its inputs // are static. // If so, whether it can be eliminated is up to whether it has an implementation. bool instr_is_truly_static = false; + if (!(_PyUop_Flags[opcode] & HAS_STATIC_FLAG)) { + reify_shadow_stack(ctx); + } #ifdef Py_DEBUG if (get_lltrace() >= 3) { @@ -609,29 +616,25 @@ partial_evaluate_uops( if (ctx->done) { break; } - // Always write these instructions for bookkeeping. - if (opcode == _CHECK_VALIDITY_AND_SET_IP || opcode == _SET_IP || opcode == _CHECK_VALIDITY) { - WRITE_OP(&trace_dest[ctx->n_trace_dest], opcode, oparg, operand); - ctx->n_trace_dest++; - } - // If the instruction is not static, - // reify the shadow stack, and write the op. - else if (!instr_is_truly_static) { - reify_shadow_stack(ctx); - WRITE_OP(&trace_dest[ctx->n_trace_dest], opcode, oparg, operand); + if (!instr_is_truly_static) { + trace_dest[ctx->n_trace_dest] = *this_instr; ctx->n_trace_dest++; + if (ctx->n_trace_dest >= UOP_MAX_TRACE_LENGTH) { + ctx->out_of_space = true; + ctx->done = true; + } } else { -//#ifdef Py_DEBUG -// if (get_lltrace() >= 3) { + // Inst is static. Nothing written :)! + assert((_PyUop_Flags[opcode] & HAS_STATIC_FLAG)); +#ifdef Py_DEBUG + if (get_lltrace() >= 3) { printf("%4d pe STATIC: ", (int) (this_instr - trace)); _PyUOpPrint(this_instr); printf("\n"); -// } -//#endif - // Inst is static. Nothing written :)! + } +#endif } - prev_instr_is_truly_static = instr_is_truly_static; } if (ctx->out_of_space) { DPRINTF(3, "\n"); @@ -658,10 +661,28 @@ partial_evaluate_uops( // That's the only time the PE's residual is valid. assert(ctx->n_trace_dest < UOP_MAX_TRACE_LENGTH); assert(is_terminator(this_instr)); - // Copy rest of trace to dest - memcpy(trace, trace_dest, ctx->n_trace_dest); + + // Copy rest of trace into trace_dest + memcpy(&trace_dest[ctx->n_trace_dest], &trace[i], (trace_len - i) * sizeof(_PyUOpInstruction )); + + printf("Optimized trace_dest (length %d):\n", ctx->n_trace_dest); + for (int x = 0; x < (trace_len - i) + ctx->n_trace_dest; x++) { + printf("%4d OPTIMIZED: ", x); + _PyUOpPrint(&trace_dest[x]); + printf("\n"); + } + + // Copy trace_dest into trace. + memcpy(trace, trace_dest, trace_len * sizeof(_PyUOpInstruction )); + printf("Optimized trace (length %d):\n", ctx->n_trace_dest); +// for (int i = 0; i < trace_len; i++) { +// printf("%4d OPTIMIZED: ", i); +// _PyUOpPrint(&trace[i]); +// printf("\n"); +// } + int trace_dest_len = ctx->n_trace_dest; _Py_uop_abstractcontext_fini(ctx); - return trace_len; + return (trace_len - i) + trace_dest_len; } error: @@ -720,7 +741,6 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } if (last->opcode == _LOAD_CONST_INLINE || last->opcode == _LOAD_CONST_INLINE_BORROW || -// last->opcode == _LOAD_FAST || last->opcode == _COPY ) { last->opcode = _NOP; @@ -787,6 +807,11 @@ _Py_uop_analyze_and_optimize( return length; } + // Help the PE by removing as many _CHECK_VALIDITY as possible, + // Since PE treats that as non-static since it can deopt arbitrarily. + + length = remove_unneeded_uops(buffer, length); + length = partial_evaluate_uops( _PyFrame_GetCode(frame), buffer, length, curr_stacklen, dependencies); @@ -795,8 +820,6 @@ _Py_uop_analyze_and_optimize( return length; } - length = remove_unneeded_uops(buffer, length); - assert(length > 0); OPT_STAT_INC(optimizer_successes); return length; diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index a5aff06f442e71..3116cbc3ff1841 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -76,6 +76,7 @@ dummy_func(void) { override op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); SET_STATIC_INST(); + value.is_virtual = true; } override op(_LOAD_FAST_AND_CLEAR, (-- value)) { @@ -95,11 +96,17 @@ dummy_func(void) { override op(_POP_TOP, (pop --)) { -// if (sym_is_virtual(pop)) { -// SET_STATIC_INST(); -// } + if (pop.is_virtual) { + SET_STATIC_INST(); + } + else { + reify_shadow_stack(ctx); + } } + override op(_NOP, (--)) { + SET_STATIC_INST(); + } override op (_CHECK_STACK_SPACE_OPERAND, ( -- )) { } diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index a9f9ea59ebece4..bd0a6a3e89d59e 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -4,6 +4,7 @@ // Do not edit! case _NOP: { + SET_STATIC_INST(); break; } @@ -40,6 +41,7 @@ _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); SET_STATIC_INST(); + value.is_virtual = true; stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -78,9 +80,13 @@ case _POP_TOP: { _Py_UopsLocalsPlusSlot pop; - // if (sym_is_virtual(pop)) { - // SET_STATIC_INST(); - // } + pop = stack_pointer[-1]; + if (pop.is_virtual) { + SET_STATIC_INST(); + } + else { + reify_shadow_stack(ctx); + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 3cc36b6b5841bd..60d7accdc40f4d 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -23,6 +23,7 @@ class Properties: has_free: bool side_exit: bool pure: bool + static: bool = False tier: int | None = None oparg_and_1: bool = False const_oparg: int = -1 @@ -674,6 +675,7 @@ def compute_properties(op: parser.InstDef) -> Properties: and not has_free, has_free=has_free, pure="pure" in op.annotations, + static="_static" in op.annotations, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index dd4057c931ca19..560a6795c95380 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -262,6 +262,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.oparg_and_1: flags.append("HAS_OPARG_AND_1_FLAG") + if p.static: + flags.append("HAS_STATIC_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index d5831593215f76..c171e0b94da5ed 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -226,6 +226,7 @@ def choice(*opts: str) -> str: "replicate", "tier1", "tier2", + "_static", } __all__ = [] diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 9b1bc98b5c08d7..58fffa3a5ac483 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -52,6 +52,7 @@ "PASSTHROUGH", "OPARG_AND_1", "ERROR_NO_POP", + "STATIC", ] From a6bc1a0dbdad294d204f167f27e14659b4cd8993 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 4 Sep 2024 03:16:25 +0800 Subject: [PATCH 07/46] dead store elimination --- Include/internal/pycore_opcode_metadata.h | 14 +++--- Include/internal/pycore_optimizer.h | 1 + Include/internal/pycore_uop_metadata.h | 22 ++++----- Lib/test/test_capi/test_opt.py | 14 ++++++ Python/bytecodes.c | 6 +-- Python/optimizer_analysis.c | 56 +++++++++-------------- Python/optimizer_bytecodes.c | 2 +- Python/optimizer_symbols.c | 6 +++ Python/partial_evaluator_bytecodes.c | 15 ++++-- Python/partial_evaluator_cases.c.h | 9 +++- 10 files changed, 84 insertions(+), 61 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index b0fa37381e9b69..960d2fd4c915f8 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1083,7 +1083,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [END_FOR] = { true, INSTR_FMT_IX, 0 }, [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1169,13 +1169,13 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, - [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [NOP] = { true, INSTR_FMT_IX, 0 }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [POP_TOP] = { true, INSTR_FMT_IX, 0 }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1228,10 +1228,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, - [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, - [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, - [SETUP_WITH] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [POP_BLOCK] = { true, -1, 0 }, + [SETUP_CLEANUP] = { true, -1, HAS_ARG_FLAG }, + [SETUP_FINALLY] = { true, -1, HAS_ARG_FLAG }, + [SETUP_WITH] = { true, -1, HAS_ARG_FLAG }, [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, }; #endif diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 906931dc2abffb..f2e15987c83b57 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -256,6 +256,7 @@ extern bool _Py_uop_sym_has_type(_Py_UopsLocalsPlusSlot sym); extern bool _Py_uop_sym_matches_type(_Py_UopsLocalsPlusSlot sym, PyTypeObject *typ); extern bool _Py_uop_sym_matches_type_version(_Py_UopsLocalsPlusSlot sym, unsigned int version); extern void _Py_uop_sym_set_locals_idx(_Py_UopsLocalsPlusSlot sym, int locals_idx); +extern int _Py_uop_sym_get_locals_idx(_Py_UopsLocalsPlusSlot sym); extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym); extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym); extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyTypeObject *typ); diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 9300d48b025a65..8d1d4af29ae7d1 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -19,7 +19,7 @@ extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { - [_NOP] = HAS_PURE_FLAG | HAS_STATIC_FLAG, + [_NOP] = HAS_STATIC_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, @@ -36,18 +36,18 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, - [_STORE_FAST_0] = HAS_LOCAL_FLAG, - [_STORE_FAST_1] = HAS_LOCAL_FLAG, - [_STORE_FAST_2] = HAS_LOCAL_FLAG, - [_STORE_FAST_3] = HAS_LOCAL_FLAG, - [_STORE_FAST_4] = HAS_LOCAL_FLAG, - [_STORE_FAST_5] = HAS_LOCAL_FLAG, - [_STORE_FAST_6] = HAS_LOCAL_FLAG, - [_STORE_FAST_7] = HAS_LOCAL_FLAG, - [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_STATIC_FLAG, [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_POP_TOP] = HAS_PURE_FLAG | HAS_STATIC_FLAG, + [_POP_TOP] = HAS_STATIC_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_SEND] = HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 449d589b984de8..302c214b74b28b 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1495,5 +1495,19 @@ def thing(a): self.assertEqual(list(iter_opnames(ex)).count("_POP_TOP"), 0) self.assertTrue(ex.is_valid()) + def test_pe_dead_store_elimination(self): + def thing(a): + x = 0 + for i in range(20): + x = x + return i + + + res, ex = self._run_with_optimizer(thing, 1) + self.assertEqual(res, 19) + self.assertIsNotNone(ex) + self.assertEqual(list(iter_opnames(ex)).count("_LOAD_FAST_1"), 0) + self.assertTrue(ex.is_valid()) + if __name__ == "__main__": unittest.main() diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a07fcf055332e7..337af0a35c821c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -140,7 +140,7 @@ dummy_func( switch (opcode) { // BEGIN BYTECODES // - pure _static inst(NOP, (--)) { + _static inst(NOP, (--)) { } family(RESUME, 0) = { @@ -261,7 +261,7 @@ dummy_func( value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); } - replicate(8) inst(STORE_FAST, (value --)) { + replicate(8) _static inst(STORE_FAST, (value --)) { SETLOCAL(oparg, value); } @@ -283,7 +283,7 @@ dummy_func( SETLOCAL(oparg2, value2); } - pure _static inst(POP_TOP, (value --)) { + _static inst(POP_TOP, (value --)) { DECREF_INPUTS(); } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index c9061eabe96bed..569ad14f161d9d 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -39,13 +39,12 @@ extern void _PyUOpPrint(const _PyUOpInstruction *uop); static const char *const DEBUG_ENV = "PYTHON_OPT_DEBUG"; static inline int get_lltrace(void) { - return 5; -// char *uop_debug = Py_GETENV(DEBUG_ENV); -// int lltrace = 0; -// if (uop_debug != NULL && *uop_debug >= '0') { -// lltrace = *uop_debug - '0'; // TODO: Parse an int and all that -// } -// return lltrace; + char *uop_debug = Py_GETENV(DEBUG_ENV); + int lltrace = 0; + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + } + return lltrace; } #define DPRINTF(level, ...) \ if (get_lltrace() >= (level)) { printf(__VA_ARGS__); } @@ -326,6 +325,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) #define sym_set_locals_idx _Py_uop_sym_set_locals_idx +#define sym_get_locals_idx _Py_uop_sym_get_locals_idx #define sym_is_bottom _Py_uop_sym_is_bottom #define sym_truthiness _Py_uop_sym_truthiness #define frame_new _Py_uop_frame_new @@ -512,16 +512,20 @@ reify_shadow_stack(_Py_UOpsContext *ctx) if (slot.is_virtual) { wrote_inst = true; if (slot.sym->const_val) { + DPRINTF(3, "reifying LOAD_CONST_INLINE\n"); WRITE_OP(&trace_dest[ctx->n_trace_dest], _Py_IsImmortal(slot.sym->const_val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uint64_t)slot.sym->const_val); + trace_dest[ctx->n_trace_dest].format = UOP_FORMAT_TARGET; + trace_dest[ctx->n_trace_dest].target = 100; } else if (slot.sym->locals_idx >= 0) { DPRINTF(3, "reifying LOAD_FAST %d\n", slot.sym->locals_idx); WRITE_OP(&trace_dest[ctx->n_trace_dest], _LOAD_FAST, slot.sym->locals_idx, 0); trace_dest[ctx->n_trace_dest].format = UOP_FORMAT_TARGET; - trace_dest[ctx->n_trace_dest].jump_target = 0; + trace_dest[ctx->n_trace_dest].target = 100; } else if (sym_is_null(slot)) { + DPRINTF(3, "reifying PUSH_NULL\n"); WRITE_OP(&trace_dest[ctx->n_trace_dest], _PUSH_NULL, 0, 0); } else { @@ -613,9 +617,6 @@ partial_evaluate_uops( DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); ctx->frame->stack_pointer = stack_pointer; assert(STACK_LEVEL() >= 0); - if (ctx->done) { - break; - } if (!instr_is_truly_static) { trace_dest[ctx->n_trace_dest] = *this_instr; ctx->n_trace_dest++; @@ -625,16 +626,17 @@ partial_evaluate_uops( } } else { - // Inst is static. Nothing written :)! - assert((_PyUop_Flags[opcode] & HAS_STATIC_FLAG)); + // Inst is static. Nothing written :)! + assert((_PyUop_Flags[opcode] & HAS_STATIC_FLAG)); #ifdef Py_DEBUG if (get_lltrace() >= 3) { - printf("%4d pe STATIC: ", (int) (this_instr - trace)); - _PyUOpPrint(this_instr); - printf("\n"); + printf("%4d pe -STATIC-\n", (int) (this_instr - trace)); } #endif } + if (ctx->done) { + break; + } } if (ctx->out_of_space) { DPRINTF(3, "\n"); @@ -661,28 +663,13 @@ partial_evaluate_uops( // That's the only time the PE's residual is valid. assert(ctx->n_trace_dest < UOP_MAX_TRACE_LENGTH); assert(is_terminator(this_instr)); - - // Copy rest of trace into trace_dest - memcpy(&trace_dest[ctx->n_trace_dest], &trace[i], (trace_len - i) * sizeof(_PyUOpInstruction )); - - printf("Optimized trace_dest (length %d):\n", ctx->n_trace_dest); - for (int x = 0; x < (trace_len - i) + ctx->n_trace_dest; x++) { - printf("%4d OPTIMIZED: ", x); - _PyUOpPrint(&trace_dest[x]); - printf("\n"); - } + assert(ctx->n_trace_dest <= trace_len); // Copy trace_dest into trace. - memcpy(trace, trace_dest, trace_len * sizeof(_PyUOpInstruction )); - printf("Optimized trace (length %d):\n", ctx->n_trace_dest); -// for (int i = 0; i < trace_len; i++) { -// printf("%4d OPTIMIZED: ", i); -// _PyUOpPrint(&trace[i]); -// printf("\n"); -// } + memcpy(trace, trace_dest, ctx->n_trace_dest * sizeof(_PyUOpInstruction )); int trace_dest_len = ctx->n_trace_dest; _Py_uop_abstractcontext_fini(ctx); - return (trace_len - i) + trace_dest_len; + return trace_dest_len; } error: @@ -741,6 +728,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } if (last->opcode == _LOAD_CONST_INLINE || last->opcode == _LOAD_CONST_INLINE_BORROW || + last->opcode == _LOAD_FAST || last->opcode == _COPY ) { last->opcode = _NOP; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 77514cfd0627ff..9877deadad2941 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -89,7 +89,7 @@ dummy_func(void) { GETLOCAL(oparg) = temp; } - op(_STORE_FAST, (value --)) { + _static op(_STORE_FAST, (value --)) { GETLOCAL(oparg) = value; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 2be4477cc11e34..22145330dd3b64 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -316,6 +316,12 @@ _Py_uop_sym_set_locals_idx(_Py_UopsLocalsPlusSlot sym, int locals_idx) sym.sym->locals_idx = locals_idx; } +int +_Py_uop_sym_get_locals_idx(_Py_UopsLocalsPlusSlot sym) +{ + return sym.sym->locals_idx; +} + int _Py_uop_sym_truthiness(_Py_UopsLocalsPlusSlot sym) { diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index 3116cbc3ff1841..d4f9a2083a61fc 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -75,6 +75,7 @@ dummy_func(void) { override op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); + sym_set_locals_idx(value, oparg); SET_STATIC_INST(); value.is_virtual = true; } @@ -84,16 +85,22 @@ dummy_func(void) { GETLOCAL(oparg) = sym_new_null(ctx); } - override op(_STORE_FAST, (value --)) { - GETLOCAL(oparg) = value; - sym_set_locals_idx(value, oparg); - } override op(_LOAD_CONST, (-- value)) { // Should've all been converted by specializer. Py_UNREACHABLE(); } + override op(_STORE_FAST, (value --)) { + // Gets rid of stores by the same load + if (value.is_virtual && oparg == sym_get_locals_idx(value)) { + SET_STATIC_INST(); + } + else { + reify_shadow_stack(ctx); + } + GETLOCAL(oparg) = value; + } override op(_POP_TOP, (pop --)) { if (pop.is_virtual) { diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index bd0a6a3e89d59e..9a1e5e75f9b250 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -40,6 +40,7 @@ case _LOAD_FAST: { _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); + sym_set_locals_idx(value, oparg); SET_STATIC_INST(); value.is_virtual = true; stack_pointer[0] = value; @@ -71,8 +72,14 @@ case _STORE_FAST: { _Py_UopsLocalsPlusSlot value; value = stack_pointer[-1]; + // Gets rid of stores by the same load + if (value.is_virtual && oparg == sym_get_locals_idx(value)) { + SET_STATIC_INST(); + } + else { + reify_shadow_stack(ctx); + } GETLOCAL(oparg) = value; - sym_set_locals_idx(value, oparg); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; From 6a6dbcec5382ccd9a949ec4979c28b28162da5a1 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 4 Sep 2024 03:28:27 +0800 Subject: [PATCH 08/46] cleanup --- Include/internal/pycore_optimizer.h | 1 - Python/bytecodes.c | 4 ++-- Python/executor_cases.c.h | 4 ++-- Python/optimizer_analysis.c | 8 ++------ Python/partial_evaluator_bytecodes.c | 18 ++---------------- Python/partial_evaluator_cases.c.h | 1 + 6 files changed, 9 insertions(+), 27 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index f2e15987c83b57..adde51eebe6423 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -192,7 +192,6 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) // handle before rejoining the rest of the program. #define MAX_CHAIN_DEPTH 4 - typedef struct _Py_UopsSymbol _Py_UopsSymbol; typedef struct _Py_UopsLocalsPlusSlot { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 337af0a35c821c..9ae41136ad95a0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4711,7 +4711,7 @@ dummy_func( if (lltrace >= 2) { printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", + printf(", exit %ld, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); @@ -4801,7 +4801,7 @@ dummy_func( if (lltrace >= 2) { printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", + printf(", exit %ld, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0de5c8a0408d8c..7751f0e751ac2c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -5277,7 +5277,7 @@ if (lltrace >= 2) { printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", + printf(", exit %ld, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); @@ -5416,7 +5416,7 @@ if (lltrace >= 2) { printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", + printf(", exit %ld, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 569ad14f161d9d..508525cae48dc4 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -502,15 +502,13 @@ optimize_uops( static void reify_shadow_stack(_Py_UOpsContext *ctx) -{; - bool wrote_inst = false; +{ _PyUOpInstruction *trace_dest = ctx->trace_dest; for (_Py_UopsLocalsPlusSlot *sp = ctx->frame->stack; sp < ctx->frame->stack_pointer; sp++) { _Py_UopsLocalsPlusSlot slot = *sp; assert(slot.sym != NULL); // Need reifying. if (slot.is_virtual) { - wrote_inst = true; if (slot.sym->const_val) { DPRINTF(3, "reifying LOAD_CONST_INLINE\n"); WRITE_OP(&trace_dest[ctx->n_trace_dest], _Py_IsImmortal(slot.sym->const_val) ? @@ -585,9 +583,7 @@ partial_evaluate_uops( int oparg = this_instr->oparg; opcode = this_instr->opcode; - uint64_t operand = this_instr->operand; _Py_UopsLocalsPlusSlot *stack_pointer = ctx->frame->stack_pointer; - _Py_UopsLocalsPlusSlot *old_sp = stack_pointer; // An instruction is candidate static if it has no escapes, and all its inputs // are static. @@ -797,8 +793,8 @@ _Py_uop_analyze_and_optimize( // Help the PE by removing as many _CHECK_VALIDITY as possible, // Since PE treats that as non-static since it can deopt arbitrarily. - length = remove_unneeded_uops(buffer, length); + assert(length > 0); length = partial_evaluate_uops( _PyFrame_GetCode(frame), buffer, diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index d4f9a2083a61fc..b6e634f0fbd7f1 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -48,21 +48,6 @@ extern PyCodeObject *get_code(_PyUOpInstruction *op); static int dummy_func(void) { - PyCodeObject *co; - int oparg; - _Py_UopsSymbol *flag; - _Py_UopsSymbol *left; - _Py_UopsSymbol *right; - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; - _Py_UopsSymbol *iter; - _Py_UopsSymbol *top; - _Py_UopsSymbol *bottom; - _Py_UOpsAbstractFrame *frame; - _Py_UOpsAbstractFrame *new_frame; - _Py_UOpsContext *ctx; - _PyUOpInstruction *this_instr; - // BEGIN BYTECODES // override op(_LOAD_FAST_CHECK, (-- value)) { @@ -115,7 +100,8 @@ dummy_func(void) { SET_STATIC_INST(); } - override op (_CHECK_STACK_SPACE_OPERAND, ( -- )) { + override op(_CHECK_STACK_SPACE_OPERAND, ( -- )) { + (void)framesize; } // END BYTECODES // diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index 9a1e5e75f9b250..89ad5fb7fa7d1f 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -2298,6 +2298,7 @@ case _CHECK_STACK_SPACE_OPERAND: { uint32_t framesize = (uint32_t)this_instr->operand; + (void)framesize; break; } From 7562c75b8679e6a2e17368a64a833145c3e05b96 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 4 Sep 2024 03:36:53 +0800 Subject: [PATCH 09/46] Create 2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst --- .../2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst b/Misc/NEWS.d/next/Core and Builtins/2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst new file mode 100644 index 00000000000000..6029db0998ebe9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst @@ -0,0 +1 @@ +Set up a tier 2 partial evaluation pass. Patch by Ken Jin. From 5200bceb3186ceae7609ccac22711badd21503eb Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 4 Sep 2024 03:39:44 +0800 Subject: [PATCH 10/46] fix tests --- Lib/test/test_generated_cases.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 7f821810aea00c..11c456e7707f69 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1196,15 +1196,15 @@ def test_overridden_abstract_args(self): """ output = """ case OP: { - _Py_UopsSymbol *arg1; - _Py_UopsSymbol *out; + _Py_UopsLocalsPlusSlot arg1; + _Py_UopsLocalsPlusSlot out; eggs(); stack_pointer[-1] = out; break; } case OP2: { - _Py_UopsSymbol *out; + _Py_UopsLocalsPlusSlot out; out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; @@ -1228,15 +1228,15 @@ def test_no_overridden_case(self): """ output = """ case OP: { - _Py_UopsSymbol *out; + _Py_UopsLocalsPlusSlot out; out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; } case OP2: { - _Py_UopsSymbol *arg1; - _Py_UopsSymbol *out; + _Py_UopsLocalsPlusSlot arg1; + _Py_UopsLocalsPlusSlot out; stack_pointer[-1] = out; break; } From 23e4b7cbf0e217f43c94abd1b6371d13c1e2e222 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 13 Sep 2024 21:03:33 +0800 Subject: [PATCH 11/46] Update partial_evaluator_cases.c.h --- Python/partial_evaluator_cases.c.h | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index 89ad5fb7fa7d1f..18945827b2b974 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -1702,11 +1702,13 @@ case _EXPAND_METHOD: { _Py_UopsLocalsPlusSlot method; - _Py_UopsLocalsPlusSlot self; + _Py_UopsLocalsPlusSlot *self; + self = &stack_pointer[-1 - oparg]; method = sym_new_not_null(ctx); - self = sym_new_not_null(ctx); + for (int _i = 1; --_i >= 0;) { + self[_i] = sym_new_not_null(ctx); + } stack_pointer[-2 - oparg] = method; - stack_pointer[-1 - oparg] = self; break; } @@ -2061,13 +2063,15 @@ case _EXPAND_METHOD_KW: { _Py_UopsLocalsPlusSlot method; - _Py_UopsLocalsPlusSlot self; + _Py_UopsLocalsPlusSlot *self; _Py_UopsLocalsPlusSlot kwnames; + self = &stack_pointer[-2 - oparg]; method = sym_new_not_null(ctx); - self = sym_new_not_null(ctx); + for (int _i = 1; --_i >= 0;) { + self[_i] = sym_new_not_null(ctx); + } kwnames = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = method; - stack_pointer[-2 - oparg] = self; stack_pointer[-1] = kwnames; break; } From 0a1d12ec98533bdae01f2e06318d6426eb8c0ee3 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 15 Sep 2024 18:11:10 +0800 Subject: [PATCH 12/46] Update partial_evaluator_cases.c.h --- Python/partial_evaluator_cases.c.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index 18945827b2b974..de42d05691572c 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -845,11 +845,13 @@ } case _LOAD_GLOBAL: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsLocalsPlusSlot *res; _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; - res = sym_new_not_null(ctx); + res = &stack_pointer[0]; + for (int _i = 1; --_i >= 0;) { + res[_i] = sym_new_not_null(ctx); + } null = sym_new_null(ctx); - stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); From bfbf6080ee4529788465aa4e03b19b76c1fc5e9d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 16 Sep 2024 00:07:06 +0800 Subject: [PATCH 13/46] reorder reifications --- Python/optimizer_analysis.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 508525cae48dc4..f8f63d82e38ea4 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -509,16 +509,16 @@ reify_shadow_stack(_Py_UOpsContext *ctx) assert(slot.sym != NULL); // Need reifying. if (slot.is_virtual) { - if (slot.sym->const_val) { - DPRINTF(3, "reifying LOAD_CONST_INLINE\n"); - WRITE_OP(&trace_dest[ctx->n_trace_dest], _Py_IsImmortal(slot.sym->const_val) ? - _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uint64_t)slot.sym->const_val); + if (slot.sym->locals_idx >= 0) { + DPRINTF(3, "reifying LOAD_FAST %d\n", slot.sym->locals_idx); + WRITE_OP(&trace_dest[ctx->n_trace_dest], _LOAD_FAST, slot.sym->locals_idx, 0); trace_dest[ctx->n_trace_dest].format = UOP_FORMAT_TARGET; trace_dest[ctx->n_trace_dest].target = 100; } - else if (slot.sym->locals_idx >= 0) { - DPRINTF(3, "reifying LOAD_FAST %d\n", slot.sym->locals_idx); - WRITE_OP(&trace_dest[ctx->n_trace_dest], _LOAD_FAST, slot.sym->locals_idx, 0); + else if (slot.sym->const_val) { + DPRINTF(3, "reifying LOAD_CONST_INLINE\n"); + WRITE_OP(&trace_dest[ctx->n_trace_dest], _Py_IsImmortal(slot.sym->const_val) ? + _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uint64_t)slot.sym->const_val); trace_dest[ctx->n_trace_dest].format = UOP_FORMAT_TARGET; trace_dest[ctx->n_trace_dest].target = 100; } From 8fe279ef6ef6fbf02704ba5c600daccd0b7dc912 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 16 Sep 2024 00:43:14 +0800 Subject: [PATCH 14/46] fix c-analzyer --- Tools/c-analyzer/cpython/_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 3a73f65f8ff7b3..6c8250f67073b4 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -84,6 +84,7 @@ def clean_lines(text): Python/generated_cases.c.h Python/executor_cases.c.h Python/optimizer_cases.c.h +Python/partial_evaluator_cases.c.h # not actually source Python/bytecodes.c From 4361821fab8c38d0d54c0d95790e79b6717bf6c6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 18 Sep 2024 02:32:13 +0800 Subject: [PATCH 15/46] remove static, remove some pure --- Include/internal/pycore_opcode_metadata.h | 34 ++++---- Include/internal/pycore_uop_metadata.h | 86 +++++++++---------- Python/bytecodes.c | 44 +++++----- Python/optimizer_analysis.c | 4 +- Python/optimizer_bytecodes.c | 2 +- Tools/cases_generator/analyzer.py | 2 - Tools/cases_generator/generators_common.py | 2 - Tools/cases_generator/lexer.py | 1 - .../opcode_metadata_generator.py | 1 - 9 files changed, 84 insertions(+), 92 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 3e9fa5588673be..f49f19cd6407a4 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -974,7 +974,6 @@ enum InstructionFormat { #define HAS_PASSTHROUGH_FLAG (4096) #define HAS_OPARG_AND_1_FLAG (8192) #define HAS_ERROR_NO_POP_FLAG (16384) -#define HAS_STATIC_FLAG (32768) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -990,7 +989,6 @@ enum InstructionFormat { #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) #define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) -#define OPCODE_HAS_STATIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_STATIC_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -1072,7 +1070,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, + [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1083,8 +1081,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [END_FOR] = { true, INSTR_FMT_IX, 0 }, - [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [END_SEND] = { true, INSTR_FMT_IX, 0 }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1147,7 +1145,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1169,15 +1167,15 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, - [NOP] = { true, INSTR_FMT_IX, 0 }, + [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_TOP] = { true, INSTR_FMT_IX, 0 }, + [POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, - [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, @@ -1197,7 +1195,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, - [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1206,7 +1204,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, + [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, @@ -1216,7 +1214,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, @@ -1227,12 +1225,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [_DO_CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [POP_BLOCK] = { true, -1, 0 }, - [SETUP_CLEANUP] = { true, -1, HAS_ARG_FLAG }, - [SETUP_FINALLY] = { true, -1, HAS_ARG_FLAG }, - [SETUP_WITH] = { true, -1, HAS_ARG_FLAG }, - [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, + [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [SETUP_WITH] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, }; #endif diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 06f51803ab13db..6616936f9ca692 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -19,39 +19,39 @@ extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { - [_NOP] = HAS_STATIC_FLAG, + [_NOP] = HAS_PURE_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, - [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_POP_TOP] = HAS_STATIC_FLAG, - [_PUSH_NULL] = HAS_PURE_FLAG, - [_END_SEND] = HAS_PURE_FLAG, + [_POP_TOP] = HAS_PURE_FLAG, + [_PUSH_NULL] = 0, + [_END_SEND] = 0, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_UNARY_NOT] = HAS_PURE_FLAG, + [_UNARY_NOT] = 0, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, [_TO_BOOL_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, @@ -63,17 +63,17 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_BOTH_INT] = HAS_EXIT_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG, + [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG, + [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG, [_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, - [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT] = 0, + [_BINARY_OP_ADD_FLOAT] = 0, + [_BINARY_OP_SUBTRACT_FLOAT] = 0, [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG, - [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -215,12 +215,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_PEP_523] = HAS_DEOPT_FLAG, [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_0] = HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_1] = HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_0] = 0, + [_INIT_CALL_PY_EXACT_ARGS_1] = 0, + [_INIT_CALL_PY_EXACT_ARGS_2] = 0, + [_INIT_CALL_PY_EXACT_ARGS_3] = 0, + [_INIT_CALL_PY_EXACT_ARGS_4] = 0, + [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG, [_PUSH_FRAME] = 0, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -252,9 +252,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_WITH_SPEC] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, + [_COPY] = HAS_ARG_FLAG, [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, + [_SWAP] = HAS_ARG_FLAG, [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, @@ -265,11 +265,11 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, - [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, - [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, - [_LOAD_CONST_INLINE_WITH_NULL] = HAS_PURE_FLAG, - [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE] = 0, + [_LOAD_CONST_INLINE_BORROW] = 0, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = 0, + [_LOAD_CONST_INLINE_WITH_NULL] = 0, + [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = 0, [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8f00fdb0158d29..6c1fd178ef9184 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -140,7 +140,7 @@ dummy_func( switch (opcode) { // BEGIN BYTECODES // - _static inst(NOP, (--)) { + pure inst(NOP, (--)) { } family(RESUME, 0) = { @@ -239,7 +239,7 @@ dummy_func( value = PyStackRef_DUP(value_s); } - replicate(8) _static inst(LOAD_FAST, (-- value)) { + replicate(8) pure inst(LOAD_FAST, (-- value)) { assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); } @@ -261,7 +261,7 @@ dummy_func( value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); } - replicate(8) _static inst(STORE_FAST, (value --)) { + replicate(8) pure inst(STORE_FAST, (value --)) { SETLOCAL(oparg, value); } @@ -283,11 +283,11 @@ dummy_func( SETLOCAL(oparg2, value2); } - _static inst(POP_TOP, (value --)) { + pure inst(POP_TOP, (value --)) { DECREF_INPUTS(); } - pure inst(PUSH_NULL, (-- res)) { + inst(PUSH_NULL, (-- res)) { res = PyStackRef_NULL; } @@ -305,7 +305,7 @@ dummy_func( DECREF_INPUTS(); } - pure inst(END_SEND, (receiver, value -- value)) { + inst(END_SEND, (receiver, value -- value)) { (void)receiver; PyStackRef_CLOSE(receiver); } @@ -328,7 +328,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - pure inst(UNARY_NOT, (value -- res)) { + inst(UNARY_NOT, (value -- res)) { assert(PyStackRef_BoolCheck(value)); res = PyStackRef_Is(value, PyStackRef_False) ? PyStackRef_True : PyStackRef_False; @@ -458,7 +458,7 @@ dummy_func( EXIT_IF(!PyLong_CheckExact(value_o)); } - pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { + op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -470,7 +470,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - pure op(_BINARY_OP_ADD_INT, (left, right -- res)) { + op(_BINARY_OP_ADD_INT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -482,7 +482,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { + op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -518,7 +518,7 @@ dummy_func( EXIT_IF(!PyFloat_CheckExact(value_o)); } - pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { + op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -531,7 +531,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { + op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -544,7 +544,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { + op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -572,7 +572,7 @@ dummy_func( EXIT_IF(!PyUnicode_CheckExact(right_o)); } - pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { + op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -3474,7 +3474,7 @@ dummy_func( DEOPT_IF(tstate->py_recursion_remaining <= 1); } - replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null[1], args[oparg] -- new_frame: _PyInterpreterFrame*)) { + replicate(5) op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null[1], args[oparg] -- new_frame: _PyInterpreterFrame*)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); @@ -4475,7 +4475,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - pure inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { + inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { assert(oparg > 0); top = PyStackRef_DUP(bottom); } @@ -4507,7 +4507,7 @@ dummy_func( macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + _BINARY_OP; - pure inst(SWAP, (bottom, unused[oparg-2], top -- + inst(SWAP, (bottom, unused[oparg-2], top -- top, unused[oparg-2], bottom)) { assert(oparg >= 2); } @@ -4747,25 +4747,25 @@ dummy_func( DEOPT_IF(!current_executor->vm_data.valid); } - tier2 pure op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + tier2 op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { value = PyStackRef_FromPyObjectNew(ptr); } - tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + tier2 op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { value = PyStackRef_FromPyObjectImmortal(ptr); } - tier2 pure op (_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { + tier2 op (_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { PyStackRef_CLOSE(pop); value = PyStackRef_FromPyObjectImmortal(ptr); } - tier2 pure op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { + tier2 op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { value = PyStackRef_FromPyObjectNew(ptr); null = PyStackRef_NULL; } - tier2 pure op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { + tier2 op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { value = PyStackRef_FromPyObjectImmortal(ptr); null = PyStackRef_NULL; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index f8f63d82e38ea4..9f63b98198eb3c 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -589,7 +589,7 @@ partial_evaluate_uops( // are static. // If so, whether it can be eliminated is up to whether it has an implementation. bool instr_is_truly_static = false; - if (!(_PyUop_Flags[opcode] & HAS_STATIC_FLAG)) { + if (!(_PyUop_Flags[opcode] & HAS_PURE_FLAG)) { reify_shadow_stack(ctx); } @@ -623,7 +623,7 @@ partial_evaluate_uops( } else { // Inst is static. Nothing written :)! - assert((_PyUop_Flags[opcode] & HAS_STATIC_FLAG)); + assert((_PyUop_Flags[opcode] & HAS_PURE_FLAG)); #ifdef Py_DEBUG if (get_lltrace() >= 3) { printf("%4d pe -STATIC-\n", (int) (this_instr - trace)); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 9877deadad2941..77514cfd0627ff 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -89,7 +89,7 @@ dummy_func(void) { GETLOCAL(oparg) = temp; } - _static op(_STORE_FAST, (value --)) { + op(_STORE_FAST, (value --)) { GETLOCAL(oparg) = value; } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 60d7accdc40f4d..3cc36b6b5841bd 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -23,7 +23,6 @@ class Properties: has_free: bool side_exit: bool pure: bool - static: bool = False tier: int | None = None oparg_and_1: bool = False const_oparg: int = -1 @@ -675,7 +674,6 @@ def compute_properties(op: parser.InstDef) -> Properties: and not has_free, has_free=has_free, pure="pure" in op.annotations, - static="_static" in op.annotations, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index c98cfce0b7adc4..2f8fccec2ea409 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -265,8 +265,6 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.oparg_and_1: flags.append("HAS_OPARG_AND_1_FLAG") - if p.static: - flags.append("HAS_STATIC_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index c171e0b94da5ed..d5831593215f76 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -226,7 +226,6 @@ def choice(*opts: str) -> str: "replicate", "tier1", "tier2", - "_static", } __all__ = [] diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 58fffa3a5ac483..9b1bc98b5c08d7 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -52,7 +52,6 @@ "PASSTHROUGH", "OPARG_AND_1", "ERROR_NO_POP", - "STATIC", ] From 5df786d61c4513d1e6fd965568de1c857c3ee51a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 18 Sep 2024 02:34:05 +0800 Subject: [PATCH 16/46] Revert "remove static, remove some pure" This reverts commit 4361821fab8c38d0d54c0d95790e79b6717bf6c6. --- Include/internal/pycore_opcode_metadata.h | 34 ++++---- Include/internal/pycore_uop_metadata.h | 86 +++++++++---------- Python/bytecodes.c | 44 +++++----- Python/optimizer_analysis.c | 4 +- Python/optimizer_bytecodes.c | 2 +- Tools/cases_generator/analyzer.py | 2 + Tools/cases_generator/generators_common.py | 2 + Tools/cases_generator/lexer.py | 1 + .../opcode_metadata_generator.py | 1 + 9 files changed, 92 insertions(+), 84 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index f49f19cd6407a4..3e9fa5588673be 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -974,6 +974,7 @@ enum InstructionFormat { #define HAS_PASSTHROUGH_FLAG (4096) #define HAS_OPARG_AND_1_FLAG (8192) #define HAS_ERROR_NO_POP_FLAG (16384) +#define HAS_STATIC_FLAG (32768) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -989,6 +990,7 @@ enum InstructionFormat { #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) #define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) +#define OPCODE_HAS_STATIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_STATIC_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -1070,7 +1072,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1081,8 +1083,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [END_SEND] = { true, INSTR_FMT_IX, 0 }, + [END_FOR] = { true, INSTR_FMT_IX, 0 }, + [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1145,7 +1147,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1167,15 +1169,15 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, - [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [NOP] = { true, INSTR_FMT_IX, 0 }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [POP_TOP] = { true, INSTR_FMT_IX, 0 }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, - [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, + [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, @@ -1195,7 +1197,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, - [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1204,7 +1206,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, @@ -1214,7 +1216,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, + [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, @@ -1225,12 +1227,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [_DO_CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, - [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, - [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, - [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, - [SETUP_WITH] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, - [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [POP_BLOCK] = { true, -1, 0 }, + [SETUP_CLEANUP] = { true, -1, HAS_ARG_FLAG }, + [SETUP_FINALLY] = { true, -1, HAS_ARG_FLAG }, + [SETUP_WITH] = { true, -1, HAS_ARG_FLAG }, + [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, }; #endif diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 6616936f9ca692..06f51803ab13db 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -19,39 +19,39 @@ extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { - [_NOP] = HAS_PURE_FLAG, + [_NOP] = HAS_STATIC_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_STATIC_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, - [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_STATIC_FLAG, [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_POP_TOP] = HAS_PURE_FLAG, - [_PUSH_NULL] = 0, - [_END_SEND] = 0, + [_POP_TOP] = HAS_STATIC_FLAG, + [_PUSH_NULL] = HAS_PURE_FLAG, + [_END_SEND] = HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_UNARY_NOT] = 0, + [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, [_TO_BOOL_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, @@ -63,17 +63,17 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_BOTH_INT] = HAS_EXIT_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG, - [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG, - [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG, + [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT] = 0, - [_BINARY_OP_ADD_FLOAT] = 0, - [_BINARY_OP_SUBTRACT_FLOAT] = 0, + [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG, - [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG, + [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -215,12 +215,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_PEP_523] = HAS_DEOPT_FLAG, [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_0] = 0, - [_INIT_CALL_PY_EXACT_ARGS_1] = 0, - [_INIT_CALL_PY_EXACT_ARGS_2] = 0, - [_INIT_CALL_PY_EXACT_ARGS_3] = 0, - [_INIT_CALL_PY_EXACT_ARGS_4] = 0, - [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_0] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_1] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = 0, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -252,9 +252,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_WITH_SPEC] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_COPY] = HAS_ARG_FLAG, + [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_SWAP] = HAS_ARG_FLAG, + [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, @@ -265,11 +265,11 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, - [_LOAD_CONST_INLINE] = 0, - [_LOAD_CONST_INLINE_BORROW] = 0, - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = 0, - [_LOAD_CONST_INLINE_WITH_NULL] = 0, - [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = 0, + [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_WITH_NULL] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6c1fd178ef9184..8f00fdb0158d29 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -140,7 +140,7 @@ dummy_func( switch (opcode) { // BEGIN BYTECODES // - pure inst(NOP, (--)) { + _static inst(NOP, (--)) { } family(RESUME, 0) = { @@ -239,7 +239,7 @@ dummy_func( value = PyStackRef_DUP(value_s); } - replicate(8) pure inst(LOAD_FAST, (-- value)) { + replicate(8) _static inst(LOAD_FAST, (-- value)) { assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); } @@ -261,7 +261,7 @@ dummy_func( value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); } - replicate(8) pure inst(STORE_FAST, (value --)) { + replicate(8) _static inst(STORE_FAST, (value --)) { SETLOCAL(oparg, value); } @@ -283,11 +283,11 @@ dummy_func( SETLOCAL(oparg2, value2); } - pure inst(POP_TOP, (value --)) { + _static inst(POP_TOP, (value --)) { DECREF_INPUTS(); } - inst(PUSH_NULL, (-- res)) { + pure inst(PUSH_NULL, (-- res)) { res = PyStackRef_NULL; } @@ -305,7 +305,7 @@ dummy_func( DECREF_INPUTS(); } - inst(END_SEND, (receiver, value -- value)) { + pure inst(END_SEND, (receiver, value -- value)) { (void)receiver; PyStackRef_CLOSE(receiver); } @@ -328,7 +328,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - inst(UNARY_NOT, (value -- res)) { + pure inst(UNARY_NOT, (value -- res)) { assert(PyStackRef_BoolCheck(value)); res = PyStackRef_Is(value, PyStackRef_False) ? PyStackRef_True : PyStackRef_False; @@ -458,7 +458,7 @@ dummy_func( EXIT_IF(!PyLong_CheckExact(value_o)); } - op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { + pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -470,7 +470,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_BINARY_OP_ADD_INT, (left, right -- res)) { + pure op(_BINARY_OP_ADD_INT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -482,7 +482,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { + pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -518,7 +518,7 @@ dummy_func( EXIT_IF(!PyFloat_CheckExact(value_o)); } - op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { + pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -531,7 +531,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { + pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -544,7 +544,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { + pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -572,7 +572,7 @@ dummy_func( EXIT_IF(!PyUnicode_CheckExact(right_o)); } - op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { + pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -3474,7 +3474,7 @@ dummy_func( DEOPT_IF(tstate->py_recursion_remaining <= 1); } - replicate(5) op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null[1], args[oparg] -- new_frame: _PyInterpreterFrame*)) { + replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null[1], args[oparg] -- new_frame: _PyInterpreterFrame*)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); @@ -4475,7 +4475,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { + pure inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { assert(oparg > 0); top = PyStackRef_DUP(bottom); } @@ -4507,7 +4507,7 @@ dummy_func( macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + _BINARY_OP; - inst(SWAP, (bottom, unused[oparg-2], top -- + pure inst(SWAP, (bottom, unused[oparg-2], top -- top, unused[oparg-2], bottom)) { assert(oparg >= 2); } @@ -4747,25 +4747,25 @@ dummy_func( DEOPT_IF(!current_executor->vm_data.valid); } - tier2 op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + tier2 pure op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { value = PyStackRef_FromPyObjectNew(ptr); } - tier2 op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { value = PyStackRef_FromPyObjectImmortal(ptr); } - tier2 op (_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { + tier2 pure op (_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { PyStackRef_CLOSE(pop); value = PyStackRef_FromPyObjectImmortal(ptr); } - tier2 op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { + tier2 pure op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { value = PyStackRef_FromPyObjectNew(ptr); null = PyStackRef_NULL; } - tier2 op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { + tier2 pure op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { value = PyStackRef_FromPyObjectImmortal(ptr); null = PyStackRef_NULL; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 9f63b98198eb3c..f8f63d82e38ea4 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -589,7 +589,7 @@ partial_evaluate_uops( // are static. // If so, whether it can be eliminated is up to whether it has an implementation. bool instr_is_truly_static = false; - if (!(_PyUop_Flags[opcode] & HAS_PURE_FLAG)) { + if (!(_PyUop_Flags[opcode] & HAS_STATIC_FLAG)) { reify_shadow_stack(ctx); } @@ -623,7 +623,7 @@ partial_evaluate_uops( } else { // Inst is static. Nothing written :)! - assert((_PyUop_Flags[opcode] & HAS_PURE_FLAG)); + assert((_PyUop_Flags[opcode] & HAS_STATIC_FLAG)); #ifdef Py_DEBUG if (get_lltrace() >= 3) { printf("%4d pe -STATIC-\n", (int) (this_instr - trace)); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 77514cfd0627ff..9877deadad2941 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -89,7 +89,7 @@ dummy_func(void) { GETLOCAL(oparg) = temp; } - op(_STORE_FAST, (value --)) { + _static op(_STORE_FAST, (value --)) { GETLOCAL(oparg) = value; } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 3cc36b6b5841bd..60d7accdc40f4d 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -23,6 +23,7 @@ class Properties: has_free: bool side_exit: bool pure: bool + static: bool = False tier: int | None = None oparg_and_1: bool = False const_oparg: int = -1 @@ -674,6 +675,7 @@ def compute_properties(op: parser.InstDef) -> Properties: and not has_free, has_free=has_free, pure="pure" in op.annotations, + static="_static" in op.annotations, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 2f8fccec2ea409..c98cfce0b7adc4 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -265,6 +265,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.oparg_and_1: flags.append("HAS_OPARG_AND_1_FLAG") + if p.static: + flags.append("HAS_STATIC_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index d5831593215f76..c171e0b94da5ed 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -226,6 +226,7 @@ def choice(*opts: str) -> str: "replicate", "tier1", "tier2", + "_static", } __all__ = [] diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 9b1bc98b5c08d7..58fffa3a5ac483 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -52,6 +52,7 @@ "PASSTHROUGH", "OPARG_AND_1", "ERROR_NO_POP", + "STATIC", ] From e8b402f60973227a84a725c32c998e2c0b891f21 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 18 Sep 2024 03:31:22 +0800 Subject: [PATCH 17/46] make LOAD_CONST static as well --- Include/internal/pycore_uop_metadata.h | 4 ++-- Python/bytecodes.c | 4 ++-- Python/optimizer_analysis.c | 12 ++++++------ Python/partial_evaluator_bytecodes.c | 22 +++++++++++++--------- Python/partial_evaluator_cases.c.h | 5 +++++ 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 06f51803ab13db..d15a72da3885bb 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -265,8 +265,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, - [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, - [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE] = HAS_STATIC_FLAG, + [_LOAD_CONST_INLINE_BORROW] = HAS_STATIC_FLAG, [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_WITH_NULL] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8f00fdb0158d29..b3c6aabe02cf1e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4747,11 +4747,11 @@ dummy_func( DEOPT_IF(!current_executor->vm_data.valid); } - tier2 pure op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + tier2 _static op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { value = PyStackRef_FromPyObjectNew(ptr); } - tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + tier2 _static op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { value = PyStackRef_FromPyObjectImmortal(ptr); } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index f8f63d82e38ea4..8edf4d75adadc9 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -509,21 +509,22 @@ reify_shadow_stack(_Py_UOpsContext *ctx) assert(slot.sym != NULL); // Need reifying. if (slot.is_virtual) { + sp->is_virtual = false; if (slot.sym->locals_idx >= 0) { - DPRINTF(3, "reifying LOAD_FAST %d\n", slot.sym->locals_idx); + DPRINTF(3, "reifying %d LOAD_FAST %d\n", (int)(sp - ctx->frame->stack), slot.sym->locals_idx); WRITE_OP(&trace_dest[ctx->n_trace_dest], _LOAD_FAST, slot.sym->locals_idx, 0); trace_dest[ctx->n_trace_dest].format = UOP_FORMAT_TARGET; - trace_dest[ctx->n_trace_dest].target = 100; + trace_dest[ctx->n_trace_dest].target = 0; } else if (slot.sym->const_val) { - DPRINTF(3, "reifying LOAD_CONST_INLINE\n"); + DPRINTF(3, "reifying %d LOAD_CONST_INLINE %p\n", (int)(sp - ctx->frame->stack), slot.sym->const_val); WRITE_OP(&trace_dest[ctx->n_trace_dest], _Py_IsImmortal(slot.sym->const_val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uint64_t)slot.sym->const_val); trace_dest[ctx->n_trace_dest].format = UOP_FORMAT_TARGET; - trace_dest[ctx->n_trace_dest].target = 100; + trace_dest[ctx->n_trace_dest].target = 0; } else if (sym_is_null(slot)) { - DPRINTF(3, "reifying PUSH_NULL\n"); + DPRINTF(3, "reifying %d PUSH_NULL\n", (int)(sp - ctx->frame->stack)); WRITE_OP(&trace_dest[ctx->n_trace_dest], _PUSH_NULL, 0, 0); } else { @@ -537,7 +538,6 @@ reify_shadow_stack(_Py_UOpsContext *ctx) ctx->done = true; return; } - sp->is_virtual = false; } } } diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index b6e634f0fbd7f1..8ef020e0b7250b 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -50,14 +50,6 @@ dummy_func(void) { // BEGIN BYTECODES // - override op(_LOAD_FAST_CHECK, (-- value)) { - value = GETLOCAL(oparg); - // We guarantee this will error - just bail and don't optimize it. - if (sym_is_null(value)) { - ctx->done = true; - } - } - override op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); sym_set_locals_idx(value, oparg); @@ -70,12 +62,23 @@ dummy_func(void) { GETLOCAL(oparg) = sym_new_null(ctx); } - override op(_LOAD_CONST, (-- value)) { // Should've all been converted by specializer. Py_UNREACHABLE(); } + override op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + value = sym_new_const(ctx, ptr); + SET_STATIC_INST(); + value.is_virtual = true; + } + + override op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + value = sym_new_const(ctx, ptr); + SET_STATIC_INST(); + value.is_virtual = true; + } + override op(_STORE_FAST, (value --)) { // Gets rid of stores by the same load if (value.is_virtual && oparg == sym_get_locals_idx(value)) { @@ -83,6 +86,7 @@ dummy_func(void) { } else { reify_shadow_stack(ctx); + value.is_virtual = false; } GETLOCAL(oparg) = value; } diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index de42d05691572c..6781a9f3a13373 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -78,6 +78,7 @@ } else { reify_shadow_stack(ctx); + value.is_virtual = false; } GETLOCAL(oparg) = value; stack_pointer += -1; @@ -2327,6 +2328,8 @@ _Py_UopsLocalsPlusSlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); + SET_STATIC_INST(); + value.is_virtual = true; stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2337,6 +2340,8 @@ _Py_UopsLocalsPlusSlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); + SET_STATIC_INST(); + value.is_virtual = true; stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); From d7d2c3cc638f775e0837aa6658e40e0a69d8d2fa Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 2 Oct 2024 23:52:10 +0800 Subject: [PATCH 18/46] fixup --- Makefile.pre.in | 5 +- .../partial_evaluator_generator.py | 234 ++++++++++++++++++ 2 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 Tools/cases_generator/partial_evaluator_generator.py diff --git a/Makefile.pre.in b/Makefile.pre.in index e98e7b8c8b22d3..9ccb735f4f0f8e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2006,7 +2006,7 @@ regen-optimizer-cases: .PHONY: regen-partial-evaluator-cases regen-partial-evaluator-cases: - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/optimizer_generator.py \ + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/partial_evaluator_generator.py \ -o $(srcdir)/Python/partial_evaluator_cases.c.h.new \ $(srcdir)/Python/optimizer_bytecodes.c \ $(srcdir)/Python/partial_evaluator_bytecodes.c \ @@ -2051,7 +2051,8 @@ Python/optimizer.o: \ Python/optimizer_analysis.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h \ $(srcdir)/Include/internal/pycore_optimizer.h \ - $(srcdir)/Python/optimizer_cases.c.h + $(srcdir)/Python/optimizer_cases.c.h \ + $(srcdir)/Python/partial_evaluator_cases.c.h Python/frozen.o: $(FROZEN_FILES_OUT) diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py new file mode 100644 index 00000000000000..8e3e6e2d6c6022 --- /dev/null +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -0,0 +1,234 @@ +"""Generate the cases for the tier 2 optimizer. +Reads the instruction definitions from bytecodes.c and optimizer_bytecodes.c +Writes the cases to optimizer_cases.c.h, which is #included in Python/optimizer_analysis.c. +""" + +import argparse + +from analyzer import ( + Analysis, + Instruction, + Uop, + analyze_files, + StackItem, + analysis_error, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + Emitter, +) +from cwriter import CWriter +from typing import TextIO, Iterator +from lexer import Token +from stack import Local, Stack, StackError + +DEFAULT_OUTPUT = ROOT / "Python/optimizer_cases.c.h" +DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/optimizer_bytecodes.c").absolute().as_posix() + + +def validate_uop(override: Uop, uop: Uop) -> None: + # To do + pass + + +def type_name(var: StackItem) -> str: + if var.is_array(): + return f"_Py_UopsLocalsPlusSlot *" + if var.type: + return var.type + return f"_Py_UopsLocalsPlusSlot " + + +def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: + variables = {"unused"} + if not skip_inputs: + for var in reversed(uop.stack.inputs): + if var.name not in variables: + variables.add(var.name) + if var.condition: + out.emit(f"{type_name(var)}{var.name} = (_Py_UopsLocalsPlusSlot){{NULL, 0}};\n") + else: + out.emit(f"{type_name(var)}{var.name};\n") + for var in uop.stack.outputs: + if var.peek: + continue + if var.name not in variables: + variables.add(var.name) + if var.condition: + out.emit(f"{type_name(var)}{var.name} = (_Py_UopsLocalsPlusSlot){{NULL, 0}};\n") + else: + out.emit(f"{type_name(var)}{var.name};\n") + + +def decref_inputs( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + out.emit_at("", tkn) + + +def emit_default(out: CWriter, uop: Uop) -> None: + for i, var in enumerate(uop.stack.outputs): + if var.name != "unused" and not var.peek: + if var.is_array(): + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") + out.emit("}\n") + elif var.name == "null": + out.emit(f"{var.name} = sym_new_null(ctx);\n") + else: + out.emit(f"{var.name} = sym_new_not_null(ctx);\n") + + +class OptimizerEmitter(Emitter): + pass + + +def write_uop( + override: Uop | None, + uop: Uop, + out: CWriter, + stack: Stack, + debug: bool, + skip_inputs: bool, +) -> None: + locals: dict[str, Local] = {} + try: + prototype = override if override else uop + is_override = override is not None + out.start_line() + for var in reversed(prototype.stack.inputs): + code, local = stack.pop(var, extract_bits=True) + if not skip_inputs: + out.emit(code) + if local.defined: + locals[local.name] = local + out.emit(stack.define_output_arrays(prototype.stack.outputs)) + if debug: + args = [] + for var in prototype.stack.inputs: + if not var.peek or is_override: + args.append(var.name) + out.emit(f'DEBUG_PRINTF({", ".join(args)});\n') + if override: + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = cast = "PyObject *" + else: + type = f"uint{cache.size*16}_t " + cast = f"uint{cache.size*16}_t" + out.emit(f"{type}{cache.name} = ({cast})this_instr->operand;\n") + if override: + emitter = OptimizerEmitter(out) + emitter.emit_tokens(override, stack, None) + else: + emit_default(out, uop) + + for var in prototype.stack.outputs: + if var.name in locals: + local = locals[var.name] + else: + local = Local.local(var) + stack.push(local) + out.start_line() + stack.flush(out, cast_type="", extract_bits=True) + except StackError as ex: + raise analysis_error(ex.args[0], uop.body[0]) + + +SKIPS = ("_EXTENDED_ARG",) + + +def generate_abstract_interpreter( + filenames: list[str], + abstract: Analysis, + base: Analysis, + outfile: TextIO, + debug: bool, +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 2, False) + out.emit("\n") + base_uop_names = set([uop.name for uop in base.uops.values()]) + for abstract_uop_name in abstract.uops: + assert ( + abstract_uop_name in base_uop_names + ), f"All abstract uops should override base uops, but {abstract_uop_name} is not." + + for uop in base.uops.values(): + override: Uop | None = None + if uop.name in abstract.uops: + override = abstract.uops[uop.name] + validate_uop(override, uop) + if uop.properties.tier == 1: + continue + if uop.replicates: + continue + if uop.is_super(): + continue + if not uop.is_viable(): + out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n") + continue + out.emit(f"case {uop.name}: {{\n") + if override: + declare_variables(override, out, skip_inputs=False) + else: + declare_variables(uop, out, skip_inputs=True) + stack = Stack() + write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) + out.start_line() + out.emit("break;\n") + out.emit("}") + out.emit("\n\n") + + +def generate_tier2_abstract_from_files( + filenames: list[str], outfilename: str, debug: bool = False +) -> None: + assert len(filenames) == 2, "Need a base file and an abstract cases file." + base = analyze_files([filenames[0]]) + abstract = analyze_files([filenames[1]]) + with open(outfilename, "w") as outfile: + generate_abstract_interpreter(filenames, abstract, base, outfile, debug) + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the tier 2 interpreter.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + + +arg_parser.add_argument("input", nargs="*", help="Abstract interpreter definition file") + +arg_parser.add_argument( + "base", nargs="*", help="The base instruction definition file(s)" +) + +arg_parser.add_argument("-d", "--debug", help="Insert debug calls", action="store_true") + +if __name__ == "__main__": + args = arg_parser.parse_args() + if not args.input: + args.base.append(DEFAULT_INPUT) + args.input.append(DEFAULT_ABSTRACT_INPUT) + else: + args.base.append(args.input[-1]) + args.input.pop() + abstract = analyze_files(args.input) + base = analyze_files(args.base) + with open(args.output, "w") as outfile: + generate_abstract_interpreter(args.input, abstract, base, outfile, args.debug) From 9a8986526cd77954eb614d4f9fffba9d5fb1800e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 3 Oct 2024 01:43:13 +0800 Subject: [PATCH 19/46] cleanup --- Include/internal/pycore_optimizer.h | 5 +-- Makefile.pre.in | 2 +- Python/optimizer_analysis.c | 60 ++++++++-------------------- Python/optimizer_symbols.c | 22 ---------- Python/partial_evaluator_bytecodes.c | 21 ---------- Python/partial_evaluator_cases.c.h | 28 +++---------- 6 files changed, 24 insertions(+), 114 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 857f9c48555ac5..60de6b2fff2a64 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -160,8 +160,7 @@ struct _Py_UopsSymbol { PyTypeObject *typ; // Borrowed reference PyObject *const_val; // Owned reference (!) unsigned int type_version; // currently stores type version - int locals_idx; - char is_static; // used for binding-time analysis + }; #define UOP_FORMAT_TARGET 0 @@ -262,8 +261,6 @@ extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_null(_Py_UOpsContext *ctx); extern bool _Py_uop_sym_has_type(_Py_UopsLocalsPlusSlot sym); extern bool _Py_uop_sym_matches_type(_Py_UopsLocalsPlusSlot sym, PyTypeObject *typ); extern bool _Py_uop_sym_matches_type_version(_Py_UopsLocalsPlusSlot sym, unsigned int version); -extern void _Py_uop_sym_set_locals_idx(_Py_UopsLocalsPlusSlot sym, int locals_idx); -extern int _Py_uop_sym_get_locals_idx(_Py_UopsLocalsPlusSlot sym); extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym); extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym); extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyTypeObject *typ); diff --git a/Makefile.pre.in b/Makefile.pre.in index 9ccb735f4f0f8e..bfbc146264d03d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2052,7 +2052,7 @@ Python/optimizer_analysis.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h \ $(srcdir)/Include/internal/pycore_optimizer.h \ $(srcdir)/Python/optimizer_cases.c.h \ - $(srcdir)/Python/partial_evaluator_cases.c.h + $(srcdir)/Python/partial_evaluator_cases.c.h Python/frozen.o: $(FROZEN_FILES_OUT) diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 0bf54f50d56d7b..ec0ba153687502 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -324,8 +324,6 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) #define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) -#define sym_set_locals_idx _Py_uop_sym_set_locals_idx -#define sym_get_locals_idx _Py_uop_sym_get_locals_idx #define sym_is_bottom _Py_uop_sym_is_bottom #define sym_truthiness _Py_uop_sym_truthiness #define frame_new _Py_uop_frame_new @@ -500,48 +498,27 @@ optimize_uops( #define SET_STATIC_INST() instr_is_truly_static = true; +static _Py_UopsLocalsPlusSlot +materialize(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot slot) +{ + return slot; +} + static void -reify_shadow_stack(_Py_UOpsContext *ctx) +materialize_whole_stack(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot *stack_start, _Py_UopsLocalsPlusSlot *stack_end) { - _PyUOpInstruction *trace_dest = ctx->trace_dest; - for (_Py_UopsLocalsPlusSlot *sp = ctx->frame->stack; sp < ctx->frame->stack_pointer; sp++) { - _Py_UopsLocalsPlusSlot slot = *sp; - assert(slot.sym != NULL); - // Need reifying. - if (slot.is_virtual) { - sp->is_virtual = false; - if (slot.sym->locals_idx >= 0) { - DPRINTF(3, "reifying %d LOAD_FAST %d\n", (int)(sp - ctx->frame->stack), slot.sym->locals_idx); - WRITE_OP(&trace_dest[ctx->n_trace_dest], _LOAD_FAST, slot.sym->locals_idx, 0); - trace_dest[ctx->n_trace_dest].format = UOP_FORMAT_TARGET; - trace_dest[ctx->n_trace_dest].target = 0; - } - else if (slot.sym->const_val) { - DPRINTF(3, "reifying %d LOAD_CONST_INLINE %p\n", (int)(sp - ctx->frame->stack), slot.sym->const_val); - WRITE_OP(&trace_dest[ctx->n_trace_dest], _Py_IsImmortal(slot.sym->const_val) ? - _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uint64_t)slot.sym->const_val); - trace_dest[ctx->n_trace_dest].format = UOP_FORMAT_TARGET; - trace_dest[ctx->n_trace_dest].target = 0; - } - else if (sym_is_null(slot)) { - DPRINTF(3, "reifying %d PUSH_NULL\n", (int)(sp - ctx->frame->stack)); - WRITE_OP(&trace_dest[ctx->n_trace_dest], _PUSH_NULL, 0, 0); - } - else { - // Is static but not a constant value of locals or NULL. - // How is that possible? - Py_UNREACHABLE(); - } - ctx->n_trace_dest++; - if (ctx->n_trace_dest >= UOP_MAX_TRACE_LENGTH) { - ctx->out_of_space = true; - ctx->done = true; - return; - } - } + while (stack_start < stack_end) { + materialize(ctx, *stack_start); + stack_start++; } } +static void +materialize_frame(_Py_UOpsContext *ctx, _Py_UOpsAbstractFrame *frame) +{ + materialize_whole_stack(ctx, frame->stack, frame->stack_pointer); +} + /* 1 for success, 0 for not ready, cannot error at the moment. */ static int partial_evaluate_uops( @@ -585,12 +562,9 @@ partial_evaluate_uops( opcode = this_instr->opcode; _Py_UopsLocalsPlusSlot *stack_pointer = ctx->frame->stack_pointer; - // An instruction is candidate static if it has no escapes, and all its inputs - // are static. - // If so, whether it can be eliminated is up to whether it has an implementation. bool instr_is_truly_static = false; if (!(_PyUop_Flags[opcode] & HAS_STATIC_FLAG)) { - reify_shadow_stack(ctx); + materialize_frame(ctx, ctx->frame); } #ifdef Py_DEBUG diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 22145330dd3b64..dfd6e843d3f775 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -82,8 +82,6 @@ sym_new(_Py_UOpsContext *ctx) self->typ = NULL; self->const_val = NULL; self->type_version = 0; - self->is_static = false; - self->locals_idx = -1; return self; } @@ -195,7 +193,6 @@ _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyObject sym.sym->typ = typ; sym.sym->const_val = Py_NewRef(const_val); } - sym.sym->is_static = true; } void @@ -205,7 +202,6 @@ _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym) sym_set_bottom(ctx, sym); } sym_set_flag(sym, IS_NULL); - sym.sym->is_static = true; } void @@ -309,19 +305,6 @@ _Py_uop_sym_matches_type_version(_Py_UopsLocalsPlusSlot sym, unsigned int versio return _Py_uop_sym_get_type_version(sym) == version; } -void -_Py_uop_sym_set_locals_idx(_Py_UopsLocalsPlusSlot sym, int locals_idx) -{ - assert(locals_idx >= 0); - sym.sym->locals_idx = locals_idx; -} - -int -_Py_uop_sym_get_locals_idx(_Py_UopsLocalsPlusSlot sym) -{ - return sym.sym->locals_idx; -} - int _Py_uop_sym_truthiness(_Py_UopsLocalsPlusSlot sym) { @@ -390,11 +373,6 @@ _Py_uop_frame_new( frame->locals[i] = _Py_uop_sym_new_unknown(ctx);; } - for (int i = 0; i < co->co_nlocalsplus; i++) { - frame->locals[i].sym->locals_idx = i; - } - - // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { frame->stack[i] = _Py_uop_sym_new_unknown(ctx); diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index 8ef020e0b7250b..ea09712238d0b3 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -52,9 +52,6 @@ dummy_func(void) { override op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); - sym_set_locals_idx(value, oparg); - SET_STATIC_INST(); - value.is_virtual = true; } override op(_LOAD_FAST_AND_CLEAR, (-- value)) { @@ -69,35 +66,17 @@ dummy_func(void) { override op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { value = sym_new_const(ctx, ptr); - SET_STATIC_INST(); - value.is_virtual = true; } override op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { value = sym_new_const(ctx, ptr); - SET_STATIC_INST(); - value.is_virtual = true; } override op(_STORE_FAST, (value --)) { - // Gets rid of stores by the same load - if (value.is_virtual && oparg == sym_get_locals_idx(value)) { - SET_STATIC_INST(); - } - else { - reify_shadow_stack(ctx); - value.is_virtual = false; - } GETLOCAL(oparg) = value; } override op(_POP_TOP, (pop --)) { - if (pop.is_virtual) { - SET_STATIC_INST(); - } - else { - reify_shadow_stack(ctx); - } } override op(_NOP, (--)) { diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index 6781a9f3a13373..d74520aaf92881 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -1,4 +1,4 @@ -// This file is generated by Tools/cases_generator/optimizer_generator.py +// This file is generated by Tools/cases_generator/partial_evaluator_generator.py // from: // Python/optimizer_bytecodes.c, Python/partial_evaluator_bytecodes.c // Do not edit! @@ -40,9 +40,6 @@ case _LOAD_FAST: { _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); - sym_set_locals_idx(value, oparg); - SET_STATIC_INST(); - value.is_virtual = true; stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -72,14 +69,6 @@ case _STORE_FAST: { _Py_UopsLocalsPlusSlot value; value = stack_pointer[-1]; - // Gets rid of stores by the same load - if (value.is_virtual && oparg == sym_get_locals_idx(value)) { - SET_STATIC_INST(); - } - else { - reify_shadow_stack(ctx); - value.is_virtual = false; - } GETLOCAL(oparg) = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -88,13 +77,6 @@ case _POP_TOP: { _Py_UopsLocalsPlusSlot pop; - pop = stack_pointer[-1]; - if (pop.is_virtual) { - SET_STATIC_INST(); - } - else { - reify_shadow_stack(ctx); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2328,8 +2310,6 @@ _Py_UopsLocalsPlusSlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); - SET_STATIC_INST(); - value.is_virtual = true; stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2340,8 +2320,6 @@ _Py_UopsLocalsPlusSlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); - SET_STATIC_INST(); - value.is_virtual = true; stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2399,6 +2377,10 @@ break; } + case _MAKE_WARM: { + break; + } + case _FATAL_ERROR: { break; } From 471d45f02ba5e404485d6b578edaf155cda102c0 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 3 Oct 2024 04:10:12 +0800 Subject: [PATCH 20/46] Use the other architecture --- Include/internal/pycore_optimizer.h | 7 +- Python/optimizer_analysis.c | 100 ++++++++++++++++----------- Python/optimizer_symbols.c | 27 +++++++- Python/partial_evaluator_bytecodes.c | 23 +++++- Python/partial_evaluator_cases.c.h | 23 +++++- 5 files changed, 132 insertions(+), 48 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 60de6b2fff2a64..ffad2b9232256c 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -59,6 +59,7 @@ typedef struct { }; }; uint64_t operand; // A cache entry + char is_virtual; // Used by tier 2 optimizer. } _PyUOpInstruction; typedef struct { @@ -160,7 +161,6 @@ struct _Py_UopsSymbol { PyTypeObject *typ; // Borrowed reference PyObject *const_val; // Owned reference (!) unsigned int type_version; // currently stores type version - }; #define UOP_FORMAT_TARGET 0 @@ -203,7 +203,7 @@ typedef struct _Py_UopsSymbol _Py_UopsSymbol; typedef struct _Py_UopsLocalsPlusSlot { _Py_UopsSymbol *sym; - char is_virtual; + _PyUOpInstruction *origin_inst; // The instruction this symbol originates from. } _Py_UopsLocalsPlusSlot; struct _Py_UOpsAbstractFrame { @@ -268,6 +268,9 @@ extern bool _Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsLocalsPlu extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyObject *const_val); extern bool _Py_uop_sym_is_bottom(_Py_UopsLocalsPlusSlot sym); extern int _Py_uop_sym_truthiness(_Py_UopsLocalsPlusSlot sym); +extern void _Py_uop_sym_set_origin_inst_override(_Py_UopsLocalsPlusSlot *sym, _PyUOpInstruction *origin); +extern bool _Py_uop_sym_is_virtual(_Py_UopsLocalsPlusSlot sym); +extern _PyUOpInstruction *_Py_uop_sym_get_origin(_Py_UopsLocalsPlusSlot sym); extern PyTypeObject *_Py_uop_sym_get_type(_Py_UopsLocalsPlusSlot sym); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index ec0ba153687502..6298fae219dc8b 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -491,32 +491,41 @@ optimize_uops( } -#define WRITE_OP(INST, OP, ARG, OPERAND) \ - (INST)->opcode = OP; \ - (INST)->oparg = ARG; \ - (INST)->operand = OPERAND; +#define MATERIALIZE_INST() (this_instr->is_virtual = false) +#define sym_set_origin_inst_override _Py_uop_sym_set_origin_inst_override +#define sym_is_virtual _Py_uop_sym_is_virtual +#define sym_get_origin _Py_uop_sym_get_origin -#define SET_STATIC_INST() instr_is_truly_static = true; - -static _Py_UopsLocalsPlusSlot -materialize(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot slot) +static void +materialize(_Py_UopsLocalsPlusSlot *slot) { - return slot; + assert(slot != NULL); + if (slot->origin_inst) { + slot->origin_inst->is_virtual = false; + } } static void -materialize_whole_stack(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot *stack_start, _Py_UopsLocalsPlusSlot *stack_end) +materialize_stack(_Py_UopsLocalsPlusSlot *stack_start, _Py_UopsLocalsPlusSlot *stack_end) { while (stack_start < stack_end) { - materialize(ctx, *stack_start); + materialize(stack_start); stack_start++; } } static void -materialize_frame(_Py_UOpsContext *ctx, _Py_UOpsAbstractFrame *frame) +materialize_frame(_Py_UOpsAbstractFrame *frame) +{ + materialize_stack(frame->stack, frame->stack_pointer); +} + +static void +materialize_ctx(_Py_UOpsContext *ctx) { - materialize_whole_stack(ctx, frame->stack, frame->stack_pointer); + for (int i = 0; i < ctx->curr_frame_depth; i++) { + materialize_frame(&ctx->frames[i]); + } } /* 1 for success, 0 for not ready, cannot error at the moment. */ @@ -529,11 +538,10 @@ partial_evaluate_uops( _PyBloomFilter *dependencies ) { - _PyUOpInstruction trace_dest[UOP_MAX_TRACE_LENGTH]; _Py_UOpsContext context; context.trace_dest = trace_dest; - context.n_trace_dest = 0; + context.n_trace_dest = trace_len; _Py_UOpsContext *ctx = &context; uint32_t opcode = UINT16_MAX; int curr_space = 0; @@ -552,29 +560,48 @@ partial_evaluate_uops( ctx->out_of_space = false; ctx->contradiction = false; + for (int i = 0; i < trace_len; i++) { + // The key part of PE --- we assume everything starts off virtual. + trace_dest[i] = trace[i]; + trace_dest[i].is_virtual = true; + } + _PyUOpInstruction *this_instr = NULL; int i = 0; for (; !ctx->done; i++) { assert(i < trace_len); - this_instr = &trace[i]; + this_instr = &trace_dest[i]; int oparg = this_instr->oparg; opcode = this_instr->opcode; _Py_UopsLocalsPlusSlot *stack_pointer = ctx->frame->stack_pointer; - bool instr_is_truly_static = false; - if (!(_PyUop_Flags[opcode] & HAS_STATIC_FLAG)) { - materialize_frame(ctx, ctx->frame); - } - #ifdef Py_DEBUG if (get_lltrace() >= 3) { - printf("%4d pe: ", (int)(this_instr - trace)); + printf("%4d pe: ", (int)(this_instr - trace_dest)); _PyUOpPrint(this_instr); printf(" "); } #endif + int is_static = (_PyUop_Flags[opcode] & HAS_STATIC_FLAG); + if (!is_static) { + MATERIALIZE_INST(); + } + if (!is_static && + // During these two opcodes, there's an abstract frame on the stack. + // Which is not a valid symbol. + (opcode != _PUSH_FRAME && opcode != _SAVE_RETURN_OFFSET)) { + // An escaping opcode means we need to materialize _everything_. + if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { + materialize_ctx(ctx); + } + else { + + materialize_frame(ctx->frame); + } + } + switch (opcode) { #include "partial_evaluator_cases.c.h" @@ -587,23 +614,6 @@ partial_evaluate_uops( DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); ctx->frame->stack_pointer = stack_pointer; assert(STACK_LEVEL() >= 0); - if (!instr_is_truly_static) { - trace_dest[ctx->n_trace_dest] = *this_instr; - ctx->n_trace_dest++; - if (ctx->n_trace_dest >= UOP_MAX_TRACE_LENGTH) { - ctx->out_of_space = true; - ctx->done = true; - } - } - else { - // Inst is static. Nothing written :)! - assert((_PyUop_Flags[opcode] & HAS_STATIC_FLAG)); -#ifdef Py_DEBUG - if (get_lltrace() >= 3) { - printf("%4d pe -STATIC-\n", (int) (this_instr - trace)); - } -#endif - } if (ctx->done) { break; } @@ -636,8 +646,18 @@ partial_evaluate_uops( assert(ctx->n_trace_dest <= trace_len); // Copy trace_dest into trace. - memcpy(trace, trace_dest, ctx->n_trace_dest * sizeof(_PyUOpInstruction )); int trace_dest_len = ctx->n_trace_dest; + // Only valid before we start inserting side exits. + assert(trace_dest_len == trace_len); + for (int x = 0; x < trace_dest_len; x++) { + // Skip all virtual instructions. + if (trace_dest[x].is_virtual) { + trace[x].opcode = _NOP; + } + else { + trace[x] = trace_dest[x]; + } + } _Py_uop_abstractcontext_fini(ctx); return trace_dest_len; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index dfd6e843d3f775..86ffa348da7153 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -57,7 +57,7 @@ static _Py_UopsSymbol NO_SPACE_SYMBOL = { static _Py_UopsLocalsPlusSlot NO_SPACE_SLOT = { .sym = &NO_SPACE_SYMBOL, - .is_virtual = 0, + .origin_inst = NULL, }; _Py_UopsLocalsPlusSlot @@ -213,11 +213,34 @@ _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym) sym_set_flag(sym, NOT_NULL); } +void +_Py_uop_sym_set_origin_inst_override(_Py_UopsLocalsPlusSlot *sym, _PyUOpInstruction *origin) +{ + sym->origin_inst = origin; +} + +_PyUOpInstruction * +_Py_uop_sym_get_origin(_Py_UopsLocalsPlusSlot sym) +{ + return sym.origin_inst; +} + +bool +_Py_uop_sym_is_virtual(_Py_UopsLocalsPlusSlot sym) +{ + if (!sym.origin_inst) { + return false; + } + else { + return sym.origin_inst->is_virtual; + } +} + _Py_UopsLocalsPlusSlot _Py_uop_sym_new_unknown(_Py_UOpsContext *ctx) { - return (_Py_UopsLocalsPlusSlot){sym_new(ctx), 0}; + return (_Py_UopsLocalsPlusSlot){sym_new(ctx), NULL}; } _Py_UopsLocalsPlusSlot diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index ea09712238d0b3..0c31e59d991232 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -52,11 +52,13 @@ dummy_func(void) { override op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); + sym_set_origin_inst_override(&value, this_instr); } override op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); GETLOCAL(oparg) = sym_new_null(ctx); + sym_set_origin_inst_override(&value, this_instr); } override op(_LOAD_CONST, (-- value)) { @@ -66,21 +68,38 @@ dummy_func(void) { override op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { value = sym_new_const(ctx, ptr); + sym_set_origin_inst_override(&value, this_instr); } override op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { value = sym_new_const(ctx, ptr); + sym_set_origin_inst_override(&value, this_instr); } override op(_STORE_FAST, (value --)) { - GETLOCAL(oparg) = value; + _PyUOpInstruction *origin = sym_get_origin(value); + // Gets rid of things like x = x. + if (sym_is_virtual(value) && + origin != NULL && + origin->opcode == _LOAD_FAST && + origin->oparg == oparg) { + // Leave it as virtual. + } + else { + materialize(&value); + MATERIALIZE_INST(); + GETLOCAL(oparg) = value; + } + } override op(_POP_TOP, (pop --)) { + if (!sym_is_virtual(pop)) { + MATERIALIZE_INST(); + } } override op(_NOP, (--)) { - SET_STATIC_INST(); } override op(_CHECK_STACK_SPACE_OPERAND, ( -- )) { diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index d74520aaf92881..d84e8312bfb690 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -4,7 +4,6 @@ // Do not edit! case _NOP: { - SET_STATIC_INST(); break; } @@ -40,6 +39,7 @@ case _LOAD_FAST: { _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); + sym_set_origin_inst_override(&value, this_instr); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -50,6 +50,7 @@ _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); GETLOCAL(oparg) = sym_new_null(ctx); + sym_set_origin_inst_override(&value, this_instr); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -69,7 +70,19 @@ case _STORE_FAST: { _Py_UopsLocalsPlusSlot value; value = stack_pointer[-1]; - GETLOCAL(oparg) = value; + _PyUOpInstruction *origin = sym_get_origin(value); + // Gets rid of things like x = x. + if (sym_is_virtual(value) && + origin != NULL && + origin->opcode == _LOAD_FAST && + origin->oparg == oparg) { + // Leave it as virtual. + } + else { + materialize(&value); + MATERIALIZE_INST(); + GETLOCAL(oparg) = value; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -77,6 +90,10 @@ case _POP_TOP: { _Py_UopsLocalsPlusSlot pop; + pop = stack_pointer[-1]; + if (!sym_is_virtual(pop)) { + MATERIALIZE_INST(); + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2310,6 +2327,7 @@ _Py_UopsLocalsPlusSlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); + sym_set_origin_inst_override(&value, this_instr); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2320,6 +2338,7 @@ _Py_UopsLocalsPlusSlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); + sym_set_origin_inst_override(&value, this_instr); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); From 9a167be9710f9c6558bdc7ef515691c8e678a5f8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 3 Oct 2024 04:20:26 +0800 Subject: [PATCH 21/46] Cleanup --- Include/internal/pycore_optimizer.h | 2 +- Python/optimizer_analysis.c | 5 ----- Tools/cases_generator/partial_evaluator_generator.py | 11 ++++++----- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index ffad2b9232256c..b1ca461fe1edda 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -165,7 +165,7 @@ struct _Py_UopsSymbol { #define UOP_FORMAT_TARGET 0 #define UOP_FORMAT_JUMP 1 -void _PyUOpPrint(const _PyUOpInstruction*); + static inline uint32_t uop_get_target(const _PyUOpInstruction *inst) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 6298fae219dc8b..0d91ad8b66ce12 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -385,11 +385,6 @@ get_code(_PyUOpInstruction *op) return co; } -static inline _Py_UopsLocalsPlusSlot -sym_to_slot(_Py_UopsSymbol *sym) -{ - return (_Py_UopsLocalsPlusSlot){sym, 0}; -} /* 1 for success, 0 for not ready, cannot error at the moment. */ static int diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py index 8e3e6e2d6c6022..22dde7acf14115 100644 --- a/Tools/cases_generator/partial_evaluator_generator.py +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -1,6 +1,6 @@ -"""Generate the cases for the tier 2 optimizer. -Reads the instruction definitions from bytecodes.c and optimizer_bytecodes.c -Writes the cases to optimizer_cases.c.h, which is #included in Python/optimizer_analysis.c. +"""Generate the cases for the tier 2 partial evaluator. +Reads the instruction definitions from bytecodes.c, optimizer_bytecodes.c and partial_evaluator_bytecodes.c +Writes the cases to partial_evaluator_cases.c.h, which is #included in Python/optimizer_analysis.c. """ import argparse @@ -24,9 +24,9 @@ from lexer import Token from stack import Local, Stack, StackError -DEFAULT_OUTPUT = ROOT / "Python/optimizer_cases.c.h" +DEFAULT_OUTPUT = ROOT / "Python/partial_evaluator_cases.c.h" DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/optimizer_bytecodes.c").absolute().as_posix() - +DEFAULT_PE_INPUT = (ROOT / "Python/partial_evaluator_bytecodes.c").absolute().as_posix() def validate_uop(override: Uop, uop: Uop) -> None: # To do @@ -225,6 +225,7 @@ def generate_tier2_abstract_from_files( if not args.input: args.base.append(DEFAULT_INPUT) args.input.append(DEFAULT_ABSTRACT_INPUT) + args.input.append(DEFAULT_PE_INPUT) else: args.base.append(args.input[-1]) args.input.pop() From 0b0f83f244648e84d5c91bdc22681578759f704c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 3 Oct 2024 04:30:40 +0800 Subject: [PATCH 22/46] Update partial_evaluator_cases.c.h --- Python/partial_evaluator_cases.c.h | 38 +++++++++++------------------- 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index d84e8312bfb690..244c06ae06dcf2 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -1679,15 +1679,18 @@ _Py_UopsLocalsPlusSlot self_or_null; _Py_UopsLocalsPlusSlot callable; _Py_UopsLocalsPlusSlot new_frame; - args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ - (void)callable; - (void)self_or_null; - (void)args; - new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; - ctx->done = true; + (void)(self_or_null); + (void)(callable); + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + new_frame.sym = frame_new(ctx, co, 0, NULL, 0); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1787,23 +1790,10 @@ (void)callable; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); - uint64_t push_operand = (this_instr + 2)->operand; - if (push_operand & 1) { - co = (PyCodeObject *)(push_operand & ~1); - DPRINTF(3, "code=%p ", co); - assert(PyCode_Check(co)); - } - else { - PyFunctionObject *func = (PyFunctionObject *)push_operand; - DPRINTF(3, "func=%p ", func); - if (func == NULL) { - DPRINTF(3, "\n"); - DPRINTF(1, "Missing function\n"); - ctx->done = true; - break; - } - co = (PyCodeObject *)func->func_code; - DPRINTF(3, "code=%p ", co); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; } assert(self_or_null.sym != NULL); assert(args != NULL); From 54b90576128de9c26c38c53271a9daf36e9a4d92 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 3 Oct 2024 23:25:09 +0800 Subject: [PATCH 23/46] Revert changes to optimizer --- Include/internal/pycore_optimizer.h | 69 +++----- Lib/test/test_generated_cases.py | 12 +- Python/optimizer_symbols.c | 172 ++++++++----------- Tools/cases_generator/optimizer_generator.py | 10 +- 4 files changed, 110 insertions(+), 153 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index b1ca461fe1edda..f92c0a0cddf906 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -59,7 +59,6 @@ typedef struct { }; }; uint64_t operand; // A cache entry - char is_virtual; // Used by tier 2 optimizer. } _PyUOpInstruction; typedef struct { @@ -166,7 +165,6 @@ struct _Py_UopsSymbol { #define UOP_FORMAT_TARGET 0 #define UOP_FORMAT_JUMP 1 - static inline uint32_t uop_get_target(const _PyUOpInstruction *inst) { assert(inst->format == UOP_FORMAT_TARGET); @@ -201,21 +199,14 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) typedef struct _Py_UopsSymbol _Py_UopsSymbol; -typedef struct _Py_UopsLocalsPlusSlot { - _Py_UopsSymbol *sym; - _PyUOpInstruction *origin_inst; // The instruction this symbol originates from. -} _Py_UopsLocalsPlusSlot; - struct _Py_UOpsAbstractFrame { // Max stacklen int stack_len; int locals_len; - _Py_UopsLocalsPlusSlot *stack_pointer; - _Py_UopsLocalsPlusSlot *stack; - _Py_UopsLocalsPlusSlot *locals; - - void *instr_ptr; + _Py_UopsSymbol **stack_pointer; + _Py_UopsSymbol **stack; + _Py_UopsSymbol **locals; }; typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; @@ -238,40 +229,34 @@ struct _Py_UOpsContext { // Arena for the symbolic types. ty_arena t_arena; - _Py_UopsLocalsPlusSlot *n_consumed; - _Py_UopsLocalsPlusSlot *limit; - _Py_UopsLocalsPlusSlot locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; - - _PyUOpInstruction *trace_dest; - int n_trace_dest; + _Py_UopsSymbol **n_consumed; + _Py_UopsSymbol **limit; + _Py_UopsSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; }; typedef struct _Py_UOpsContext _Py_UOpsContext; -extern bool _Py_uop_sym_is_null(_Py_UopsLocalsPlusSlot sym); -extern bool _Py_uop_sym_is_not_null(_Py_UopsLocalsPlusSlot sym); -extern bool _Py_uop_sym_is_const(_Py_UopsLocalsPlusSlot sym); -extern PyObject *_Py_uop_sym_get_const(_Py_UopsLocalsPlusSlot sym); -extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_unknown(_Py_UOpsContext *ctx); -extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_not_null(_Py_UOpsContext *ctx); -extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_type( +extern bool _Py_uop_sym_is_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_is_not_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_is_const(_Py_UopsSymbol *sym); +extern PyObject *_Py_uop_sym_get_const(_Py_UopsSymbol *sym); +extern _Py_UopsSymbol *_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx); +extern _Py_UopsSymbol *_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx); +extern _Py_UopsSymbol *_Py_uop_sym_new_type( _Py_UOpsContext *ctx, PyTypeObject *typ); -extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); -extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_null(_Py_UOpsContext *ctx); -extern bool _Py_uop_sym_has_type(_Py_UopsLocalsPlusSlot sym); -extern bool _Py_uop_sym_matches_type(_Py_UopsLocalsPlusSlot sym, PyTypeObject *typ); -extern bool _Py_uop_sym_matches_type_version(_Py_UopsLocalsPlusSlot sym, unsigned int version); -extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym); -extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym); -extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyTypeObject *typ); -extern bool _Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, unsigned int version); -extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyObject *const_val); -extern bool _Py_uop_sym_is_bottom(_Py_UopsLocalsPlusSlot sym); -extern int _Py_uop_sym_truthiness(_Py_UopsLocalsPlusSlot sym); -extern void _Py_uop_sym_set_origin_inst_override(_Py_UopsLocalsPlusSlot *sym, _PyUOpInstruction *origin); -extern bool _Py_uop_sym_is_virtual(_Py_UopsLocalsPlusSlot sym); -extern _PyUOpInstruction *_Py_uop_sym_get_origin(_Py_UopsLocalsPlusSlot sym); -extern PyTypeObject *_Py_uop_sym_get_type(_Py_UopsLocalsPlusSlot sym); +extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); +extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); +extern bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version); +extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); +extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); +extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version); +extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val); +extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); +extern int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym); +extern PyTypeObject *_Py_uop_sym_get_type(_Py_UopsSymbol *sym); extern void _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); @@ -281,7 +266,7 @@ extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( _Py_UOpsContext *ctx, PyCodeObject *co, int curr_stackentries, - _Py_UopsLocalsPlusSlot *args, + _Py_UopsSymbol **args, int arg_len); extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index e8d9d1861adf3a..214e53dde64bbf 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1264,15 +1264,15 @@ def test_overridden_abstract_args(self): """ output = """ case OP: { - _Py_UopsLocalsPlusSlot arg1; - _Py_UopsLocalsPlusSlot out; + _Py_UopsSymbol *arg1; + _Py_UopsSymbol *out; eggs(); stack_pointer[-1] = out; break; } case OP2: { - _Py_UopsLocalsPlusSlot out; + _Py_UopsSymbol *out; out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; @@ -1296,15 +1296,15 @@ def test_no_overridden_case(self): """ output = """ case OP: { - _Py_UopsLocalsPlusSlot out; + _Py_UopsSymbol *out; out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; } case OP2: { - _Py_UopsLocalsPlusSlot arg1; - _Py_UopsLocalsPlusSlot out; + _Py_UopsSymbol *arg1; + _Py_UopsSymbol *out; stack_pointer[-1] = out; break; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 86ffa348da7153..40cbf95e3d6d39 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -55,17 +55,12 @@ static _Py_UopsSymbol NO_SPACE_SYMBOL = { .type_version = 0, }; -static _Py_UopsLocalsPlusSlot NO_SPACE_SLOT = { - .sym = &NO_SPACE_SYMBOL, - .origin_inst = NULL, -}; - -_Py_UopsLocalsPlusSlot +_Py_UopsSymbol * out_of_space(_Py_UOpsContext *ctx) { ctx->done = true; ctx->out_of_space = true; - return NO_SPACE_SLOT; + return &NO_SPACE_SYMBOL; } static _Py_UopsSymbol * @@ -87,25 +82,24 @@ sym_new(_Py_UOpsContext *ctx) } static inline void -sym_set_flag(_Py_UopsLocalsPlusSlot sym, int flag) +sym_set_flag(_Py_UopsSymbol *sym, int flag) { - sym.sym->flags |= flag; + sym->flags |= flag; } static inline void -sym_set_bottom(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym) +sym_set_bottom(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) { sym_set_flag(sym, IS_NULL | NOT_NULL); - sym.sym->typ = NULL; - Py_CLEAR(sym.sym->const_val); + sym->typ = NULL; + Py_CLEAR(sym->const_val); ctx->done = true; ctx->contradiction = true; } bool -_Py_uop_sym_is_bottom(_Py_UopsLocalsPlusSlot sym_l) +_Py_uop_sym_is_bottom(_Py_UopsSymbol *sym) { - _Py_UopsSymbol *sym = sym_l.sym; if ((sym->flags & IS_NULL) && (sym->flags & NOT_NULL)) { assert(sym->flags == (IS_NULL | NOT_NULL)); assert(sym->typ == NULL); @@ -116,87 +110,87 @@ _Py_uop_sym_is_bottom(_Py_UopsLocalsPlusSlot sym_l) } bool -_Py_uop_sym_is_not_null(_Py_UopsLocalsPlusSlot sym) +_Py_uop_sym_is_not_null(_Py_UopsSymbol *sym) { - return sym.sym->flags == NOT_NULL; + return sym->flags == NOT_NULL; } bool -_Py_uop_sym_is_null(_Py_UopsLocalsPlusSlot sym) +_Py_uop_sym_is_null(_Py_UopsSymbol *sym) { - return sym.sym->flags == IS_NULL; + return sym->flags == IS_NULL; } bool -_Py_uop_sym_is_const(_Py_UopsLocalsPlusSlot sym) +_Py_uop_sym_is_const(_Py_UopsSymbol *sym) { - return sym.sym->const_val != NULL; + return sym->const_val != NULL; } PyObject * -_Py_uop_sym_get_const(_Py_UopsLocalsPlusSlot sym) +_Py_uop_sym_get_const(_Py_UopsSymbol *sym) { - return sym.sym->const_val; + return sym->const_val; } void -_Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyTypeObject *typ) +_Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); - if (sym.sym->flags & IS_NULL) { + if (sym->flags & IS_NULL) { sym_set_bottom(ctx, sym); return; } - if (sym.sym->typ != NULL) { - if (sym.sym->typ != typ) { + if (sym->typ != NULL) { + if (sym->typ != typ) { sym_set_bottom(ctx, sym); return; } } else { sym_set_flag(sym, NOT_NULL); - sym.sym->typ = typ; + sym->typ = typ; } } bool -_Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, unsigned int version) +_Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version) { // if the type version was already set, then it must be different and we should set it to bottom - if (sym.sym->type_version) { + if (sym->type_version) { sym_set_bottom(ctx, sym); return false; } - sym.sym->type_version = version; + sym->type_version = version; return true; } void -_Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyObject *const_val) +_Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val) { assert(const_val != NULL); - if (sym.sym->flags & IS_NULL) { + if (sym->flags & IS_NULL) { sym_set_bottom(ctx, sym); } PyTypeObject *typ = Py_TYPE(const_val); - if (sym.sym->typ != NULL && sym.sym->typ != typ) { + if (sym->typ != NULL && sym->typ != typ) { sym_set_bottom(ctx, sym); } - if (sym.sym->const_val != NULL) { - if (sym.sym->const_val != const_val) { + if (sym->const_val != NULL) { + if (sym->const_val != const_val) { // TODO: What if they're equal? sym_set_bottom(ctx, sym); } } else { sym_set_flag(sym, NOT_NULL); - sym.sym->typ = typ; - sym.sym->const_val = Py_NewRef(const_val); + sym->typ = typ; + sym->const_val = Py_NewRef(const_val); } } void -_Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym) +_Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) { if (_Py_uop_sym_is_not_null(sym)) { sym_set_bottom(ctx, sym); @@ -205,7 +199,7 @@ _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym) } void -_Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym) +_Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) { if (_Py_uop_sym_is_null(sym)) { sym_set_bottom(ctx, sym); @@ -213,61 +207,37 @@ _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym) sym_set_flag(sym, NOT_NULL); } -void -_Py_uop_sym_set_origin_inst_override(_Py_UopsLocalsPlusSlot *sym, _PyUOpInstruction *origin) -{ - sym->origin_inst = origin; -} -_PyUOpInstruction * -_Py_uop_sym_get_origin(_Py_UopsLocalsPlusSlot sym) -{ - return sym.origin_inst; -} - -bool -_Py_uop_sym_is_virtual(_Py_UopsLocalsPlusSlot sym) -{ - if (!sym.origin_inst) { - return false; - } - else { - return sym.origin_inst->is_virtual; - } -} - - -_Py_UopsLocalsPlusSlot +_Py_UopsSymbol * _Py_uop_sym_new_unknown(_Py_UOpsContext *ctx) { - return (_Py_UopsLocalsPlusSlot){sym_new(ctx), NULL}; + return sym_new(ctx); } -_Py_UopsLocalsPlusSlot +_Py_UopsSymbol * _Py_uop_sym_new_not_null(_Py_UOpsContext *ctx) { - _Py_UopsLocalsPlusSlot res = _Py_uop_sym_new_unknown(ctx); - if (res.sym == NULL) { + _Py_UopsSymbol *res = _Py_uop_sym_new_unknown(ctx); + if (res == NULL) { return out_of_space(ctx); } sym_set_flag(res, NOT_NULL); return res; } -_Py_UopsLocalsPlusSlot +_Py_UopsSymbol * _Py_uop_sym_new_type(_Py_UOpsContext *ctx, PyTypeObject *typ) { _Py_UopsSymbol *res = sym_new(ctx); if (res == NULL) { return out_of_space(ctx); } - _Py_UopsLocalsPlusSlot sym = {res, 0}; - _Py_uop_sym_set_type(ctx, sym, typ); - return sym; + _Py_uop_sym_set_type(ctx, res, typ); + return res; } // Adds a new reference to const_val, owned by the symbol. -_Py_UopsLocalsPlusSlot +_Py_UopsSymbol * _Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) { assert(const_val != NULL); @@ -275,16 +245,15 @@ _Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) if (res == NULL) { return out_of_space(ctx); } - _Py_UopsLocalsPlusSlot sym = {res, 0}; - _Py_uop_sym_set_const(ctx, sym, const_val); - return sym; + _Py_uop_sym_set_const(ctx, res, const_val); + return res; } -_Py_UopsLocalsPlusSlot +_Py_UopsSymbol * _Py_uop_sym_new_null(_Py_UOpsContext *ctx) { - _Py_UopsLocalsPlusSlot null_sym = _Py_uop_sym_new_unknown(ctx); - if (null_sym.sym == NULL) { + _Py_UopsSymbol *null_sym = _Py_uop_sym_new_unknown(ctx); + if (null_sym == NULL) { return out_of_space(ctx); } _Py_uop_sym_set_null(ctx, null_sym); @@ -292,44 +261,45 @@ _Py_uop_sym_new_null(_Py_UOpsContext *ctx) } PyTypeObject * -_Py_uop_sym_get_type(_Py_UopsLocalsPlusSlot sym) +_Py_uop_sym_get_type(_Py_UopsSymbol *sym) { if (_Py_uop_sym_is_bottom(sym)) { return NULL; } - return sym.sym->typ; + return sym->typ; } unsigned int -_Py_uop_sym_get_type_version(_Py_UopsLocalsPlusSlot sym) +_Py_uop_sym_get_type_version(_Py_UopsSymbol *sym) { - return sym.sym->type_version; + return sym->type_version; } bool -_Py_uop_sym_has_type(_Py_UopsLocalsPlusSlot sym) +_Py_uop_sym_has_type(_Py_UopsSymbol *sym) { if (_Py_uop_sym_is_bottom(sym)) { return false; } - return sym.sym->typ != NULL; + return sym->typ != NULL; } bool -_Py_uop_sym_matches_type(_Py_UopsLocalsPlusSlot sym, PyTypeObject *typ) +_Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); return _Py_uop_sym_get_type(sym) == typ; } bool -_Py_uop_sym_matches_type_version(_Py_UopsLocalsPlusSlot sym, unsigned int version) +_Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version) { return _Py_uop_sym_get_type_version(sym) == version; } + int -_Py_uop_sym_truthiness(_Py_UopsLocalsPlusSlot sym) +_Py_uop_sym_truthiness(_Py_UopsSymbol *sym) { /* There are some non-constant values for * which `bool(val)` always evaluates to @@ -368,7 +338,7 @@ _Py_uop_frame_new( _Py_UOpsContext *ctx, PyCodeObject *co, int curr_stackentries, - _Py_UopsLocalsPlusSlot *args, + _Py_UopsSymbol **args, int arg_len) { assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); @@ -393,12 +363,15 @@ _Py_uop_frame_new( } for (int i = arg_len; i < co->co_nlocalsplus; i++) { - frame->locals[i] = _Py_uop_sym_new_unknown(ctx);; + _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx); + frame->locals[i] = local; } + // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { - frame->stack[i] = _Py_uop_sym_new_unknown(ctx); + _Py_UopsSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); + frame->stack[i] = stackvar; } return frame; @@ -424,8 +397,7 @@ _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) ctx->n_consumed = ctx->locals_and_stack; #ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { - _Py_UopsLocalsPlusSlot slot = {NULL, 0}; - ctx->locals_and_stack[i] = slot; + ctx->locals_and_stack[i] = NULL; } #endif @@ -459,10 +431,10 @@ do { \ } \ } while (0) -static _Py_UopsLocalsPlusSlot +static _Py_UopsSymbol * make_bottom(_Py_UOpsContext *ctx) { - _Py_UopsLocalsPlusSlot sym = _Py_uop_sym_new_unknown(ctx); + _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); _Py_uop_sym_set_null(ctx, sym); _Py_uop_sym_set_non_null(ctx, sym); return sym; @@ -478,8 +450,8 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) PyObject *val_43 = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. - _Py_UopsLocalsPlusSlot sym = _Py_uop_sym_new_unknown(ctx); - if (sym.sym == NULL) { + _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); + if (sym == NULL) { goto fail; } TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); @@ -490,7 +462,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); sym = make_bottom(ctx); - if (sym.sym == NULL) { + if (sym == NULL) { goto fail; } TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); @@ -501,7 +473,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym.sym == NULL) { + if (sym == NULL) { goto fail; } TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); @@ -526,7 +498,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) assert(_Py_IsImmortal(val_43)); sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym.sym == NULL) { + if (sym == NULL) { goto fail; } _Py_uop_sym_set_const(ctx, sym, val_42); @@ -547,7 +519,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym.sym == NULL) { + if (sym == NULL) { goto fail; } _Py_uop_sym_set_const(ctx, sym, val_42); diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 8e3e6e2d6c6022..b74f627235ad84 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -35,10 +35,10 @@ def validate_uop(override: Uop, uop: Uop) -> None: def type_name(var: StackItem) -> str: if var.is_array(): - return f"_Py_UopsLocalsPlusSlot *" + return f"_Py_UopsSymbol **" if var.type: return var.type - return f"_Py_UopsLocalsPlusSlot " + return f"_Py_UopsSymbol *" def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: @@ -48,7 +48,7 @@ def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: if var.name not in variables: variables.add(var.name) if var.condition: - out.emit(f"{type_name(var)}{var.name} = (_Py_UopsLocalsPlusSlot){{NULL, 0}};\n") + out.emit(f"{type_name(var)}{var.name} = NULL;\n") else: out.emit(f"{type_name(var)}{var.name};\n") for var in uop.stack.outputs: @@ -57,7 +57,7 @@ def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: if var.name not in variables: variables.add(var.name) if var.condition: - out.emit(f"{type_name(var)}{var.name} = (_Py_UopsLocalsPlusSlot){{NULL, 0}};\n") + out.emit(f"{type_name(var)}{var.name} = NULL;\n") else: out.emit(f"{type_name(var)}{var.name};\n") @@ -141,7 +141,7 @@ def write_uop( local = Local.local(var) stack.push(local) out.start_line() - stack.flush(out, cast_type="", extract_bits=True) + stack.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=True) except StackError as ex: raise analysis_error(ex.args[0], uop.body[0]) From cea3f2ccbb2a7c837e5e895533fd17ed048a4815 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 3 Oct 2024 23:40:36 +0800 Subject: [PATCH 24/46] Create separate pe files --- .github/CODEOWNERS | 2 + Makefile.pre.in | 2 +- PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/optimizer_analysis.c | 198 +---- Python/optimizer_cases.c.h | 504 +++++------ Python/partial_evaluator.c | 320 +++++++ Python/partial_evaluator_bytecodes.c | 26 +- Python/partial_evaluator_cases.c.h | 792 ++---------------- .../partial_evaluator_generator.py | 4 +- 12 files changed, 697 insertions(+), 1159 deletions(-) create mode 100644 Python/partial_evaluator.c diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7e9c3caf23f079..dfc32d2193a4bd 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -42,6 +42,8 @@ Python/bytecodes.c @markshannon Python/optimizer*.c @markshannon Python/optimizer_analysis.c @Fidget-Spinner Python/optimizer_bytecodes.c @Fidget-Spinner +Python/partial_evaluator.c @Fidget-Spinner +Python/partial_evaluator_bytecodes.c @Fidget-Spinner Python/symtable.c @JelleZijlstra @carljm Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv Lib/test/test_patma.py @brandtbucher diff --git a/Makefile.pre.in b/Makefile.pre.in index bfbc146264d03d..531acc91721ab0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -470,6 +470,7 @@ PYTHON_OBJS= \ Python/optimizer_analysis.o \ Python/optimizer_symbols.o \ Python/parking_lot.o \ + Python/partial_evaluator.o \ Python/pathconfig.o \ Python/preconfig.o \ Python/pyarena.o \ @@ -2008,7 +2009,6 @@ regen-optimizer-cases: regen-partial-evaluator-cases: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/partial_evaluator_generator.py \ -o $(srcdir)/Python/partial_evaluator_cases.c.h.new \ - $(srcdir)/Python/optimizer_bytecodes.c \ $(srcdir)/Python/partial_evaluator_bytecodes.c \ $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/partial_evaluator_cases.c.h $(srcdir)/Python/partial_evaluator_cases.c.h.new diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index a3c2d32c454e04..baf72c491944ff 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -239,6 +239,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 91b1d75fb8df5e..bad40a1d52512c 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -326,6 +326,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 3b33c6bf6bb91d..6419d9dd350b9f 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -628,6 +628,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index ee2930b10439a9..2806311d94968f 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1424,6 +1424,9 @@ Python + + Python + Python diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 5489b81e73314f..06826ff942a761 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -333,8 +333,8 @@ static int optimize_to_bool( _PyUOpInstruction *this_instr, _Py_UOpsContext *ctx, - _Py_UopsLocalsPlusSlot value, - _Py_UopsLocalsPlusSlot *result_ptr) + _Py_UopsSymbol *value, + _Py_UopsSymbol **result_ptr) { if (sym_matches_type(value, &PyBool_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -446,7 +446,7 @@ optimize_uops( int oparg = this_instr->oparg; opcode = this_instr->opcode; - _Py_UopsLocalsPlusSlot *stack_pointer = ctx->frame->stack_pointer; + _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; #ifdef Py_DEBUG if (get_lltrace() >= 3) { @@ -509,187 +509,6 @@ optimize_uops( } -#define MATERIALIZE_INST() (this_instr->is_virtual = false) -#define sym_set_origin_inst_override _Py_uop_sym_set_origin_inst_override -#define sym_is_virtual _Py_uop_sym_is_virtual -#define sym_get_origin _Py_uop_sym_get_origin - -static void -materialize(_Py_UopsLocalsPlusSlot *slot) -{ - assert(slot != NULL); - if (slot->origin_inst) { - slot->origin_inst->is_virtual = false; - } -} - -static void -materialize_stack(_Py_UopsLocalsPlusSlot *stack_start, _Py_UopsLocalsPlusSlot *stack_end) -{ - while (stack_start < stack_end) { - materialize(stack_start); - stack_start++; - } -} - -static void -materialize_frame(_Py_UOpsAbstractFrame *frame) -{ - materialize_stack(frame->stack, frame->stack_pointer); -} - -static void -materialize_ctx(_Py_UOpsContext *ctx) -{ - for (int i = 0; i < ctx->curr_frame_depth; i++) { - materialize_frame(&ctx->frames[i]); - } -} - -/* 1 for success, 0 for not ready, cannot error at the moment. */ -static int -partial_evaluate_uops( - PyCodeObject *co, - _PyUOpInstruction *trace, - int trace_len, - int curr_stacklen, - _PyBloomFilter *dependencies -) -{ - _PyUOpInstruction trace_dest[UOP_MAX_TRACE_LENGTH]; - _Py_UOpsContext context; - context.trace_dest = trace_dest; - context.n_trace_dest = trace_len; - _Py_UOpsContext *ctx = &context; - uint32_t opcode = UINT16_MAX; - int curr_space = 0; - int max_space = 0; - _PyUOpInstruction *first_valid_check_stack = NULL; - _PyUOpInstruction *corresponding_check_stack = NULL; - - _Py_uop_abstractcontext_init(ctx); - _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, curr_stacklen, NULL, 0); - if (frame == NULL) { - return -1; - } - ctx->curr_frame_depth++; - ctx->frame = frame; - ctx->done = false; - ctx->out_of_space = false; - ctx->contradiction = false; - - for (int i = 0; i < trace_len; i++) { - // The key part of PE --- we assume everything starts off virtual. - trace_dest[i] = trace[i]; - trace_dest[i].is_virtual = true; - } - - _PyUOpInstruction *this_instr = NULL; - int i = 0; - for (; !ctx->done; i++) { - assert(i < trace_len); - this_instr = &trace_dest[i]; - - int oparg = this_instr->oparg; - opcode = this_instr->opcode; - _Py_UopsLocalsPlusSlot *stack_pointer = ctx->frame->stack_pointer; - -#ifdef Py_DEBUG - if (get_lltrace() >= 3) { - printf("%4d pe: ", (int)(this_instr - trace_dest)); - _PyUOpPrint(this_instr); - printf(" "); - } -#endif - - int is_static = (_PyUop_Flags[opcode] & HAS_STATIC_FLAG); - if (!is_static) { - MATERIALIZE_INST(); - } - if (!is_static && - // During these two opcodes, there's an abstract frame on the stack. - // Which is not a valid symbol. - (opcode != _PUSH_FRAME && opcode != _SAVE_RETURN_OFFSET)) { - // An escaping opcode means we need to materialize _everything_. - if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { - materialize_ctx(ctx); - } - else { - - materialize_frame(ctx->frame); - } - } - - switch (opcode) { - -#include "partial_evaluator_cases.c.h" - - default: - DPRINTF(1, "\nUnknown opcode in pe's abstract interpreter\n"); - Py_UNREACHABLE(); - } - assert(ctx->frame != NULL); - DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); - ctx->frame->stack_pointer = stack_pointer; - assert(STACK_LEVEL() >= 0); - if (ctx->done) { - break; - } - } - if (ctx->out_of_space) { - DPRINTF(3, "\n"); - DPRINTF(1, "Out of space in pe's abstract interpreter\n"); - } - if (ctx->contradiction) { - // Attempted to push a "bottom" (contradiction) symbol onto the stack. - // This means that the abstract interpreter has hit unreachable code. - // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting - // bottom indicates type instability, so we are probably better off - // retrying later. - DPRINTF(3, "\n"); - DPRINTF(1, "Hit bottom in pe's abstract interpreter\n"); - _Py_uop_abstractcontext_fini(ctx); - return 0; - } - - if (ctx->out_of_space || !is_terminator(this_instr)) { - _Py_uop_abstractcontext_fini(ctx); - return trace_len; - } - else { - // We MUST not have bailed early here. - // That's the only time the PE's residual is valid. - assert(ctx->n_trace_dest < UOP_MAX_TRACE_LENGTH); - assert(is_terminator(this_instr)); - assert(ctx->n_trace_dest <= trace_len); - - // Copy trace_dest into trace. - int trace_dest_len = ctx->n_trace_dest; - // Only valid before we start inserting side exits. - assert(trace_dest_len == trace_len); - for (int x = 0; x < trace_dest_len; x++) { - // Skip all virtual instructions. - if (trace_dest[x].is_virtual) { - trace[x].opcode = _NOP; - } - else { - trace[x] = trace_dest[x]; - } - } - _Py_uop_abstractcontext_fini(ctx); - return trace_dest_len; - } - -error: - DPRINTF(3, "\n"); - DPRINTF(1, "Encountered error in pe's abstract interpreter\n"); - if (opcode <= MAX_UOP_ID) { - OPT_ERROR_IN_OPCODE(opcode); - } - _Py_uop_abstractcontext_fini(ctx); - return -1; - -} static int remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) @@ -803,20 +622,9 @@ _Py_uop_analyze_and_optimize( return length; } - // Help the PE by removing as many _CHECK_VALIDITY as possible, - // Since PE treats that as non-static since it can deopt arbitrarily. length = remove_unneeded_uops(buffer, length); assert(length > 0); - length = partial_evaluate_uops( - _PyFrame_GetCode(frame), buffer, - length, curr_stacklen, dependencies); - - if (length <= 0) { - return length; - } - - OPT_STAT_INC(optimizer_successes); return length; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index c6298214614251..1ca85c2243f5f4 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -24,7 +24,7 @@ /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsSymbol *value; value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. if (sym_is_null(value)) { @@ -37,7 +37,7 @@ } case _LOAD_FAST: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsSymbol *value; value = GETLOCAL(oparg); stack_pointer[0] = value; stack_pointer += 1; @@ -46,7 +46,7 @@ } case _LOAD_FAST_AND_CLEAR: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsSymbol *value; value = GETLOCAL(oparg); _Py_UopsLocalsPlusSlot temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; @@ -57,7 +57,7 @@ } case _LOAD_CONST: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsSymbol *value; PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); @@ -69,7 +69,7 @@ } case _STORE_FAST: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsSymbol *value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; stack_pointer += -1; @@ -84,7 +84,7 @@ } case _PUSH_NULL: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -93,7 +93,7 @@ } case _END_SEND: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsSymbol *value; value = sym_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; @@ -102,22 +102,22 @@ } case _UNARY_NEGATIVE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_NOT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL: { - _Py_UopsLocalsPlusSlot value; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { res = sym_new_type(ctx, &PyBool_Type); @@ -127,8 +127,8 @@ } case _TO_BOOL_BOOL: { - _Py_UopsLocalsPlusSlot value; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyBool_Type); @@ -139,8 +139,8 @@ } case _TO_BOOL_INT: { - _Py_UopsLocalsPlusSlot value; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyLong_Type); @@ -151,8 +151,8 @@ } case _TO_BOOL_LIST: { - _Py_UopsLocalsPlusSlot value; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyList_Type); @@ -163,8 +163,8 @@ } case _TO_BOOL_NONE: { - _Py_UopsLocalsPlusSlot value; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_const(value, Py_None); @@ -175,8 +175,8 @@ } case _TO_BOOL_STR: { - _Py_UopsLocalsPlusSlot value; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { res = sym_new_type(ctx, &PyBool_Type); @@ -187,22 +187,22 @@ } case _REPLACE_WITH_TRUE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_INVERT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _GUARD_BOTH_INT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyLong_Type)) { @@ -232,9 +232,9 @@ } case _BINARY_OP_MULTIPLY_INT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -262,9 +262,9 @@ } case _BINARY_OP_ADD_INT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -292,9 +292,9 @@ } case _BINARY_OP_SUBTRACT_INT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -322,8 +322,8 @@ } case _GUARD_BOTH_FLOAT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyFloat_Type)) { @@ -353,9 +353,9 @@ } case _BINARY_OP_MULTIPLY_FLOAT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -384,9 +384,9 @@ } case _BINARY_OP_ADD_FLOAT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -415,9 +415,9 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -446,8 +446,8 @@ } case _GUARD_BOTH_UNICODE: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyUnicode_Type) && @@ -460,9 +460,9 @@ } case _BINARY_OP_ADD_UNICODE: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -490,7 +490,7 @@ } case _BINARY_SUBSCR: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -499,7 +499,7 @@ } case _BINARY_SLICE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -514,7 +514,7 @@ } case _BINARY_SUBSCR_LIST_INT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -523,7 +523,7 @@ } case _BINARY_SUBSCR_STR_INT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -532,7 +532,7 @@ } case _BINARY_SUBSCR_TUPLE_INT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -541,7 +541,7 @@ } case _BINARY_SUBSCR_DICT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -554,9 +554,9 @@ } case _BINARY_SUBSCR_INIT_CALL: { - _Py_UopsLocalsPlusSlot sub; - _Py_UopsLocalsPlusSlot container; - _Py_UopsLocalsPlusSlot new_frame; + _Py_UopsSymbol *sub; + _Py_UopsSymbol *container; + _Py_UopsSymbol *new_frame; sub = stack_pointer[-1]; container = stack_pointer[-2]; (void)container; @@ -606,14 +606,14 @@ } case _CALL_INTRINSIC_1: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -622,8 +622,8 @@ } case _RETURN_VALUE: { - _Py_UopsLocalsPlusSlot retval; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *retval; + _Py_UopsSymbol *res; retval = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -650,14 +650,14 @@ } case _GET_AITER: { - _Py_UopsLocalsPlusSlot iter; + _Py_UopsSymbol *iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - _Py_UopsLocalsPlusSlot awaitable; + _Py_UopsSymbol *awaitable; awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; @@ -666,7 +666,7 @@ } case _GET_AWAITABLE: { - _Py_UopsLocalsPlusSlot iter; + _Py_UopsSymbol *iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -681,7 +681,7 @@ } case _YIELD_VALUE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); stack_pointer[-1] = res; break; @@ -694,7 +694,7 @@ } case _LOAD_COMMON_CONSTANT: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsSymbol *value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -703,7 +703,7 @@ } case _LOAD_BUILD_CLASS: { - _Py_UopsLocalsPlusSlot bc; + _Py_UopsSymbol *bc; bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; @@ -722,8 +722,8 @@ } case _UNPACK_SEQUENCE: { - _Py_UopsLocalsPlusSlot seq; - _Py_UopsLocalsPlusSlot *values; + _Py_UopsSymbol *seq; + _Py_UopsSymbol **values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ @@ -737,8 +737,8 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { - _Py_UopsLocalsPlusSlot val1; - _Py_UopsLocalsPlusSlot val0; + _Py_UopsSymbol *val1; + _Py_UopsSymbol *val0; val1 = sym_new_not_null(ctx); val0 = sym_new_not_null(ctx); stack_pointer[-1] = val1; @@ -749,7 +749,7 @@ } case _UNPACK_SEQUENCE_TUPLE: { - _Py_UopsLocalsPlusSlot *values; + _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); @@ -760,7 +760,7 @@ } case _UNPACK_SEQUENCE_LIST: { - _Py_UopsLocalsPlusSlot *values; + _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); @@ -771,8 +771,8 @@ } case _UNPACK_EX: { - _Py_UopsLocalsPlusSlot seq; - _Py_UopsLocalsPlusSlot *values; + _Py_UopsSymbol *seq; + _Py_UopsSymbol **values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ @@ -809,7 +809,7 @@ } case _LOAD_LOCALS: { - _Py_UopsLocalsPlusSlot locals; + _Py_UopsSymbol *locals; locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; @@ -820,7 +820,7 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ case _LOAD_NAME: { - _Py_UopsLocalsPlusSlot v; + _Py_UopsSymbol *v; v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; @@ -829,8 +829,8 @@ } case _LOAD_GLOBAL: { - _Py_UopsLocalsPlusSlot *res; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsSymbol **res; + _Py_UopsSymbol *null = NULL; res = &stack_pointer[0]; for (int _i = 1; --_i >= 0;) { res[_i] = sym_new_not_null(ctx); @@ -851,8 +851,8 @@ } case _LOAD_GLOBAL_MODULE: { - _Py_UopsLocalsPlusSlot res; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -863,8 +863,8 @@ } case _LOAD_GLOBAL_BUILTINS: { - _Py_UopsLocalsPlusSlot res; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -887,14 +887,14 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsSymbol *value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsSymbol *value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -913,7 +913,7 @@ } case _BUILD_STRING: { - _Py_UopsLocalsPlusSlot str; + _Py_UopsSymbol *str; str = sym_new_not_null(ctx); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -922,7 +922,7 @@ } case _BUILD_TUPLE: { - _Py_UopsLocalsPlusSlot tup; + _Py_UopsSymbol *tup; tup = sym_new_not_null(ctx); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; @@ -931,7 +931,7 @@ } case _BUILD_LIST: { - _Py_UopsLocalsPlusSlot list; + _Py_UopsSymbol *list; list = sym_new_not_null(ctx); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -952,7 +952,7 @@ } case _BUILD_SET: { - _Py_UopsLocalsPlusSlot set; + _Py_UopsSymbol *set; set = sym_new_not_null(ctx); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -961,7 +961,7 @@ } case _BUILD_MAP: { - _Py_UopsLocalsPlusSlot map; + _Py_UopsSymbol *map; map = sym_new_not_null(ctx); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -994,7 +994,7 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ case _LOAD_SUPER_ATTR_ATTR: { - _Py_UopsLocalsPlusSlot attr_st; + _Py_UopsSymbol *attr_st; attr_st = sym_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -1003,8 +1003,8 @@ } case _LOAD_SUPER_ATTR_METHOD: { - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); stack_pointer[-3] = attr; @@ -1015,9 +1015,9 @@ } case _LOAD_ATTR: { - _Py_UopsLocalsPlusSlot owner; - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot self_or_null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null = NULL; owner = stack_pointer[-1]; (void)owner; attr = sym_new_not_null(ctx); @@ -1032,7 +1032,7 @@ } case _GUARD_TYPE_VERSION: { - _Py_UopsLocalsPlusSlot owner; + _Py_UopsSymbol *owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand; assert(type_version); @@ -1062,9 +1062,9 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - _Py_UopsLocalsPlusSlot owner; - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t offset = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1079,7 +1079,7 @@ } case _CHECK_ATTR_MODULE: { - _Py_UopsLocalsPlusSlot owner; + _Py_UopsSymbol *owner; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand; (void)dict_version; @@ -1100,9 +1100,9 @@ } case _LOAD_ATTR_MODULE: { - _Py_UopsLocalsPlusSlot owner; - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; (void)index; @@ -1136,9 +1136,9 @@ } case _LOAD_ATTR_WITH_HINT: { - _Py_UopsLocalsPlusSlot owner; - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t hint = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1153,9 +1153,9 @@ } case _LOAD_ATTR_SLOT: { - _Py_UopsLocalsPlusSlot owner; - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1174,9 +1174,9 @@ } case _LOAD_ATTR_CLASS: { - _Py_UopsLocalsPlusSlot owner; - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1191,8 +1191,8 @@ } case _LOAD_ATTR_PROPERTY_FRAME: { - _Py_UopsLocalsPlusSlot owner; - _Py_UopsLocalsPlusSlot new_frame; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *new_frame; owner = stack_pointer[-1]; PyObject *fget = (PyObject *)this_instr->operand; (void)fget; @@ -1228,9 +1228,9 @@ } case _COMPARE_OP: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1248,9 +1248,9 @@ } case _COMPARE_OP_FLOAT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1263,9 +1263,9 @@ } case _COMPARE_OP_INT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1278,9 +1278,9 @@ } case _COMPARE_OP_STR: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1293,9 +1293,9 @@ } case _IS_OP: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1308,9 +1308,9 @@ } case _CONTAINS_OP: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1323,7 +1323,7 @@ } case _CONTAINS_OP_SET: { - _Py_UopsLocalsPlusSlot b; + _Py_UopsSymbol *b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -1332,7 +1332,7 @@ } case _CONTAINS_OP_DICT: { - _Py_UopsLocalsPlusSlot b; + _Py_UopsSymbol *b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -1341,8 +1341,8 @@ } case _CHECK_EG_MATCH: { - _Py_UopsLocalsPlusSlot rest; - _Py_UopsLocalsPlusSlot match; + _Py_UopsSymbol *rest; + _Py_UopsSymbol *match; rest = sym_new_not_null(ctx); match = sym_new_not_null(ctx); stack_pointer[-2] = rest; @@ -1351,14 +1351,14 @@ } case _CHECK_EXC_MATCH: { - _Py_UopsLocalsPlusSlot b; + _Py_UopsSymbol *b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -1367,7 +1367,7 @@ } case _IMPORT_FROM: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1380,14 +1380,14 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - _Py_UopsLocalsPlusSlot b; + _Py_UopsSymbol *b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _GET_LEN: { - _Py_UopsLocalsPlusSlot len; + _Py_UopsSymbol *len; len = sym_new_not_null(ctx); stack_pointer[0] = len; stack_pointer += 1; @@ -1396,7 +1396,7 @@ } case _MATCH_CLASS: { - _Py_UopsLocalsPlusSlot attrs; + _Py_UopsSymbol *attrs; attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1405,7 +1405,7 @@ } case _MATCH_MAPPING: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1414,7 +1414,7 @@ } case _MATCH_SEQUENCE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1423,7 +1423,7 @@ } case _MATCH_KEYS: { - _Py_UopsLocalsPlusSlot values_or_none; + _Py_UopsSymbol *values_or_none; values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1432,14 +1432,14 @@ } case _GET_ITER: { - _Py_UopsLocalsPlusSlot iter; + _Py_UopsSymbol *iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_YIELD_FROM_ITER: { - _Py_UopsLocalsPlusSlot iter; + _Py_UopsSymbol *iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -1448,7 +1448,7 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - _Py_UopsLocalsPlusSlot next; + _Py_UopsSymbol *next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1469,7 +1469,7 @@ } case _ITER_NEXT_LIST: { - _Py_UopsLocalsPlusSlot next; + _Py_UopsSymbol *next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1488,7 +1488,7 @@ } case _ITER_NEXT_TUPLE: { - _Py_UopsLocalsPlusSlot next; + _Py_UopsSymbol *next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1507,8 +1507,8 @@ } case _ITER_NEXT_RANGE: { - _Py_UopsLocalsPlusSlot iter; - _Py_UopsLocalsPlusSlot next; + _Py_UopsSymbol *iter; + _Py_UopsSymbol *next; iter = stack_pointer[-1]; next = sym_new_type(ctx, &PyLong_Type); (void)iter; @@ -1525,9 +1525,9 @@ } case _LOAD_SPECIAL: { - _Py_UopsLocalsPlusSlot owner; - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null; owner = stack_pointer[-1]; (void)owner; attr = sym_new_not_null(ctx); @@ -1540,7 +1540,7 @@ } case _WITH_EXCEPT_START: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1549,8 +1549,8 @@ } case _PUSH_EXC_INFO: { - _Py_UopsLocalsPlusSlot prev_exc; - _Py_UopsLocalsPlusSlot new_exc; + _Py_UopsSymbol *prev_exc; + _Py_UopsSymbol *new_exc; prev_exc = sym_new_not_null(ctx); new_exc = sym_new_not_null(ctx); stack_pointer[-1] = prev_exc; @@ -1569,9 +1569,9 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - _Py_UopsLocalsPlusSlot owner; - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; @@ -1585,9 +1585,9 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { - _Py_UopsLocalsPlusSlot owner; - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; @@ -1601,14 +1601,14 @@ } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - _Py_UopsLocalsPlusSlot attr; + _Py_UopsSymbol *attr; attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - _Py_UopsLocalsPlusSlot attr; + _Py_UopsSymbol *attr; attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; @@ -1619,9 +1619,9 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - _Py_UopsLocalsPlusSlot owner; - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; @@ -1635,11 +1635,11 @@ } case _MAYBE_EXPAND_METHOD: { - _Py_UopsLocalsPlusSlot *args; - _Py_UopsLocalsPlusSlot self_or_null; - _Py_UopsLocalsPlusSlot callable; - _Py_UopsLocalsPlusSlot func; - _Py_UopsLocalsPlusSlot maybe_self; + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UopsSymbol *func; + _Py_UopsSymbol *maybe_self; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1659,10 +1659,10 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { - _Py_UopsLocalsPlusSlot *args; - _Py_UopsLocalsPlusSlot self_or_null; - _Py_UopsLocalsPlusSlot callable; - _Py_UopsLocalsPlusSlot new_frame; + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UopsSymbol *new_frame; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; (void)(self_or_null); @@ -1690,8 +1690,8 @@ } case _EXPAND_METHOD: { - _Py_UopsLocalsPlusSlot method; - _Py_UopsLocalsPlusSlot *self; + _Py_UopsSymbol *method; + _Py_UopsSymbol **self; self = &stack_pointer[-1 - oparg]; method = sym_new_not_null(ctx); for (int _i = 1; --_i >= 0;) { @@ -1706,7 +1706,7 @@ } case _CALL_NON_PY_GENERAL: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1715,8 +1715,8 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UopsLocalsPlusSlot null; - _Py_UopsLocalsPlusSlot callable; + _Py_UopsSymbol *null; + _Py_UopsSymbol *callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; sym_set_null(null); @@ -1725,9 +1725,9 @@ } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UopsLocalsPlusSlot callable; - _Py_UopsLocalsPlusSlot func; - _Py_UopsLocalsPlusSlot self; + _Py_UopsSymbol *callable; + _Py_UopsSymbol *func; + _Py_UopsSymbol *self; callable = stack_pointer[-2 - oparg]; (void)callable; func = sym_new_not_null(ctx); @@ -1747,8 +1747,8 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - _Py_UopsLocalsPlusSlot self_or_null; - _Py_UopsLocalsPlusSlot callable; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; sym_set_type(callable, &PyFunction_Type); @@ -1763,10 +1763,10 @@ } case _INIT_CALL_PY_EXACT_ARGS: { - _Py_UopsLocalsPlusSlot *args; - _Py_UopsLocalsPlusSlot self_or_null; - _Py_UopsLocalsPlusSlot callable; - _Py_UopsLocalsPlusSlot new_frame; + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UopsSymbol *new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1798,7 +1798,7 @@ } case _PUSH_FRAME: { - _Py_UopsLocalsPlusSlot new_frame; + _Py_UopsSymbol *new_frame; new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1834,7 +1834,7 @@ } case _CALL_TYPE_1: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1843,7 +1843,7 @@ } case _CALL_STR_1: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1852,7 +1852,7 @@ } case _CALL_TUPLE_1: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1861,11 +1861,11 @@ } case _CHECK_AND_ALLOCATE_OBJECT: { - _Py_UopsLocalsPlusSlot *args; - _Py_UopsLocalsPlusSlot null; - _Py_UopsLocalsPlusSlot callable; - _Py_UopsLocalsPlusSlot self; - _Py_UopsLocalsPlusSlot init; + _Py_UopsSymbol **args; + _Py_UopsSymbol *null; + _Py_UopsSymbol *callable; + _Py_UopsSymbol *self; + _Py_UopsSymbol *init; args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1883,10 +1883,10 @@ } case _CREATE_INIT_FRAME: { - _Py_UopsLocalsPlusSlot *args; - _Py_UopsLocalsPlusSlot init; - _Py_UopsLocalsPlusSlot self; - _Py_UopsLocalsPlusSlot init_frame; + _Py_UopsSymbol **args; + _Py_UopsSymbol *init; + _Py_UopsSymbol *self; + _Py_UopsSymbol *init_frame; args = &stack_pointer[-oparg]; init = stack_pointer[-1 - oparg]; self = stack_pointer[-2 - oparg]; @@ -1908,7 +1908,7 @@ } case _CALL_BUILTIN_CLASS: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1917,7 +1917,7 @@ } case _CALL_BUILTIN_O: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1926,7 +1926,7 @@ } case _CALL_BUILTIN_FAST: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1935,7 +1935,7 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1944,7 +1944,7 @@ } case _CALL_LEN: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1953,7 +1953,7 @@ } case _CALL_ISINSTANCE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1968,7 +1968,7 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1977,7 +1977,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1986,7 +1986,7 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1995,7 +1995,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2008,11 +2008,11 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { - _Py_UopsLocalsPlusSlot kwnames; - _Py_UopsLocalsPlusSlot *args; - _Py_UopsLocalsPlusSlot self_or_null; - _Py_UopsLocalsPlusSlot callable; - _Py_UopsLocalsPlusSlot new_frame; + _Py_UopsSymbol *kwnames; + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UopsSymbol *new_frame; kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; @@ -2038,9 +2038,9 @@ } case _EXPAND_METHOD_KW: { - _Py_UopsLocalsPlusSlot method; - _Py_UopsLocalsPlusSlot *self; - _Py_UopsLocalsPlusSlot kwnames; + _Py_UopsSymbol *method; + _Py_UopsSymbol **self; + _Py_UopsSymbol *kwnames; self = &stack_pointer[-2 - oparg]; method = sym_new_not_null(ctx); for (int _i = 1; --_i >= 0;) { @@ -2057,7 +2057,7 @@ } case _CALL_KW_NON_PY: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; @@ -2070,14 +2070,14 @@ /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - _Py_UopsLocalsPlusSlot func; + _Py_UopsSymbol *func; func = sym_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - _Py_UopsLocalsPlusSlot func_st; + _Py_UopsSymbol *func_st; func_st = sym_new_not_null(ctx); stack_pointer[-2] = func_st; stack_pointer += -1; @@ -2086,7 +2086,7 @@ } case _RETURN_GENERATOR: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -2110,7 +2110,7 @@ } case _BUILD_SLICE: { - _Py_UopsLocalsPlusSlot slice; + _Py_UopsSymbol *slice; slice = sym_new_not_null(ctx); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -2119,21 +2119,21 @@ } case _CONVERT_VALUE: { - _Py_UopsLocalsPlusSlot result; + _Py_UopsSymbol *result; result = sym_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -2142,8 +2142,8 @@ } case _COPY: { - _Py_UopsLocalsPlusSlot bottom; - _Py_UopsLocalsPlusSlot top; + _Py_UopsSymbol *bottom; + _Py_UopsSymbol *top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; @@ -2154,9 +2154,9 @@ } case _BINARY_OP: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; PyTypeObject *ltype = sym_get_type(left); @@ -2182,8 +2182,8 @@ } case _SWAP: { - _Py_UopsLocalsPlusSlot top; - _Py_UopsLocalsPlusSlot bottom; + _Py_UopsSymbol *top; + _Py_UopsSymbol *bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; stack_pointer[-2 - (oparg-2)] = top; @@ -2208,7 +2208,7 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { - _Py_UopsLocalsPlusSlot flag; + _Py_UopsSymbol *flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2221,7 +2221,7 @@ } case _GUARD_IS_FALSE_POP: { - _Py_UopsLocalsPlusSlot flag; + _Py_UopsSymbol *flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2234,7 +2234,7 @@ } case _GUARD_IS_NONE_POP: { - _Py_UopsLocalsPlusSlot flag; + _Py_UopsSymbol *flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2251,7 +2251,7 @@ } case _GUARD_IS_NOT_NONE_POP: { - _Py_UopsLocalsPlusSlot flag; + _Py_UopsSymbol *flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2301,7 +2301,7 @@ } case _LOAD_CONST_INLINE: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsSymbol *value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); stack_pointer[0] = value; @@ -2311,7 +2311,7 @@ } case _LOAD_CONST_INLINE_BORROW: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsSymbol *value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); stack_pointer[0] = value; @@ -2321,15 +2321,15 @@ } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsSymbol *value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_CONST_INLINE_WITH_NULL: { - _Py_UopsLocalsPlusSlot value; - _Py_UopsLocalsPlusSlot null; + _Py_UopsSymbol *value; + _Py_UopsSymbol *null; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); null = sym_new_null(ctx); @@ -2341,8 +2341,8 @@ } case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { - _Py_UopsLocalsPlusSlot value; - _Py_UopsLocalsPlusSlot null; + _Py_UopsSymbol *value; + _Py_UopsSymbol *null; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); null = sym_new_null(ctx); diff --git a/Python/partial_evaluator.c b/Python/partial_evaluator.c new file mode 100644 index 00000000000000..a1facb3307c37d --- /dev/null +++ b/Python/partial_evaluator.c @@ -0,0 +1,320 @@ +#ifdef _Py_TIER2 + +/* + * This file contains the support code for CPython's partial evaluator. + * It performs an abstract interpretation[1] over the trace of uops. + * Using the information gained, it chooses to emit, or skip certain instructions + * if possible. + * + * [1] For information on abstract interpertation and partial evaluation, please see + * https://en.wikipedia.org/wiki/Abstract_interpretation + * https://en.wikipedia.org/wiki/Partial_evaluation + * + * */ +#include "Python.h" +#include "opcode.h" +#include "pycore_dict.h" +#include "pycore_interp.h" +#include "pycore_opcode_metadata.h" +#include "pycore_opcode_utils.h" +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_uop_metadata.h" +#include "pycore_dict.h" +#include "pycore_long.h" +#include "pycore_optimizer.h" +#include "pycore_object.h" +#include "pycore_dict.h" +#include "pycore_function.h" +#include "pycore_uop_metadata.h" +#include "pycore_uop_ids.h" +#include "pycore_range.h" + +#include +#include +#include +#include + +#ifdef Py_DEBUG + extern const char *_PyUOpName(int index); + extern void _PyUOpPrint(const _PyUOpInstruction *uop); + static const char *const DEBUG_ENV = "PYTHON_OPT_DEBUG"; + static inline int get_lltrace(void) { + char *uop_debug = Py_GETENV(DEBUG_ENV); + int lltrace = 0; + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + } + return lltrace; + } + #define DPRINTF(level, ...) \ + if (get_lltrace() >= (level)) { printf(__VA_ARGS__); } +#else + #define DPRINTF(level, ...) +#endif + +#define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) +#define STACK_SIZE() ((int)(ctx->frame->stack_len)) + +#define WITHIN_STACK_BOUNDS() \ + (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE()) + + +#define GETLOCAL(idx) ((ctx->frame->locals[idx])) + + +/* _PUSH_FRAME/_RETURN_VALUE's operand can be 0, a PyFunctionObject *, or a + * PyCodeObject *. Retrieve the code object if possible. + */ +static PyCodeObject * +get_code(_PyUOpInstruction *op) +{ + assert(op->opcode == _PUSH_FRAME || op->opcode == _RETURN_VALUE || op->opcode == _RETURN_GENERATOR); + PyCodeObject *co = NULL; + uint64_t operand = op->operand; + if (operand == 0) { + return NULL; + } + if (operand & 1) { + co = (PyCodeObject *)(operand & ~1); + } + else { + PyFunctionObject *func = (PyFunctionObject *)operand; + assert(PyFunction_Check(func)); + co = (PyCodeObject *)func->func_code; + } + assert(PyCode_Check(co)); + return co; +} + +static PyCodeObject * +get_code_with_logging(_PyUOpInstruction *op) +{ + PyCodeObject *co = NULL; + uint64_t push_operand = op->operand; + if (push_operand & 1) { + co = (PyCodeObject *)(push_operand & ~1); + DPRINTF(3, "code=%p ", co); + assert(PyCode_Check(co)); + } + else { + PyFunctionObject *func = (PyFunctionObject *)push_operand; + DPRINTF(3, "func=%p ", func); + if (func == NULL) { + DPRINTF(3, "\n"); + DPRINTF(1, "Missing function\n"); + return NULL; + } + co = (PyCodeObject *)func->func_code; + DPRINTF(3, "code=%p ", co); + } + return co; +} + +#define MATERIALIZE_INST() (this_instr->is_virtual = false) +#define sym_set_origin_inst_override _Py_uop_sym_set_origin_inst_override +#define sym_is_virtual _Py_uop_sym_is_virtual +#define sym_get_origin _Py_uop_sym_get_origin + +static void +materialize(_Py_UopsLocalsPlusSlot *slot) +{ + assert(slot != NULL); + if (slot->origin_inst) { + slot->origin_inst->is_virtual = false; + } +} + +static void +materialize_stack(_Py_UopsLocalsPlusSlot *stack_start, _Py_UopsLocalsPlusSlot *stack_end) +{ + while (stack_start < stack_end) { + materialize(stack_start); + stack_start++; + } +} + +static void +materialize_frame(_Py_UOpsAbstractFrame *frame) +{ + materialize_stack(frame->stack, frame->stack_pointer); +} + +static void +materialize_ctx(_Py_UOpsContext *ctx) +{ + for (int i = 0; i < ctx->curr_frame_depth; i++) { + materialize_frame(&ctx->frames[i]); + } +} + +/* 1 for success, 0 for not ready, cannot error at the moment. */ +static int +partial_evaluate_uops( + PyCodeObject *co, + _PyUOpInstruction *trace, + int trace_len, + int curr_stacklen, + _PyBloomFilter *dependencies +) +{ + _PyUOpInstruction trace_dest[UOP_MAX_TRACE_LENGTH]; + _Py_UOpsContext context; + context.trace_dest = trace_dest; + context.n_trace_dest = trace_len; + _Py_UOpsContext *ctx = &context; + uint32_t opcode = UINT16_MAX; + int curr_space = 0; + int max_space = 0; + _PyUOpInstruction *first_valid_check_stack = NULL; + _PyUOpInstruction *corresponding_check_stack = NULL; + + _Py_uop_abstractcontext_init(ctx); + _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, curr_stacklen, NULL, 0); + if (frame == NULL) { + return -1; + } + ctx->curr_frame_depth++; + ctx->frame = frame; + ctx->done = false; + ctx->out_of_space = false; + ctx->contradiction = false; + + for (int i = 0; i < trace_len; i++) { + // The key part of PE --- we assume everything starts off virtual. + trace_dest[i] = trace[i]; + trace_dest[i].is_virtual = true; + } + + _PyUOpInstruction *this_instr = NULL; + int i = 0; + for (; !ctx->done; i++) { + assert(i < trace_len); + this_instr = &trace_dest[i]; + + int oparg = this_instr->oparg; + opcode = this_instr->opcode; + _Py_UopsLocalsPlusSlot *stack_pointer = ctx->frame->stack_pointer; + +#ifdef Py_DEBUG + if (get_lltrace() >= 3) { + printf("%4d pe: ", (int)(this_instr - trace_dest)); + _PyUOpPrint(this_instr); + printf(" "); + } +#endif + + int is_static = (_PyUop_Flags[opcode] & HAS_STATIC_FLAG); + if (!is_static) { + MATERIALIZE_INST(); + } + if (!is_static && + // During these two opcodes, there's an abstract frame on the stack. + // Which is not a valid symbol. + (opcode != _PUSH_FRAME && opcode != _SAVE_RETURN_OFFSET)) { + // An escaping opcode means we need to materialize _everything_. + if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { + materialize_ctx(ctx); + } + else { + + materialize_frame(ctx->frame); + } + } + + switch (opcode) { + +#include "partial_evaluator_cases.c.h" + + default: + DPRINTF(1, "\nUnknown opcode in pe's abstract interpreter\n"); + Py_UNREACHABLE(); + } + assert(ctx->frame != NULL); + DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); + ctx->frame->stack_pointer = stack_pointer; + assert(STACK_LEVEL() >= 0); + if (ctx->done) { + break; + } + } + if (ctx->out_of_space) { + DPRINTF(3, "\n"); + DPRINTF(1, "Out of space in pe's abstract interpreter\n"); + } + if (ctx->contradiction) { + // Attempted to push a "bottom" (contradiction) symbol onto the stack. + // This means that the abstract interpreter has hit unreachable code. + // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting + // bottom indicates type instability, so we are probably better off + // retrying later. + DPRINTF(3, "\n"); + DPRINTF(1, "Hit bottom in pe's abstract interpreter\n"); + _Py_uop_abstractcontext_fini(ctx); + return 0; + } + + if (ctx->out_of_space || !is_terminator(this_instr)) { + _Py_uop_abstractcontext_fini(ctx); + return trace_len; + } + else { + // We MUST not have bailed early here. + // That's the only time the PE's residual is valid. + assert(ctx->n_trace_dest < UOP_MAX_TRACE_LENGTH); + assert(is_terminator(this_instr)); + assert(ctx->n_trace_dest <= trace_len); + + // Copy trace_dest into trace. + int trace_dest_len = ctx->n_trace_dest; + // Only valid before we start inserting side exits. + assert(trace_dest_len == trace_len); + for (int x = 0; x < trace_dest_len; x++) { + // Skip all virtual instructions. + if (trace_dest[x].is_virtual) { + trace[x].opcode = _NOP; + } + else { + trace[x] = trace_dest[x]; + } + } + _Py_uop_abstractcontext_fini(ctx); + return trace_dest_len; + } + +error: + DPRINTF(3, "\n"); + DPRINTF(1, "Encountered error in pe's abstract interpreter\n"); + if (opcode <= MAX_UOP_ID) { + OPT_ERROR_IN_OPCODE(opcode); + } + _Py_uop_abstractcontext_fini(ctx); + return -1; + +} + + +// 0 - failure, no error raised, just fall back to Tier 1 +// -1 - failure, and raise error +// > 0 - length of optimized trace +int +_Py_uop_partial_evaluate( + _PyInterpreterFrame *frame, + _PyUOpInstruction *buffer, + int length, + int curr_stacklen, + _PyBloomFilter *dependencies +) +{ + + length = partial_evaluate_uops( + _PyFrame_GetCode(frame), buffer, + length, curr_stacklen, dependencies); + + if (length <= 0) { + return length; + } + + return length; +} + +#endif /* _Py_TIER2 */ diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index 0c31e59d991232..d5b83ed1bcc032 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -50,33 +50,41 @@ dummy_func(void) { // BEGIN BYTECODES // - override op(_LOAD_FAST, (-- value)) { + op(_LOAD_FAST_CHECK, (-- value)) { + value = GETLOCAL(oparg); + // We guarantee this will error - just bail and don't optimize it. + if (sym_is_null(value)) { + ctx->done = true; + } + } + + op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); sym_set_origin_inst_override(&value, this_instr); } - override op(_LOAD_FAST_AND_CLEAR, (-- value)) { + op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); GETLOCAL(oparg) = sym_new_null(ctx); sym_set_origin_inst_override(&value, this_instr); } - override op(_LOAD_CONST, (-- value)) { + op(_LOAD_CONST, (-- value)) { // Should've all been converted by specializer. Py_UNREACHABLE(); } - override op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { value = sym_new_const(ctx, ptr); sym_set_origin_inst_override(&value, this_instr); } - override op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { value = sym_new_const(ctx, ptr); sym_set_origin_inst_override(&value, this_instr); } - override op(_STORE_FAST, (value --)) { + op(_STORE_FAST, (value --)) { _PyUOpInstruction *origin = sym_get_origin(value); // Gets rid of things like x = x. if (sym_is_virtual(value) && @@ -93,16 +101,16 @@ dummy_func(void) { } - override op(_POP_TOP, (pop --)) { + op(_POP_TOP, (pop --)) { if (!sym_is_virtual(pop)) { MATERIALIZE_INST(); } } - override op(_NOP, (--)) { + op(_NOP, (--)) { } - override op(_CHECK_STACK_SPACE_OPERAND, ( -- )) { + op(_CHECK_STACK_SPACE_OPERAND, ( -- )) { (void)framesize; } diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index 244c06ae06dcf2..344d20ed11758a 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -1,6 +1,6 @@ // This file is generated by Tools/cases_generator/partial_evaluator_generator.py // from: -// Python/optimizer_bytecodes.c, Python/partial_evaluator_bytecodes.c +// Python/partial_evaluator_bytecodes.c // Do not edit! case _NOP: { @@ -101,7 +101,7 @@ case _PUSH_NULL: { _Py_UopsLocalsPlusSlot res; - res = sym_new_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -132,72 +132,40 @@ } case _TO_BOOL: { - _Py_UopsLocalsPlusSlot value; _Py_UopsLocalsPlusSlot res; - value = stack_pointer[-1]; - if (!optimize_to_bool(this_instr, ctx, value, &res)) { - res = sym_new_type(ctx, &PyBool_Type); - } + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL_BOOL: { - _Py_UopsLocalsPlusSlot value; - _Py_UopsLocalsPlusSlot res; - value = stack_pointer[-1]; - if (!optimize_to_bool(this_instr, ctx, value, &res)) { - sym_set_type(value, &PyBool_Type); - res = value; - } - stack_pointer[-1] = res; break; } case _TO_BOOL_INT: { - _Py_UopsLocalsPlusSlot value; _Py_UopsLocalsPlusSlot res; - value = stack_pointer[-1]; - if (!optimize_to_bool(this_instr, ctx, value, &res)) { - sym_set_type(value, &PyLong_Type); - res = sym_new_type(ctx, &PyBool_Type); - } + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL_LIST: { - _Py_UopsLocalsPlusSlot value; _Py_UopsLocalsPlusSlot res; - value = stack_pointer[-1]; - if (!optimize_to_bool(this_instr, ctx, value, &res)) { - sym_set_type(value, &PyList_Type); - res = sym_new_type(ctx, &PyBool_Type); - } + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL_NONE: { - _Py_UopsLocalsPlusSlot value; _Py_UopsLocalsPlusSlot res; - value = stack_pointer[-1]; - if (!optimize_to_bool(this_instr, ctx, value, &res)) { - sym_set_const(value, Py_None); - res = sym_new_const(ctx, Py_False); - } + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL_STR: { - _Py_UopsLocalsPlusSlot value; _Py_UopsLocalsPlusSlot res; - value = stack_pointer[-1]; - if (!optimize_to_bool(this_instr, ctx, value, &res)) { - res = sym_new_type(ctx, &PyBool_Type); - sym_set_type(value, &PyUnicode_Type); - } + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } @@ -217,25 +185,6 @@ } case _GUARD_BOTH_INT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_matches_type(left, &PyLong_Type)) { - if (sym_matches_type(right, &PyLong_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); - } - else { - REPLACE_OP(this_instr, _GUARD_TOS_INT, 0, 0); - } - } - else { - if (sym_matches_type(right, &PyLong_Type)) { - REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0); - } - } - sym_set_type(left, &PyLong_Type); - sym_set_type(right, &PyLong_Type); break; } @@ -248,29 +197,8 @@ } case _BINARY_OP_MULTIPLY_INT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && - sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) - { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_type(ctx, &PyLong_Type); - } + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -278,29 +206,8 @@ } case _BINARY_OP_ADD_INT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && - sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) - { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_type(ctx, &PyLong_Type); - } + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -308,29 +215,8 @@ } case _BINARY_OP_SUBTRACT_INT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && - sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) - { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_type(ctx, &PyLong_Type); - } + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -338,25 +224,6 @@ } case _GUARD_BOTH_FLOAT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_matches_type(left, &PyFloat_Type)) { - if (sym_matches_type(right, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); - } - else { - REPLACE_OP(this_instr, _GUARD_TOS_FLOAT, 0, 0); - } - } - else { - if (sym_matches_type(right, &PyFloat_Type)) { - REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0); - } - } - sym_set_type(left, &PyFloat_Type); - sym_set_type(right, &PyFloat_Type); break; } @@ -369,30 +236,8 @@ } case _BINARY_OP_MULTIPLY_FLOAT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && - sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) - { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) * - PyFloat_AS_DOUBLE(sym_get_const(right))); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and update tests! - } - else { - res = sym_new_type(ctx, &PyFloat_Type); - } + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -400,30 +245,8 @@ } case _BINARY_OP_ADD_FLOAT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && - sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) - { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) + - PyFloat_AS_DOUBLE(sym_get_const(right))); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and update tests! - } - else { - res = sym_new_type(ctx, &PyFloat_Type); - } + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -431,30 +254,8 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && - sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) - { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) - - PyFloat_AS_DOUBLE(sym_get_const(right))); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and update tests! - } - else { - res = sym_new_type(ctx, &PyFloat_Type); - } + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -462,37 +263,12 @@ } case _GUARD_BOTH_UNICODE: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_matches_type(left, &PyUnicode_Type) && - sym_matches_type(right, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); - } - sym_set_type(left, &PyUnicode_Type); - sym_set_type(left, &PyUnicode_Type); break; } case _BINARY_OP_ADD_UNICODE: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && - sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { - PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - } - else { - res = sym_new_type(ctx, &PyUnicode_Type); - } + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -570,15 +346,8 @@ } case _BINARY_SUBSCR_INIT_CALL: { - _Py_UopsLocalsPlusSlot sub; - _Py_UopsLocalsPlusSlot container; - _Py_UopsLocalsPlusSlot new_frame; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - (void)container; - (void)sub; - new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; - ctx->done = true; + _PyInterpreterFrame *new_frame; + new_frame = sym_new_not_null(ctx); stack_pointer[-2] = new_frame; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -638,30 +407,9 @@ } case _RETURN_VALUE: { - _Py_UopsLocalsPlusSlot retval; _Py_UopsLocalsPlusSlot res; - retval = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); - stack_pointer = ctx->frame->stack_pointer; - res = retval; - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; - co = get_code(this_instr); - if (co == NULL) { - // might be impossible, but bailing is still safe - ctx->done = true; - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; break; } @@ -691,15 +439,16 @@ /* _SEND is not a viable micro-op for tier 2 */ case _SEND_GEN_FRAME: { - // We are about to hit the end of the trace: - ctx->done = true; + _PyInterpreterFrame *gen_frame; + gen_frame = sym_new_not_null(ctx); + stack_pointer[-1] = gen_frame; break; } case _YIELD_VALUE: { - _Py_UopsLocalsPlusSlot res; - res = sym_new_unknown(ctx); - stack_pointer[-1] = res; + _Py_UopsLocalsPlusSlot value; + value = sym_new_not_null(ctx); + stack_pointer[-1] = value; break; } @@ -738,14 +487,10 @@ } case _UNPACK_SEQUENCE: { - _Py_UopsLocalsPlusSlot seq; - _Py_UopsLocalsPlusSlot *values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - /* This has to be done manually */ - (void)seq; - for (int i = 0; i < oparg; i++) { - values[i] = sym_new_unknown(ctx); + _Py_UopsLocalsPlusSlot *output; + output = &stack_pointer[-1]; + for (int _i = oparg; --_i >= 0;) { + output[_i] = sym_new_not_null(ctx); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -787,15 +532,14 @@ } case _UNPACK_EX: { - _Py_UopsLocalsPlusSlot seq; - _Py_UopsLocalsPlusSlot *values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - /* This has to be done manually */ - (void)seq; - int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; - for (int i = 0; i < totalargs; i++) { - values[i] = sym_new_unknown(ctx); + _Py_UopsLocalsPlusSlot *left; + _Py_UopsLocalsPlusSlot *right; + right = &stack_pointer[(oparg & 0xFF)]; + for (int _i = oparg & 0xFF; --_i >= 0;) { + left[_i] = sym_new_not_null(ctx); + } + for (int _i = oparg >> 8; --_i >= 0;) { + right[_i] = sym_new_not_null(ctx); } stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); @@ -1031,15 +775,10 @@ } case _LOAD_ATTR: { - _Py_UopsLocalsPlusSlot owner; _Py_UopsLocalsPlusSlot attr; _Py_UopsLocalsPlusSlot self_or_null = (_Py_UopsLocalsPlusSlot){NULL, 0}; - owner = stack_pointer[-1]; - (void)owner; attr = sym_new_not_null(ctx); - if (oparg & 1) { - self_or_null = sym_new_unknown(ctx); - } + self_or_null = sym_new_not_null(ctx); stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; stack_pointer += (oparg & 1); @@ -1048,28 +787,6 @@ } case _GUARD_TYPE_VERSION: { - _Py_UopsLocalsPlusSlot owner; - owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)this_instr->operand; - assert(type_version); - if (sym_matches_type_version(owner, type_version)) { - REPLACE_OP(this_instr, _NOP, 0, 0); - } else { - // add watcher so that whenever the type changes we invalidate this - PyTypeObject *type = _PyType_LookupByVersion(type_version); - // if the type is null, it was not found in the cache (there was a conflict) - // with the key, in which case we can't trust the version - if (type) { - // if the type version was set properly, then add a watcher - // if it wasn't this means that the type version was previously set to something else - // and we set the owner to bottom, so we don't need to add a watcher because we must have - // already added one earlier. - if (sym_set_type_version(owner, type_version)) { - PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); - _Py_BloomFilter_Add(dependencies, type); - } - } - } break; } @@ -1078,15 +795,10 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - _Py_UopsLocalsPlusSlot owner; _Py_UopsLocalsPlusSlot attr; _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; - owner = stack_pointer[-1]; - uint16_t offset = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); - (void)offset; - (void)owner; stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -1095,51 +807,14 @@ } case _CHECK_ATTR_MODULE: { - _Py_UopsLocalsPlusSlot owner; - owner = stack_pointer[-1]; - uint32_t dict_version = (uint32_t)this_instr->operand; - (void)dict_version; - if (sym_is_const(owner)) { - PyObject *cnst = sym_get_const(owner); - if (PyModule_CheckExact(cnst)) { - PyModuleObject *mod = (PyModuleObject *)cnst; - PyObject *dict = mod->md_dict; - uint64_t watched_mutations = get_mutations(dict); - if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { - PyDict_Watch(GLOBALS_WATCHER_ID, dict); - _Py_BloomFilter_Add(dependencies, dict); - this_instr->opcode = _NOP; - } - } - } break; } case _LOAD_ATTR_MODULE: { - _Py_UopsLocalsPlusSlot owner; _Py_UopsLocalsPlusSlot attr; _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; - owner = stack_pointer[-1]; - uint16_t index = (uint16_t)this_instr->operand; - (void)index; + attr = sym_new_not_null(ctx); null = sym_new_null(ctx); - attr = (_Py_UopsLocalsPlusSlot){NULL, 0}; - if (this_instr[-1].opcode == _NOP) { - // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. - assert(sym_is_const(owner)); - PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); - assert(PyModule_CheckExact(mod)); - PyObject *dict = mod->md_dict; - PyObject *res = convert_global_to_const(this_instr, dict); - if (res != NULL) { - this_instr[-1].opcode = _POP_TOP; - attr = sym_new_const(ctx, res); - } - } - if (attr.sym == NULL) { - /* No conversion made. We don't know what `attr` is. */ - attr = sym_new_not_null(ctx); - } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -1152,15 +827,10 @@ } case _LOAD_ATTR_WITH_HINT: { - _Py_UopsLocalsPlusSlot owner; _Py_UopsLocalsPlusSlot attr; _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; - owner = stack_pointer[-1]; - uint16_t hint = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); - (void)hint; - (void)owner; stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -1169,15 +839,10 @@ } case _LOAD_ATTR_SLOT: { - _Py_UopsLocalsPlusSlot owner; _Py_UopsLocalsPlusSlot attr; _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; - owner = stack_pointer[-1]; - uint16_t index = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); - (void)index; - (void)owner; stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -1190,15 +855,10 @@ } case _LOAD_ATTR_CLASS: { - _Py_UopsLocalsPlusSlot owner; _Py_UopsLocalsPlusSlot attr; _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); - (void)descr; - (void)owner; stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -1207,14 +867,8 @@ } case _LOAD_ATTR_PROPERTY_FRAME: { - _Py_UopsLocalsPlusSlot owner; - _Py_UopsLocalsPlusSlot new_frame; - owner = stack_pointer[-1]; - PyObject *fget = (PyObject *)this_instr->operand; - (void)fget; - (void)owner; - new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; - ctx->done = true; + _PyInterpreterFrame *new_frame; + new_frame = sym_new_not_null(ctx); stack_pointer[-1] = new_frame; break; } @@ -1244,19 +898,8 @@ } case _COMPARE_OP: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - (void)left; - (void)right; - if (oparg & 16) { - res = sym_new_type(ctx, &PyBool_Type); - } - else { - res = _Py_uop_sym_new_not_null(ctx); - } + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1264,14 +907,8 @@ } case _COMPARE_OP_FLOAT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - (void)left; - (void)right; - res = sym_new_type(ctx, &PyBool_Type); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1279,14 +916,8 @@ } case _COMPARE_OP_INT: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - (void)left; - (void)right; - res = sym_new_type(ctx, &PyBool_Type); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1294,14 +925,8 @@ } case _COMPARE_OP_STR: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - (void)left; - (void)right; - res = sym_new_type(ctx, &PyBool_Type); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1309,30 +934,18 @@ } case _IS_OP: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - (void)left; - (void)right; - res = sym_new_type(ctx, &PyBool_Type); - stack_pointer[-2] = res; + _Py_UopsLocalsPlusSlot b; + b = sym_new_not_null(ctx); + stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _CONTAINS_OP: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; - _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - (void)left; - (void)right; - res = sym_new_type(ctx, &PyBool_Type); - stack_pointer[-2] = res; + _Py_UopsLocalsPlusSlot b; + b = sym_new_not_null(ctx); + stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1523,11 +1136,8 @@ } case _ITER_NEXT_RANGE: { - _Py_UopsLocalsPlusSlot iter; _Py_UopsLocalsPlusSlot next; - iter = stack_pointer[-1]; - next = sym_new_type(ctx, &PyLong_Type); - (void)iter; + next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1535,19 +1145,19 @@ } case _FOR_ITER_GEN_FRAME: { - /* We are about to hit the end of the trace */ - ctx->done = true; + _PyInterpreterFrame *gen_frame; + gen_frame = sym_new_not_null(ctx); + stack_pointer[0] = gen_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_SPECIAL: { - _Py_UopsLocalsPlusSlot owner; _Py_UopsLocalsPlusSlot attr; _Py_UopsLocalsPlusSlot self_or_null; - owner = stack_pointer[-1]; - (void)owner; attr = sym_new_not_null(ctx); - self_or_null = sym_new_unknown(ctx); + self_or_null = sym_new_not_null(ctx); stack_pointer[-1] = attr; stack_pointer[0] = self_or_null; stack_pointer += 1; @@ -1585,14 +1195,10 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - _Py_UopsLocalsPlusSlot owner; _Py_UopsLocalsPlusSlot attr; _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; - (void)descr; attr = sym_new_not_null(ctx); - self = owner; + self = sym_new_not_null(ctx); stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; @@ -1601,14 +1207,10 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { - _Py_UopsLocalsPlusSlot owner; _Py_UopsLocalsPlusSlot attr; _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; - (void)descr; attr = sym_new_not_null(ctx); - self = owner; + self = sym_new_not_null(ctx); stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; @@ -1635,14 +1237,10 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - _Py_UopsLocalsPlusSlot owner; _Py_UopsLocalsPlusSlot attr; _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; - (void)descr; attr = sym_new_not_null(ctx); - self = owner; + self = sym_new_not_null(ctx); stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; @@ -1651,22 +1249,18 @@ } case _MAYBE_EXPAND_METHOD: { - _Py_UopsLocalsPlusSlot *args; - _Py_UopsLocalsPlusSlot self_or_null; - _Py_UopsLocalsPlusSlot callable; _Py_UopsLocalsPlusSlot func; - _Py_UopsLocalsPlusSlot maybe_self; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - args = &stack_pointer[-oparg]; - (void)callable; - (void)self_or_null; - (void)args; + _Py_UopsLocalsPlusSlot *maybe_self; + _Py_UopsLocalsPlusSlot *args; + maybe_self = &stack_pointer[-1 - oparg]; func = sym_new_not_null(ctx); - maybe_self = sym_new_not_null(ctx); + for (int _i = 1; --_i >= 0;) { + maybe_self[_i] = sym_new_not_null(ctx); + } + for (int _i = oparg; --_i >= 0;) { + args[_i] = sym_new_not_null(ctx); + } stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; break; } @@ -1675,22 +1269,8 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { - _Py_UopsLocalsPlusSlot *args; - _Py_UopsLocalsPlusSlot self_or_null; - _Py_UopsLocalsPlusSlot callable; - _Py_UopsLocalsPlusSlot new_frame; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - (void)(self_or_null); - (void)(callable); - PyCodeObject *co = NULL; - assert((this_instr + 2)->opcode == _PUSH_FRAME); - co = get_code_with_logging((this_instr + 2)); - if (co == NULL) { - ctx->done = true; - break; - } - new_frame.sym = frame_new(ctx, co, 0, NULL, 0); + _PyInterpreterFrame *new_frame; + new_frame = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1731,82 +1311,36 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UopsLocalsPlusSlot null; - _Py_UopsLocalsPlusSlot callable; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - sym_set_null(null); - sym_set_type(callable, &PyMethod_Type); break; } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UopsLocalsPlusSlot callable; _Py_UopsLocalsPlusSlot func; - _Py_UopsLocalsPlusSlot self; - callable = stack_pointer[-2 - oparg]; - (void)callable; + _Py_UopsLocalsPlusSlot *self; + self = &stack_pointer[-1 - oparg]; func = sym_new_not_null(ctx); - self = sym_new_not_null(ctx); + for (int _i = 1; --_i >= 0;) { + self[_i] = sym_new_not_null(ctx); + } stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = self; break; } case _CHECK_PEP_523: { - /* Setting the eval frame function invalidates - * all executors, so no need to check dynamically */ - if (_PyInterpreterState_GET()->eval_frame == NULL) { - REPLACE_OP(this_instr, _NOP, 0 ,0); - } break; } case _CHECK_FUNCTION_EXACT_ARGS: { - _Py_UopsLocalsPlusSlot self_or_null; - _Py_UopsLocalsPlusSlot callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - sym_set_type(callable, &PyFunction_Type); - (void)self_or_null; break; } case _CHECK_STACK_SPACE: { - assert(corresponding_check_stack == NULL); - corresponding_check_stack = this_instr; break; } case _INIT_CALL_PY_EXACT_ARGS: { - _Py_UopsLocalsPlusSlot *args; - _Py_UopsLocalsPlusSlot self_or_null; - _Py_UopsLocalsPlusSlot callable; - _Py_UopsLocalsPlusSlot new_frame; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - (void)callable; - PyCodeObject *co = NULL; - assert((this_instr + 2)->opcode == _PUSH_FRAME); - co = get_code_with_logging((this_instr + 2)); - if (co == NULL) { - ctx->done = true; - break; - } - assert(self_or_null.sym != NULL); - assert(args != NULL); - if (sym_is_not_null(self_or_null)) { - // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM - args--; - argcount++; - } - if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, args, argcount); - } else { - new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, NULL, 0); - } + _PyInterpreterFrame *new_frame; + new_frame = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1814,38 +1348,8 @@ } case _PUSH_FRAME: { - _Py_UopsLocalsPlusSlot new_frame; - new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - ctx->frame->stack_pointer = stack_pointer; - ctx->frame = (_Py_UOpsAbstractFrame *)new_frame.sym; - ctx->curr_frame_depth++; - stack_pointer = ((_Py_UOpsAbstractFrame *)new_frame.sym)->stack_pointer; - co = get_code(this_instr); - if (co == NULL) { - // should be about to _EXIT_TRACE anyway - ctx->done = true; - break; - } - /* Stack space handling */ - int framesize = co->co_framesize; - assert(framesize > 0); - curr_space += framesize; - if (curr_space < 0 || curr_space > INT32_MAX) { - // won't fit in signed 32-bit int - ctx->done = true; - break; - } - max_space = curr_space > max_space ? curr_space : max_space; - if (first_valid_check_stack == NULL) { - first_valid_check_stack = corresponding_check_stack; - } - else if (corresponding_check_stack) { - // delete all but the first valid _CHECK_STACK_SPACE - corresponding_check_stack->opcode = _NOP; - } - corresponding_check_stack = NULL; break; } @@ -1877,40 +1381,22 @@ } case _CHECK_AND_ALLOCATE_OBJECT: { - _Py_UopsLocalsPlusSlot *args; - _Py_UopsLocalsPlusSlot null; - _Py_UopsLocalsPlusSlot callable; _Py_UopsLocalsPlusSlot self; _Py_UopsLocalsPlusSlot init; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - args = &stack_pointer[-oparg]; - uint32_t type_version = (uint32_t)this_instr->operand; - (void)type_version; - (void)callable; - (void)null; - (void)args; + _Py_UopsLocalsPlusSlot *args; self = sym_new_not_null(ctx); init = sym_new_not_null(ctx); + for (int _i = oparg; --_i >= 0;) { + args[_i] = sym_new_not_null(ctx); + } stack_pointer[-2 - oparg] = self; stack_pointer[-1 - oparg] = init; break; } case _CREATE_INIT_FRAME: { - _Py_UopsLocalsPlusSlot *args; - _Py_UopsLocalsPlusSlot init; - _Py_UopsLocalsPlusSlot self; - _Py_UopsLocalsPlusSlot init_frame; - args = &stack_pointer[-oparg]; - init = stack_pointer[-1 - oparg]; - self = stack_pointer[-2 - oparg]; - (void)self; - (void)init; - (void)args; - init_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; - ctx->done = true; + _PyInterpreterFrame *init_frame; + init_frame = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2024,21 +1510,8 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { - _Py_UopsLocalsPlusSlot kwnames; - _Py_UopsLocalsPlusSlot *args; - _Py_UopsLocalsPlusSlot self_or_null; - _Py_UopsLocalsPlusSlot callable; - _Py_UopsLocalsPlusSlot new_frame; - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - (void)callable; - (void)self_or_null; - (void)args; - (void)kwnames; - new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; - ctx->done = true; + _PyInterpreterFrame *new_frame; + new_frame = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2103,22 +1576,7 @@ case _RETURN_GENERATOR: { _Py_UopsLocalsPlusSlot res; - ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); - stack_pointer = ctx->frame->stack_pointer; - res = sym_new_unknown(ctx); - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; - co = get_code(this_instr); - if (co == NULL) { - // might be impossible, but bailing is still safe - ctx->done = true; - } + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2158,11 +1616,8 @@ } case _COPY: { - _Py_UopsLocalsPlusSlot bottom; _Py_UopsLocalsPlusSlot top; - bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); - top = bottom; + top = sym_new_not_null(ctx); stack_pointer[0] = top; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2170,27 +1625,8 @@ } case _BINARY_OP: { - _Py_UopsLocalsPlusSlot right; - _Py_UopsLocalsPlusSlot left; _Py_UopsLocalsPlusSlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyTypeObject *ltype = sym_get_type(left); - PyTypeObject *rtype = sym_get_type(right); - if (ltype != NULL && (ltype == &PyLong_Type || ltype == &PyFloat_Type) && - rtype != NULL && (rtype == &PyLong_Type || rtype == &PyFloat_Type)) - { - if (oparg != NB_TRUE_DIVIDE && oparg != NB_INPLACE_TRUE_DIVIDE && - ltype == &PyLong_Type && rtype == &PyLong_Type) { - /* If both inputs are ints and the op is not division the result is an int */ - res = sym_new_type(ctx, &PyLong_Type); - } - else { - /* For any other op combining ints/floats the result is a float */ - res = sym_new_type(ctx, &PyFloat_Type); - } - } - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2200,8 +1636,8 @@ case _SWAP: { _Py_UopsLocalsPlusSlot top; _Py_UopsLocalsPlusSlot bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; + top = sym_new_not_null(ctx); + bottom = sym_new_not_null(ctx); stack_pointer[-2 - (oparg-2)] = top; stack_pointer[-1] = bottom; break; @@ -2224,67 +1660,30 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { - _Py_UopsLocalsPlusSlot flag; - flag = stack_pointer[-1]; - if (sym_is_const(flag)) { - PyObject *value = sym_get_const(flag); - assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_True); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_FALSE_POP: { - _Py_UopsLocalsPlusSlot flag; - flag = stack_pointer[-1]; - if (sym_is_const(flag)) { - PyObject *value = sym_get_const(flag); - assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_False); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NONE_POP: { - _Py_UopsLocalsPlusSlot flag; - flag = stack_pointer[-1]; - if (sym_is_const(flag)) { - PyObject *value = sym_get_const(flag); - assert(value != NULL); - eliminate_pop_guard(this_instr, !Py_IsNone(value)); - } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); - eliminate_pop_guard(this_instr, true); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NOT_NONE_POP: { - _Py_UopsLocalsPlusSlot flag; - flag = stack_pointer[-1]; - if (sym_is_const(flag)) { - PyObject *value = sym_get_const(flag); - assert(value != NULL); - eliminate_pop_guard(this_instr, Py_IsNone(value)); - } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); - eliminate_pop_guard(this_instr, false); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _JUMP_TO_TOP: { - ctx->done = true; break; } @@ -2303,9 +1702,6 @@ } case _EXIT_TRACE: { - PyObject *exit_p = (PyObject *)this_instr->operand; - (void)exit_p; - ctx->done = true; break; } @@ -2345,8 +1741,7 @@ case _LOAD_CONST_INLINE_WITH_NULL: { _Py_UopsLocalsPlusSlot value; _Py_UopsLocalsPlusSlot null; - PyObject *ptr = (PyObject *)this_instr->operand; - value = sym_new_const(ctx, ptr); + value = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = value; stack_pointer[1] = null; @@ -2358,8 +1753,7 @@ case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { _Py_UopsLocalsPlusSlot value; _Py_UopsLocalsPlusSlot null; - PyObject *ptr = (PyObject *)this_instr->operand; - value = sym_new_const(ctx, ptr); + value = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = value; stack_pointer[1] = null; diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py index 22dde7acf14115..30b31dcd74a4ea 100644 --- a/Tools/cases_generator/partial_evaluator_generator.py +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -25,8 +25,7 @@ from stack import Local, Stack, StackError DEFAULT_OUTPUT = ROOT / "Python/partial_evaluator_cases.c.h" -DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/optimizer_bytecodes.c").absolute().as_posix() -DEFAULT_PE_INPUT = (ROOT / "Python/partial_evaluator_bytecodes.c").absolute().as_posix() +DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/partial_evaluator_bytecodes.c").absolute().as_posix() def validate_uop(override: Uop, uop: Uop) -> None: # To do @@ -225,7 +224,6 @@ def generate_tier2_abstract_from_files( if not args.input: args.base.append(DEFAULT_INPUT) args.input.append(DEFAULT_ABSTRACT_INPUT) - args.input.append(DEFAULT_PE_INPUT) else: args.base.append(args.input[-1]) args.input.pop() From a4e13d4874002155af87084c0dd7272f32519c53 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 3 Oct 2024 23:59:26 +0800 Subject: [PATCH 25/46] add sourcefile and makefile --- Include/internal/pycore_optimizer.h | 53 ++ Makefile.pre.in | 1 + PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/optimizer_bytecodes.c | 12 +- Python/optimizer_cases.c.h | 12 +- Python/partial_evaluator.c | 16 +- Python/partial_evaluator_cases.c.h | 320 +++++------ Python/partial_evaluator_symbols.c | 538 ++++++++++++++++++ .../partial_evaluator_generator.py | 8 +- 12 files changed, 782 insertions(+), 186 deletions(-) create mode 100644 Python/partial_evaluator_symbols.c diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index f92c0a0cddf906..e615f49b5efa56 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -59,6 +59,7 @@ typedef struct { }; }; uint64_t operand; // A cache entry + char is_virtual; // Used for tier2 optimization. } _PyUOpInstruction; typedef struct { @@ -155,6 +156,8 @@ extern PyTypeObject _PyUOpOptimizer_Type; /* Symbols */ /* See explanation in optimizer_symbols.c */ +// Specializer. + struct _Py_UopsSymbol { int flags; // 0 bits: Top; 2 or more bits: Bottom PyTypeObject *typ; // Borrowed reference @@ -284,6 +287,56 @@ static inline int is_terminator(const _PyUOpInstruction *uop) ); } +// Partial evaluator. +struct _Py_UopsPESymbol { + int flags; // 0 bits: Top; 2 or more bits: Bottom + PyObject *const_val; // Owned reference (!) +}; + +typedef struct _Py_UopsPESlot { + _Py_UopsSymbol *sym; + _PyUOpInstruction *origin_inst; // The instruction this symbol originates from. +} _Py_UopsPESlot; + +typedef struct _Py_UopsPESymbol _Py_UopsPESymbol; + +struct _Py_UOpsPEAbstractFrame { + // Max stacklen + int stack_len; + int locals_len; + + _Py_UopsPESlot *stack_pointer; + _Py_UopsPESlot *stack; + _Py_UopsPESlot *locals; +}; + +typedef struct _Py_UOpsPEAbstractFrame _Py_UOpsPEAbstractFrame; + +typedef struct pe_arena { + int sym_curr_number; + int sym_max_number; + _Py_UopsSymbol arena[TY_ARENA_SIZE]; +} pe_arena; + +struct _Py_UOpsPEContext { + char done; + char out_of_space; + bool contradiction; + // The current "executing" frame. + _Py_UOpsPEAbstractFrame *frame; + _Py_UOpsPEAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; + int curr_frame_depth; + + // Arena for the symbolic information. + pe_arena t_arena; + + _Py_UopsPESymbol **n_consumed; + _Py_UopsPESymbol **limit; + _Py_UopsPESymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; +}; + +typedef struct _Py_UOpsPEContext _Py_UOpsPEContext; + #ifdef __cplusplus } #endif diff --git a/Makefile.pre.in b/Makefile.pre.in index 531acc91721ab0..07ed191eeff5fb 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -471,6 +471,7 @@ PYTHON_OBJS= \ Python/optimizer_symbols.o \ Python/parking_lot.o \ Python/partial_evaluator.o \ + Python/partial_evaluator_symbols.o \ Python/pathconfig.o \ Python/preconfig.o \ Python/pyarena.o \ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index baf72c491944ff..a976e808f8b891 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -240,6 +240,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index bad40a1d52512c..fe0eef59fb06cf 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -329,6 +329,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 6419d9dd350b9f..d3cec7efa3bb6b 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -629,6 +629,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 2806311d94968f..cf5d9fab439215 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1427,6 +1427,9 @@ Python + + Python + Python diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 2372b844dc990d..799ad55d3a75a6 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -85,7 +85,7 @@ dummy_func(void) { op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - _Py_UopsLocalsPlusSlot temp = sym_new_null(ctx); + _Py_UopsPESlot temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; } @@ -332,7 +332,7 @@ dummy_func(void) { op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame)) { (void)container; (void)sub; - new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; + new_frame = (_Py_UopsPESlot){NULL, 0}; ctx->done = true; } @@ -487,7 +487,7 @@ dummy_func(void) { op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { (void)index; null = sym_new_null(ctx); - attr = (_Py_UopsLocalsPlusSlot){NULL, 0}; + attr = (_Py_UopsPESlot){NULL, 0}; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. assert(sym_is_const(owner)); @@ -548,7 +548,7 @@ dummy_func(void) { op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { (void)fget; (void)owner; - new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; + new_frame = (_Py_UopsPESlot){NULL, 0}; ctx->done = true; } @@ -624,7 +624,7 @@ dummy_func(void) { (void)self_or_null; (void)args; (void)kwnames; - new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; + new_frame = (_Py_UopsPESlot){NULL, 0}; ctx->done = true; } @@ -641,7 +641,7 @@ dummy_func(void) { (void)self; (void)init; (void)args; - init_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; + init_frame = (_Py_UopsPESlot){NULL, 0}; ctx->done = true; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 1ca85c2243f5f4..2a49d9497349eb 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -48,7 +48,7 @@ case _LOAD_FAST_AND_CLEAR: { _Py_UopsSymbol *value; value = GETLOCAL(oparg); - _Py_UopsLocalsPlusSlot temp = sym_new_null(ctx); + _Py_UopsPESlot temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; @@ -561,7 +561,7 @@ container = stack_pointer[-2]; (void)container; (void)sub; - new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; + new_frame = (_Py_UopsPESlot){NULL, 0}; ctx->done = true; stack_pointer[-2] = new_frame; stack_pointer += -1; @@ -1107,7 +1107,7 @@ uint16_t index = (uint16_t)this_instr->operand; (void)index; null = sym_new_null(ctx); - attr = (_Py_UopsLocalsPlusSlot){NULL, 0}; + attr = (_Py_UopsPESlot){NULL, 0}; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. assert(sym_is_const(owner)); @@ -1197,7 +1197,7 @@ PyObject *fget = (PyObject *)this_instr->operand; (void)fget; (void)owner; - new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; + new_frame = (_Py_UopsPESlot){NULL, 0}; ctx->done = true; stack_pointer[-1] = new_frame; break; @@ -1893,7 +1893,7 @@ (void)self; (void)init; (void)args; - init_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; + init_frame = (_Py_UopsPESlot){NULL, 0}; ctx->done = true; stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; @@ -2021,7 +2021,7 @@ (void)self_or_null; (void)args; (void)kwnames; - new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; + new_frame = (_Py_UopsPESlot){NULL, 0}; ctx->done = true; stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; diff --git a/Python/partial_evaluator.c b/Python/partial_evaluator.c index a1facb3307c37d..1af1703b735210 100644 --- a/Python/partial_evaluator.c +++ b/Python/partial_evaluator.c @@ -116,7 +116,7 @@ get_code_with_logging(_PyUOpInstruction *op) #define sym_get_origin _Py_uop_sym_get_origin static void -materialize(_Py_UopsLocalsPlusSlot *slot) +materialize(_Py_UopsPESlot *slot) { assert(slot != NULL); if (slot->origin_inst) { @@ -125,7 +125,7 @@ materialize(_Py_UopsLocalsPlusSlot *slot) } static void -materialize_stack(_Py_UopsLocalsPlusSlot *stack_start, _Py_UopsLocalsPlusSlot *stack_end) +materialize_stack(_Py_UopsPESlot *stack_start, _Py_UopsPESlot *stack_end) { while (stack_start < stack_end) { materialize(stack_start); @@ -158,10 +158,8 @@ partial_evaluate_uops( ) { _PyUOpInstruction trace_dest[UOP_MAX_TRACE_LENGTH]; - _Py_UOpsContext context; - context.trace_dest = trace_dest; - context.n_trace_dest = trace_len; - _Py_UOpsContext *ctx = &context; + _Py_UOpsPEContext context; + _Py_UOpsPEContext *ctx = &context; uint32_t opcode = UINT16_MAX; int curr_space = 0; int max_space = 0; @@ -193,7 +191,7 @@ partial_evaluate_uops( int oparg = this_instr->oparg; opcode = this_instr->opcode; - _Py_UopsLocalsPlusSlot *stack_pointer = ctx->frame->stack_pointer; + _Py_UopsPESlot *stack_pointer = ctx->frame->stack_pointer; #ifdef Py_DEBUG if (get_lltrace() >= 3) { @@ -260,12 +258,10 @@ partial_evaluate_uops( else { // We MUST not have bailed early here. // That's the only time the PE's residual is valid. - assert(ctx->n_trace_dest < UOP_MAX_TRACE_LENGTH); assert(is_terminator(this_instr)); - assert(ctx->n_trace_dest <= trace_len); // Copy trace_dest into trace. - int trace_dest_len = ctx->n_trace_dest; + int trace_dest_len = trace_len; // Only valid before we start inserting side exits. assert(trace_dest_len == trace_len); for (int x = 0; x < trace_dest_len; x++) { diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index 344d20ed11758a..c7d8d2252f444a 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -24,7 +24,7 @@ /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsPESlot value; value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. if (sym_is_null(value)) { @@ -37,7 +37,7 @@ } case _LOAD_FAST: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsPESlot value; value = GETLOCAL(oparg); sym_set_origin_inst_override(&value, this_instr); stack_pointer[0] = value; @@ -47,7 +47,7 @@ } case _LOAD_FAST_AND_CLEAR: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsPESlot value; value = GETLOCAL(oparg); GETLOCAL(oparg) = sym_new_null(ctx); sym_set_origin_inst_override(&value, this_instr); @@ -58,7 +58,7 @@ } case _LOAD_CONST: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsPESlot value; // Should've all been converted by specializer. Py_UNREACHABLE(); stack_pointer[0] = value; @@ -68,7 +68,7 @@ } case _STORE_FAST: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsPESlot value; value = stack_pointer[-1]; _PyUOpInstruction *origin = sym_get_origin(value); // Gets rid of things like x = x. @@ -89,7 +89,7 @@ } case _POP_TOP: { - _Py_UopsLocalsPlusSlot pop; + _Py_UopsPESlot pop; pop = stack_pointer[-1]; if (!sym_is_virtual(pop)) { MATERIALIZE_INST(); @@ -100,7 +100,7 @@ } case _PUSH_NULL: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -109,7 +109,7 @@ } case _END_SEND: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsPESlot value; value = sym_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; @@ -118,21 +118,21 @@ } case _UNARY_NEGATIVE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_NOT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; @@ -143,42 +143,42 @@ } case _TO_BOOL_INT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL_LIST: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL_NONE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL_STR: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _REPLACE_WITH_TRUE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_INVERT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; @@ -197,7 +197,7 @@ } case _BINARY_OP_MULTIPLY_INT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -206,7 +206,7 @@ } case _BINARY_OP_ADD_INT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -215,7 +215,7 @@ } case _BINARY_OP_SUBTRACT_INT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -236,7 +236,7 @@ } case _BINARY_OP_MULTIPLY_FLOAT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -245,7 +245,7 @@ } case _BINARY_OP_ADD_FLOAT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -254,7 +254,7 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -267,7 +267,7 @@ } case _BINARY_OP_ADD_UNICODE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -282,7 +282,7 @@ } case _BINARY_SUBSCR: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -291,7 +291,7 @@ } case _BINARY_SLICE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -306,7 +306,7 @@ } case _BINARY_SUBSCR_LIST_INT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -315,7 +315,7 @@ } case _BINARY_SUBSCR_STR_INT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -324,7 +324,7 @@ } case _BINARY_SUBSCR_TUPLE_INT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -333,7 +333,7 @@ } case _BINARY_SUBSCR_DICT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -391,14 +391,14 @@ } case _CALL_INTRINSIC_1: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -407,21 +407,21 @@ } case _RETURN_VALUE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _GET_AITER: { - _Py_UopsLocalsPlusSlot iter; + _Py_UopsPESlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - _Py_UopsLocalsPlusSlot awaitable; + _Py_UopsPESlot awaitable; awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; @@ -430,7 +430,7 @@ } case _GET_AWAITABLE: { - _Py_UopsLocalsPlusSlot iter; + _Py_UopsPESlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -446,7 +446,7 @@ } case _YIELD_VALUE: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsPESlot value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; @@ -459,7 +459,7 @@ } case _LOAD_COMMON_CONSTANT: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsPESlot value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -468,7 +468,7 @@ } case _LOAD_BUILD_CLASS: { - _Py_UopsLocalsPlusSlot bc; + _Py_UopsPESlot bc; bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; @@ -487,7 +487,7 @@ } case _UNPACK_SEQUENCE: { - _Py_UopsLocalsPlusSlot *output; + _Py_UopsPESlot *output; output = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { output[_i] = sym_new_not_null(ctx); @@ -498,8 +498,8 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { - _Py_UopsLocalsPlusSlot val1; - _Py_UopsLocalsPlusSlot val0; + _Py_UopsPESlot val1; + _Py_UopsPESlot val0; val1 = sym_new_not_null(ctx); val0 = sym_new_not_null(ctx); stack_pointer[-1] = val1; @@ -510,7 +510,7 @@ } case _UNPACK_SEQUENCE_TUPLE: { - _Py_UopsLocalsPlusSlot *values; + _Py_UopsPESlot *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); @@ -521,7 +521,7 @@ } case _UNPACK_SEQUENCE_LIST: { - _Py_UopsLocalsPlusSlot *values; + _Py_UopsPESlot *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); @@ -532,8 +532,8 @@ } case _UNPACK_EX: { - _Py_UopsLocalsPlusSlot *left; - _Py_UopsLocalsPlusSlot *right; + _Py_UopsPESlot *left; + _Py_UopsPESlot *right; right = &stack_pointer[(oparg & 0xFF)]; for (int _i = oparg & 0xFF; --_i >= 0;) { left[_i] = sym_new_not_null(ctx); @@ -569,7 +569,7 @@ } case _LOAD_LOCALS: { - _Py_UopsLocalsPlusSlot locals; + _Py_UopsPESlot locals; locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; @@ -580,7 +580,7 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ case _LOAD_NAME: { - _Py_UopsLocalsPlusSlot v; + _Py_UopsPESlot v; v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; @@ -589,8 +589,8 @@ } case _LOAD_GLOBAL: { - _Py_UopsLocalsPlusSlot *res; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsPESlot *res; + _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; res = &stack_pointer[0]; for (int _i = 1; --_i >= 0;) { res[_i] = sym_new_not_null(ctx); @@ -611,8 +611,8 @@ } case _LOAD_GLOBAL_MODULE: { - _Py_UopsLocalsPlusSlot res; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsPESlot res; + _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -623,8 +623,8 @@ } case _LOAD_GLOBAL_BUILTINS: { - _Py_UopsLocalsPlusSlot res; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsPESlot res; + _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -647,14 +647,14 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsPESlot value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsPESlot value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -673,7 +673,7 @@ } case _BUILD_STRING: { - _Py_UopsLocalsPlusSlot str; + _Py_UopsPESlot str; str = sym_new_not_null(ctx); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -682,7 +682,7 @@ } case _BUILD_TUPLE: { - _Py_UopsLocalsPlusSlot tup; + _Py_UopsPESlot tup; tup = sym_new_not_null(ctx); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; @@ -691,7 +691,7 @@ } case _BUILD_LIST: { - _Py_UopsLocalsPlusSlot list; + _Py_UopsPESlot list; list = sym_new_not_null(ctx); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -712,7 +712,7 @@ } case _BUILD_SET: { - _Py_UopsLocalsPlusSlot set; + _Py_UopsPESlot set; set = sym_new_not_null(ctx); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -721,7 +721,7 @@ } case _BUILD_MAP: { - _Py_UopsLocalsPlusSlot map; + _Py_UopsPESlot map; map = sym_new_not_null(ctx); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -754,7 +754,7 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ case _LOAD_SUPER_ATTR_ATTR: { - _Py_UopsLocalsPlusSlot attr_st; + _Py_UopsPESlot attr_st; attr_st = sym_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -763,8 +763,8 @@ } case _LOAD_SUPER_ATTR_METHOD: { - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsPESlot attr; + _Py_UopsPESlot self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); stack_pointer[-3] = attr; @@ -775,8 +775,8 @@ } case _LOAD_ATTR: { - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot self_or_null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsPESlot attr; + _Py_UopsPESlot self_or_null = (_Py_UopsPESlot){NULL, 0}; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); stack_pointer[-1] = attr; @@ -795,8 +795,8 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsPESlot attr; + _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[-1] = attr; @@ -811,8 +811,8 @@ } case _LOAD_ATTR_MODULE: { - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsPESlot attr; + _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[-1] = attr; @@ -827,8 +827,8 @@ } case _LOAD_ATTR_WITH_HINT: { - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsPESlot attr; + _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[-1] = attr; @@ -839,8 +839,8 @@ } case _LOAD_ATTR_SLOT: { - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsPESlot attr; + _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[-1] = attr; @@ -855,8 +855,8 @@ } case _LOAD_ATTR_CLASS: { - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsPESlot attr; + _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[-1] = attr; @@ -898,7 +898,7 @@ } case _COMPARE_OP: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -907,7 +907,7 @@ } case _COMPARE_OP_FLOAT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -916,7 +916,7 @@ } case _COMPARE_OP_INT: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -925,7 +925,7 @@ } case _COMPARE_OP_STR: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -934,7 +934,7 @@ } case _IS_OP: { - _Py_UopsLocalsPlusSlot b; + _Py_UopsPESlot b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -943,7 +943,7 @@ } case _CONTAINS_OP: { - _Py_UopsLocalsPlusSlot b; + _Py_UopsPESlot b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -952,7 +952,7 @@ } case _CONTAINS_OP_SET: { - _Py_UopsLocalsPlusSlot b; + _Py_UopsPESlot b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -961,7 +961,7 @@ } case _CONTAINS_OP_DICT: { - _Py_UopsLocalsPlusSlot b; + _Py_UopsPESlot b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -970,8 +970,8 @@ } case _CHECK_EG_MATCH: { - _Py_UopsLocalsPlusSlot rest; - _Py_UopsLocalsPlusSlot match; + _Py_UopsPESlot rest; + _Py_UopsPESlot match; rest = sym_new_not_null(ctx); match = sym_new_not_null(ctx); stack_pointer[-2] = rest; @@ -980,14 +980,14 @@ } case _CHECK_EXC_MATCH: { - _Py_UopsLocalsPlusSlot b; + _Py_UopsPESlot b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -996,7 +996,7 @@ } case _IMPORT_FROM: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1009,14 +1009,14 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - _Py_UopsLocalsPlusSlot b; + _Py_UopsPESlot b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _GET_LEN: { - _Py_UopsLocalsPlusSlot len; + _Py_UopsPESlot len; len = sym_new_not_null(ctx); stack_pointer[0] = len; stack_pointer += 1; @@ -1025,7 +1025,7 @@ } case _MATCH_CLASS: { - _Py_UopsLocalsPlusSlot attrs; + _Py_UopsPESlot attrs; attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1034,7 +1034,7 @@ } case _MATCH_MAPPING: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1043,7 +1043,7 @@ } case _MATCH_SEQUENCE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1052,7 +1052,7 @@ } case _MATCH_KEYS: { - _Py_UopsLocalsPlusSlot values_or_none; + _Py_UopsPESlot values_or_none; values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1061,14 +1061,14 @@ } case _GET_ITER: { - _Py_UopsLocalsPlusSlot iter; + _Py_UopsPESlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_YIELD_FROM_ITER: { - _Py_UopsLocalsPlusSlot iter; + _Py_UopsPESlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -1077,7 +1077,7 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - _Py_UopsLocalsPlusSlot next; + _Py_UopsPESlot next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1098,7 +1098,7 @@ } case _ITER_NEXT_LIST: { - _Py_UopsLocalsPlusSlot next; + _Py_UopsPESlot next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1117,7 +1117,7 @@ } case _ITER_NEXT_TUPLE: { - _Py_UopsLocalsPlusSlot next; + _Py_UopsPESlot next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1136,7 +1136,7 @@ } case _ITER_NEXT_RANGE: { - _Py_UopsLocalsPlusSlot next; + _Py_UopsPESlot next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1154,8 +1154,8 @@ } case _LOAD_SPECIAL: { - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsPESlot attr; + _Py_UopsPESlot self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); stack_pointer[-1] = attr; @@ -1166,7 +1166,7 @@ } case _WITH_EXCEPT_START: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1175,8 +1175,8 @@ } case _PUSH_EXC_INFO: { - _Py_UopsLocalsPlusSlot prev_exc; - _Py_UopsLocalsPlusSlot new_exc; + _Py_UopsPESlot prev_exc; + _Py_UopsPESlot new_exc; prev_exc = sym_new_not_null(ctx); new_exc = sym_new_not_null(ctx); stack_pointer[-1] = prev_exc; @@ -1195,8 +1195,8 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsPESlot attr; + _Py_UopsPESlot self = (_Py_UopsPESlot){NULL, 0}; attr = sym_new_not_null(ctx); self = sym_new_not_null(ctx); stack_pointer[-1] = attr; @@ -1207,8 +1207,8 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsPESlot attr; + _Py_UopsPESlot self = (_Py_UopsPESlot){NULL, 0}; attr = sym_new_not_null(ctx); self = sym_new_not_null(ctx); stack_pointer[-1] = attr; @@ -1219,14 +1219,14 @@ } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - _Py_UopsLocalsPlusSlot attr; + _Py_UopsPESlot attr; attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - _Py_UopsLocalsPlusSlot attr; + _Py_UopsPESlot attr; attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; @@ -1237,8 +1237,8 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - _Py_UopsLocalsPlusSlot attr; - _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; + _Py_UopsPESlot attr; + _Py_UopsPESlot self = (_Py_UopsPESlot){NULL, 0}; attr = sym_new_not_null(ctx); self = sym_new_not_null(ctx); stack_pointer[-1] = attr; @@ -1249,9 +1249,9 @@ } case _MAYBE_EXPAND_METHOD: { - _Py_UopsLocalsPlusSlot func; - _Py_UopsLocalsPlusSlot *maybe_self; - _Py_UopsLocalsPlusSlot *args; + _Py_UopsPESlot func; + _Py_UopsPESlot *maybe_self; + _Py_UopsPESlot *args; maybe_self = &stack_pointer[-1 - oparg]; func = sym_new_not_null(ctx); for (int _i = 1; --_i >= 0;) { @@ -1286,8 +1286,8 @@ } case _EXPAND_METHOD: { - _Py_UopsLocalsPlusSlot method; - _Py_UopsLocalsPlusSlot *self; + _Py_UopsPESlot method; + _Py_UopsPESlot *self; self = &stack_pointer[-1 - oparg]; method = sym_new_not_null(ctx); for (int _i = 1; --_i >= 0;) { @@ -1302,7 +1302,7 @@ } case _CALL_NON_PY_GENERAL: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1315,8 +1315,8 @@ } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UopsLocalsPlusSlot func; - _Py_UopsLocalsPlusSlot *self; + _Py_UopsPESlot func; + _Py_UopsPESlot *self; self = &stack_pointer[-1 - oparg]; func = sym_new_not_null(ctx); for (int _i = 1; --_i >= 0;) { @@ -1354,7 +1354,7 @@ } case _CALL_TYPE_1: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1363,7 +1363,7 @@ } case _CALL_STR_1: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1372,7 +1372,7 @@ } case _CALL_TUPLE_1: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1381,9 +1381,9 @@ } case _CHECK_AND_ALLOCATE_OBJECT: { - _Py_UopsLocalsPlusSlot self; - _Py_UopsLocalsPlusSlot init; - _Py_UopsLocalsPlusSlot *args; + _Py_UopsPESlot self; + _Py_UopsPESlot init; + _Py_UopsPESlot *args; self = sym_new_not_null(ctx); init = sym_new_not_null(ctx); for (int _i = oparg; --_i >= 0;) { @@ -1410,7 +1410,7 @@ } case _CALL_BUILTIN_CLASS: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1419,7 +1419,7 @@ } case _CALL_BUILTIN_O: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1428,7 +1428,7 @@ } case _CALL_BUILTIN_FAST: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1437,7 +1437,7 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1446,7 +1446,7 @@ } case _CALL_LEN: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1455,7 +1455,7 @@ } case _CALL_ISINSTANCE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1470,7 +1470,7 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1479,7 +1479,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1488,7 +1488,7 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1497,7 +1497,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1527,9 +1527,9 @@ } case _EXPAND_METHOD_KW: { - _Py_UopsLocalsPlusSlot method; - _Py_UopsLocalsPlusSlot *self; - _Py_UopsLocalsPlusSlot kwnames; + _Py_UopsPESlot method; + _Py_UopsPESlot *self; + _Py_UopsPESlot kwnames; self = &stack_pointer[-2 - oparg]; method = sym_new_not_null(ctx); for (int _i = 1; --_i >= 0;) { @@ -1546,7 +1546,7 @@ } case _CALL_KW_NON_PY: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; @@ -1559,14 +1559,14 @@ /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - _Py_UopsLocalsPlusSlot func; + _Py_UopsPESlot func; func = sym_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - _Py_UopsLocalsPlusSlot func_st; + _Py_UopsPESlot func_st; func_st = sym_new_not_null(ctx); stack_pointer[-2] = func_st; stack_pointer += -1; @@ -1575,7 +1575,7 @@ } case _RETURN_GENERATOR: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1584,7 +1584,7 @@ } case _BUILD_SLICE: { - _Py_UopsLocalsPlusSlot slice; + _Py_UopsPESlot slice; slice = sym_new_not_null(ctx); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -1593,21 +1593,21 @@ } case _CONVERT_VALUE: { - _Py_UopsLocalsPlusSlot result; + _Py_UopsPESlot result; result = sym_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -1616,7 +1616,7 @@ } case _COPY: { - _Py_UopsLocalsPlusSlot top; + _Py_UopsPESlot top; top = sym_new_not_null(ctx); stack_pointer[0] = top; stack_pointer += 1; @@ -1625,7 +1625,7 @@ } case _BINARY_OP: { - _Py_UopsLocalsPlusSlot res; + _Py_UopsPESlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -1634,8 +1634,8 @@ } case _SWAP: { - _Py_UopsLocalsPlusSlot top; - _Py_UopsLocalsPlusSlot bottom; + _Py_UopsPESlot top; + _Py_UopsPESlot bottom; top = sym_new_not_null(ctx); bottom = sym_new_not_null(ctx); stack_pointer[-2 - (oparg-2)] = top; @@ -1710,7 +1710,7 @@ } case _LOAD_CONST_INLINE: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsPESlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); sym_set_origin_inst_override(&value, this_instr); @@ -1721,7 +1721,7 @@ } case _LOAD_CONST_INLINE_BORROW: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsPESlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); sym_set_origin_inst_override(&value, this_instr); @@ -1732,15 +1732,15 @@ } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - _Py_UopsLocalsPlusSlot value; + _Py_UopsPESlot value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_CONST_INLINE_WITH_NULL: { - _Py_UopsLocalsPlusSlot value; - _Py_UopsLocalsPlusSlot null; + _Py_UopsPESlot value; + _Py_UopsPESlot null; value = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = value; @@ -1751,8 +1751,8 @@ } case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { - _Py_UopsLocalsPlusSlot value; - _Py_UopsLocalsPlusSlot null; + _Py_UopsPESlot value; + _Py_UopsPESlot null; value = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = value; diff --git a/Python/partial_evaluator_symbols.c b/Python/partial_evaluator_symbols.c new file mode 100644 index 00000000000000..f87dc9128401d0 --- /dev/null +++ b/Python/partial_evaluator_symbols.c @@ -0,0 +1,538 @@ +#ifdef _Py_TIER2 + +#include "Python.h" + +#include "pycore_code.h" +#include "pycore_frame.h" +#include "pycore_long.h" +#include "pycore_optimizer.h" + +#include +#include +#include + +/* Symbols + ======= + + Documentation TODO gh-120619. + */ + +// Flags for below. +#define IS_NULL 1 << 0 +#define NOT_NULL 1 << 1 +#define NO_SPACE 1 << 2 + +#ifdef Py_DEBUG +static inline int get_lltrace(void) { + char *uop_debug = Py_GETENV("PYTHON_OPT_DEBUG"); + int lltrace = 0; + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + } + return lltrace; +} +#define DPRINTF(level, ...) \ + if (get_lltrace() >= (level)) { printf(__VA_ARGS__); } +#else +#define DPRINTF(level, ...) +#endif + +static _Py_UopsSymbol NO_SPACE_SYMBOL = { + .flags = IS_NULL | NOT_NULL | NO_SPACE, + .typ = NULL, + .const_val = NULL, + .type_version = 0, +}; + +_Py_UopsSymbol * +out_of_space(_Py_UOpsContext *ctx) +{ + ctx->done = true; + ctx->out_of_space = true; + return &NO_SPACE_SYMBOL; +} + +static _Py_UopsSymbol * +sym_new(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *self = &ctx->t_arena.arena[ctx->t_arena.ty_curr_number]; + if (ctx->t_arena.ty_curr_number >= ctx->t_arena.ty_max_number) { + OPT_STAT_INC(optimizer_failure_reason_no_memory); + DPRINTF(1, "out of space for symbolic expression type\n"); + return NULL; + } + ctx->t_arena.ty_curr_number++; + self->flags = 0; + self->typ = NULL; + self->const_val = NULL; + self->type_version = 0; + + return self; +} + +static inline void +sym_set_flag(_Py_UopsSymbol *sym, int flag) +{ + sym->flags |= flag; +} + +static inline void +sym_set_bottom(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +{ + sym_set_flag(sym, IS_NULL | NOT_NULL); + sym->typ = NULL; + Py_CLEAR(sym->const_val); + ctx->done = true; + ctx->contradiction = true; +} + +bool +_Py_uop_sym_is_bottom(_Py_UopsSymbol *sym) +{ + if ((sym->flags & IS_NULL) && (sym->flags & NOT_NULL)) { + assert(sym->flags == (IS_NULL | NOT_NULL)); + assert(sym->typ == NULL); + assert(sym->const_val == NULL); + return true; + } + return false; +} + +bool +_Py_uop_sym_is_not_null(_Py_UopsSymbol *sym) +{ + return sym->flags == NOT_NULL; +} + +bool +_Py_uop_sym_is_null(_Py_UopsSymbol *sym) +{ + return sym->flags == IS_NULL; +} + +bool +_Py_uop_sym_is_const(_Py_UopsSymbol *sym) +{ + return sym->const_val != NULL; +} + +PyObject * +_Py_uop_sym_get_const(_Py_UopsSymbol *sym) +{ + return sym->const_val; +} + +void +_Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ) +{ + assert(typ != NULL && PyType_Check(typ)); + if (sym->flags & IS_NULL) { + sym_set_bottom(ctx, sym); + return; + } + if (sym->typ != NULL) { + if (sym->typ != typ) { + sym_set_bottom(ctx, sym); + return; + } + } + else { + sym_set_flag(sym, NOT_NULL); + sym->typ = typ; + } +} + +bool +_Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version) +{ + // if the type version was already set, then it must be different and we should set it to bottom + if (sym->type_version) { + sym_set_bottom(ctx, sym); + return false; + } + sym->type_version = version; + return true; +} + +void +_Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val) +{ + assert(const_val != NULL); + if (sym->flags & IS_NULL) { + sym_set_bottom(ctx, sym); + } + PyTypeObject *typ = Py_TYPE(const_val); + if (sym->typ != NULL && sym->typ != typ) { + sym_set_bottom(ctx, sym); + } + if (sym->const_val != NULL) { + if (sym->const_val != const_val) { + // TODO: What if they're equal? + sym_set_bottom(ctx, sym); + } + } + else { + sym_set_flag(sym, NOT_NULL); + sym->typ = typ; + sym->const_val = Py_NewRef(const_val); + } +} + +void +_Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +{ + if (_Py_uop_sym_is_not_null(sym)) { + sym_set_bottom(ctx, sym); + } + sym_set_flag(sym, IS_NULL); +} + +void +_Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +{ + if (_Py_uop_sym_is_null(sym)) { + sym_set_bottom(ctx, sym); + } + sym_set_flag(sym, NOT_NULL); +} + + +_Py_UopsSymbol * +_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx) +{ + return sym_new(ctx); +} + +_Py_UopsSymbol * +_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *res = _Py_uop_sym_new_unknown(ctx); + if (res == NULL) { + return out_of_space(ctx); + } + sym_set_flag(res, NOT_NULL); + return res; +} + +_Py_UopsSymbol * +_Py_uop_sym_new_type(_Py_UOpsContext *ctx, PyTypeObject *typ) +{ + _Py_UopsSymbol *res = sym_new(ctx); + if (res == NULL) { + return out_of_space(ctx); + } + _Py_uop_sym_set_type(ctx, res, typ); + return res; +} + +// Adds a new reference to const_val, owned by the symbol. +_Py_UopsSymbol * +_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) +{ + assert(const_val != NULL); + _Py_UopsSymbol *res = sym_new(ctx); + if (res == NULL) { + return out_of_space(ctx); + } + _Py_uop_sym_set_const(ctx, res, const_val); + return res; +} + +_Py_UopsSymbol * +_Py_uop_sym_new_null(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *null_sym = _Py_uop_sym_new_unknown(ctx); + if (null_sym == NULL) { + return out_of_space(ctx); + } + _Py_uop_sym_set_null(ctx, null_sym); + return null_sym; +} + +PyTypeObject * +_Py_uop_sym_get_type(_Py_UopsSymbol *sym) +{ + if (_Py_uop_sym_is_bottom(sym)) { + return NULL; + } + return sym->typ; +} + +unsigned int +_Py_uop_sym_get_type_version(_Py_UopsSymbol *sym) +{ + return sym->type_version; +} + +bool +_Py_uop_sym_has_type(_Py_UopsSymbol *sym) +{ + if (_Py_uop_sym_is_bottom(sym)) { + return false; + } + return sym->typ != NULL; +} + +bool +_Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) +{ + assert(typ != NULL && PyType_Check(typ)); + return _Py_uop_sym_get_type(sym) == typ; +} + +bool +_Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version) +{ + return _Py_uop_sym_get_type_version(sym) == version; +} + + +int +_Py_uop_sym_truthiness(_Py_UopsSymbol *sym) +{ + /* There are some non-constant values for + * which `bool(val)` always evaluates to + * True or False, such as tuples with known + * length, but unknown contents, or bound-methods. + * This function will need updating + * should we support those values. + */ + if (_Py_uop_sym_is_bottom(sym)) { + return -1; + } + if (!_Py_uop_sym_is_const(sym)) { + return -1; + } + PyObject *value = _Py_uop_sym_get_const(sym); + if (value == Py_None) { + return 0; + } + /* Only handle a few known safe types */ + PyTypeObject *tp = Py_TYPE(value); + if (tp == &PyLong_Type) { + return !_PyLong_IsZero((PyLongObject *)value); + } + if (tp == &PyUnicode_Type) { + return value != &_Py_STR(empty); + } + if (tp == &PyBool_Type) { + return value == Py_True; + } + return -1; +} + +// 0 on success, -1 on error. +_Py_UOpsAbstractFrame * +_Py_uop_frame_new( + _Py_UOpsContext *ctx, + PyCodeObject *co, + int curr_stackentries, + _Py_UopsSymbol **args, + int arg_len) +{ + assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); + _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; + + frame->stack_len = co->co_stacksize; + frame->locals_len = co->co_nlocalsplus; + + frame->locals = ctx->n_consumed; + frame->stack = frame->locals + co->co_nlocalsplus; + frame->stack_pointer = frame->stack + curr_stackentries; + ctx->n_consumed = ctx->n_consumed + (co->co_nlocalsplus + co->co_stacksize); + if (ctx->n_consumed >= ctx->limit) { + ctx->done = true; + ctx->out_of_space = true; + return NULL; + } + + // Initialize with the initial state of all local variables + for (int i = 0; i < arg_len; i++) { + frame->locals[i] = args[i]; + } + + for (int i = arg_len; i < co->co_nlocalsplus; i++) { + _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx); + frame->locals[i] = local; + } + + + // Initialize the stack as well + for (int i = 0; i < curr_stackentries; i++) { + _Py_UopsSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); + frame->stack[i] = stackvar; + } + + return frame; +} + +void +_Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx) +{ + if (ctx == NULL) { + return; + } + ctx->curr_frame_depth = 0; + int tys = ctx->t_arena.ty_curr_number; + for (int i = 0; i < tys; i++) { + Py_CLEAR(ctx->t_arena.arena[i].const_val); + } +} + +void +_Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) +{ + ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; + ctx->n_consumed = ctx->locals_and_stack; +#ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. + for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { + ctx->locals_and_stack[i] = NULL; + } +#endif + + // Setup the arena for sym expressions. + ctx->t_arena.ty_curr_number = 0; + ctx->t_arena.ty_max_number = TY_ARENA_SIZE; + + // Frame setup + ctx->curr_frame_depth = 0; +} + +int +_Py_uop_frame_pop(_Py_UOpsContext *ctx) +{ + _Py_UOpsAbstractFrame *frame = ctx->frame; + ctx->n_consumed = frame->locals; + ctx->curr_frame_depth--; + assert(ctx->curr_frame_depth >= 1); + ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + return 0; +} + +#define TEST_PREDICATE(PRED, MSG) \ +do { \ + if (!(PRED)) { \ + PyErr_SetString( \ + PyExc_AssertionError, \ + (MSG)); \ + goto fail; \ + } \ +} while (0) + +static _Py_UopsSymbol * +make_bottom(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); + _Py_uop_sym_set_null(ctx, sym); + _Py_uop_sym_set_non_null(ctx, sym); + return sym; +} + +PyObject * +_Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) +{ + _Py_UOpsContext context; + _Py_UOpsContext *ctx = &context; + _Py_uop_abstractcontext_init(ctx); + PyObject *val_42 = NULL; + PyObject *val_43 = NULL; + + // Use a single 'sym' variable so copy-pasting tests is easier. + _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "top is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); + + sym = make_bottom(ctx); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "bottom is not NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "bottom matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "int isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "int isn't int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "int matches float"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "int is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "int as constant is not NULL"); + + _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); + + _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(int and float) isn't bottom"); + + val_42 = PyLong_FromLong(42); + assert(val_42 != NULL); + assert(_Py_IsImmortal(val_42)); + + val_43 = PyLong_FromLong(43); + assert(val_43 != NULL); + assert(_Py_IsImmortal(val_43)); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + _Py_uop_sym_set_const(ctx, sym, val_42); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 1, "bool(42) is not True"); + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "42 matches float"); + TEST_PREDICATE(_Py_uop_sym_is_const(sym), "42 is not a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) != NULL, "42 as constant is NULL"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "42 as constant isn't 42"); + + _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "(42 and 42) as constant isn't 42"); + + _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + _Py_uop_sym_set_const(ctx, sym, val_42); + _Py_uop_sym_set_const(ctx, sym, val_43); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); + + + sym = _Py_uop_sym_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(None) is not False"); + sym = _Py_uop_sym_new_const(ctx, Py_False); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(False) is not False"); + sym = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(0) is not False"); + + _Py_uop_abstractcontext_fini(ctx); + Py_DECREF(val_42); + Py_DECREF(val_43); + Py_RETURN_NONE; + +fail: + _Py_uop_abstractcontext_fini(ctx); + Py_XDECREF(val_42); + Py_XDECREF(val_43); + return NULL; +} + +#endif /* _Py_TIER2 */ diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py index 30b31dcd74a4ea..cf92b585b76288 100644 --- a/Tools/cases_generator/partial_evaluator_generator.py +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -34,10 +34,10 @@ def validate_uop(override: Uop, uop: Uop) -> None: def type_name(var: StackItem) -> str: if var.is_array(): - return f"_Py_UopsLocalsPlusSlot *" + return f"_Py_UopsPESlot *" if var.type: return var.type - return f"_Py_UopsLocalsPlusSlot " + return f"_Py_UopsPESlot " def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: @@ -47,7 +47,7 @@ def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: if var.name not in variables: variables.add(var.name) if var.condition: - out.emit(f"{type_name(var)}{var.name} = (_Py_UopsLocalsPlusSlot){{NULL, 0}};\n") + out.emit(f"{type_name(var)}{var.name} = (_Py_UopsPESlot){{NULL, 0}};\n") else: out.emit(f"{type_name(var)}{var.name};\n") for var in uop.stack.outputs: @@ -56,7 +56,7 @@ def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: if var.name not in variables: variables.add(var.name) if var.condition: - out.emit(f"{type_name(var)}{var.name} = (_Py_UopsLocalsPlusSlot){{NULL, 0}};\n") + out.emit(f"{type_name(var)}{var.name} = (_Py_UopsPESlot){{NULL, 0}};\n") else: out.emit(f"{type_name(var)}{var.name};\n") From 32057d9dcf7f50db04f994cb933791c796712310 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 4 Oct 2024 00:44:51 +0800 Subject: [PATCH 26/46] fix up compile problems --- Include/internal/pycore_optimizer.h | 48 +++- Python/optimizer_bytecodes.c | 43 +-- Python/optimizer_cases.c.h | 54 ++-- Python/partial_evaluator.c | 29 +- Python/partial_evaluator_bytecodes.c | 224 +++++++++++++--- Python/partial_evaluator_cases.c.h | 198 +++++++++++--- Python/partial_evaluator_symbols.c | 381 +++++++++------------------ 7 files changed, 593 insertions(+), 384 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index e615f49b5efa56..d2765356140144 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -293,13 +293,13 @@ struct _Py_UopsPESymbol { PyObject *const_val; // Owned reference (!) }; +typedef struct _Py_UopsPESymbol _Py_UopsPESymbol; + typedef struct _Py_UopsPESlot { - _Py_UopsSymbol *sym; + _Py_UopsPESymbol *sym; _PyUOpInstruction *origin_inst; // The instruction this symbol originates from. } _Py_UopsPESlot; -typedef struct _Py_UopsPESymbol _Py_UopsPESymbol; - struct _Py_UOpsPEAbstractFrame { // Max stacklen int stack_len; @@ -315,7 +315,7 @@ typedef struct _Py_UOpsPEAbstractFrame _Py_UOpsPEAbstractFrame; typedef struct pe_arena { int sym_curr_number; int sym_max_number; - _Py_UopsSymbol arena[TY_ARENA_SIZE]; + _Py_UopsPESymbol arena[TY_ARENA_SIZE]; } pe_arena; struct _Py_UOpsPEContext { @@ -328,15 +328,47 @@ struct _Py_UOpsPEContext { int curr_frame_depth; // Arena for the symbolic information. - pe_arena t_arena; + pe_arena sym_arena; - _Py_UopsPESymbol **n_consumed; - _Py_UopsPESymbol **limit; - _Py_UopsPESymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; + _Py_UopsPESlot *n_consumed; + _Py_UopsPESlot *limit; + _Py_UopsPESlot locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; }; typedef struct _Py_UOpsPEContext _Py_UOpsPEContext; +extern bool _Py_uop_pe_sym_is_null(_Py_UopsPESlot *sym); +extern bool _Py_uop_pe_sym_is_not_null(_Py_UopsPESlot *sym); +extern bool _Py_uop_pe_sym_is_const(_Py_UopsPESlot *sym); +extern PyObject *_Py_uop_pe_sym_get_const(_Py_UopsPESlot *sym); +extern _Py_UopsPESlot _Py_uop_pe_sym_new_unknown(_Py_UOpsPEContext *ctx); +extern _Py_UopsPESlot _Py_uop_pe_sym_new_not_null(_Py_UOpsPEContext *ctx); +extern _Py_UopsPESlot _Py_uop_pe_sym_new_const(_Py_UOpsPEContext *ctx, PyObject *const_val); +extern _Py_UopsPESlot _Py_uop_pe_sym_new_null(_Py_UOpsPEContext *ctx); +extern void _Py_uop_pe_sym_set_null(_Py_UOpsPEContext *ctx, _Py_UopsPESlot *sym); +extern void _Py_uop_pe_sym_set_non_null(_Py_UOpsPEContext *ctx, _Py_UopsPESlot *sym); +extern void _Py_uop_pe_sym_set_const(_Py_UOpsPEContext *ctx, _Py_UopsPESlot *sym, PyObject *const_val); +extern bool _Py_uop_pe_sym_is_bottom(_Py_UopsPESlot *sym); +extern int _Py_uop_pe_sym_truthiness(_Py_UopsPESlot *sym); +extern void _Py_uop_sym_set_origin_inst_override(_Py_UopsPESlot *sym, _PyUOpInstruction *origin); +extern _PyUOpInstruction *_Py_uop_sym_get_origin(_Py_UopsPESlot *sym); +extern bool _Py_uop_sym_is_virtual(_Py_UopsPESlot *sym); + + +extern _Py_UOpsPEAbstractFrame * +_Py_uop_pe_frame_new( + _Py_UOpsPEContext *ctx, + PyCodeObject *co, + int curr_stackentries, + _Py_UopsPESlot *args, + int arg_len); + +int _Py_uop_pe_frame_pop(_Py_UOpsPEContext *ctx); + +extern void _Py_uop_pe_abstractcontext_init(_Py_UOpsPEContext *ctx); +extern void _Py_uop_pe_abstractcontext_fini(_Py_UOpsPEContext *ctx); + + #ifdef __cplusplus } #endif diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 799ad55d3a75a6..bf8f0753f800c0 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -85,11 +85,11 @@ dummy_func(void) { op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - _Py_UopsPESlot temp = sym_new_null(ctx); + _Py_UopsSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; } - _static op(_STORE_FAST, (value --)) { + op(_STORE_FAST, (value --)) { GETLOCAL(oparg) = value; } @@ -329,10 +329,10 @@ dummy_func(void) { } } - op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame)) { + op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame: _Py_UOpsAbstractFrame *)) { (void)container; (void)sub; - new_frame = (_Py_UopsPESlot){NULL, 0}; + new_frame = NULL; ctx->done = true; } @@ -487,7 +487,7 @@ dummy_func(void) { op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { (void)index; null = sym_new_null(ctx); - attr = (_Py_UopsPESlot){NULL, 0}; + attr = NULL; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. assert(sym_is_const(owner)); @@ -500,7 +500,7 @@ dummy_func(void) { attr = sym_new_const(ctx, res); } } - if (attr.sym == NULL) { + if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ attr = sym_new_not_null(ctx); } @@ -545,10 +545,10 @@ dummy_func(void) { self = owner; } - op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { + op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _Py_UOpsAbstractFrame *)) { (void)fget; (void)owner; - new_frame = (_Py_UopsPESlot){NULL, 0}; + new_frame = NULL; ctx->done = true; } @@ -568,7 +568,7 @@ dummy_func(void) { sym_set_type(callable, &PyMethod_Type); } - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { int argcount = oparg; (void)callable; @@ -581,7 +581,8 @@ dummy_func(void) { break; } - assert(self_or_null.sym != NULL); + + assert(self_or_null != NULL); assert(args != NULL); if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM @@ -590,9 +591,9 @@ dummy_func(void) { } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, args, argcount); + new_frame = frame_new(ctx, co, 0, args, argcount); } else { - new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = frame_new(ctx, co, 0, NULL, 0); } } @@ -605,7 +606,7 @@ dummy_func(void) { maybe_self = sym_new_not_null(ctx); } - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { (void)(self_or_null); (void)(callable); PyCodeObject *co = NULL; @@ -616,15 +617,15 @@ dummy_func(void) { break; } - new_frame.sym = frame_new(ctx, co, 0, NULL, 0); + new_frame = frame_new(ctx, co, 0, NULL, 0); } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { (void)callable; (void)self_or_null; (void)args; (void)kwnames; - new_frame = (_Py_UopsPESlot){NULL, 0}; + new_frame = NULL; ctx->done = true; } @@ -637,11 +638,11 @@ dummy_func(void) { init = sym_new_not_null(ctx); } - op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame)) { + op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { (void)self; (void)init; (void)args; - init_frame = (_Py_UopsPESlot){NULL, 0}; + init_frame = NULL; ctx->done = true; } @@ -715,12 +716,12 @@ dummy_func(void) { Py_UNREACHABLE(); } - op(_PUSH_FRAME, (new_frame -- unused if (0))) { + op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = (_Py_UOpsAbstractFrame *)new_frame.sym; + ctx->frame = new_frame; ctx->curr_frame_depth++; - stack_pointer = ((_Py_UOpsAbstractFrame *)new_frame.sym)->stack_pointer; + stack_pointer = new_frame->stack_pointer; co = get_code(this_instr); if (co == NULL) { // should be about to _EXIT_TRACE anyway diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 2a49d9497349eb..fc0c0eff01d4c1 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -48,7 +48,7 @@ case _LOAD_FAST_AND_CLEAR: { _Py_UopsSymbol *value; value = GETLOCAL(oparg); - _Py_UopsPESlot temp = sym_new_null(ctx); + _Py_UopsSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; @@ -556,14 +556,14 @@ case _BINARY_SUBSCR_INIT_CALL: { _Py_UopsSymbol *sub; _Py_UopsSymbol *container; - _Py_UopsSymbol *new_frame; + _Py_UOpsAbstractFrame *new_frame; sub = stack_pointer[-1]; container = stack_pointer[-2]; (void)container; (void)sub; - new_frame = (_Py_UopsPESlot){NULL, 0}; + new_frame = NULL; ctx->done = true; - stack_pointer[-2] = new_frame; + stack_pointer[-2] = (_Py_UopsSymbol *)new_frame; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1107,7 +1107,7 @@ uint16_t index = (uint16_t)this_instr->operand; (void)index; null = sym_new_null(ctx); - attr = (_Py_UopsPESlot){NULL, 0}; + attr = NULL; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. assert(sym_is_const(owner)); @@ -1120,7 +1120,7 @@ attr = sym_new_const(ctx, res); } } - if (attr.sym == NULL) { + if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ attr = sym_new_not_null(ctx); } @@ -1192,14 +1192,14 @@ case _LOAD_ATTR_PROPERTY_FRAME: { _Py_UopsSymbol *owner; - _Py_UopsSymbol *new_frame; + _Py_UOpsAbstractFrame *new_frame; owner = stack_pointer[-1]; PyObject *fget = (PyObject *)this_instr->operand; (void)fget; (void)owner; - new_frame = (_Py_UopsPESlot){NULL, 0}; + new_frame = NULL; ctx->done = true; - stack_pointer[-1] = new_frame; + stack_pointer[-1] = (_Py_UopsSymbol *)new_frame; break; } @@ -1662,7 +1662,7 @@ _Py_UopsSymbol **args; _Py_UopsSymbol *self_or_null; _Py_UopsSymbol *callable; - _Py_UopsSymbol *new_frame; + _Py_UOpsAbstractFrame *new_frame; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; (void)(self_or_null); @@ -1674,8 +1674,8 @@ ctx->done = true; break; } - new_frame.sym = frame_new(ctx, co, 0, NULL, 0); - stack_pointer[-2 - oparg] = new_frame; + new_frame = frame_new(ctx, co, 0, NULL, 0); + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -1766,7 +1766,7 @@ _Py_UopsSymbol **args; _Py_UopsSymbol *self_or_null; _Py_UopsSymbol *callable; - _Py_UopsSymbol *new_frame; + _Py_UOpsAbstractFrame *new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1779,7 +1779,7 @@ ctx->done = true; break; } - assert(self_or_null.sym != NULL); + assert(self_or_null != NULL); assert(args != NULL); if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM @@ -1787,25 +1787,25 @@ argcount++; } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, args, argcount); + new_frame = frame_new(ctx, co, 0, args, argcount); } else { - new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = frame_new(ctx, co, 0, NULL, 0); } - stack_pointer[-2 - oparg] = new_frame; + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_FRAME: { - _Py_UopsSymbol *new_frame; - new_frame = stack_pointer[-1]; + _Py_UOpsAbstractFrame *new_frame; + new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = (_Py_UOpsAbstractFrame *)new_frame.sym; + ctx->frame = new_frame; ctx->curr_frame_depth++; - stack_pointer = ((_Py_UOpsAbstractFrame *)new_frame.sym)->stack_pointer; + stack_pointer = new_frame->stack_pointer; co = get_code(this_instr); if (co == NULL) { // should be about to _EXIT_TRACE anyway @@ -1886,16 +1886,16 @@ _Py_UopsSymbol **args; _Py_UopsSymbol *init; _Py_UopsSymbol *self; - _Py_UopsSymbol *init_frame; + _Py_UOpsAbstractFrame *init_frame; args = &stack_pointer[-oparg]; init = stack_pointer[-1 - oparg]; self = stack_pointer[-2 - oparg]; (void)self; (void)init; (void)args; - init_frame = (_Py_UopsPESlot){NULL, 0}; + init_frame = NULL; ctx->done = true; - stack_pointer[-2 - oparg] = init_frame; + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)init_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -2012,7 +2012,7 @@ _Py_UopsSymbol **args; _Py_UopsSymbol *self_or_null; _Py_UopsSymbol *callable; - _Py_UopsSymbol *new_frame; + _Py_UOpsAbstractFrame *new_frame; kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; @@ -2021,9 +2021,9 @@ (void)self_or_null; (void)args; (void)kwnames; - new_frame = (_Py_UopsPESlot){NULL, 0}; + new_frame = NULL; ctx->done = true; - stack_pointer[-3 - oparg] = new_frame; + stack_pointer[-3 - oparg] = (_Py_UopsSymbol *)new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); break; diff --git a/Python/partial_evaluator.c b/Python/partial_evaluator.c index 1af1703b735210..18baab86c1b2ed 100644 --- a/Python/partial_evaluator.c +++ b/Python/partial_evaluator.c @@ -110,6 +110,21 @@ get_code_with_logging(_PyUOpInstruction *op) return co; } +#define sym_is_not_null _Py_uop_pe_sym_is_not_null +#define sym_is_const _Py_uop_pe_sym_is_const +#define sym_get_const _Py_uop_pe_sym_get_const +#define sym_new_unknown _Py_uop_pe_sym_new_unknown +#define sym_new_not_null _Py_uop_pe_sym_new_not_null +#define sym_is_null _Py_uop_pe_sym_is_null +#define sym_new_const _Py_uop_pe_sym_new_const +#define sym_new_null _Py_uop_pe_sym_new_null +#define sym_set_null(SYM) _Py_uop_pe_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_pe_sym_set_non_null(ctx, SYM) +#define sym_set_const(SYM, CNST) _Py_uop_pe_sym_set_const(ctx, SYM, CNST) +#define sym_is_bottom _Py_uop_pe_sym_is_bottom +#define frame_new _Py_uop_pe_frame_new +#define frame_pop _Py_uop_pe_frame_pop + #define MATERIALIZE_INST() (this_instr->is_virtual = false) #define sym_set_origin_inst_override _Py_uop_sym_set_origin_inst_override #define sym_is_virtual _Py_uop_sym_is_virtual @@ -134,13 +149,13 @@ materialize_stack(_Py_UopsPESlot *stack_start, _Py_UopsPESlot *stack_end) } static void -materialize_frame(_Py_UOpsAbstractFrame *frame) +materialize_frame(_Py_UOpsPEAbstractFrame *frame) { materialize_stack(frame->stack, frame->stack_pointer); } static void -materialize_ctx(_Py_UOpsContext *ctx) +materialize_ctx(_Py_UOpsPEContext *ctx) { for (int i = 0; i < ctx->curr_frame_depth; i++) { materialize_frame(&ctx->frames[i]); @@ -167,7 +182,7 @@ partial_evaluate_uops( _PyUOpInstruction *corresponding_check_stack = NULL; _Py_uop_abstractcontext_init(ctx); - _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, curr_stacklen, NULL, 0); + _Py_UOpsPEAbstractFrame *frame = _Py_uop_pe_frame_new(ctx, co, curr_stacklen, NULL, 0); if (frame == NULL) { return -1; } @@ -247,12 +262,12 @@ partial_evaluate_uops( // retrying later. DPRINTF(3, "\n"); DPRINTF(1, "Hit bottom in pe's abstract interpreter\n"); - _Py_uop_abstractcontext_fini(ctx); + _Py_uop_pe_abstractcontext_fini(ctx); return 0; } if (ctx->out_of_space || !is_terminator(this_instr)) { - _Py_uop_abstractcontext_fini(ctx); + _Py_uop_pe_abstractcontext_fini(ctx); return trace_len; } else { @@ -273,7 +288,7 @@ partial_evaluate_uops( trace[x] = trace_dest[x]; } } - _Py_uop_abstractcontext_fini(ctx); + _Py_uop_pe_abstractcontext_fini(ctx); return trace_dest_len; } @@ -283,7 +298,7 @@ partial_evaluate_uops( if (opcode <= MAX_UOP_ID) { OPT_ERROR_IN_OPCODE(opcode); } - _Py_uop_abstractcontext_fini(ctx); + _Py_uop_pe_abstractcontext_fini(ctx); return -1; } diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index d5b83ed1bcc032..44be57f8da530a 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -6,42 +6,25 @@ #define op(name, ...) /* NAME is ignored */ -typedef struct _Py_UopsSymbol _Py_UopsSymbol; -typedef struct _Py_UOpsContext _Py_UOpsContext; -typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; +typedef struct _Py_UopsPESymbol _Py_UopsPESymbol; +typedef struct _Py_UOpsPEContext _Py_UOpsPEContext; +typedef struct _Py_UOpsPEAbstractFrame _Py_UOpsPEAbstractFrame; /* Shortened forms for convenience */ -#define sym_is_not_null _Py_uop_sym_is_not_null -#define sym_is_const _Py_uop_sym_is_const -#define sym_get_const _Py_uop_sym_get_const -#define sym_new_unknown _Py_uop_sym_new_unknown -#define sym_new_not_null _Py_uop_sym_new_not_null -#define sym_new_type _Py_uop_sym_new_type -#define sym_is_null _Py_uop_sym_is_null -#define sym_new_const _Py_uop_sym_new_const -#define sym_new_null _Py_uop_sym_new_null -#define sym_matches_type _Py_uop_sym_matches_type -#define sym_matches_type_version _Py_uop_sym_matches_type_version -#define sym_get_type _Py_uop_sym_get_type -#define sym_has_type _Py_uop_sym_has_type -#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) -#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) -#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) -#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) -#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) -#define sym_is_bottom _Py_uop_sym_is_bottom -#define frame_new _Py_uop_frame_new -#define frame_pop _Py_uop_frame_pop - -extern int -optimize_to_bool( - _PyUOpInstruction *this_instr, - _Py_UOpsContext *ctx, - _Py_UopsSymbol *value, - _Py_UopsSymbol **result_ptr); - -extern void -eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit); +#define sym_is_not_null _Py_uop_pe_sym_is_not_null +#define sym_is_const _Py_uop_pe_sym_is_const +#define sym_get_const _Py_uop_pe_sym_get_const +#define sym_new_unknown _Py_uop_pe_sym_new_unknown +#define sym_new_not_null _Py_uop_pe_sym_new_not_null +#define sym_is_null _Py_uop_pe_sym_is_null +#define sym_new_const _Py_uop_pe_sym_new_const +#define sym_new_null _Py_uop_pe_sym_new_null +#define sym_set_null(SYM) _Py_uop_pe_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_pe_sym_set_non_null(ctx, SYM) +#define sym_set_const(SYM, CNST) _Py_uop_pe_sym_set_const(ctx, SYM, CNST) +#define sym_is_bottom _Py_uop_pe_sym_is_bottom +#define frame_new _Py_uop_pe_frame_new +#define frame_pop _Py_uop_pe_frame_pop extern PyCodeObject *get_code(_PyUOpInstruction *op); @@ -53,7 +36,7 @@ dummy_func(void) { op(_LOAD_FAST_CHECK, (-- value)) { value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. - if (sym_is_null(value)) { + if (sym_is_null(&value)) { ctx->done = true; } } @@ -85,9 +68,9 @@ dummy_func(void) { } op(_STORE_FAST, (value --)) { - _PyUOpInstruction *origin = sym_get_origin(value); + _PyUOpInstruction *origin = sym_get_origin(&value); // Gets rid of things like x = x. - if (sym_is_virtual(value) && + if (sym_is_virtual(&value) && origin != NULL && origin->opcode == _LOAD_FAST && origin->oparg == oparg) { @@ -102,7 +85,7 @@ dummy_func(void) { } op(_POP_TOP, (pop --)) { - if (!sym_is_virtual(pop)) { + if (!sym_is_virtual(&pop)) { MATERIALIZE_INST(); } } @@ -114,6 +97,171 @@ dummy_func(void) { (void)framesize; } + op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame)) { + (void)container; + (void)sub; + new_frame = (_Py_UopsPESlot){NULL, NULL}; + ctx->done = true; + } + + op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { + (void)fget; + (void)owner; + new_frame = (_Py_UopsPESlot){NULL, NULL}; + ctx->done = true; + } + + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { + int argcount = oparg; + + (void)callable; + + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + + + assert(self_or_null.sym != NULL); + assert(args != NULL); + if (sym_is_not_null(&self_or_null)) { + // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM + args--; + argcount++; + } + + if (sym_is_null(&self_or_null) || sym_is_not_null(&self_or_null)) { + new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, args, argcount), NULL}; + } else { + new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL}; + + } + } + + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { + (void)(self_or_null); + (void)(callable); + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + + new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL}; + } + + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { + (void)callable; + (void)self_or_null; + (void)args; + (void)kwnames; + new_frame = (_Py_UopsPESlot){NULL, NULL}; + ctx->done = true; + } + + op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame)) { + (void)self; + (void)init; + (void)args; + init_frame = (_Py_UopsPESlot){NULL, NULL}; + ctx->done = true; + } + + op(_FOR_ITER_GEN_FRAME, ( -- )) { + /* We are about to hit the end of the trace */ + ctx->done = true; + } + + op(_SEND_GEN_FRAME, ( -- )) { + // We are about to hit the end of the trace: + ctx->done = true; + } + + op(_PUSH_FRAME, (new_frame -- unused if (0))) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + ctx->frame = (_Py_UOpsPEAbstractFrame *)new_frame.sym; + ctx->curr_frame_depth++; + stack_pointer = ((_Py_UOpsPEAbstractFrame *)new_frame.sym)->stack_pointer; + co = get_code(this_instr); + if (co == NULL) { + // should be about to _EXIT_TRACE anyway + ctx->done = true; + break; + } + + /* Stack space handling */ + int framesize = co->co_framesize; + assert(framesize > 0); + curr_space += framesize; + if (curr_space < 0 || curr_space > INT32_MAX) { + // won't fit in signed 32-bit int + ctx->done = true; + break; + } + max_space = curr_space > max_space ? curr_space : max_space; + if (first_valid_check_stack == NULL) { + first_valid_check_stack = corresponding_check_stack; + } + else if (corresponding_check_stack) { + // delete all but the first valid _CHECK_STACK_SPACE + corresponding_check_stack->opcode = _NOP; + } + corresponding_check_stack = NULL; + } + + op(_RETURN_VALUE, (retval -- res)) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = retval; + + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + } + + op(_RETURN_GENERATOR, ( -- res)) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = sym_new_unknown(ctx); + + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + } + + op(_YIELD_VALUE, (unused -- res)) { + res = sym_new_unknown(ctx); + } // END BYTECODES // } diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index c7d8d2252f444a..ec2fdc6fc6d42c 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -27,7 +27,7 @@ _Py_UopsPESlot value; value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. - if (sym_is_null(value)) { + if (sym_is_null(&value)) { ctx->done = true; } stack_pointer[0] = value; @@ -70,9 +70,9 @@ case _STORE_FAST: { _Py_UopsPESlot value; value = stack_pointer[-1]; - _PyUOpInstruction *origin = sym_get_origin(value); + _PyUOpInstruction *origin = sym_get_origin(&value); // Gets rid of things like x = x. - if (sym_is_virtual(value) && + if (sym_is_virtual(&value) && origin != NULL && origin->opcode == _LOAD_FAST && origin->oparg == oparg) { @@ -91,7 +91,7 @@ case _POP_TOP: { _Py_UopsPESlot pop; pop = stack_pointer[-1]; - if (!sym_is_virtual(pop)) { + if (!sym_is_virtual(&pop)) { MATERIALIZE_INST(); } stack_pointer += -1; @@ -346,8 +346,15 @@ } case _BINARY_SUBSCR_INIT_CALL: { - _PyInterpreterFrame *new_frame; - new_frame = sym_new_not_null(ctx); + _Py_UopsPESlot sub; + _Py_UopsPESlot container; + _Py_UopsPESlot new_frame; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + (void)container; + (void)sub; + new_frame = (_Py_UopsPESlot){NULL, NULL}; + ctx->done = true; stack_pointer[-2] = new_frame; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -407,9 +414,30 @@ } case _RETURN_VALUE: { + _Py_UopsPESlot retval; _Py_UopsPESlot res; - res = sym_new_not_null(ctx); - stack_pointer[-1] = res; + retval = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = retval; + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -439,16 +467,15 @@ /* _SEND is not a viable micro-op for tier 2 */ case _SEND_GEN_FRAME: { - _PyInterpreterFrame *gen_frame; - gen_frame = sym_new_not_null(ctx); - stack_pointer[-1] = gen_frame; + // We are about to hit the end of the trace: + ctx->done = true; break; } case _YIELD_VALUE: { - _Py_UopsPESlot value; - value = sym_new_not_null(ctx); - stack_pointer[-1] = value; + _Py_UopsPESlot res; + res = sym_new_unknown(ctx); + stack_pointer[-1] = res; break; } @@ -867,8 +894,14 @@ } case _LOAD_ATTR_PROPERTY_FRAME: { - _PyInterpreterFrame *new_frame; - new_frame = sym_new_not_null(ctx); + _Py_UopsPESlot owner; + _Py_UopsPESlot new_frame; + owner = stack_pointer[-1]; + PyObject *fget = (PyObject *)this_instr->operand; + (void)fget; + (void)owner; + new_frame = (_Py_UopsPESlot){NULL, NULL}; + ctx->done = true; stack_pointer[-1] = new_frame; break; } @@ -1145,11 +1178,8 @@ } case _FOR_ITER_GEN_FRAME: { - _PyInterpreterFrame *gen_frame; - gen_frame = sym_new_not_null(ctx); - stack_pointer[0] = gen_frame; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + /* We are about to hit the end of the trace */ + ctx->done = true; break; } @@ -1269,8 +1299,22 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { - _PyInterpreterFrame *new_frame; - new_frame = sym_new_not_null(ctx); + _Py_UopsPESlot *args; + _Py_UopsPESlot self_or_null; + _Py_UopsPESlot callable; + _Py_UopsPESlot new_frame; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + (void)(self_or_null); + (void)(callable); + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL}; stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1339,8 +1383,34 @@ } case _INIT_CALL_PY_EXACT_ARGS: { - _PyInterpreterFrame *new_frame; - new_frame = sym_new_not_null(ctx); + _Py_UopsPESlot *args; + _Py_UopsPESlot self_or_null; + _Py_UopsPESlot callable; + _Py_UopsPESlot new_frame; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + (void)callable; + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + assert(self_or_null.sym != NULL); + assert(args != NULL); + if (sym_is_not_null(&self_or_null)) { + // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM + args--; + argcount++; + } + if (sym_is_null(&self_or_null) || sym_is_not_null(&self_or_null)) { + new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, args, argcount), NULL}; + } else { + new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL}; + } stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1348,8 +1418,38 @@ } case _PUSH_FRAME: { + _Py_UopsPESlot new_frame; + new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + ctx->frame->stack_pointer = stack_pointer; + ctx->frame = (_Py_UOpsPEAbstractFrame *)new_frame.sym; + ctx->curr_frame_depth++; + stack_pointer = ((_Py_UOpsPEAbstractFrame *)new_frame.sym)->stack_pointer; + co = get_code(this_instr); + if (co == NULL) { + // should be about to _EXIT_TRACE anyway + ctx->done = true; + break; + } + /* Stack space handling */ + int framesize = co->co_framesize; + assert(framesize > 0); + curr_space += framesize; + if (curr_space < 0 || curr_space > INT32_MAX) { + // won't fit in signed 32-bit int + ctx->done = true; + break; + } + max_space = curr_space > max_space ? curr_space : max_space; + if (first_valid_check_stack == NULL) { + first_valid_check_stack = corresponding_check_stack; + } + else if (corresponding_check_stack) { + // delete all but the first valid _CHECK_STACK_SPACE + corresponding_check_stack->opcode = _NOP; + } + corresponding_check_stack = NULL; break; } @@ -1395,8 +1495,18 @@ } case _CREATE_INIT_FRAME: { - _PyInterpreterFrame *init_frame; - init_frame = sym_new_not_null(ctx); + _Py_UopsPESlot *args; + _Py_UopsPESlot init; + _Py_UopsPESlot self; + _Py_UopsPESlot init_frame; + args = &stack_pointer[-oparg]; + init = stack_pointer[-1 - oparg]; + self = stack_pointer[-2 - oparg]; + (void)self; + (void)init; + (void)args; + init_frame = (_Py_UopsPESlot){NULL, NULL}; + ctx->done = true; stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1510,8 +1620,21 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { - _PyInterpreterFrame *new_frame; - new_frame = sym_new_not_null(ctx); + _Py_UopsPESlot kwnames; + _Py_UopsPESlot *args; + _Py_UopsPESlot self_or_null; + _Py_UopsPESlot callable; + _Py_UopsPESlot new_frame; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + (void)callable; + (void)self_or_null; + (void)args; + (void)kwnames; + new_frame = (_Py_UopsPESlot){NULL, NULL}; + ctx->done = true; stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1576,7 +1699,22 @@ case _RETURN_GENERATOR: { _Py_UopsPESlot res; - res = sym_new_not_null(ctx); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = sym_new_unknown(ctx); + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/partial_evaluator_symbols.c b/Python/partial_evaluator_symbols.c index f87dc9128401d0..ff58a0c71a2ab7 100644 --- a/Python/partial_evaluator_symbols.c +++ b/Python/partial_evaluator_symbols.c @@ -14,7 +14,7 @@ /* Symbols ======= - Documentation TODO gh-120619. + Documentation TODO gh-120619 */ // Flags for below. @@ -37,258 +37,166 @@ static inline int get_lltrace(void) { #define DPRINTF(level, ...) #endif -static _Py_UopsSymbol NO_SPACE_SYMBOL = { +static _Py_UopsPESymbol NO_SPACE_SYMBOL = { .flags = IS_NULL | NOT_NULL | NO_SPACE, - .typ = NULL, .const_val = NULL, - .type_version = 0, }; -_Py_UopsSymbol * -out_of_space(_Py_UOpsContext *ctx) +_Py_UopsPESlot +out_of_space(_Py_UOpsPEContext *ctx) { ctx->done = true; ctx->out_of_space = true; - return &NO_SPACE_SYMBOL; + return (_Py_UopsPESlot){&NO_SPACE_SYMBOL, NULL}; } -static _Py_UopsSymbol * -sym_new(_Py_UOpsContext *ctx) +static _Py_UopsPESlot +sym_new(_Py_UOpsPEContext *ctx) { - _Py_UopsSymbol *self = &ctx->t_arena.arena[ctx->t_arena.ty_curr_number]; - if (ctx->t_arena.ty_curr_number >= ctx->t_arena.ty_max_number) { + _Py_UopsPESymbol *self = &ctx->sym_arena.arena[ctx->sym_arena.sym_curr_number]; + if (ctx->sym_arena.sym_curr_number >= ctx->sym_arena.sym_max_number) { OPT_STAT_INC(optimizer_failure_reason_no_memory); DPRINTF(1, "out of space for symbolic expression type\n"); - return NULL; + return (_Py_UopsPESlot){NULL, NULL}; } - ctx->t_arena.ty_curr_number++; + ctx->sym_arena.sym_curr_number++; self->flags = 0; - self->typ = NULL; self->const_val = NULL; - self->type_version = 0; - return self; + return (_Py_UopsPESlot){self, NULL}; } static inline void -sym_set_flag(_Py_UopsSymbol *sym, int flag) +sym_set_flag(_Py_UopsPESlot *sym, int flag) { - sym->flags |= flag; + sym->sym->flags |= flag; } static inline void -sym_set_bottom(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +sym_set_bottom(_Py_UOpsPEContext *ctx, _Py_UopsPESlot *sym) { sym_set_flag(sym, IS_NULL | NOT_NULL); - sym->typ = NULL; - Py_CLEAR(sym->const_val); + Py_CLEAR(sym->sym->const_val); ctx->done = true; ctx->contradiction = true; } bool -_Py_uop_sym_is_bottom(_Py_UopsSymbol *sym) +_Py_uop_pe_sym_is_bottom(_Py_UopsPESlot *sym) { - if ((sym->flags & IS_NULL) && (sym->flags & NOT_NULL)) { - assert(sym->flags == (IS_NULL | NOT_NULL)); - assert(sym->typ == NULL); - assert(sym->const_val == NULL); + if ((sym->sym->flags & IS_NULL) && (sym->sym->flags & NOT_NULL)) { + assert(sym->sym->flags == (IS_NULL | NOT_NULL)); + assert(sym->sym->const_val == NULL); return true; } return false; } bool -_Py_uop_sym_is_not_null(_Py_UopsSymbol *sym) +_Py_uop_pe_sym_is_not_null(_Py_UopsPESlot *sym) { - return sym->flags == NOT_NULL; + return sym->sym->flags == NOT_NULL; } bool -_Py_uop_sym_is_null(_Py_UopsSymbol *sym) +_Py_uop_pe_sym_is_null(_Py_UopsPESlot *sym) { - return sym->flags == IS_NULL; + return sym->sym->flags == IS_NULL; } bool -_Py_uop_sym_is_const(_Py_UopsSymbol *sym) +_Py_uop_pe_sym_is_const(_Py_UopsPESlot *sym) { - return sym->const_val != NULL; + return sym->sym->const_val != NULL; } PyObject * -_Py_uop_sym_get_const(_Py_UopsSymbol *sym) -{ - return sym->const_val; -} - -void -_Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ) -{ - assert(typ != NULL && PyType_Check(typ)); - if (sym->flags & IS_NULL) { - sym_set_bottom(ctx, sym); - return; - } - if (sym->typ != NULL) { - if (sym->typ != typ) { - sym_set_bottom(ctx, sym); - return; - } - } - else { - sym_set_flag(sym, NOT_NULL); - sym->typ = typ; - } -} - -bool -_Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version) +_Py_uop_pe_sym_get_const(_Py_UopsPESlot *sym) { - // if the type version was already set, then it must be different and we should set it to bottom - if (sym->type_version) { - sym_set_bottom(ctx, sym); - return false; - } - sym->type_version = version; - return true; + return sym->sym->const_val; } void -_Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val) +_Py_uop_pe_sym_set_const(_Py_UOpsPEContext *ctx, _Py_UopsPESlot *sym, PyObject *const_val) { assert(const_val != NULL); - if (sym->flags & IS_NULL) { - sym_set_bottom(ctx, sym); - } - PyTypeObject *typ = Py_TYPE(const_val); - if (sym->typ != NULL && sym->typ != typ) { + if (sym->sym->flags & IS_NULL) { sym_set_bottom(ctx, sym); } - if (sym->const_val != NULL) { - if (sym->const_val != const_val) { + if (sym->sym->const_val != NULL) { + if (sym->sym->const_val != const_val) { // TODO: What if they're equal? sym_set_bottom(ctx, sym); } } else { sym_set_flag(sym, NOT_NULL); - sym->typ = typ; - sym->const_val = Py_NewRef(const_val); + sym->sym->const_val = Py_NewRef(const_val); } } void -_Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +_Py_uop_pe_sym_set_null(_Py_UOpsPEContext *ctx, _Py_UopsPESlot *sym) { - if (_Py_uop_sym_is_not_null(sym)) { + if (_Py_uop_pe_sym_is_not_null(sym)) { sym_set_bottom(ctx, sym); } sym_set_flag(sym, IS_NULL); } void -_Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +_Py_uop_pe_sym_set_non_null(_Py_UOpsPEContext *ctx, _Py_UopsPESlot *sym) { - if (_Py_uop_sym_is_null(sym)) { + if (_Py_uop_pe_sym_is_null(sym)) { sym_set_bottom(ctx, sym); } sym_set_flag(sym, NOT_NULL); } -_Py_UopsSymbol * -_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx) +_Py_UopsPESlot +_Py_uop_pe_sym_new_unknown(_Py_UOpsPEContext *ctx) { return sym_new(ctx); } -_Py_UopsSymbol * -_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx) -{ - _Py_UopsSymbol *res = _Py_uop_sym_new_unknown(ctx); - if (res == NULL) { - return out_of_space(ctx); - } - sym_set_flag(res, NOT_NULL); - return res; -} - -_Py_UopsSymbol * -_Py_uop_sym_new_type(_Py_UOpsContext *ctx, PyTypeObject *typ) +_Py_UopsPESlot +_Py_uop_pe_sym_new_not_null(_Py_UOpsPEContext *ctx) { - _Py_UopsSymbol *res = sym_new(ctx); - if (res == NULL) { + _Py_UopsPESlot res = _Py_uop_pe_sym_new_unknown(ctx); + if (res.sym == NULL) { return out_of_space(ctx); } - _Py_uop_sym_set_type(ctx, res, typ); + sym_set_flag(&res, NOT_NULL); return res; } // Adds a new reference to const_val, owned by the symbol. -_Py_UopsSymbol * -_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) +_Py_UopsPESlot +_Py_uop_pe_sym_new_const(_Py_UOpsPEContext *ctx, PyObject *const_val) { assert(const_val != NULL); - _Py_UopsSymbol *res = sym_new(ctx); - if (res == NULL) { + _Py_UopsPESlot res = sym_new(ctx); + if (res.sym == NULL) { return out_of_space(ctx); } - _Py_uop_sym_set_const(ctx, res, const_val); + _Py_uop_pe_sym_set_const(ctx, &res, const_val); return res; } -_Py_UopsSymbol * -_Py_uop_sym_new_null(_Py_UOpsContext *ctx) +_Py_UopsPESlot +_Py_uop_pe_sym_new_null(_Py_UOpsPEContext *ctx) { - _Py_UopsSymbol *null_sym = _Py_uop_sym_new_unknown(ctx); - if (null_sym == NULL) { + _Py_UopsPESlot null_sym = _Py_uop_pe_sym_new_unknown(ctx); + if (null_sym.sym == NULL) { return out_of_space(ctx); } - _Py_uop_sym_set_null(ctx, null_sym); + _Py_uop_pe_sym_set_null(ctx, &null_sym); return null_sym; } -PyTypeObject * -_Py_uop_sym_get_type(_Py_UopsSymbol *sym) -{ - if (_Py_uop_sym_is_bottom(sym)) { - return NULL; - } - return sym->typ; -} - -unsigned int -_Py_uop_sym_get_type_version(_Py_UopsSymbol *sym) -{ - return sym->type_version; -} - -bool -_Py_uop_sym_has_type(_Py_UopsSymbol *sym) -{ - if (_Py_uop_sym_is_bottom(sym)) { - return false; - } - return sym->typ != NULL; -} - -bool -_Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) -{ - assert(typ != NULL && PyType_Check(typ)); - return _Py_uop_sym_get_type(sym) == typ; -} - -bool -_Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version) -{ - return _Py_uop_sym_get_type_version(sym) == version; -} - - int -_Py_uop_sym_truthiness(_Py_UopsSymbol *sym) +_Py_uop_pe_sym_truthiness(_Py_UopsPESlot *sym) { /* There are some non-constant values for * which `bool(val)` always evaluates to @@ -297,13 +205,13 @@ _Py_uop_sym_truthiness(_Py_UopsSymbol *sym) * This function will need updating * should we support those values. */ - if (_Py_uop_sym_is_bottom(sym)) { + if (_Py_uop_pe_sym_is_bottom(sym)) { return -1; } - if (!_Py_uop_sym_is_const(sym)) { + if (!_Py_uop_pe_sym_is_const(sym)) { return -1; } - PyObject *value = _Py_uop_sym_get_const(sym); + PyObject *value = _Py_uop_pe_sym_get_const(sym); if (value == Py_None) { return 0; } @@ -321,17 +229,40 @@ _Py_uop_sym_truthiness(_Py_UopsSymbol *sym) return -1; } +void +_Py_uop_sym_set_origin_inst_override(_Py_UopsPESlot *sym, _PyUOpInstruction *origin) +{ + sym->origin_inst = origin; +} + +_PyUOpInstruction * +_Py_uop_sym_get_origin(_Py_UopsPESlot *sym) +{ + return sym->origin_inst; +} + +bool +_Py_uop_sym_is_virtual(_Py_UopsPESlot *sym) +{ + if (!sym->origin_inst) { + return false; + } + else { + return sym->origin_inst->is_virtual; + } +} + // 0 on success, -1 on error. -_Py_UOpsAbstractFrame * -_Py_uop_frame_new( - _Py_UOpsContext *ctx, +_Py_UOpsPEAbstractFrame * +_Py_uop_pe_frame_new( + _Py_UOpsPEContext *ctx, PyCodeObject *co, int curr_stackentries, - _Py_UopsSymbol **args, + _Py_UopsPESlot *args, int arg_len) { assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); - _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; + _Py_UOpsPEAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; frame->stack_len = co->co_stacksize; frame->locals_len = co->co_nlocalsplus; @@ -352,14 +283,14 @@ _Py_uop_frame_new( } for (int i = arg_len; i < co->co_nlocalsplus; i++) { - _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx); + _Py_UopsPESlot local = _Py_uop_pe_sym_new_unknown(ctx); frame->locals[i] = local; } // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { - _Py_UopsSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); + _Py_UopsPESlot stackvar = _Py_uop_pe_sym_new_unknown(ctx); frame->stack[i] = stackvar; } @@ -367,41 +298,41 @@ _Py_uop_frame_new( } void -_Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx) +_Py_uop_pe_abstractcontext_fini(_Py_UOpsPEContext *ctx) { if (ctx == NULL) { return; } ctx->curr_frame_depth = 0; - int tys = ctx->t_arena.ty_curr_number; + int tys = ctx->sym_arena.sym_curr_number; for (int i = 0; i < tys; i++) { - Py_CLEAR(ctx->t_arena.arena[i].const_val); + Py_CLEAR(ctx->sym_arena.arena[i].const_val); } } void -_Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) +_Py_uop_pe_abstractcontext_init(_Py_UOpsPEContext *ctx) { ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; ctx->n_consumed = ctx->locals_and_stack; #ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { - ctx->locals_and_stack[i] = NULL; + ctx->locals_and_stack[i] = (_Py_UopsPESlot){NULL, NULL}; } #endif // Setup the arena for sym expressions. - ctx->t_arena.ty_curr_number = 0; - ctx->t_arena.ty_max_number = TY_ARENA_SIZE; + ctx->sym_arena.sym_curr_number = 0; + ctx->sym_arena.sym_max_number = TY_ARENA_SIZE; // Frame setup ctx->curr_frame_depth = 0; } int -_Py_uop_frame_pop(_Py_UOpsContext *ctx) +_Py_uop_pe_frame_pop(_Py_UOpsPEContext *ctx) { - _Py_UOpsAbstractFrame *frame = ctx->frame; + _Py_UOpsPEAbstractFrame *frame = ctx->frame; ctx->n_consumed = frame->locals; ctx->curr_frame_depth--; assert(ctx->curr_frame_depth >= 1); @@ -410,6 +341,7 @@ _Py_uop_frame_pop(_Py_UOpsContext *ctx) return 0; } + #define TEST_PREDICATE(PRED, MSG) \ do { \ if (!(PRED)) { \ @@ -420,116 +352,59 @@ do { \ } \ } while (0) -static _Py_UopsSymbol * -make_bottom(_Py_UOpsContext *ctx) +static _Py_UopsPESlot +make_bottom(_Py_UOpsPEContext *ctx) { - _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); - _Py_uop_sym_set_null(ctx, sym); - _Py_uop_sym_set_non_null(ctx, sym); + _Py_UopsPESlot sym = _Py_uop_pe_sym_new_unknown(ctx); + _Py_uop_pe_sym_set_null(ctx, &sym); + _Py_uop_pe_sym_set_non_null(ctx, &sym); return sym; } PyObject * -_Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) +_Py_uop_pe_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { - _Py_UOpsContext context; - _Py_UOpsContext *ctx = &context; - _Py_uop_abstractcontext_init(ctx); + _Py_UOpsPEContext context; + _Py_UOpsPEContext *ctx = &context; + _Py_uop_pe_abstractcontext_init(ctx); PyObject *val_42 = NULL; PyObject *val_43 = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. - _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); - if (sym == NULL) { + _Py_UopsPESlot sym = _Py_uop_pe_sym_new_unknown(ctx); + if (sym.sym == NULL) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "top is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "top as constant is not NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); + TEST_PREDICATE(!_Py_uop_pe_sym_is_null(&sym), "top is NULL"); + TEST_PREDICATE(!_Py_uop_pe_sym_is_not_null(&sym), "top is not NULL"); + TEST_PREDICATE(!_Py_uop_pe_sym_is_const(&sym), "top is a constant"); + TEST_PREDICATE(_Py_uop_pe_sym_get_const(&sym) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_pe_sym_is_bottom(&sym), "top is bottom"); sym = make_bottom(ctx); - if (sym == NULL) { - goto fail; - } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); - TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "bottom is not NULL is not false"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "bottom matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "bottom is a constant is not false"); - TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "bottom as constant is not NULL"); - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); - - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + if (sym.sym == NULL) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); - TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "int isn't not NULL"); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "int isn't int"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "int matches float"); - TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "int is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "int as constant is not NULL"); - - _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); - - _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(int and float) isn't bottom"); - - val_42 = PyLong_FromLong(42); - assert(val_42 != NULL); - assert(_Py_IsImmortal(val_42)); - - val_43 = PyLong_FromLong(43); - assert(val_43 != NULL); - assert(_Py_IsImmortal(val_43)); - - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { - goto fail; - } - _Py_uop_sym_set_const(ctx, sym, val_42); - TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 1, "bool(42) is not True"); - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); - TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "42 matches float"); - TEST_PREDICATE(_Py_uop_sym_is_const(sym), "42 is not a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(sym) != NULL, "42 as constant is NULL"); - TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "42 as constant isn't 42"); - - _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); - TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "(42 and 42) as constant isn't 42"); - - _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); - - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { - goto fail; - } - _Py_uop_sym_set_const(ctx, sym, val_42); - _Py_uop_sym_set_const(ctx, sym, val_43); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); - + TEST_PREDICATE(!_Py_uop_pe_sym_is_null(&sym), "bottom is NULL is not false"); + TEST_PREDICATE(!_Py_uop_pe_sym_is_not_null(&sym), "bottom is not NULL is not false"); + TEST_PREDICATE(!_Py_uop_pe_sym_is_const(&sym), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_pe_sym_get_const(&sym) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(_Py_uop_pe_sym_is_bottom(&sym), "bottom isn't bottom"); - sym = _Py_uop_sym_new_const(ctx, Py_None); - TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(None) is not False"); - sym = _Py_uop_sym_new_const(ctx, Py_False); - TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(False) is not False"); - sym = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); - TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(0) is not False"); + sym = _Py_uop_pe_sym_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_pe_sym_truthiness(&sym) == 0, "bool(None) is not False"); + sym = _Py_uop_pe_sym_new_const(ctx, Py_False); + TEST_PREDICATE(_Py_uop_pe_sym_truthiness(&sym) == 0, "bool(False) is not False"); + sym = _Py_uop_pe_sym_new_const(ctx, PyLong_FromLong(0)); + TEST_PREDICATE(_Py_uop_pe_sym_truthiness(&sym) == 0, "bool(0) is not False"); - _Py_uop_abstractcontext_fini(ctx); + _Py_uop_pe_abstractcontext_fini(ctx); Py_DECREF(val_42); Py_DECREF(val_43); Py_RETURN_NONE; fail: - _Py_uop_abstractcontext_fini(ctx); + _Py_uop_pe_abstractcontext_fini(ctx); Py_XDECREF(val_42); Py_XDECREF(val_43); return NULL; From 6cd8a8ca15cc8ccdd69224a39c23802b734e190c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 4 Oct 2024 00:47:05 +0800 Subject: [PATCH 27/46] fix compilation --- Python/optimizer_symbols.c | 2 +- Python/partial_evaluator_symbols.c | 2 +- Python/pylifecycle.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 40cbf95e3d6d39..78b83737a2bd5d 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -55,7 +55,7 @@ static _Py_UopsSymbol NO_SPACE_SYMBOL = { .type_version = 0, }; -_Py_UopsSymbol * +static _Py_UopsSymbol * out_of_space(_Py_UOpsContext *ctx) { ctx->done = true; diff --git a/Python/partial_evaluator_symbols.c b/Python/partial_evaluator_symbols.c index ff58a0c71a2ab7..96f950fcc2003a 100644 --- a/Python/partial_evaluator_symbols.c +++ b/Python/partial_evaluator_symbols.c @@ -42,7 +42,7 @@ static _Py_UopsPESymbol NO_SPACE_SYMBOL = { .const_val = NULL, }; -_Py_UopsPESlot +static _Py_UopsPESlot out_of_space(_Py_UOpsPEContext *ctx) { ctx->done = true; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ebeee4f41d795d..fed1afa39d4f48 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1290,7 +1290,7 @@ init_interp_main(PyThreadState *tstate) // This is also needed when the JIT is enabled #ifdef _Py_TIER2 if (is_main_interp) { - int enabled = 1; + int enabled = 0; #if _Py_TIER2 & 2 enabled = 0; #endif From 8477c5af47388fe2e93a7a2b6203883571c84b3d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 4 Oct 2024 00:56:34 +0800 Subject: [PATCH 28/46] fix rest --- Include/internal/pycore_optimizer.h | 9 +++++ Python/optimizer.c | 4 +++ Python/partial_evaluator.c | 2 +- Python/partial_evaluator_bytecodes.c | 35 ++++++++++++++++++++ Python/partial_evaluator_cases.c.h | 49 +++++++++++++++++++--------- 5 files changed, 82 insertions(+), 17 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index d2765356140144..45ea4a836b0679 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -147,6 +147,15 @@ int _Py_uop_analyze_and_optimize(struct _PyInterpreterFrame *frame, _PyUOpInstruction *trace, int trace_len, int curr_stackentries, _PyBloomFilter *dependencies); +int +_Py_uop_partial_evaluate( + _PyInterpreterFrame *frame, + _PyUOpInstruction *buffer, + int length, + int curr_stacklen, + _PyBloomFilter *dependencies +); + extern PyTypeObject _PyCounterExecutor_Type; extern PyTypeObject _PyCounterOptimizer_Type; extern PyTypeObject _PyDefaultOptimizer_Type; diff --git a/Python/optimizer.c b/Python/optimizer.c index b876b6c2bd72fd..1b9cd36323023a 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1253,6 +1253,10 @@ uop_optimize( if (length <= 0) { return length; } + length = _Py_uop_partial_evaluate(frame, buffer, length, curr_stackentries, &dependencies); + if (length <= 0) { + return length; + } } assert(length < UOP_MAX_TRACE_LENGTH); assert(length >= 1); diff --git a/Python/partial_evaluator.c b/Python/partial_evaluator.c index 18baab86c1b2ed..24d217125d0647 100644 --- a/Python/partial_evaluator.c +++ b/Python/partial_evaluator.c @@ -181,7 +181,7 @@ partial_evaluate_uops( _PyUOpInstruction *first_valid_check_stack = NULL; _PyUOpInstruction *corresponding_check_stack = NULL; - _Py_uop_abstractcontext_init(ctx); + _Py_uop_pe_abstractcontext_init(ctx); _Py_UOpsPEAbstractFrame *frame = _Py_uop_pe_frame_new(ctx, co, curr_stacklen, NULL, 0); if (frame == NULL) { return -1; diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index 44be57f8da530a..c5dba244de2c60 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -164,6 +164,15 @@ dummy_func(void) { ctx->done = true; } + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { + (void)type_version; + (void)callable; + (void)null; + (void)args; + self = sym_new_not_null(ctx); + init = sym_new_not_null(ctx); + } + op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame)) { (void)self; (void)init; @@ -262,6 +271,32 @@ dummy_func(void) { op(_YIELD_VALUE, (unused -- res)) { res = sym_new_unknown(ctx); } + + op(_JUMP_TO_TOP, (--)) { + ctx->done = true; + } + + op(_EXIT_TRACE, (exit_p/4 --)) { + (void)exit_p; + ctx->done = true; + } + + op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { + /* This has to be done manually */ + (void)seq; + for (int i = 0; i < oparg; i++) { + values[i] = sym_new_unknown(ctx); + } + } + + op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8])) { + /* This has to be done manually */ + (void)seq; + int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; + for (int i = 0; i < totalargs; i++) { + values[i] = sym_new_unknown(ctx); + } + } // END BYTECODES // } diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index ec2fdc6fc6d42c..8832982f33d148 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -514,10 +514,14 @@ } case _UNPACK_SEQUENCE: { - _Py_UopsPESlot *output; - output = &stack_pointer[-1]; - for (int _i = oparg; --_i >= 0;) { - output[_i] = sym_new_not_null(ctx); + _Py_UopsPESlot seq; + _Py_UopsPESlot *values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + /* This has to be done manually */ + (void)seq; + for (int i = 0; i < oparg; i++) { + values[i] = sym_new_unknown(ctx); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -559,14 +563,15 @@ } case _UNPACK_EX: { - _Py_UopsPESlot *left; - _Py_UopsPESlot *right; - right = &stack_pointer[(oparg & 0xFF)]; - for (int _i = oparg & 0xFF; --_i >= 0;) { - left[_i] = sym_new_not_null(ctx); - } - for (int _i = oparg >> 8; --_i >= 0;) { - right[_i] = sym_new_not_null(ctx); + _Py_UopsPESlot seq; + _Py_UopsPESlot *values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + /* This has to be done manually */ + (void)seq; + int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; + for (int i = 0; i < totalargs; i++) { + values[i] = sym_new_unknown(ctx); } stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); @@ -1481,14 +1486,22 @@ } case _CHECK_AND_ALLOCATE_OBJECT: { + _Py_UopsPESlot *args; + _Py_UopsPESlot null; + _Py_UopsPESlot callable; _Py_UopsPESlot self; _Py_UopsPESlot init; - _Py_UopsPESlot *args; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-oparg]; + uint32_t type_version = (uint32_t)this_instr->operand; + (void)type_version; + (void)callable; + (void)null; + (void)args; self = sym_new_not_null(ctx); init = sym_new_not_null(ctx); - for (int _i = oparg; --_i >= 0;) { - args[_i] = sym_new_not_null(ctx); - } stack_pointer[-2 - oparg] = self; stack_pointer[-1 - oparg] = init; break; @@ -1822,6 +1835,7 @@ } case _JUMP_TO_TOP: { + ctx->done = true; break; } @@ -1840,6 +1854,9 @@ } case _EXIT_TRACE: { + PyObject *exit_p = (PyObject *)this_instr->operand; + (void)exit_p; + ctx->done = true; break; } From d2b0d8235a381e3eebf921ecc806e840bfcbaad0 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 4 Oct 2024 00:59:48 +0800 Subject: [PATCH 29/46] fix everything for real --- Python/partial_evaluator_bytecodes.c | 8 ++++++++ Python/partial_evaluator_cases.c.h | 22 +++++++++++++--------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index c5dba244de2c60..50767275376dea 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -297,6 +297,14 @@ dummy_func(void) { values[i] = sym_new_unknown(ctx); } } + + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- func, maybe_self, args[oparg])) { + (void)callable; + (void)self_or_null; + (void)args; + func = sym_new_not_null(ctx); + maybe_self = sym_new_not_null(ctx); + } // END BYTECODES // } diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index 8832982f33d148..5dd098a8a19358 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -1284,18 +1284,22 @@ } case _MAYBE_EXPAND_METHOD: { - _Py_UopsPESlot func; - _Py_UopsPESlot *maybe_self; _Py_UopsPESlot *args; - maybe_self = &stack_pointer[-1 - oparg]; + _Py_UopsPESlot self_or_null; + _Py_UopsPESlot callable; + _Py_UopsPESlot func; + _Py_UopsPESlot maybe_self; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-oparg]; + (void)callable; + (void)self_or_null; + (void)args; func = sym_new_not_null(ctx); - for (int _i = 1; --_i >= 0;) { - maybe_self[_i] = sym_new_not_null(ctx); - } - for (int _i = oparg; --_i >= 0;) { - args[_i] = sym_new_not_null(ctx); - } + maybe_self = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; break; } From 96d371ed28133e62bfb013bd5d26919c88800c1b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 4 Oct 2024 01:03:59 +0800 Subject: [PATCH 30/46] partially address review --- PCbuild/regen.targets | 2 ++ Tools/cases_generator/README.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 416241d9d0df10..8ea0ea1f8e3717 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -104,6 +104,8 @@ WorkingDirectory="$(PySourcePath)" /> + Date: Fri, 4 Oct 2024 01:05:24 +0800 Subject: [PATCH 31/46] turn JIT back on --- Python/pylifecycle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index fed1afa39d4f48..ebeee4f41d795d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1290,7 +1290,7 @@ init_interp_main(PyThreadState *tstate) // This is also needed when the JIT is enabled #ifdef _Py_TIER2 if (is_main_interp) { - int enabled = 0; + int enabled = 1; #if _Py_TIER2 & 2 enabled = 0; #endif From 865bd26a4772bd73a42d1cfb702193d37cc9da45 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 4 Oct 2024 01:26:23 +0800 Subject: [PATCH 32/46] fix Windows build --- PCbuild/_freeze_module.vcxproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index a976e808f8b891..bc943a9be6b3aa 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -240,7 +240,7 @@ - + From 1fb67924899d65e5e44e5b58c7d11f174285252f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 4 Oct 2024 01:58:42 +0800 Subject: [PATCH 33/46] Fix c analyzer --- Tools/c-analyzer/cpython/_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 6c8250f67073b4..0e4b865ac1957b 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -89,6 +89,7 @@ def clean_lines(text): # not actually source Python/bytecodes.c Python/optimizer_bytecodes.c +Python/partial_evaluator_bytecodes.c # mimalloc Objects/mimalloc/*.c From 365082f913d318af77abd82a1f6108fcff703c39 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:48:37 +0800 Subject: [PATCH 34/46] Address review --- Include/internal/pycore_opcode_metadata.h | 24 +- Include/internal/pycore_optimizer.h | 4 +- Include/internal/pycore_stackref.h | 2 +- Include/internal/pycore_uop_metadata.h | 44 +- Python/bytecodes.c | 12 +- Python/partial_evaluator.c | 18 - Python/partial_evaluator_bytecodes.c | 94 +- Python/partial_evaluator_cases.c.h | 1570 ++++++++++++++++- Tools/cases_generator/analyzer.py | 2 - Tools/cases_generator/generators_common.py | 2 - Tools/cases_generator/lexer.py | 1 - .../opcode_metadata_generator.py | 1 - .../partial_evaluator_generator.py | 143 +- Tools/cases_generator/stack.py | 6 +- 14 files changed, 1716 insertions(+), 207 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 2a6dabf4f97c70..0952a2c5f302d3 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -984,7 +984,6 @@ enum InstructionFormat { #define HAS_PASSTHROUGH_FLAG (4096) #define HAS_OPARG_AND_1_FLAG (8192) #define HAS_ERROR_NO_POP_FLAG (16384) -#define HAS_STATIC_FLAG (32768) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1000,7 +999,6 @@ enum InstructionFormat { #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) #define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) -#define OPCODE_HAS_STATIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_STATIC_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -1093,7 +1091,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [END_FOR] = { true, INSTR_FMT_IX, 0 }, + [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1157,7 +1155,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1179,13 +1177,13 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, - [NOP] = { true, INSTR_FMT_IX, 0 }, + [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_TOP] = { true, INSTR_FMT_IX, 0 }, + [POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1207,7 +1205,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, - [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1239,12 +1237,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_TRUE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [POP_BLOCK] = { true, -1, 0 }, - [SETUP_CLEANUP] = { true, -1, HAS_ARG_FLAG }, - [SETUP_FINALLY] = { true, -1, HAS_ARG_FLAG }, - [SETUP_WITH] = { true, -1, HAS_ARG_FLAG }, - [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, + [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [SETUP_WITH] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, }; #endif diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 45ea4a836b0679..4754fd34ff3514 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -48,8 +48,9 @@ typedef struct { * uint16_t error_target; */ typedef struct { - uint16_t opcode:15; + uint16_t opcode:14; uint16_t format:1; + uint16_t is_virtual:1; // Used for tier2 optimization. uint16_t oparg; union { uint32_t target; @@ -59,7 +60,6 @@ typedef struct { }; }; uint64_t operand; // A cache entry - char is_virtual; // Used for tier2 optimization. } _PyUOpInstruction; typedef struct { diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index b5b6993812057d..8d8e17fce60582 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -193,7 +193,7 @@ PyStackRef_FromPyObjectImmortal(PyObject *obj) # define PyStackRef_CLOSE(REF) \ do { \ _PyStackRef _close_tmp = (REF); \ - if (!PyStackRef_IsDeferred(_close_tmp)) { \ + if (!i(_close_tmp)) { \ Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \ } \ } while (0) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index d6e977000e17f5..1cecc87e6b5718 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -19,35 +19,35 @@ extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { - [_NOP] = HAS_STATIC_FLAG, + [_NOP] = HAS_PURE_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, - [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, - [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_POP_TOP] = HAS_STATIC_FLAG, + [_POP_TOP] = HAS_PURE_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_SEND] = HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -265,8 +265,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, - [_LOAD_CONST_INLINE] = HAS_STATIC_FLAG, - [_LOAD_CONST_INLINE_BORROW] = HAS_STATIC_FLAG, + [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_WITH_NULL] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index eef07a7e734f7a..4b41990b30ece6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -140,7 +140,7 @@ dummy_func( switch (opcode) { // BEGIN BYTECODES // - _static inst(NOP, (--)) { + pure inst(NOP, (--)) { } family(RESUME, 0) = { @@ -239,7 +239,7 @@ dummy_func( value = PyStackRef_DUP(value_s); } - replicate(8) _static inst(LOAD_FAST, (-- value)) { + replicate(8) pure inst(LOAD_FAST, (-- value)) { assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); } @@ -261,7 +261,7 @@ dummy_func( value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); } - replicate(8) _static inst(STORE_FAST, (value --)) { + replicate(8) pure inst(STORE_FAST, (value --)) { SETLOCAL(oparg, value); } @@ -283,7 +283,7 @@ dummy_func( SETLOCAL(oparg2, value2); } - _static inst(POP_TOP, (value --)) { + pure inst(POP_TOP, (value --)) { DECREF_INPUTS(); } @@ -4749,11 +4749,11 @@ dummy_func( DEOPT_IF(!current_executor->vm_data.valid); } - tier2 _static op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + tier2 pure op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { value = PyStackRef_FromPyObjectNew(ptr); } - tier2 _static op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { value = PyStackRef_FromPyObjectImmortal(ptr); } diff --git a/Python/partial_evaluator.c b/Python/partial_evaluator.c index 24d217125d0647..6c8986040a59a2 100644 --- a/Python/partial_evaluator.c +++ b/Python/partial_evaluator.c @@ -216,24 +216,6 @@ partial_evaluate_uops( } #endif - int is_static = (_PyUop_Flags[opcode] & HAS_STATIC_FLAG); - if (!is_static) { - MATERIALIZE_INST(); - } - if (!is_static && - // During these two opcodes, there's an abstract frame on the stack. - // Which is not a valid symbol. - (opcode != _PUSH_FRAME && opcode != _SAVE_RETURN_OFFSET)) { - // An escaping opcode means we need to materialize _everything_. - if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { - materialize_ctx(ctx); - } - else { - - materialize_frame(ctx->frame); - } - } - switch (opcode) { #include "partial_evaluator_cases.c.h" diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index 50767275376dea..3968999a6e78a3 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -34,6 +34,7 @@ dummy_func(void) { // BEGIN BYTECODES // op(_LOAD_FAST_CHECK, (-- value)) { + MATERIALIZE_INST(); value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. if (sym_is_null(&value)) { @@ -47,6 +48,7 @@ dummy_func(void) { } op(_LOAD_FAST_AND_CLEAR, (-- value)) { + MATERIALIZE_INST(); value = GETLOCAL(oparg); GETLOCAL(oparg) = sym_new_null(ctx); sym_set_origin_inst_override(&value, this_instr); @@ -58,11 +60,13 @@ dummy_func(void) { } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + MATERIALIZE_INST(); value = sym_new_const(ctx, ptr); sym_set_origin_inst_override(&value, this_instr); } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + MATERIALIZE_INST(); value = sym_new_const(ctx, ptr); sym_set_origin_inst_override(&value, this_instr); } @@ -84,8 +88,8 @@ dummy_func(void) { } - op(_POP_TOP, (pop --)) { - if (!sym_is_virtual(&pop)) { + op(_POP_TOP, (value --)) { + if (!sym_is_virtual(&value)) { MATERIALIZE_INST(); } } @@ -94,27 +98,29 @@ dummy_func(void) { } op(_CHECK_STACK_SPACE_OPERAND, ( -- )) { + MATERIALIZE_INST(); (void)framesize; } op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame)) { - (void)container; - (void)sub; + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); new_frame = (_Py_UopsPESlot){NULL, NULL}; ctx->done = true; } op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { - (void)fget; - (void)owner; + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); new_frame = (_Py_UopsPESlot){NULL, NULL}; ctx->done = true; } - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { - int argcount = oparg; + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null[1], args[oparg] -- new_frame)) { + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); - (void)callable; + int argcount = oparg; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); @@ -125,15 +131,15 @@ dummy_func(void) { } - assert(self_or_null.sym != NULL); + assert(self_or_null->sym != NULL); assert(args != NULL); - if (sym_is_not_null(&self_or_null)) { + if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM args--; argcount++; } - if (sym_is_null(&self_or_null) || sym_is_not_null(&self_or_null)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, args, argcount), NULL}; } else { new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL}; @@ -141,9 +147,9 @@ dummy_func(void) { } } - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { - (void)(self_or_null); - (void)(callable); + op(_PY_FRAME_GENERAL, (callable, self_or_null[1], args[oparg] -- new_frame)) { + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); co = get_code_with_logging((this_instr + 2)); @@ -155,43 +161,42 @@ dummy_func(void) { new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL}; } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { - (void)callable; - (void)self_or_null; - (void)args; - (void)kwnames; + op(_PY_FRAME_KW, (callable, self_or_null[1], args[oparg], kwnames -- new_frame)) { + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); new_frame = (_Py_UopsPESlot){NULL, NULL}; ctx->done = true; } op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { (void)type_version; - (void)callable; - (void)null; - (void)args; + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); self = sym_new_not_null(ctx); init = sym_new_not_null(ctx); } op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame)) { - (void)self; - (void)init; - (void)args; + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); init_frame = (_Py_UopsPESlot){NULL, NULL}; ctx->done = true; } op(_FOR_ITER_GEN_FRAME, ( -- )) { + MATERIALIZE_INST(); /* We are about to hit the end of the trace */ ctx->done = true; } op(_SEND_GEN_FRAME, ( -- )) { + MATERIALIZE_INST(); // We are about to hit the end of the trace: ctx->done = true; } op(_PUSH_FRAME, (new_frame -- unused if (0))) { + MATERIALIZE_INST(); SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; ctx->frame = (_Py_UOpsPEAbstractFrame *)new_frame.sym; @@ -225,6 +230,8 @@ dummy_func(void) { } op(_RETURN_VALUE, (retval -- res)) { + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); @@ -247,6 +254,7 @@ dummy_func(void) { } op(_RETURN_GENERATOR, ( -- res)) { + MATERIALIZE_INST(); SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); @@ -268,42 +276,50 @@ dummy_func(void) { } } - op(_YIELD_VALUE, (unused -- res)) { - res = sym_new_unknown(ctx); + op(_YIELD_VALUE, (retval -- value)) { + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); + value = sym_new_unknown(ctx); } op(_JUMP_TO_TOP, (--)) { + MATERIALIZE_INST(); + materialize_ctx(ctx); ctx->done = true; } op(_EXIT_TRACE, (exit_p/4 --)) { + MATERIALIZE_INST(); + materialize_ctx(ctx); (void)exit_p; ctx->done = true; } - op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { + op(_UNPACK_SEQUENCE, (seq -- output[oparg])) { /* This has to be done manually */ - (void)seq; + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); for (int i = 0; i < oparg; i++) { - values[i] = sym_new_unknown(ctx); + output[i] = sym_new_unknown(ctx); } } - op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8])) { + op(_UNPACK_EX, (seq -- left[oparg & 0xFF], unused, right[oparg >> 8])) { /* This has to be done manually */ - (void)seq; + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - values[i] = sym_new_unknown(ctx); + left[i] = sym_new_unknown(ctx); } + (void)right; } - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- func, maybe_self, args[oparg])) { - (void)callable; - (void)self_or_null; - (void)args; + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null[1], args[oparg] -- func, maybe_self[1], args[oparg])) { + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); func = sym_new_not_null(ctx); - maybe_self = sym_new_not_null(ctx); + maybe_self[0] = sym_new_not_null(ctx); } // END BYTECODES // diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index 5dd098a8a19358..f812a878436636 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -8,16 +8,21 @@ } case _CHECK_PERIODIC: { + MATERIALIZE_INST(); + materialize_ctx(ctx); break; } case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { + MATERIALIZE_INST(); + materialize_ctx(ctx); break; } /* _QUICKEN_RESUME is not a viable micro-op for tier 2 */ case _RESUME_CHECK: { + MATERIALIZE_INST(); break; } @@ -25,6 +30,7 @@ case _LOAD_FAST_CHECK: { _Py_UopsPESlot value; + MATERIALIZE_INST(); value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. if (sym_is_null(&value)) { @@ -48,6 +54,7 @@ case _LOAD_FAST_AND_CLEAR: { _Py_UopsPESlot value; + MATERIALIZE_INST(); value = GETLOCAL(oparg); GETLOCAL(oparg) = sym_new_null(ctx); sym_set_origin_inst_override(&value, this_instr); @@ -89,9 +96,9 @@ } case _POP_TOP: { - _Py_UopsPESlot pop; - pop = stack_pointer[-1]; - if (!sym_is_virtual(&pop)) { + _Py_UopsPESlot value; + value = stack_pointer[-1]; + if (!sym_is_virtual(&value)) { MATERIALIZE_INST(); } stack_pointer += -1; @@ -101,6 +108,7 @@ case _PUSH_NULL: { _Py_UopsPESlot res; + MATERIALIZE_INST(); res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -110,6 +118,12 @@ case _END_SEND: { _Py_UopsPESlot value; + _Py_UopsPESlot receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&value); + materialize(&receiver); value = sym_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; @@ -118,86 +132,162 @@ } case _UNARY_NEGATIVE: { + _Py_UopsPESlot value; _Py_UopsPESlot res; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)value; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_NOT: { + _Py_UopsPESlot value; _Py_UopsPESlot res; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&value); res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL: { + _Py_UopsPESlot value; _Py_UopsPESlot res; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)value; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL_BOOL: { + _Py_UopsPESlot value; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&value); break; } case _TO_BOOL_INT: { + _Py_UopsPESlot value; _Py_UopsPESlot res; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)value; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL_LIST: { + _Py_UopsPESlot value; _Py_UopsPESlot res; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&value); res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL_NONE: { + _Py_UopsPESlot value; _Py_UopsPESlot res; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&value); res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL_STR: { + _Py_UopsPESlot value; _Py_UopsPESlot res; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)value; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _REPLACE_WITH_TRUE: { + _Py_UopsPESlot value; _Py_UopsPESlot res; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&value); res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_INVERT: { + _Py_UopsPESlot value; _Py_UopsPESlot res; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)value; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _GUARD_BOTH_INT: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); break; } case _GUARD_NOS_INT: { + _Py_UopsPESlot unused_0; + _Py_UopsPESlot left; + _Py_UopsPESlot unused_1; + unused_0 = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&unused_0); + materialize(&left); break; } case _GUARD_TOS_INT: { + _Py_UopsPESlot value; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&value); break; } case _BINARY_OP_MULTIPLY_INT: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -206,7 +296,14 @@ } case _BINARY_OP_ADD_INT: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -215,7 +312,14 @@ } case _BINARY_OP_SUBTRACT_INT: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -224,19 +328,45 @@ } case _GUARD_BOTH_FLOAT: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); break; } case _GUARD_NOS_FLOAT: { + _Py_UopsPESlot unused_0; + _Py_UopsPESlot left; + _Py_UopsPESlot unused_1; + unused_0 = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&unused_0); + materialize(&left); break; } case _GUARD_TOS_FLOAT: { + _Py_UopsPESlot value; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&value); break; } case _BINARY_OP_MULTIPLY_FLOAT: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -245,7 +375,14 @@ } case _BINARY_OP_ADD_FLOAT: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -254,7 +391,14 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -263,11 +407,25 @@ } case _GUARD_BOTH_UNICODE: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); break; } case _BINARY_OP_ADD_UNICODE: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -276,13 +434,30 @@ } case _BINARY_OP_INPLACE_ADD_UNICODE: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)right; + (void)left; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR: { + _Py_UopsPESlot sub; + _Py_UopsPESlot container; _Py_UopsPESlot res; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)sub; + (void)container; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -291,7 +466,19 @@ } case _BINARY_SLICE: { + _Py_UopsPESlot stop; + _Py_UopsPESlot start; + _Py_UopsPESlot container; _Py_UopsPESlot res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)stop; + (void)start; + (void)container; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -300,13 +487,34 @@ } case _STORE_SLICE: { + _Py_UopsPESlot stop; + _Py_UopsPESlot start; + _Py_UopsPESlot container; + _Py_UopsPESlot v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)stop; + (void)start; + (void)container; + (void)v; stack_pointer += -4; assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR_LIST_INT: { + _Py_UopsPESlot sub_st; + _Py_UopsPESlot list_st; _Py_UopsPESlot res; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&sub_st); + materialize(&list_st); res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -315,7 +523,14 @@ } case _BINARY_SUBSCR_STR_INT: { + _Py_UopsPESlot sub_st; + _Py_UopsPESlot str_st; _Py_UopsPESlot res; + sub_st = stack_pointer[-1]; + str_st = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&sub_st); + materialize(&str_st); res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -324,7 +539,14 @@ } case _BINARY_SUBSCR_TUPLE_INT: { + _Py_UopsPESlot sub_st; + _Py_UopsPESlot tuple_st; _Py_UopsPESlot res; + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&sub_st); + materialize(&tuple_st); res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -333,7 +555,16 @@ } case _BINARY_SUBSCR_DICT: { + _Py_UopsPESlot sub_st; + _Py_UopsPESlot dict_st; _Py_UopsPESlot res; + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)sub_st; + (void)dict_st; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -342,6 +573,17 @@ } case _BINARY_SUBSCR_CHECK_FUNC: { + _Py_UopsPESlot unused_0; + _Py_UopsPESlot container; + _Py_UopsPESlot unused_1; + unused_0 = stack_pointer[-1]; + container = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)unused_0; + (void)container; + (void)container; + (void)unused_1; break; } @@ -351,8 +593,9 @@ _Py_UopsPESlot new_frame; sub = stack_pointer[-1]; container = stack_pointer[-2]; - (void)container; - (void)sub; + MATERIALIZE_INST(); + materialize(&container); + materialize(&sub); new_frame = (_Py_UopsPESlot){NULL, NULL}; ctx->done = true; stack_pointer[-2] = new_frame; @@ -362,50 +605,132 @@ } case _LIST_APPEND: { + _Py_UopsPESlot v; + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot list; + _Py_UopsPESlot *unused_1; + v = stack_pointer[-1]; + unused_0 = &stack_pointer[-1 - (oparg-1)]; + list = stack_pointer[-2 - (oparg-1)]; + MATERIALIZE_INST(); + materialize(&v); + for (int _i = oparg-1; --_i >= 0;) { + materialize(&unused_0[_i]); + } + materialize(&list); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _SET_ADD: { + _Py_UopsPESlot v; + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot set; + _Py_UopsPESlot *unused_1; + v = stack_pointer[-1]; + unused_0 = &stack_pointer[-1 - (oparg-1)]; + set = stack_pointer[-2 - (oparg-1)]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)v; + (void)unused_0; + (void)set; + (void)set; + (void)unused_1; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_SUBSCR: { + _Py_UopsPESlot sub; + _Py_UopsPESlot container; + _Py_UopsPESlot v; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + v = stack_pointer[-3]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)sub; + (void)container; + (void)v; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_SUBSCR_LIST_INT: { + _Py_UopsPESlot sub_st; + _Py_UopsPESlot list_st; + _Py_UopsPESlot value; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + value = stack_pointer[-3]; + MATERIALIZE_INST(); + materialize(&sub_st); + materialize(&list_st); + materialize(&value); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_SUBSCR_DICT: { + _Py_UopsPESlot sub; + _Py_UopsPESlot dict_st; + _Py_UopsPESlot value; + sub = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + value = stack_pointer[-3]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)sub; + (void)dict_st; + (void)value; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; } case _DELETE_SUBSCR: { + _Py_UopsPESlot sub; + _Py_UopsPESlot container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)sub; + (void)container; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_INTRINSIC_1: { + _Py_UopsPESlot value; _Py_UopsPESlot res; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)value; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { + _Py_UopsPESlot value1_st; + _Py_UopsPESlot value2_st; _Py_UopsPESlot res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)value1_st; + (void)value2_st; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -417,6 +742,8 @@ _Py_UopsPESlot retval; _Py_UopsPESlot res; retval = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&retval); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; @@ -442,14 +769,27 @@ } case _GET_AITER: { + _Py_UopsPESlot obj; _Py_UopsPESlot iter; + obj = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)obj; + (void)iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { + _Py_UopsPESlot aiter; _Py_UopsPESlot awaitable; + aiter = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)aiter; + (void)aiter; + (void)awaitable; awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; @@ -458,7 +798,13 @@ } case _GET_AWAITABLE: { + _Py_UopsPESlot iterable; _Py_UopsPESlot iter; + iterable = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)iterable; + (void)iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -467,19 +813,32 @@ /* _SEND is not a viable micro-op for tier 2 */ case _SEND_GEN_FRAME: { + _Py_UopsPESlot v; + _Py_UopsPESlot receiver; + _Py_UopsPESlot gen_frame; + MATERIALIZE_INST(); // We are about to hit the end of the trace: ctx->done = true; break; } case _YIELD_VALUE: { - _Py_UopsPESlot res; - res = sym_new_unknown(ctx); - stack_pointer[-1] = res; + _Py_UopsPESlot retval; + _Py_UopsPESlot value; + retval = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&retval); + value = sym_new_unknown(ctx); + stack_pointer[-1] = value; break; } case _POP_EXCEPT: { + _Py_UopsPESlot exc_value; + exc_value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)exc_value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -487,6 +846,7 @@ case _LOAD_COMMON_CONSTANT: { _Py_UopsPESlot value; + MATERIALIZE_INST(); value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -496,6 +856,9 @@ case _LOAD_BUILD_CLASS: { _Py_UopsPESlot bc; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)bc; bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; @@ -504,24 +867,32 @@ } case _STORE_NAME: { + _Py_UopsPESlot v; + v = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)v; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _DELETE_NAME: { + MATERIALIZE_INST(); + materialize_ctx(ctx); break; } case _UNPACK_SEQUENCE: { _Py_UopsPESlot seq; - _Py_UopsPESlot *values; + _Py_UopsPESlot *output; seq = stack_pointer[-1]; - values = &stack_pointer[-1]; + output = &stack_pointer[-1]; /* This has to be done manually */ - (void)seq; + MATERIALIZE_INST(); + materialize(&seq); for (int i = 0; i < oparg; i++) { - values[i] = sym_new_unknown(ctx); + output[i] = sym_new_unknown(ctx); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -529,8 +900,12 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { + _Py_UopsPESlot seq; _Py_UopsPESlot val1; _Py_UopsPESlot val0; + seq = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&seq); val1 = sym_new_not_null(ctx); val0 = sym_new_not_null(ctx); stack_pointer[-1] = val1; @@ -541,8 +916,12 @@ } case _UNPACK_SEQUENCE_TUPLE: { + _Py_UopsPESlot seq; _Py_UopsPESlot *values; + seq = stack_pointer[-1]; values = &stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&seq); for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); } @@ -552,8 +931,12 @@ } case _UNPACK_SEQUENCE_LIST: { + _Py_UopsPESlot seq; _Py_UopsPESlot *values; + seq = stack_pointer[-1]; values = &stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&seq); for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); } @@ -564,44 +947,73 @@ case _UNPACK_EX: { _Py_UopsPESlot seq; - _Py_UopsPESlot *values; + _Py_UopsPESlot *left; + _Py_UopsPESlot unused_0; + _Py_UopsPESlot *right; seq = stack_pointer[-1]; - values = &stack_pointer[-1]; + left = &stack_pointer[-1]; + +right = &stack_pointer[(oparg & 0xFF)]; /* This has to be done manually */ - (void)seq; + MATERIALIZE_INST(); + materialize(&seq); int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - values[i] = sym_new_unknown(ctx); + left[i] = sym_new_unknown(ctx); } + (void)right; stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_ATTR: { + _Py_UopsPESlot owner; + _Py_UopsPESlot v; + owner = stack_pointer[-1]; + v = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)owner; + (void)v; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } case _DELETE_ATTR: { + _Py_UopsPESlot owner; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)owner; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_GLOBAL: { + _Py_UopsPESlot v; + v = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)v; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _DELETE_GLOBAL: { + MATERIALIZE_INST(); + materialize_ctx(ctx); break; } case _LOAD_LOCALS: { _Py_UopsPESlot locals; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)locals; locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; @@ -613,6 +1025,9 @@ case _LOAD_NAME: { _Py_UopsPESlot v; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)v; v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; @@ -624,6 +1039,10 @@ _Py_UopsPESlot *res; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; res = &stack_pointer[0]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)res; + (void)null; for (int _i = 1; --_i >= 0;) { res[_i] = sym_new_not_null(ctx); } @@ -635,16 +1054,19 @@ } case _GUARD_GLOBALS_VERSION: { + MATERIALIZE_INST(); break; } case _GUARD_BUILTINS_VERSION: { + MATERIALIZE_INST(); break; } case _LOAD_GLOBAL_MODULE: { _Py_UopsPESlot res; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; + MATERIALIZE_INST(); res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -657,6 +1079,7 @@ case _LOAD_GLOBAL_BUILTINS: { _Py_UopsPESlot res; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; + MATERIALIZE_INST(); res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -667,19 +1090,30 @@ } case _DELETE_FAST: { + MATERIALIZE_INST(); + materialize_ctx(ctx); break; } case _MAKE_CELL: { + MATERIALIZE_INST(); break; } case _DELETE_DEREF: { + MATERIALIZE_INST(); + materialize_ctx(ctx); break; } case _LOAD_FROM_DICT_OR_DEREF: { + _Py_UopsPESlot class_dict_st; _Py_UopsPESlot value; + class_dict_st = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)class_dict_st; + (void)value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; @@ -687,6 +1121,9 @@ case _LOAD_DEREF: { _Py_UopsPESlot value; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -695,17 +1132,29 @@ } case _STORE_DEREF: { + _Py_UopsPESlot v; + v = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)v; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _COPY_FREE_VARS: { + MATERIALIZE_INST(); break; } case _BUILD_STRING: { + _Py_UopsPESlot *pieces; _Py_UopsPESlot str; + pieces = &stack_pointer[-oparg]; + MATERIALIZE_INST(); + for (int _i = oparg; --_i >= 0;) { + materialize(&pieces[_i]); + } str = sym_new_not_null(ctx); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -714,7 +1163,13 @@ } case _BUILD_TUPLE: { + _Py_UopsPESlot *values; _Py_UopsPESlot tup; + values = &stack_pointer[-oparg]; + MATERIALIZE_INST(); + for (int _i = oparg; --_i >= 0;) { + materialize(&values[_i]); + } tup = sym_new_not_null(ctx); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; @@ -723,7 +1178,13 @@ } case _BUILD_LIST: { + _Py_UopsPESlot *values; _Py_UopsPESlot list; + values = &stack_pointer[-oparg]; + MATERIALIZE_INST(); + for (int _i = oparg; --_i >= 0;) { + materialize(&values[_i]); + } list = sym_new_not_null(ctx); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -732,19 +1193,53 @@ } case _LIST_EXTEND: { + _Py_UopsPESlot iterable_st; + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot list_st; + _Py_UopsPESlot *unused_1; + iterable_st = stack_pointer[-1]; + unused_0 = &stack_pointer[-1 - (oparg-1)]; + list_st = stack_pointer[-2 - (oparg-1)]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)iterable_st; + (void)unused_0; + (void)list_st; + (void)list_st; + (void)unused_1; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _SET_UPDATE: { + _Py_UopsPESlot iterable; + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot set; + _Py_UopsPESlot *unused_1; + iterable = stack_pointer[-1]; + unused_0 = &stack_pointer[-1 - (oparg-1)]; + set = stack_pointer[-2 - (oparg-1)]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)iterable; + (void)unused_0; + (void)set; + (void)set; + (void)unused_1; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _BUILD_SET: { + _Py_UopsPESlot *values; _Py_UopsPESlot set; + values = &stack_pointer[-oparg]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)values; + (void)set; set = sym_new_not_null(ctx); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -753,7 +1248,13 @@ } case _BUILD_MAP: { + _Py_UopsPESlot *values; _Py_UopsPESlot map; + values = &stack_pointer[-oparg*2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)values; + (void)map; map = sym_new_not_null(ctx); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -762,22 +1263,83 @@ } case _SETUP_ANNOTATIONS: { + MATERIALIZE_INST(); + materialize_ctx(ctx); break; } case _DICT_UPDATE: { + _Py_UopsPESlot update; + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot dict; + _Py_UopsPESlot *unused_1; + update = stack_pointer[-1]; + unused_0 = &stack_pointer[-1 - (oparg - 1)]; + dict = stack_pointer[-2 - (oparg - 1)]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)update; + (void)unused_0; + (void)dict; + (void)dict; + (void)unused_1; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _DICT_MERGE: { + _Py_UopsPESlot update; + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot dict; + _Py_UopsPESlot unused_1; + _Py_UopsPESlot unused_2; + _Py_UopsPESlot callable; + _Py_UopsPESlot unused_3; + _Py_UopsPESlot unused_4; + _Py_UopsPESlot *unused_5; + update = stack_pointer[-1]; + unused_0 = &stack_pointer[-1 - (oparg - 1)]; + dict = stack_pointer[-2 - (oparg - 1)]; + unused_1 = stack_pointer[-3 - (oparg - 1)]; + unused_2 = stack_pointer[-4 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)update; + (void)unused_0; + (void)dict; + (void)unused_1; + (void)unused_2; + (void)callable; + (void)callable; + (void)unused_3; + (void)unused_4; + (void)dict; + (void)unused_5; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _MAP_ADD: { + _Py_UopsPESlot value; + _Py_UopsPESlot key; + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot dict_st; + _Py_UopsPESlot *unused_1; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + unused_0 = &stack_pointer[-2 - (oparg - 1)]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)value; + (void)key; + (void)unused_0; + (void)dict_st; + (void)dict_st; + (void)unused_1; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -786,7 +1348,21 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ case _LOAD_SUPER_ATTR_ATTR: { + _Py_UopsPESlot self_st; + _Py_UopsPESlot class_st; + _Py_UopsPESlot global_super_st; _Py_UopsPESlot attr_st; + _Py_UopsPESlot unused_0 = (_Py_UopsPESlot){NULL, 0}; + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)self_st; + (void)class_st; + (void)global_super_st; + (void)attr_st; + (void)unused_0; attr_st = sym_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -795,8 +1371,21 @@ } case _LOAD_SUPER_ATTR_METHOD: { + _Py_UopsPESlot self_st; + _Py_UopsPESlot class_st; + _Py_UopsPESlot global_super_st; _Py_UopsPESlot attr; _Py_UopsPESlot self_or_null; + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)self_st; + (void)class_st; + (void)global_super_st; + (void)attr; + (void)self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); stack_pointer[-3] = attr; @@ -807,8 +1396,15 @@ } case _LOAD_ATTR: { + _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot self_or_null = (_Py_UopsPESlot){NULL, 0}; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)owner; + (void)attr; + (void)self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); stack_pointer[-1] = attr; @@ -819,16 +1415,28 @@ } case _GUARD_TYPE_VERSION: { + _Py_UopsPESlot owner; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); break; } case _CHECK_MANAGED_OBJECT_HAS_VALUES: { + _Py_UopsPESlot owner; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); break; } case _LOAD_ATTR_INSTANCE_VALUE: { + _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); attr = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[-1] = attr; @@ -839,12 +1447,20 @@ } case _CHECK_ATTR_MODULE: { + _Py_UopsPESlot owner; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); break; } case _LOAD_ATTR_MODULE: { + _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); attr = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[-1] = attr; @@ -855,12 +1471,20 @@ } case _CHECK_ATTR_WITH_HINT: { + _Py_UopsPESlot owner; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); break; } case _LOAD_ATTR_WITH_HINT: { + _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); attr = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[-1] = attr; @@ -871,8 +1495,12 @@ } case _LOAD_ATTR_SLOT: { + _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); attr = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[-1] = attr; @@ -883,12 +1511,20 @@ } case _CHECK_ATTR_CLASS: { + _Py_UopsPESlot owner; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); break; } case _LOAD_ATTR_CLASS: { + _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); attr = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[-1] = attr; @@ -903,8 +1539,8 @@ _Py_UopsPESlot new_frame; owner = stack_pointer[-1]; PyObject *fget = (PyObject *)this_instr->operand; - (void)fget; - (void)owner; + MATERIALIZE_INST(); + materialize(&owner); new_frame = (_Py_UopsPESlot){NULL, NULL}; ctx->done = true; stack_pointer[-1] = new_frame; @@ -914,29 +1550,64 @@ /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ case _GUARD_DORV_NO_DICT: { + _Py_UopsPESlot owner; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); break; } case _STORE_ATTR_INSTANCE_VALUE: { + _Py_UopsPESlot owner; + _Py_UopsPESlot value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&owner); + materialize(&value); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_ATTR_WITH_HINT: { + _Py_UopsPESlot owner; + _Py_UopsPESlot value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)owner; + (void)value; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_ATTR_SLOT: { + _Py_UopsPESlot owner; + _Py_UopsPESlot value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&owner); + materialize(&value); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } case _COMPARE_OP: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)right; + (void)left; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -945,7 +1616,14 @@ } case _COMPARE_OP_FLOAT: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -954,7 +1632,14 @@ } case _COMPARE_OP_INT: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -963,7 +1648,14 @@ } case _COMPARE_OP_STR: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -972,7 +1664,14 @@ } case _IS_OP: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize(&right); + materialize(&left); b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -981,7 +1680,16 @@ } case _CONTAINS_OP: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)right; + (void)left; + (void)b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -990,7 +1698,16 @@ } case _CONTAINS_OP_SET: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)right; + (void)left; + (void)b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -999,7 +1716,16 @@ } case _CONTAINS_OP_DICT: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)right; + (void)left; + (void)b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -1008,8 +1734,18 @@ } case _CHECK_EG_MATCH: { + _Py_UopsPESlot match_type_st; + _Py_UopsPESlot exc_value_st; _Py_UopsPESlot rest; _Py_UopsPESlot match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)match_type_st; + (void)exc_value_st; + (void)rest; + (void)match; rest = sym_new_not_null(ctx); match = sym_new_not_null(ctx); stack_pointer[-2] = rest; @@ -1018,14 +1754,33 @@ } case _CHECK_EXC_MATCH: { + _Py_UopsPESlot right; + _Py_UopsPESlot left; _Py_UopsPESlot b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)right; + (void)left; + (void)left; + (void)b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { + _Py_UopsPESlot fromlist; + _Py_UopsPESlot level; _Py_UopsPESlot res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)fromlist; + (void)level; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -1034,7 +1789,14 @@ } case _IMPORT_FROM: { + _Py_UopsPESlot from; _Py_UopsPESlot res; + from = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)from; + (void)from; + (void)res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1047,14 +1809,25 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { + _Py_UopsPESlot value; _Py_UopsPESlot b; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&value); b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _GET_LEN: { + _Py_UopsPESlot obj; _Py_UopsPESlot len; + obj = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)obj; + (void)obj; + (void)len; len = sym_new_not_null(ctx); stack_pointer[0] = len; stack_pointer += 1; @@ -1063,7 +1836,19 @@ } case _MATCH_CLASS: { + _Py_UopsPESlot names; + _Py_UopsPESlot type; + _Py_UopsPESlot subject; _Py_UopsPESlot attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)names; + (void)type; + (void)subject; + (void)attrs; attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1072,7 +1857,11 @@ } case _MATCH_MAPPING: { + _Py_UopsPESlot subject; _Py_UopsPESlot res; + subject = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&subject); res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1081,7 +1870,11 @@ } case _MATCH_SEQUENCE: { + _Py_UopsPESlot subject; _Py_UopsPESlot res; + subject = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&subject); res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1090,7 +1883,18 @@ } case _MATCH_KEYS: { + _Py_UopsPESlot keys; + _Py_UopsPESlot subject; _Py_UopsPESlot values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)keys; + (void)subject; + (void)subject; + (void)keys; + (void)values_or_none; values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1099,14 +1903,26 @@ } case _GET_ITER: { + _Py_UopsPESlot iterable; _Py_UopsPESlot iter; + iterable = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)iterable; + (void)iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_YIELD_FROM_ITER: { + _Py_UopsPESlot iterable; _Py_UopsPESlot iter; + iterable = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)iterable; + (void)iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -1115,7 +1931,14 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { + _Py_UopsPESlot iter; _Py_UopsPESlot next; + iter = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)iter; + (void)iter; + (void)next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1126,17 +1949,29 @@ /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 */ case _ITER_CHECK_LIST: { + _Py_UopsPESlot iter; + iter = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&iter); break; } /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 */ case _GUARD_NOT_EXHAUSTED_LIST: { + _Py_UopsPESlot iter; + iter = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&iter); break; } case _ITER_NEXT_LIST: { + _Py_UopsPESlot iter; _Py_UopsPESlot next; + iter = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&iter); next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1145,17 +1980,29 @@ } case _ITER_CHECK_TUPLE: { + _Py_UopsPESlot iter; + iter = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&iter); break; } /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 */ case _GUARD_NOT_EXHAUSTED_TUPLE: { + _Py_UopsPESlot iter; + iter = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&iter); break; } case _ITER_NEXT_TUPLE: { + _Py_UopsPESlot iter; _Py_UopsPESlot next; + iter = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&iter); next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1164,17 +2011,29 @@ } case _ITER_CHECK_RANGE: { + _Py_UopsPESlot iter; + iter = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&iter); break; } /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 */ case _GUARD_NOT_EXHAUSTED_RANGE: { + _Py_UopsPESlot iter; + iter = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&iter); break; } case _ITER_NEXT_RANGE: { + _Py_UopsPESlot iter; _Py_UopsPESlot next; + iter = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&iter); next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1183,14 +2042,24 @@ } case _FOR_ITER_GEN_FRAME: { + _Py_UopsPESlot iter; + _Py_UopsPESlot gen_frame; + MATERIALIZE_INST(); /* We are about to hit the end of the trace */ ctx->done = true; break; } case _LOAD_SPECIAL: { + _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot self_or_null; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)owner; + (void)attr; + (void)self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); stack_pointer[-1] = attr; @@ -1201,7 +2070,31 @@ } case _WITH_EXCEPT_START: { + _Py_UopsPESlot val; + _Py_UopsPESlot unused_0; + _Py_UopsPESlot lasti; + _Py_UopsPESlot exit_self; + _Py_UopsPESlot exit_func; + _Py_UopsPESlot unused_1; _Py_UopsPESlot res; + val = stack_pointer[-1]; + unused_0 = stack_pointer[-2]; + lasti = stack_pointer[-3]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)val; + (void)unused_0; + (void)lasti; + (void)exit_self; + (void)exit_func; + (void)exit_func; + (void)exit_self; + (void)lasti; + (void)unused_1; + (void)val; + (void)res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1210,8 +2103,11 @@ } case _PUSH_EXC_INFO: { - _Py_UopsPESlot prev_exc; _Py_UopsPESlot new_exc; + _Py_UopsPESlot prev_exc; + new_exc = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&new_exc); prev_exc = sym_new_not_null(ctx); new_exc = sym_new_not_null(ctx); stack_pointer[-1] = prev_exc; @@ -1222,16 +2118,28 @@ } case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { + _Py_UopsPESlot owner; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); break; } case _GUARD_KEYS_VERSION: { + _Py_UopsPESlot owner; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); break; } case _LOAD_ATTR_METHOD_WITH_VALUES: { + _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot self = (_Py_UopsPESlot){NULL, 0}; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); attr = sym_new_not_null(ctx); self = sym_new_not_null(ctx); stack_pointer[-1] = attr; @@ -1242,8 +2150,12 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { + _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot self = (_Py_UopsPESlot){NULL, 0}; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); attr = sym_new_not_null(ctx); self = sym_new_not_null(ctx); stack_pointer[-1] = attr; @@ -1254,26 +2166,44 @@ } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { + _Py_UopsPESlot owner; _Py_UopsPESlot attr; + _Py_UopsPESlot unused_0 = (_Py_UopsPESlot){NULL, 0}; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { + _Py_UopsPESlot owner; _Py_UopsPESlot attr; + _Py_UopsPESlot unused_0 = (_Py_UopsPESlot){NULL, 0}; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; } case _CHECK_ATTR_METHOD_LAZY_DICT: { + _Py_UopsPESlot owner; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); break; } case _LOAD_ATTR_METHOD_LAZY_DICT: { + _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot self = (_Py_UopsPESlot){NULL, 0}; + owner = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&owner); attr = sym_new_not_null(ctx); self = sym_new_not_null(ctx); stack_pointer[-1] = attr; @@ -1285,21 +2215,23 @@ case _MAYBE_EXPAND_METHOD: { _Py_UopsPESlot *args; - _Py_UopsPESlot self_or_null; + _Py_UopsPESlot *self_or_null; _Py_UopsPESlot callable; _Py_UopsPESlot func; - _Py_UopsPESlot maybe_self; + _Py_UopsPESlot *maybe_self; args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - args = &stack_pointer[-oparg]; - (void)callable; - (void)self_or_null; - (void)args; + maybe_self = &stack_pointer[-1 - oparg]; + MATERIALIZE_INST(); + materialize(&callable); + materialize(&self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } func = sym_new_not_null(ctx); - maybe_self = sym_new_not_null(ctx); + maybe_self[0] = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; break; } @@ -1309,13 +2241,18 @@ case _PY_FRAME_GENERAL: { _Py_UopsPESlot *args; - _Py_UopsPESlot self_or_null; + _Py_UopsPESlot *self_or_null; _Py_UopsPESlot callable; _Py_UopsPESlot new_frame; - self_or_null = stack_pointer[-1 - oparg]; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - (void)(self_or_null); - (void)(callable); + MATERIALIZE_INST(); + materialize(&callable); + materialize(&self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); co = get_code_with_logging((this_instr + 2)); @@ -1331,17 +2268,62 @@ } case _CHECK_FUNCTION_VERSION: { + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; + _Py_UopsPESlot *unused_1; + unused_0 = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + for (int _i = oparg; --_i >= 0;) { + materialize(&unused_0[_i]); + } + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + materialize(&callable); break; } case _CHECK_METHOD_VERSION: { + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot *null; + _Py_UopsPESlot callable; + _Py_UopsPESlot *unused_1; + unused_0 = &stack_pointer[-oparg]; + null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + for (int _i = oparg; --_i >= 0;) { + materialize(&unused_0[_i]); + } + for (int _i = 1; --_i >= 0;) { + materialize(&null[_i]); + } + materialize(&callable); break; } case _EXPAND_METHOD: { + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot *null; + _Py_UopsPESlot callable; _Py_UopsPESlot method; _Py_UopsPESlot *self; + _Py_UopsPESlot *unused_1; + unused_0 = &stack_pointer[-oparg]; + null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; + MATERIALIZE_INST(); + for (int _i = oparg; --_i >= 0;) { + materialize(&unused_0[_i]); + } + for (int _i = 1; --_i >= 0;) { + materialize(&null[_i]); + } + materialize(&callable); method = sym_new_not_null(ctx); for (int _i = 1; --_i >= 0;) { self[_i] = sym_new_not_null(ctx); @@ -1351,11 +2333,39 @@ } case _CHECK_IS_NOT_PY_CALLABLE: { + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot *unused_1; + _Py_UopsPESlot callable; + _Py_UopsPESlot *unused_2; + _Py_UopsPESlot *unused_3; + unused_0 = &stack_pointer[-oparg]; + unused_1 = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + for (int _i = oparg; --_i >= 0;) { + materialize(&unused_0[_i]); + } + for (int _i = 1; --_i >= 0;) { + materialize(&unused_1[_i]); + } + materialize(&callable); break; } case _CALL_NON_PY_GENERAL: { + _Py_UopsPESlot *args; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)args; + (void)self_or_null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1364,13 +2374,43 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot *null; + _Py_UopsPESlot callable; + _Py_UopsPESlot *unused_1; + unused_0 = &stack_pointer[-oparg]; + null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + for (int _i = oparg; --_i >= 0;) { + materialize(&unused_0[_i]); + } + for (int _i = 1; --_i >= 0;) { + materialize(&null[_i]); + } + materialize(&callable); break; } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot *null; + _Py_UopsPESlot callable; _Py_UopsPESlot func; _Py_UopsPESlot *self; + _Py_UopsPESlot *unused_1; + unused_0 = &stack_pointer[-oparg]; + null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; + MATERIALIZE_INST(); + for (int _i = oparg; --_i >= 0;) { + materialize(&unused_0[_i]); + } + for (int _i = 1; --_i >= 0;) { + materialize(&null[_i]); + } + materialize(&callable); func = sym_new_not_null(ctx); for (int _i = 1; --_i >= 0;) { self[_i] = sym_new_not_null(ctx); @@ -1380,27 +2420,63 @@ } case _CHECK_PEP_523: { + MATERIALIZE_INST(); break; } case _CHECK_FUNCTION_EXACT_ARGS: { + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; + _Py_UopsPESlot *unused_1; + unused_0 = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + for (int _i = oparg; --_i >= 0;) { + materialize(&unused_0[_i]); + } + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + materialize(&callable); break; } case _CHECK_STACK_SPACE: { + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; + _Py_UopsPESlot *unused_1; + unused_0 = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + for (int _i = oparg; --_i >= 0;) { + materialize(&unused_0[_i]); + } + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + materialize(&callable); break; } case _INIT_CALL_PY_EXACT_ARGS: { _Py_UopsPESlot *args; - _Py_UopsPESlot self_or_null; + _Py_UopsPESlot *self_or_null; _Py_UopsPESlot callable; _Py_UopsPESlot new_frame; args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + materialize(&callable); + materialize(&self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } int argcount = oparg; - (void)callable; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); co = get_code_with_logging((this_instr + 2)); @@ -1408,14 +2484,14 @@ ctx->done = true; break; } - assert(self_or_null.sym != NULL); + assert(self_or_null->sym != NULL); assert(args != NULL); - if (sym_is_not_null(&self_or_null)) { + if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM args--; argcount++; } - if (sym_is_null(&self_or_null) || sym_is_not_null(&self_or_null)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, args, argcount), NULL}; } else { new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL}; @@ -1429,6 +2505,7 @@ case _PUSH_FRAME: { _Py_UopsPESlot new_frame; new_frame = stack_pointer[-1]; + MATERIALIZE_INST(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; @@ -1463,7 +2540,17 @@ } case _CALL_TYPE_1: { + _Py_UopsPESlot arg; + _Py_UopsPESlot null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + MATERIALIZE_INST(); + materialize(&arg); + materialize(&null); + materialize(&callable); res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1472,7 +2559,19 @@ } case _CALL_STR_1: { + _Py_UopsPESlot arg; + _Py_UopsPESlot null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)arg; + (void)null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1481,7 +2580,19 @@ } case _CALL_TUPLE_1: { + _Py_UopsPESlot arg; + _Py_UopsPESlot null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)arg; + (void)null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1498,12 +2609,14 @@ args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - args = &stack_pointer[-oparg]; uint32_t type_version = (uint32_t)this_instr->operand; (void)type_version; - (void)callable; - (void)null; - (void)args; + MATERIALIZE_INST(); + materialize(&callable); + materialize(&null); + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } self = sym_new_not_null(ctx); init = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = self; @@ -1519,9 +2632,12 @@ args = &stack_pointer[-oparg]; init = stack_pointer[-1 - oparg]; self = stack_pointer[-2 - oparg]; - (void)self; - (void)init; - (void)args; + MATERIALIZE_INST(); + materialize(&self); + materialize(&init); + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } init_frame = (_Py_UopsPESlot){NULL, NULL}; ctx->done = true; stack_pointer[-2 - oparg] = init_frame; @@ -1531,13 +2647,30 @@ } case _EXIT_INIT_CHECK: { + _Py_UopsPESlot should_be_none; + should_be_none = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)should_be_none; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_BUILTIN_CLASS: { + _Py_UopsPESlot *args; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)args; + (void)self_or_null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1546,7 +2679,19 @@ } case _CALL_BUILTIN_O: { + _Py_UopsPESlot *args; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)args; + (void)self_or_null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1555,7 +2700,19 @@ } case _CALL_BUILTIN_FAST: { + _Py_UopsPESlot *args; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)args; + (void)self_or_null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1564,7 +2721,19 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { + _Py_UopsPESlot *args; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)args; + (void)self_or_null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1573,7 +2742,19 @@ } case _CALL_LEN: { + _Py_UopsPESlot *args; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)args; + (void)self_or_null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1582,7 +2763,19 @@ } case _CALL_ISINSTANCE: { + _Py_UopsPESlot *args; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)args; + (void)self_or_null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1591,13 +2784,35 @@ } case _CALL_LIST_APPEND: { + _Py_UopsPESlot arg; + _Py_UopsPESlot self; + _Py_UopsPESlot callable; + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; + MATERIALIZE_INST(); + materialize(&arg); + materialize(&self); + materialize(&callable); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_METHOD_DESCRIPTOR_O: { + _Py_UopsPESlot *args; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)args; + (void)self_or_null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1606,7 +2821,19 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { + _Py_UopsPESlot *args; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)args; + (void)self_or_null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1615,7 +2842,19 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { + _Py_UopsPESlot *args; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)args; + (void)self_or_null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1624,7 +2863,19 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { + _Py_UopsPESlot *args; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)args; + (void)self_or_null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1639,17 +2890,20 @@ case _PY_FRAME_KW: { _Py_UopsPESlot kwnames; _Py_UopsPESlot *args; - _Py_UopsPESlot self_or_null; + _Py_UopsPESlot *self_or_null; _Py_UopsPESlot callable; _Py_UopsPESlot new_frame; kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; - (void)callable; - (void)self_or_null; - (void)args; - (void)kwnames; + MATERIALIZE_INST(); + materialize(&callable); + materialize(&self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } + materialize(&kwnames); new_frame = (_Py_UopsPESlot){NULL, NULL}; ctx->done = true; stack_pointer[-3 - oparg] = new_frame; @@ -1659,18 +2913,71 @@ } case _CHECK_FUNCTION_VERSION_KW: { + _Py_UopsPESlot kwnames; + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; + _Py_UopsPESlot *unused_1; + kwnames = stack_pointer[-1]; + unused_0 = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + MATERIALIZE_INST(); + materialize(&kwnames); + for (int _i = oparg; --_i >= 0;) { + materialize(&unused_0[_i]); + } + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + materialize(&callable); break; } case _CHECK_METHOD_VERSION_KW: { + _Py_UopsPESlot kwnames; + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot *null; + _Py_UopsPESlot callable; + _Py_UopsPESlot *unused_1; + kwnames = stack_pointer[-1]; + unused_0 = &stack_pointer[-1 - oparg]; + null = &stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + MATERIALIZE_INST(); + materialize(&kwnames); + for (int _i = oparg; --_i >= 0;) { + materialize(&unused_0[_i]); + } + for (int _i = 1; --_i >= 0;) { + materialize(&null[_i]); + } + materialize(&callable); break; } case _EXPAND_METHOD_KW: { + _Py_UopsPESlot kwnames; + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot *null; + _Py_UopsPESlot callable; _Py_UopsPESlot method; _Py_UopsPESlot *self; - _Py_UopsPESlot kwnames; + _Py_UopsPESlot *unused_1; + kwnames = stack_pointer[-1]; + unused_0 = &stack_pointer[-1 - oparg]; + null = &stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; self = &stack_pointer[-2 - oparg]; + MATERIALIZE_INST(); + materialize(&kwnames); + for (int _i = oparg; --_i >= 0;) { + materialize(&unused_0[_i]); + } + for (int _i = 1; --_i >= 0;) { + materialize(&null[_i]); + } + materialize(&callable); method = sym_new_not_null(ctx); for (int _i = 1; --_i >= 0;) { self[_i] = sym_new_not_null(ctx); @@ -1682,11 +2989,45 @@ } case _CHECK_IS_NOT_PY_CALLABLE_KW: { + _Py_UopsPESlot kwnames; + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot *unused_1; + _Py_UopsPESlot callable; + _Py_UopsPESlot *unused_2; + _Py_UopsPESlot *unused_3; + kwnames = stack_pointer[-1]; + unused_0 = &stack_pointer[-1 - oparg]; + unused_1 = &stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + MATERIALIZE_INST(); + materialize(&kwnames); + for (int _i = oparg; --_i >= 0;) { + materialize(&unused_0[_i]); + } + for (int _i = 1; --_i >= 0;) { + materialize(&unused_1[_i]); + } + materialize(&callable); break; } case _CALL_KW_NON_PY: { + _Py_UopsPESlot kwnames; + _Py_UopsPESlot *args; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot callable; _Py_UopsPESlot res; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)kwnames; + (void)args; + (void)self_or_null; + (void)callable; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; @@ -1699,7 +3040,13 @@ /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { + _Py_UopsPESlot codeobj_st; _Py_UopsPESlot func; + codeobj_st = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)codeobj_st; + (void)func; func = sym_new_not_null(ctx); stack_pointer[-1] = func; break; @@ -1707,6 +3054,14 @@ case _SET_FUNCTION_ATTRIBUTE: { _Py_UopsPESlot func_st; + _Py_UopsPESlot attr_st; + func_st = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)func_st; + (void)attr_st; + (void)func_st; func_st = sym_new_not_null(ctx); stack_pointer[-2] = func_st; stack_pointer += -1; @@ -1716,6 +3071,7 @@ case _RETURN_GENERATOR: { _Py_UopsPESlot res; + MATERIALIZE_INST(); ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -1739,7 +3095,17 @@ } case _BUILD_SLICE: { + _Py_UopsPESlot step = (_Py_UopsPESlot){NULL, 0}; + _Py_UopsPESlot stop; + _Py_UopsPESlot start; _Py_UopsPESlot slice; + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; + MATERIALIZE_INST(); + materialize(&step); + materialize(&stop); + materialize(&start); slice = sym_new_not_null(ctx); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -1748,21 +3114,40 @@ } case _CONVERT_VALUE: { + _Py_UopsPESlot value; _Py_UopsPESlot result; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&value); result = sym_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { + _Py_UopsPESlot value; _Py_UopsPESlot res; + value = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)value; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { + _Py_UopsPESlot fmt_spec; + _Py_UopsPESlot value; _Py_UopsPESlot res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)fmt_spec; + (void)value; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -1771,7 +3156,17 @@ } case _COPY: { + _Py_UopsPESlot *unused_0; + _Py_UopsPESlot bottom; + _Py_UopsPESlot *unused_1; _Py_UopsPESlot top; + unused_0 = &stack_pointer[-(oparg-1)]; + bottom = stack_pointer[-1 - (oparg-1)]; + MATERIALIZE_INST(); + for (int _i = oparg-1; --_i >= 0;) { + materialize(&unused_0[_i]); + } + materialize(&bottom); top = sym_new_not_null(ctx); stack_pointer[0] = top; stack_pointer += 1; @@ -1780,7 +3175,16 @@ } case _BINARY_OP: { + _Py_UopsPESlot rhs; + _Py_UopsPESlot lhs; _Py_UopsPESlot res; + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + MATERIALIZE_INST(); + materialize_ctx(ctx); + (void)rhs; + (void)lhs; + (void)res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -1790,7 +3194,18 @@ case _SWAP: { _Py_UopsPESlot top; + _Py_UopsPESlot *unused_0; _Py_UopsPESlot bottom; + _Py_UopsPESlot *unused_1; + top = stack_pointer[-1]; + unused_0 = &stack_pointer[-1 - (oparg-2)]; + bottom = stack_pointer[-2 - (oparg-2)]; + MATERIALIZE_INST(); + materialize(&top); + for (int _i = oparg-2; --_i >= 0;) { + materialize(&unused_0[_i]); + } + materialize(&bottom); top = sym_new_not_null(ctx); bottom = sym_new_not_null(ctx); stack_pointer[-2 - (oparg-2)] = top; @@ -1815,62 +3230,87 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { + _Py_UopsPESlot flag; + flag = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&flag); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_FALSE_POP: { + _Py_UopsPESlot flag; + flag = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&flag); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NONE_POP: { + _Py_UopsPESlot val; + val = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&val); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NOT_NONE_POP: { + _Py_UopsPESlot val; + val = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&val); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _JUMP_TO_TOP: { + MATERIALIZE_INST(); + materialize_ctx(ctx); ctx->done = true; break; } case _SET_IP: { + MATERIALIZE_INST(); break; } case _CHECK_STACK_SPACE_OPERAND: { uint32_t framesize = (uint32_t)this_instr->operand; + MATERIALIZE_INST(); (void)framesize; break; } case _SAVE_RETURN_OFFSET: { + MATERIALIZE_INST(); break; } case _EXIT_TRACE: { PyObject *exit_p = (PyObject *)this_instr->operand; + MATERIALIZE_INST(); + materialize_ctx(ctx); (void)exit_p; ctx->done = true; break; } case _CHECK_VALIDITY: { + MATERIALIZE_INST(); break; } case _LOAD_CONST_INLINE: { _Py_UopsPESlot value; PyObject *ptr = (PyObject *)this_instr->operand; + MATERIALIZE_INST(); value = sym_new_const(ctx, ptr); sym_set_origin_inst_override(&value, this_instr); stack_pointer[0] = value; @@ -1882,6 +3322,7 @@ case _LOAD_CONST_INLINE_BORROW: { _Py_UopsPESlot value; PyObject *ptr = (PyObject *)this_instr->operand; + MATERIALIZE_INST(); value = sym_new_const(ctx, ptr); sym_set_origin_inst_override(&value, this_instr); stack_pointer[0] = value; @@ -1891,7 +3332,11 @@ } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { + _Py_UopsPESlot pop; _Py_UopsPESlot value; + pop = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&pop); value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; @@ -1900,6 +3345,7 @@ case _LOAD_CONST_INLINE_WITH_NULL: { _Py_UopsPESlot value; _Py_UopsPESlot null; + MATERIALIZE_INST(); value = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = value; @@ -1912,6 +3358,7 @@ case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { _Py_UopsPESlot value; _Py_UopsPESlot null; + MATERIALIZE_INST(); value = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = value; @@ -1922,46 +3369,65 @@ } case _CHECK_FUNCTION: { + MATERIALIZE_INST(); break; } case _INTERNAL_INCREMENT_OPT_COUNTER: { + _Py_UopsPESlot opt; + opt = stack_pointer[-1]; + MATERIALIZE_INST(); + materialize(&opt); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _DYNAMIC_EXIT: { + MATERIALIZE_INST(); + materialize_ctx(ctx); break; } case _START_EXECUTOR: { + MATERIALIZE_INST(); break; } case _MAKE_WARM: { + MATERIALIZE_INST(); break; } case _FATAL_ERROR: { + MATERIALIZE_INST(); break; } case _CHECK_VALIDITY_AND_SET_IP: { + MATERIALIZE_INST(); break; } case _DEOPT: { + MATERIALIZE_INST(); break; } case _ERROR_POP_N: { + _Py_UopsPESlot *unused_0; + unused_0 = &stack_pointer[-oparg]; + MATERIALIZE_INST(); + for (int _i = oparg; --_i >= 0;) { + materialize(&unused_0[_i]); + } stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _TIER2_RESUME_CHECK: { + MATERIALIZE_INST(); break; } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 1d17c80ed0a67c..a4ce207703edcd 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -23,7 +23,6 @@ class Properties: has_free: bool side_exit: bool pure: bool - static: bool = False tier: int | None = None oparg_and_1: bool = False const_oparg: int = -1 @@ -689,7 +688,6 @@ def compute_properties(op: parser.InstDef) -> Properties: and not has_free, has_free=has_free, pure="pure" in op.annotations, - static="_static" in op.annotations, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 682b7be6e50de1..4cfd4ad3d05988 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -266,8 +266,6 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.oparg_and_1: flags.append("HAS_OPARG_AND_1_FLAG") - if p.static: - flags.append("HAS_STATIC_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index c171e0b94da5ed..d5831593215f76 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -226,7 +226,6 @@ def choice(*opts: str) -> str: "replicate", "tier1", "tier2", - "_static", } __all__ = [] diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 176596719453cb..2ad7604af9cc0d 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -52,7 +52,6 @@ "PASSTHROUGH", "OPARG_AND_1", "ERROR_NO_POP", - "STATIC", ] diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py index cf92b585b76288..69221667cc6ebe 100644 --- a/Tools/cases_generator/partial_evaluator_generator.py +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -28,55 +28,77 @@ DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/partial_evaluator_bytecodes.c").absolute().as_posix() def validate_uop(override: Uop, uop: Uop) -> None: - # To do - pass + for override_input, uop_input in zip(override.stack.inputs, uop.stack.inputs): + if override_input.name != uop_input.name: + assert False, f"Uop {uop.name} input names don't match." + if override_input.size != uop_input.size: + assert False, f"Uop {uop.name} input sizes don't match." + for override_output, uop_output in zip(override.stack.outputs, uop.stack.outputs): + if override_output.name != uop_output.name: + assert False, f"Uop {uop.name} output names don't match." + if override_output.size != uop_output.size: + assert False, f"Uop {uop.name} output sizes don't match." + def type_name(var: StackItem) -> str: if var.is_array(): return f"_Py_UopsPESlot *" - if var.type: - return var.type return f"_Py_UopsPESlot " +def var_name(var: StackItem, unused_count: int) -> tuple[str, int]: + if var.name == "unused": + var = f"unused_{unused_count}" + unused_count += 1 + else: + var = var.name + return var, unused_count -def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: - variables = {"unused"} - if not skip_inputs: - for var in reversed(uop.stack.inputs): - if var.name not in variables: - variables.add(var.name) - if var.condition: - out.emit(f"{type_name(var)}{var.name} = (_Py_UopsPESlot){{NULL, 0}};\n") - else: - out.emit(f"{type_name(var)}{var.name};\n") + +def declare_variables(uop: Uop, out: CWriter) -> None: + variables = set() + unused_count = 0 + for var in reversed(uop.stack.inputs): + if var.name not in variables: + name, unused_count = var_name(var, unused_count) + variables.add(name) + if var.condition: + out.emit(f"{type_name(var)}{name} = (_Py_UopsPESlot){{NULL, 0}};\n") + else: + out.emit(f"{type_name(var)}{name};\n") for var in uop.stack.outputs: - if var.peek: - continue if var.name not in variables: - variables.add(var.name) + name, unused_count = var_name(var, unused_count) + variables.add(name) if var.condition: - out.emit(f"{type_name(var)}{var.name} = (_Py_UopsPESlot){{NULL, 0}};\n") + out.emit(f"{type_name(var)}{name} = (_Py_UopsPESlot){{NULL, 0}};\n") else: - out.emit(f"{type_name(var)}{var.name};\n") + out.emit(f"{type_name(var)}{name};\n") -def decref_inputs( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - out.emit_at("", tkn) def emit_default(out: CWriter, uop: Uop) -> None: - for i, var in enumerate(uop.stack.outputs): + out.emit("MATERIALIZE_INST();\n") + unused_count = 0 + if uop.properties.escapes: + out.emit("materialize_ctx(ctx);\n") + for var in reversed(uop.stack.inputs): + name, unused_count = var_name(var, unused_count) + out.emit(f"(void){name};\n") + for var in uop.stack.outputs: + name, unused_count = var_name(var, unused_count) + out.emit(f"(void){name};\n") + else: + for var in reversed(uop.stack.inputs): + name, unused_count = var_name(var, unused_count) + if var.is_array(): + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"materialize(&{name}[_i]);\n") + out.emit("}\n") + else: + out.emit(f"materialize(&{name});\n") + for var in uop.stack.outputs: if var.name != "unused" and not var.peek: if var.is_array(): out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") @@ -88,8 +110,38 @@ def emit_default(out: CWriter, uop: Uop) -> None: out.emit(f"{var.name} = sym_new_not_null(ctx);\n") -class OptimizerEmitter(Emitter): - pass +class Tier2PEEmitter(Emitter): + def __init__(self, out: CWriter): + super().__init__(out) + self._replacers["MATERIALIZE_INPUTS"] = self.materialize_inputs + + def materialize_inputs( + self, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, + ) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + self.out.emit_at("", tkn) + for var in uop.stack.inputs: + if var.size: + if var.size == "1": + self.out.emit(f"materialize(&{var.name}[0]);\n") + else: + self.out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + self.out.emit(f"materialize(&{var.name}[_i]);\n") + self.out.emit("}\n") + elif var.condition: + if var.condition == "1": + self.out.emit(f"materialize(&{var.name});\n") + elif var.condition != "0": + self.out.emit(f"materialize(&{var.name});\n") + else: + self.out.emit(f"materialize(&{var.name});\n") def write_uop( @@ -98,19 +150,22 @@ def write_uop( out: CWriter, stack: Stack, debug: bool, - skip_inputs: bool, ) -> None: locals: dict[str, Local] = {} + unused_count = 0 try: prototype = override if override else uop is_override = override is not None out.start_line() for var in reversed(prototype.stack.inputs): - code, local = stack.pop(var, extract_bits=True) - if not skip_inputs: - out.emit(code) + name, unused_count = var_name(var, unused_count) + old_name = var.name + var.name = name + code, local = stack.pop(var, extract_bits=True, assign_unused=True) + var.name = old_name + out.emit(code) if local.defined: - locals[local.name] = local + locals[name] = local out.emit(stack.define_output_arrays(prototype.stack.outputs)) if debug: args = [] @@ -128,11 +183,12 @@ def write_uop( cast = f"uint{cache.size*16}_t" out.emit(f"{type}{cache.name} = ({cast})this_instr->operand;\n") if override: - emitter = OptimizerEmitter(out) + emitter = Tier2PEEmitter(out) emitter.emit_tokens(override, stack, None) else: emit_default(out, uop) + for var in prototype.stack.outputs: if var.name in locals: local = locals[var.name] @@ -179,12 +235,9 @@ def generate_abstract_interpreter( out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n") continue out.emit(f"case {uop.name}: {{\n") - if override: - declare_variables(override, out, skip_inputs=False) - else: - declare_variables(uop, out, skip_inputs=True) + declare_variables(uop, out) stack = Stack() - write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) + write_uop(override, uop, out, stack, debug) out.start_line() out.emit("break;\n") out.emit("}") diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index f7623a2c60fe64..11366e3caa5e6d 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -174,7 +174,7 @@ def __init__(self) -> None: self.variables: list[Local] = [] self.defined: set[str] = set() - def pop(self, var: StackItem, extract_bits: bool = False) -> tuple[str, Local]: + def pop(self, var: StackItem, extract_bits: bool = False, assign_unused: bool = False) -> tuple[str, Local]: self.top_offset.pop(var) indirect = "&" if var.is_array() else "" if self.variables: @@ -189,7 +189,7 @@ def pop(self, var: StackItem, extract_bits: bool = False) -> tuple[str, Local]: f"Size mismatch when popping '{popped.name}' from stack to assign to '{var.name}'. " f"Expected {var_size(var)} got {var_size(popped.item)}" ) - if var.name in UNUSED: + if var.name in UNUSED and not assign_unused: if popped.name not in UNUSED and popped.name in self.defined: raise StackError( f"Value is declared unused, but is already cached by prior operation" @@ -211,7 +211,7 @@ def pop(self, var: StackItem, extract_bits: bool = False) -> tuple[str, Local]: return defn, Local.redefinition(var, popped) self.base_offset.pop(var) - if var.name in UNUSED or not var.used: + if not assign_unused and (var.name in UNUSED or not var.used): return "", Local.unused(var) self.defined.add(var.name) cast = f"({var.type})" if (not indirect and var.type) else "" From b2f4fa687f3eb56b07cb244e533ef24e57cce265 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 11 Oct 2024 21:05:17 +0800 Subject: [PATCH 35/46] update --- Python/partial_evaluator_cases.c.h.new | 70 +++++++++++++++++++ .../partial_evaluator_generator.py | 51 ++++++-------- 2 files changed, 90 insertions(+), 31 deletions(-) create mode 100644 Python/partial_evaluator_cases.c.h.new diff --git a/Python/partial_evaluator_cases.c.h.new b/Python/partial_evaluator_cases.c.h.new new file mode 100644 index 00000000000000..893ac915a979c2 --- /dev/null +++ b/Python/partial_evaluator_cases.c.h.new @@ -0,0 +1,70 @@ +// This file is generated by Tools/cases_generator/partial_evaluator_generator.py +// from: +// Python/partial_evaluator_bytecodes.c +// Do not edit! + + case _NOP: { + break; + } + + case _CHECK_PERIODIC: { + MATERIALIZE_INST(); + materialize_ctx(ctx); + break; + } + + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { + MATERIALIZE_INST(); + materialize_ctx(ctx); + break; + } + + /* _QUICKEN_RESUME is not a viable micro-op for tier 2 */ + + case _RESUME_CHECK: { + MATERIALIZE_INST(); + break; + } + + /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ + + case _LOAD_FAST_CHECK: { + _Py_UopsPESlot value; + MATERIALIZE_INST(); + value = GETLOCAL(oparg); + // We guarantee this will error - just bail and don't optimize it. + if (sym_is_null(&value)) { + ctx->done = true; + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST: { + _Py_UopsPESlot value; + value = GETLOCAL(oparg); + sym_set_origin_inst_override(&value, this_instr); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST_AND_CLEAR: { + _Py_UopsPESlot value; + MATERIALIZE_INST(); + value = GETLOCAL(oparg); + GETLOCAL(oparg) = sym_new_null(ctx); + sym_set_origin_inst_override(&value, this_instr); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST: { + _Py_UopsPESlot value; + // Should've all been converted by specializer. + Py_UNREACHABLE(); \ No newline at end of file diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py index 69221667cc6ebe..57df99f6506a3e 100644 --- a/Tools/cases_generator/partial_evaluator_generator.py +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -22,7 +22,7 @@ from cwriter import CWriter from typing import TextIO, Iterator from lexer import Token -from stack import Local, Stack, StackError +from stack import Local, Stack, StackError, Storage DEFAULT_OUTPUT = ROOT / "Python/partial_evaluator_cases.c.h" DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/partial_evaluator_bytecodes.c").absolute().as_posix() @@ -78,7 +78,7 @@ def declare_variables(uop: Uop, out: CWriter) -> None: -def emit_default(out: CWriter, uop: Uop) -> None: +def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: out.emit("MATERIALIZE_INST();\n") unused_count = 0 if uop.properties.escapes: @@ -152,26 +152,18 @@ def write_uop( debug: bool, ) -> None: locals: dict[str, Local] = {} - unused_count = 0 + prototype = override if override else uop try: - prototype = override if override else uop - is_override = override is not None out.start_line() - for var in reversed(prototype.stack.inputs): - name, unused_count = var_name(var, unused_count) - old_name = var.name - var.name = name - code, local = stack.pop(var, extract_bits=True, assign_unused=True) - var.name = old_name - out.emit(code) - if local.defined: - locals[name] = local - out.emit(stack.define_output_arrays(prototype.stack.outputs)) + if override: + code_list, storage = Storage.for_uop(stack, prototype, extract_bits=False) + for code in code_list: + out.emit(code) if debug: args = [] - for var in prototype.stack.inputs: - if not var.peek or is_override: - args.append(var.name) + for input in prototype.stack.inputs: + if not input.peek or override: + args.append(input.name) out.emit(f'DEBUG_PRINTF({", ".join(args)});\n') if override: for cache in uop.caches: @@ -184,21 +176,18 @@ def write_uop( out.emit(f"{type}{cache.name} = ({cast})this_instr->operand;\n") if override: emitter = Tier2PEEmitter(out) - emitter.emit_tokens(override, stack, None) + # No reference management of inputs needed. + for var in storage.inputs: # type: ignore[possibly-undefined] + var.defined = False + storage = emitter.emit_tokens(override, storage, None) + out.start_line() + storage.flush(out, cast_type="", extract_bits=False) else: - emit_default(out, uop) - - - for var in prototype.stack.outputs: - if var.name in locals: - local = locals[var.name] - else: - local = Local.local(var) - stack.push(local) - out.start_line() - stack.flush(out, cast_type="", extract_bits=True) + emit_default(out, uop, stack) + out.start_line() + stack.flush(out, cast_type="", extract_bits=False) except StackError as ex: - raise analysis_error(ex.args[0], uop.body[0]) + raise analysis_error(ex.args[0], prototype.body[0]) # from None SKIPS = ("_EXTENDED_ARG",) From 195bb888e23ecd3a337046419b401c51bbdf2e28 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 11 Oct 2024 22:26:37 +0800 Subject: [PATCH 36/46] fix problems from merge --- Python/bytecodes.c | 4 +- Python/executor_cases.c.h | 4 +- Python/partial_evaluator_bytecodes.c | 51 +- Python/partial_evaluator_cases.c.h | 1555 +++++++++-------- Python/partial_evaluator_cases.c.h.new | 70 - .../partial_evaluator_generator.py | 62 +- Tools/cases_generator/stack.py | 2 +- 7 files changed, 928 insertions(+), 820 deletions(-) delete mode 100644 Python/partial_evaluator_cases.c.h.new diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7d772bb74ccc10..8049785dc629b0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4811,7 +4811,7 @@ dummy_func( printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %ld, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); } @@ -4921,7 +4921,7 @@ dummy_func( printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %ld, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index dedd033b3f70db..dc8daf5eb4b637 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -5609,7 +5609,7 @@ printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %ld, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5795,7 +5795,7 @@ printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %ld, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); stack_pointer = _PyFrame_GetStackPointer(frame); diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index 3968999a6e78a3..333bb69260deb3 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -57,6 +57,8 @@ dummy_func(void) { op(_LOAD_CONST, (-- value)) { // Should've all been converted by specializer. Py_UNREACHABLE(); + // Just to please the code generator that value is defined. + value = sym_new_const(ctx, NULL); } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { @@ -99,7 +101,6 @@ dummy_func(void) { op(_CHECK_STACK_SPACE_OPERAND, ( -- )) { MATERIALIZE_INST(); - (void)framesize; } op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame)) { @@ -116,7 +117,7 @@ dummy_func(void) { ctx->done = true; } - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null[1], args[oparg] -- new_frame)) { + op(_INIT_CALL_PY_EXACT_ARGS, (callable[1], self_or_null[1], args[oparg] -- new_frame)) { MATERIALIZE_INST(); MATERIALIZE_INPUTS(); @@ -139,15 +140,20 @@ dummy_func(void) { argcount++; } + _Py_UopsPESlot temp; if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, args, argcount), NULL}; + temp = (_Py_UopsPESlot){ + (_Py_UopsPESymbol *)frame_new(ctx, co, 0, args, argcount), NULL + }; } else { - new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL}; - + temp = (_Py_UopsPESlot){ + (_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL + }; } + new_frame = temp; } - op(_PY_FRAME_GENERAL, (callable, self_or_null[1], args[oparg] -- new_frame)) { + op(_PY_FRAME_GENERAL, (callable[1], self_or_null[1], args[oparg] -- new_frame)) { MATERIALIZE_INST(); MATERIALIZE_INPUTS(); PyCodeObject *co = NULL; @@ -158,25 +164,26 @@ dummy_func(void) { break; } - new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL}; + _Py_UopsPESlot temp = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL}; + new_frame = temp; } - op(_PY_FRAME_KW, (callable, self_or_null[1], args[oparg], kwnames -- new_frame)) { + op(_PY_FRAME_KW, (callable[1], self_or_null[1], args[oparg], kwnames -- new_frame)) { MATERIALIZE_INST(); MATERIALIZE_INPUTS(); new_frame = (_Py_UopsPESlot){NULL, NULL}; ctx->done = true; } - op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable[1], null[1], args[oparg] -- init[1], self[1], args[oparg])) { (void)type_version; MATERIALIZE_INST(); MATERIALIZE_INPUTS(); - self = sym_new_not_null(ctx); - init = sym_new_not_null(ctx); + self[0] = sym_new_not_null(ctx); + init[0] = sym_new_not_null(ctx); } - op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame)) { + op(_CREATE_INIT_FRAME, (init[1], self[1], args[oparg] -- init_frame)) { MATERIALIZE_INST(); MATERIALIZE_INPUTS(); init_frame = (_Py_UopsPESlot){NULL, NULL}; @@ -315,12 +322,28 @@ dummy_func(void) { (void)right; } - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null[1], args[oparg] -- func, maybe_self[1], args[oparg])) { + op(_MAYBE_EXPAND_METHOD, (callable[1], self_or_null[1], args[oparg] -- func[1], maybe_self[1], args[oparg])) { MATERIALIZE_INST(); MATERIALIZE_INPUTS(); - func = sym_new_not_null(ctx); + func[0] = sym_new_not_null(ctx); maybe_self[0] = sym_new_not_null(ctx); } + + op(_LOAD_GLOBAL_MODULE_FROM_KEYS, (index/1, globals_keys -- res, null if (oparg & 1))) { + (void)index; + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + } + + op(_LOAD_GLOBAL_BUILTINS_FROM_KEYS, (index/1, builtins_keys -- res, null if (oparg & 1))) { + (void)index; + MATERIALIZE_INST(); + MATERIALIZE_INPUTS(); + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + } // END BYTECODES // } diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index f812a878436636..8a87f4b7e45c3c 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -68,6 +68,8 @@ _Py_UopsPESlot value; // Should've all been converted by specializer. Py_UNREACHABLE(); + // Just to please the code generator that value is defined. + value = sym_new_const(ctx, NULL); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -86,9 +88,13 @@ // Leave it as virtual. } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); materialize(&value); MATERIALIZE_INST(); GETLOCAL(oparg) = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -119,13 +125,14 @@ case _END_SEND: { _Py_UopsPESlot value; _Py_UopsPESlot receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; + _Py_UopsPESlot val; MATERIALIZE_INST(); + value = stack_pointer[-1]; materialize(&value); + receiver = stack_pointer[-2]; materialize(&receiver); - value = sym_new_not_null(ctx); - stack_pointer[-2] = value; + val = sym_new_not_null(ctx); + stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -134,12 +141,11 @@ case _UNARY_NEGATIVE: { _Py_UopsPESlot value; _Py_UopsPESlot res; - value = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)value; - (void)res; + value = stack_pointer[-1]; + materialize(&value); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = res; break; } @@ -147,8 +153,8 @@ case _UNARY_NOT: { _Py_UopsPESlot value; _Py_UopsPESlot res; - value = stack_pointer[-1]; MATERIALIZE_INST(); + value = stack_pointer[-1]; materialize(&value); res = sym_new_not_null(ctx); stack_pointer[-1] = res; @@ -158,20 +164,19 @@ case _TO_BOOL: { _Py_UopsPESlot value; _Py_UopsPESlot res; - value = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)value; - (void)res; + value = stack_pointer[-1]; + materialize(&value); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL_BOOL: { _Py_UopsPESlot value; - value = stack_pointer[-1]; MATERIALIZE_INST(); + value = stack_pointer[-1]; materialize(&value); break; } @@ -179,11 +184,9 @@ case _TO_BOOL_INT: { _Py_UopsPESlot value; _Py_UopsPESlot res; - value = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)value; - (void)res; + value = stack_pointer[-1]; + materialize(&value); res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; @@ -192,8 +195,8 @@ case _TO_BOOL_LIST: { _Py_UopsPESlot value; _Py_UopsPESlot res; - value = stack_pointer[-1]; MATERIALIZE_INST(); + value = stack_pointer[-1]; materialize(&value); res = sym_new_not_null(ctx); stack_pointer[-1] = res; @@ -203,8 +206,8 @@ case _TO_BOOL_NONE: { _Py_UopsPESlot value; _Py_UopsPESlot res; - value = stack_pointer[-1]; MATERIALIZE_INST(); + value = stack_pointer[-1]; materialize(&value); res = sym_new_not_null(ctx); stack_pointer[-1] = res; @@ -214,11 +217,9 @@ case _TO_BOOL_STR: { _Py_UopsPESlot value; _Py_UopsPESlot res; - value = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)value; - (void)res; + value = stack_pointer[-1]; + materialize(&value); res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; @@ -227,8 +228,8 @@ case _REPLACE_WITH_TRUE: { _Py_UopsPESlot value; _Py_UopsPESlot res; - value = stack_pointer[-1]; MATERIALIZE_INST(); + value = stack_pointer[-1]; materialize(&value); res = sym_new_not_null(ctx); stack_pointer[-1] = res; @@ -238,12 +239,11 @@ case _UNARY_INVERT: { _Py_UopsPESlot value; _Py_UopsPESlot res; - value = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)value; - (void)res; + value = stack_pointer[-1]; + materialize(&value); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = res; break; } @@ -251,10 +251,10 @@ case _GUARD_BOTH_INT: { _Py_UopsPESlot right; _Py_UopsPESlot left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); break; } @@ -263,18 +263,18 @@ _Py_UopsPESlot unused_0; _Py_UopsPESlot left; _Py_UopsPESlot unused_1; - unused_0 = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + unused_0 = stack_pointer[-1]; materialize(&unused_0); + left = stack_pointer[-2]; materialize(&left); break; } case _GUARD_TOS_INT: { _Py_UopsPESlot value; - value = stack_pointer[-1]; MATERIALIZE_INST(); + value = stack_pointer[-1]; materialize(&value); break; } @@ -283,10 +283,10 @@ _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -299,10 +299,10 @@ _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -315,10 +315,10 @@ _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -330,10 +330,10 @@ case _GUARD_BOTH_FLOAT: { _Py_UopsPESlot right; _Py_UopsPESlot left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); break; } @@ -342,18 +342,18 @@ _Py_UopsPESlot unused_0; _Py_UopsPESlot left; _Py_UopsPESlot unused_1; - unused_0 = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + unused_0 = stack_pointer[-1]; materialize(&unused_0); + left = stack_pointer[-2]; materialize(&left); break; } case _GUARD_TOS_FLOAT: { _Py_UopsPESlot value; - value = stack_pointer[-1]; MATERIALIZE_INST(); + value = stack_pointer[-1]; materialize(&value); break; } @@ -362,10 +362,10 @@ _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -378,10 +378,10 @@ _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -394,10 +394,10 @@ _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -409,10 +409,10 @@ case _GUARD_BOTH_UNICODE: { _Py_UopsPESlot right; _Py_UopsPESlot left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); break; } @@ -421,10 +421,10 @@ _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -436,12 +436,11 @@ case _BINARY_OP_INPLACE_ADD_UNICODE: { _Py_UopsPESlot right; _Py_UopsPESlot left; + MATERIALIZE_INST(); right = stack_pointer[-1]; + materialize(&right); left = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)right; - (void)left; + materialize(&left); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -451,14 +450,13 @@ _Py_UopsPESlot sub; _Py_UopsPESlot container; _Py_UopsPESlot res; + MATERIALIZE_INST(); sub = stack_pointer[-1]; + materialize(&sub); container = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)sub; - (void)container; - (void)res; + materialize(&container); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -470,16 +468,15 @@ _Py_UopsPESlot start; _Py_UopsPESlot container; _Py_UopsPESlot res; + MATERIALIZE_INST(); stop = stack_pointer[-1]; + materialize(&stop); start = stack_pointer[-2]; + materialize(&start); container = stack_pointer[-3]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)stop; - (void)start; - (void)container; - (void)res; + materialize(&container); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -491,16 +488,16 @@ _Py_UopsPESlot start; _Py_UopsPESlot container; _Py_UopsPESlot v; + MATERIALIZE_INST(); stop = stack_pointer[-1]; + materialize(&stop); start = stack_pointer[-2]; + materialize(&start); container = stack_pointer[-3]; + materialize(&container); v = stack_pointer[-4]; - MATERIALIZE_INST(); + materialize(&v); materialize_ctx(ctx); - (void)stop; - (void)start; - (void)container; - (void)v; stack_pointer += -4; assert(WITHIN_STACK_BOUNDS()); break; @@ -510,10 +507,10 @@ _Py_UopsPESlot sub_st; _Py_UopsPESlot list_st; _Py_UopsPESlot res; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; MATERIALIZE_INST(); + sub_st = stack_pointer[-1]; materialize(&sub_st); + list_st = stack_pointer[-2]; materialize(&list_st); res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -526,10 +523,10 @@ _Py_UopsPESlot sub_st; _Py_UopsPESlot str_st; _Py_UopsPESlot res; - sub_st = stack_pointer[-1]; - str_st = stack_pointer[-2]; MATERIALIZE_INST(); + sub_st = stack_pointer[-1]; materialize(&sub_st); + str_st = stack_pointer[-2]; materialize(&str_st); res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -542,10 +539,10 @@ _Py_UopsPESlot sub_st; _Py_UopsPESlot tuple_st; _Py_UopsPESlot res; - sub_st = stack_pointer[-1]; - tuple_st = stack_pointer[-2]; MATERIALIZE_INST(); + sub_st = stack_pointer[-1]; materialize(&sub_st); + tuple_st = stack_pointer[-2]; materialize(&tuple_st); res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -558,14 +555,13 @@ _Py_UopsPESlot sub_st; _Py_UopsPESlot dict_st; _Py_UopsPESlot res; + MATERIALIZE_INST(); sub_st = stack_pointer[-1]; + materialize(&sub_st); dict_st = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)sub_st; - (void)dict_st; - (void)res; + materialize(&dict_st); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -576,14 +572,11 @@ _Py_UopsPESlot unused_0; _Py_UopsPESlot container; _Py_UopsPESlot unused_1; + MATERIALIZE_INST(); unused_0 = stack_pointer[-1]; + materialize(&unused_0); container = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)unused_0; - (void)container; - (void)container; - (void)unused_1; + materialize(&container); break; } @@ -591,8 +584,6 @@ _Py_UopsPESlot sub; _Py_UopsPESlot container; _Py_UopsPESlot new_frame; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; MATERIALIZE_INST(); materialize(&container); materialize(&sub); @@ -609,14 +600,14 @@ _Py_UopsPESlot *unused_0; _Py_UopsPESlot list; _Py_UopsPESlot *unused_1; - v = stack_pointer[-1]; - unused_0 = &stack_pointer[-1 - (oparg-1)]; - list = stack_pointer[-2 - (oparg-1)]; MATERIALIZE_INST(); + v = stack_pointer[-1]; materialize(&v); + unused_0 = &stack_pointer[-1 - (oparg-1)]; for (int _i = oparg-1; --_i >= 0;) { materialize(&unused_0[_i]); } + list = stack_pointer[-2 - (oparg-1)]; materialize(&list); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -628,16 +619,16 @@ _Py_UopsPESlot *unused_0; _Py_UopsPESlot set; _Py_UopsPESlot *unused_1; + MATERIALIZE_INST(); v = stack_pointer[-1]; + materialize(&v); unused_0 = &stack_pointer[-1 - (oparg-1)]; + for (int _i = oparg-1; --_i >= 0;) { + materialize(&unused_0[_i]); + } set = stack_pointer[-2 - (oparg-1)]; - MATERIALIZE_INST(); + materialize(&set); materialize_ctx(ctx); - (void)v; - (void)unused_0; - (void)set; - (void)set; - (void)unused_1; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -647,14 +638,14 @@ _Py_UopsPESlot sub; _Py_UopsPESlot container; _Py_UopsPESlot v; + MATERIALIZE_INST(); sub = stack_pointer[-1]; + materialize(&sub); container = stack_pointer[-2]; + materialize(&container); v = stack_pointer[-3]; - MATERIALIZE_INST(); + materialize(&v); materialize_ctx(ctx); - (void)sub; - (void)container; - (void)v; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; @@ -664,12 +655,12 @@ _Py_UopsPESlot sub_st; _Py_UopsPESlot list_st; _Py_UopsPESlot value; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - value = stack_pointer[-3]; MATERIALIZE_INST(); + sub_st = stack_pointer[-1]; materialize(&sub_st); + list_st = stack_pointer[-2]; materialize(&list_st); + value = stack_pointer[-3]; materialize(&value); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -680,14 +671,14 @@ _Py_UopsPESlot sub; _Py_UopsPESlot dict_st; _Py_UopsPESlot value; + MATERIALIZE_INST(); sub = stack_pointer[-1]; + materialize(&sub); dict_st = stack_pointer[-2]; + materialize(&dict_st); value = stack_pointer[-3]; - MATERIALIZE_INST(); + materialize(&value); materialize_ctx(ctx); - (void)sub; - (void)dict_st; - (void)value; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; @@ -696,12 +687,12 @@ case _DELETE_SUBSCR: { _Py_UopsPESlot sub; _Py_UopsPESlot container; + MATERIALIZE_INST(); sub = stack_pointer[-1]; + materialize(&sub); container = stack_pointer[-2]; - MATERIALIZE_INST(); + materialize(&container); materialize_ctx(ctx); - (void)sub; - (void)container; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -710,12 +701,11 @@ case _CALL_INTRINSIC_1: { _Py_UopsPESlot value; _Py_UopsPESlot res; - value = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)value; - (void)res; + value = stack_pointer[-1]; + materialize(&value); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = res; break; } @@ -724,14 +714,13 @@ _Py_UopsPESlot value1_st; _Py_UopsPESlot value2_st; _Py_UopsPESlot res; + MATERIALIZE_INST(); value1_st = stack_pointer[-1]; + materialize(&value1_st); value2_st = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)value1_st; - (void)value2_st; - (void)res; + materialize(&value2_st); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -757,26 +746,25 @@ assert(framesize > 0); assert(framesize <= curr_space); curr_space -= framesize; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); co = get_code(this_instr); if (co == NULL) { // might be impossible, but bailing is still safe ctx->done = true; } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); break; } case _GET_AITER: { _Py_UopsPESlot obj; _Py_UopsPESlot iter; - obj = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)obj; - (void)iter; + obj = stack_pointer[-1]; + materialize(&obj); iter = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = iter; break; } @@ -784,13 +772,11 @@ case _GET_ANEXT: { _Py_UopsPESlot aiter; _Py_UopsPESlot awaitable; - aiter = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)aiter; - (void)aiter; - (void)awaitable; + aiter = stack_pointer[-1]; + materialize(&aiter); awaitable = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -800,12 +786,11 @@ case _GET_AWAITABLE: { _Py_UopsPESlot iterable; _Py_UopsPESlot iter; - iterable = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)iterable; - (void)iter; + iterable = stack_pointer[-1]; + materialize(&iterable); iter = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = iter; break; } @@ -825,7 +810,6 @@ case _YIELD_VALUE: { _Py_UopsPESlot retval; _Py_UopsPESlot value; - retval = stack_pointer[-1]; MATERIALIZE_INST(); materialize(&retval); value = sym_new_unknown(ctx); @@ -835,10 +819,10 @@ case _POP_EXCEPT: { _Py_UopsPESlot exc_value; - exc_value = stack_pointer[-1]; MATERIALIZE_INST(); + exc_value = stack_pointer[-1]; + materialize(&exc_value); materialize_ctx(ctx); - (void)exc_value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -857,9 +841,8 @@ case _LOAD_BUILD_CLASS: { _Py_UopsPESlot bc; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)bc; bc = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[0] = bc; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -868,10 +851,10 @@ case _STORE_NAME: { _Py_UopsPESlot v; - v = stack_pointer[-1]; MATERIALIZE_INST(); + v = stack_pointer[-1]; + materialize(&v); materialize_ctx(ctx); - (void)v; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -886,7 +869,6 @@ case _UNPACK_SEQUENCE: { _Py_UopsPESlot seq; _Py_UopsPESlot *output; - seq = stack_pointer[-1]; output = &stack_pointer[-1]; /* This has to be done manually */ MATERIALIZE_INST(); @@ -903,8 +885,8 @@ _Py_UopsPESlot seq; _Py_UopsPESlot val1; _Py_UopsPESlot val0; - seq = stack_pointer[-1]; MATERIALIZE_INST(); + seq = stack_pointer[-1]; materialize(&seq); val1 = sym_new_not_null(ctx); val0 = sym_new_not_null(ctx); @@ -918,10 +900,10 @@ case _UNPACK_SEQUENCE_TUPLE: { _Py_UopsPESlot seq; _Py_UopsPESlot *values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; MATERIALIZE_INST(); + seq = stack_pointer[-1]; materialize(&seq); + values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); } @@ -933,10 +915,10 @@ case _UNPACK_SEQUENCE_LIST: { _Py_UopsPESlot seq; _Py_UopsPESlot *values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; MATERIALIZE_INST(); + seq = stack_pointer[-1]; materialize(&seq); + values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); } @@ -950,10 +932,8 @@ _Py_UopsPESlot *left; _Py_UopsPESlot unused_0; _Py_UopsPESlot *right; - seq = stack_pointer[-1]; left = &stack_pointer[-1]; - -right = &stack_pointer[(oparg & 0xFF)]; + right = &stack_pointer[(oparg & 0xFF)]; /* This has to be done manually */ MATERIALIZE_INST(); materialize(&seq); @@ -970,12 +950,12 @@ right = &stack_pointer[(oparg & 0xFF)]; case _STORE_ATTR: { _Py_UopsPESlot owner; _Py_UopsPESlot v; + MATERIALIZE_INST(); owner = stack_pointer[-1]; + materialize(&owner); v = stack_pointer[-2]; - MATERIALIZE_INST(); + materialize(&v); materialize_ctx(ctx); - (void)owner; - (void)v; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -983,10 +963,10 @@ right = &stack_pointer[(oparg & 0xFF)]; case _DELETE_ATTR: { _Py_UopsPESlot owner; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; + materialize(&owner); materialize_ctx(ctx); - (void)owner; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -994,10 +974,10 @@ right = &stack_pointer[(oparg & 0xFF)]; case _STORE_GLOBAL: { _Py_UopsPESlot v; - v = stack_pointer[-1]; MATERIALIZE_INST(); + v = stack_pointer[-1]; + materialize(&v); materialize_ctx(ctx); - (void)v; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1012,9 +992,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _LOAD_LOCALS: { _Py_UopsPESlot locals; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)locals; locals = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[0] = locals; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1026,9 +1005,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _LOAD_NAME: { _Py_UopsPESlot v; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)v; v = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[0] = v; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1038,15 +1016,11 @@ right = &stack_pointer[(oparg & 0xFF)]; case _LOAD_GLOBAL: { _Py_UopsPESlot *res; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; - res = &stack_pointer[0]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)res; - (void)null; - for (int _i = 1; --_i >= 0;) { - res[_i] = sym_new_not_null(ctx); - } + res = &stack_pointer[0]; + res[0] = sym_new_not_null(ctx); null = sym_new_null(ctx); + materialize_ctx(ctx); if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -1058,33 +1032,56 @@ right = &stack_pointer[(oparg & 0xFF)]; break; } - case _GUARD_BUILTINS_VERSION: { + case _GUARD_GLOBALS_VERSION_PUSH_KEYS: { + _Py_UopsPESlot globals_keys; + MATERIALIZE_INST(); + globals_keys = sym_new_not_null(ctx); + stack_pointer[0] = globals_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_BUILTINS_VERSION_PUSH_KEYS: { + _Py_UopsPESlot builtins_keys; MATERIALIZE_INST(); + builtins_keys = sym_new_not_null(ctx); + stack_pointer[0] = builtins_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_GLOBAL_MODULE: { + case _LOAD_GLOBAL_MODULE_FROM_KEYS: { + _Py_UopsPESlot globals_keys; _Py_UopsPESlot res; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; + uint16_t index = (uint16_t)this_instr->operand; + (void)index; MATERIALIZE_INST(); + materialize(&globals_keys); res = sym_new_not_null(ctx); null = sym_new_null(ctx); - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer[-1] = res; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_GLOBAL_BUILTINS: { + case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: { + _Py_UopsPESlot builtins_keys; _Py_UopsPESlot res; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; + uint16_t index = (uint16_t)this_instr->operand; + (void)index; MATERIALIZE_INST(); + materialize(&builtins_keys); res = sym_new_not_null(ctx); null = sym_new_null(ctx); - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer[-1] = res; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } @@ -1109,12 +1106,11 @@ right = &stack_pointer[(oparg & 0xFF)]; case _LOAD_FROM_DICT_OR_DEREF: { _Py_UopsPESlot class_dict_st; _Py_UopsPESlot value; - class_dict_st = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)class_dict_st; - (void)value; + class_dict_st = stack_pointer[-1]; + materialize(&class_dict_st); value = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = value; break; } @@ -1122,9 +1118,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _LOAD_DEREF: { _Py_UopsPESlot value; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)value; value = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1133,10 +1128,10 @@ right = &stack_pointer[(oparg & 0xFF)]; case _STORE_DEREF: { _Py_UopsPESlot v; - v = stack_pointer[-1]; MATERIALIZE_INST(); + v = stack_pointer[-1]; + materialize(&v); materialize_ctx(ctx); - (void)v; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1150,8 +1145,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _BUILD_STRING: { _Py_UopsPESlot *pieces; _Py_UopsPESlot str; - pieces = &stack_pointer[-oparg]; MATERIALIZE_INST(); + pieces = &stack_pointer[-oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&pieces[_i]); } @@ -1165,8 +1160,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _BUILD_TUPLE: { _Py_UopsPESlot *values; _Py_UopsPESlot tup; - values = &stack_pointer[-oparg]; MATERIALIZE_INST(); + values = &stack_pointer[-oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&values[_i]); } @@ -1180,8 +1175,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _BUILD_LIST: { _Py_UopsPESlot *values; _Py_UopsPESlot list; - values = &stack_pointer[-oparg]; MATERIALIZE_INST(); + values = &stack_pointer[-oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&values[_i]); } @@ -1197,16 +1192,16 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot *unused_0; _Py_UopsPESlot list_st; _Py_UopsPESlot *unused_1; + MATERIALIZE_INST(); iterable_st = stack_pointer[-1]; + materialize(&iterable_st); unused_0 = &stack_pointer[-1 - (oparg-1)]; + for (int _i = oparg-1; --_i >= 0;) { + materialize(&unused_0[_i]); + } list_st = stack_pointer[-2 - (oparg-1)]; - MATERIALIZE_INST(); + materialize(&list_st); materialize_ctx(ctx); - (void)iterable_st; - (void)unused_0; - (void)list_st; - (void)list_st; - (void)unused_1; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1217,16 +1212,16 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot *unused_0; _Py_UopsPESlot set; _Py_UopsPESlot *unused_1; + MATERIALIZE_INST(); iterable = stack_pointer[-1]; + materialize(&iterable); unused_0 = &stack_pointer[-1 - (oparg-1)]; + for (int _i = oparg-1; --_i >= 0;) { + materialize(&unused_0[_i]); + } set = stack_pointer[-2 - (oparg-1)]; - MATERIALIZE_INST(); + materialize(&set); materialize_ctx(ctx); - (void)iterable; - (void)unused_0; - (void)set; - (void)set; - (void)unused_1; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1235,12 +1230,13 @@ right = &stack_pointer[(oparg & 0xFF)]; case _BUILD_SET: { _Py_UopsPESlot *values; _Py_UopsPESlot set; - values = &stack_pointer[-oparg]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)values; - (void)set; + values = &stack_pointer[-oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&values[_i]); + } set = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1250,12 +1246,13 @@ right = &stack_pointer[(oparg & 0xFF)]; case _BUILD_MAP: { _Py_UopsPESlot *values; _Py_UopsPESlot map; - values = &stack_pointer[-oparg*2]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)values; - (void)map; + values = &stack_pointer[-oparg*2]; + for (int _i = oparg*2; --_i >= 0;) { + materialize(&values[_i]); + } map = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; assert(WITHIN_STACK_BOUNDS()); @@ -1273,16 +1270,16 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot *unused_0; _Py_UopsPESlot dict; _Py_UopsPESlot *unused_1; + MATERIALIZE_INST(); update = stack_pointer[-1]; + materialize(&update); unused_0 = &stack_pointer[-1 - (oparg - 1)]; + for (int _i = oparg - 1; --_i >= 0;) { + materialize(&unused_0[_i]); + } dict = stack_pointer[-2 - (oparg - 1)]; - MATERIALIZE_INST(); + materialize(&dict); materialize_ctx(ctx); - (void)update; - (void)unused_0; - (void)dict; - (void)dict; - (void)unused_1; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1298,25 +1295,22 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot unused_3; _Py_UopsPESlot unused_4; _Py_UopsPESlot *unused_5; + MATERIALIZE_INST(); update = stack_pointer[-1]; + materialize(&update); unused_0 = &stack_pointer[-1 - (oparg - 1)]; + for (int _i = oparg - 1; --_i >= 0;) { + materialize(&unused_0[_i]); + } dict = stack_pointer[-2 - (oparg - 1)]; + materialize(&dict); unused_1 = stack_pointer[-3 - (oparg - 1)]; + materialize(&unused_1); unused_2 = stack_pointer[-4 - (oparg - 1)]; + materialize(&unused_2); callable = stack_pointer[-5 - (oparg - 1)]; - MATERIALIZE_INST(); + materialize(&callable); materialize_ctx(ctx); - (void)update; - (void)unused_0; - (void)dict; - (void)unused_1; - (void)unused_2; - (void)callable; - (void)callable; - (void)unused_3; - (void)unused_4; - (void)dict; - (void)unused_5; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1328,18 +1322,18 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot *unused_0; _Py_UopsPESlot dict_st; _Py_UopsPESlot *unused_1; + MATERIALIZE_INST(); value = stack_pointer[-1]; + materialize(&value); key = stack_pointer[-2]; + materialize(&key); unused_0 = &stack_pointer[-2 - (oparg - 1)]; + for (int _i = oparg - 1; --_i >= 0;) { + materialize(&unused_0[_i]); + } dict_st = stack_pointer[-3 - (oparg - 1)]; - MATERIALIZE_INST(); + materialize(&dict_st); materialize_ctx(ctx); - (void)value; - (void)key; - (void)unused_0; - (void)dict_st; - (void)dict_st; - (void)unused_1; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -1353,17 +1347,15 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot global_super_st; _Py_UopsPESlot attr_st; _Py_UopsPESlot unused_0 = (_Py_UopsPESlot){NULL, 0}; + MATERIALIZE_INST(); self_st = stack_pointer[-1]; + materialize(&self_st); class_st = stack_pointer[-2]; + materialize(&class_st); global_super_st = stack_pointer[-3]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)self_st; - (void)class_st; - (void)global_super_st; - (void)attr_st; - (void)unused_0; + materialize(&global_super_st); attr_st = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1376,18 +1368,16 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot global_super_st; _Py_UopsPESlot attr; _Py_UopsPESlot self_or_null; + MATERIALIZE_INST(); self_st = stack_pointer[-1]; + materialize(&self_st); class_st = stack_pointer[-2]; + materialize(&class_st); global_super_st = stack_pointer[-3]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)self_st; - (void)class_st; - (void)global_super_st; - (void)attr; - (void)self_or_null; + materialize(&global_super_st); attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; @@ -1399,14 +1389,12 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot self_or_null = (_Py_UopsPESlot){NULL, 0}; - owner = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)owner; - (void)attr; - (void)self_or_null; + owner = stack_pointer[-1]; + materialize(&owner); attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; stack_pointer += (oparg & 1); @@ -1416,16 +1404,16 @@ right = &stack_pointer[(oparg & 0xFF)]; case _GUARD_TYPE_VERSION: { _Py_UopsPESlot owner; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); break; } case _CHECK_MANAGED_OBJECT_HAS_VALUES: { _Py_UopsPESlot owner; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); break; } @@ -1434,8 +1422,8 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); attr = sym_new_not_null(ctx); null = sym_new_null(ctx); @@ -1448,8 +1436,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CHECK_ATTR_MODULE: { _Py_UopsPESlot owner; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); break; } @@ -1458,8 +1446,8 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); attr = sym_new_not_null(ctx); null = sym_new_null(ctx); @@ -1472,8 +1460,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CHECK_ATTR_WITH_HINT: { _Py_UopsPESlot owner; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); break; } @@ -1482,8 +1470,8 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); attr = sym_new_not_null(ctx); null = sym_new_null(ctx); @@ -1498,8 +1486,8 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); attr = sym_new_not_null(ctx); null = sym_new_null(ctx); @@ -1512,8 +1500,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CHECK_ATTR_CLASS: { _Py_UopsPESlot owner; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); break; } @@ -1522,8 +1510,8 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); attr = sym_new_not_null(ctx); null = sym_new_null(ctx); @@ -1537,7 +1525,6 @@ right = &stack_pointer[(oparg & 0xFF)]; case _LOAD_ATTR_PROPERTY_FRAME: { _Py_UopsPESlot owner; _Py_UopsPESlot new_frame; - owner = stack_pointer[-1]; PyObject *fget = (PyObject *)this_instr->operand; MATERIALIZE_INST(); materialize(&owner); @@ -1551,8 +1538,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _GUARD_DORV_NO_DICT: { _Py_UopsPESlot owner; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); break; } @@ -1560,10 +1547,10 @@ right = &stack_pointer[(oparg & 0xFF)]; case _STORE_ATTR_INSTANCE_VALUE: { _Py_UopsPESlot owner; _Py_UopsPESlot value; - owner = stack_pointer[-1]; - value = stack_pointer[-2]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); + value = stack_pointer[-2]; materialize(&value); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1573,12 +1560,12 @@ right = &stack_pointer[(oparg & 0xFF)]; case _STORE_ATTR_WITH_HINT: { _Py_UopsPESlot owner; _Py_UopsPESlot value; + MATERIALIZE_INST(); owner = stack_pointer[-1]; + materialize(&owner); value = stack_pointer[-2]; - MATERIALIZE_INST(); + materialize(&value); materialize_ctx(ctx); - (void)owner; - (void)value; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -1587,10 +1574,10 @@ right = &stack_pointer[(oparg & 0xFF)]; case _STORE_ATTR_SLOT: { _Py_UopsPESlot owner; _Py_UopsPESlot value; - owner = stack_pointer[-1]; - value = stack_pointer[-2]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); + value = stack_pointer[-2]; materialize(&value); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1601,14 +1588,13 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot res; + MATERIALIZE_INST(); right = stack_pointer[-1]; + materialize(&right); left = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)right; - (void)left; - (void)res; + materialize(&left); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1619,10 +1605,10 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -1635,10 +1621,10 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -1651,10 +1637,10 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -1667,10 +1653,10 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; MATERIALIZE_INST(); + right = stack_pointer[-1]; materialize(&right); + left = stack_pointer[-2]; materialize(&left); b = sym_new_not_null(ctx); stack_pointer[-2] = b; @@ -1683,14 +1669,13 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot b; + MATERIALIZE_INST(); right = stack_pointer[-1]; + materialize(&right); left = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)right; - (void)left; - (void)b; + materialize(&left); b = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1701,14 +1686,13 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot b; + MATERIALIZE_INST(); right = stack_pointer[-1]; + materialize(&right); left = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)right; - (void)left; - (void)b; + materialize(&left); b = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1719,14 +1703,13 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot b; + MATERIALIZE_INST(); right = stack_pointer[-1]; + materialize(&right); left = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)right; - (void)left; - (void)b; + materialize(&left); b = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1738,16 +1721,14 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot exc_value_st; _Py_UopsPESlot rest; _Py_UopsPESlot match; + MATERIALIZE_INST(); match_type_st = stack_pointer[-1]; + materialize(&match_type_st); exc_value_st = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)match_type_st; - (void)exc_value_st; - (void)rest; - (void)match; + materialize(&exc_value_st); rest = sym_new_not_null(ctx); match = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2] = rest; stack_pointer[-1] = match; break; @@ -1757,15 +1738,13 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot right; _Py_UopsPESlot left; _Py_UopsPESlot b; + MATERIALIZE_INST(); right = stack_pointer[-1]; + materialize(&right); left = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)right; - (void)left; - (void)left; - (void)b; + materialize(&left); b = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = b; break; } @@ -1774,14 +1753,13 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot fromlist; _Py_UopsPESlot level; _Py_UopsPESlot res; + MATERIALIZE_INST(); fromlist = stack_pointer[-1]; + materialize(&fromlist); level = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)fromlist; - (void)level; - (void)res; + materialize(&level); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1791,13 +1769,11 @@ right = &stack_pointer[(oparg & 0xFF)]; case _IMPORT_FROM: { _Py_UopsPESlot from; _Py_UopsPESlot res; - from = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)from; - (void)from; - (void)res; + from = stack_pointer[-1]; + materialize(&from); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1811,8 +1787,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _IS_NONE: { _Py_UopsPESlot value; _Py_UopsPESlot b; - value = stack_pointer[-1]; MATERIALIZE_INST(); + value = stack_pointer[-1]; materialize(&value); b = sym_new_not_null(ctx); stack_pointer[-1] = b; @@ -1822,13 +1798,11 @@ right = &stack_pointer[(oparg & 0xFF)]; case _GET_LEN: { _Py_UopsPESlot obj; _Py_UopsPESlot len; - obj = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)obj; - (void)obj; - (void)len; + obj = stack_pointer[-1]; + materialize(&obj); len = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[0] = len; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1840,16 +1814,15 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot type; _Py_UopsPESlot subject; _Py_UopsPESlot attrs; + MATERIALIZE_INST(); names = stack_pointer[-1]; + materialize(&names); type = stack_pointer[-2]; + materialize(&type); subject = stack_pointer[-3]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)names; - (void)type; - (void)subject; - (void)attrs; + materialize(&subject); attrs = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1859,8 +1832,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _MATCH_MAPPING: { _Py_UopsPESlot subject; _Py_UopsPESlot res; - subject = stack_pointer[-1]; MATERIALIZE_INST(); + subject = stack_pointer[-1]; materialize(&subject); res = sym_new_not_null(ctx); stack_pointer[0] = res; @@ -1872,8 +1845,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _MATCH_SEQUENCE: { _Py_UopsPESlot subject; _Py_UopsPESlot res; - subject = stack_pointer[-1]; MATERIALIZE_INST(); + subject = stack_pointer[-1]; materialize(&subject); res = sym_new_not_null(ctx); stack_pointer[0] = res; @@ -1886,16 +1859,13 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot keys; _Py_UopsPESlot subject; _Py_UopsPESlot values_or_none; + MATERIALIZE_INST(); keys = stack_pointer[-1]; + materialize(&keys); subject = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)keys; - (void)subject; - (void)subject; - (void)keys; - (void)values_or_none; + materialize(&subject); values_or_none = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1905,12 +1875,11 @@ right = &stack_pointer[(oparg & 0xFF)]; case _GET_ITER: { _Py_UopsPESlot iterable; _Py_UopsPESlot iter; - iterable = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)iterable; - (void)iter; + iterable = stack_pointer[-1]; + materialize(&iterable); iter = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = iter; break; } @@ -1918,12 +1887,11 @@ right = &stack_pointer[(oparg & 0xFF)]; case _GET_YIELD_FROM_ITER: { _Py_UopsPESlot iterable; _Py_UopsPESlot iter; - iterable = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)iterable; - (void)iter; + iterable = stack_pointer[-1]; + materialize(&iterable); iter = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = iter; break; } @@ -1933,13 +1901,11 @@ right = &stack_pointer[(oparg & 0xFF)]; case _FOR_ITER_TIER_TWO: { _Py_UopsPESlot iter; _Py_UopsPESlot next; - iter = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)iter; - (void)iter; - (void)next; + iter = stack_pointer[-1]; + materialize(&iter); next = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1950,8 +1916,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _ITER_CHECK_LIST: { _Py_UopsPESlot iter; - iter = stack_pointer[-1]; MATERIALIZE_INST(); + iter = stack_pointer[-1]; materialize(&iter); break; } @@ -1960,8 +1926,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _GUARD_NOT_EXHAUSTED_LIST: { _Py_UopsPESlot iter; - iter = stack_pointer[-1]; MATERIALIZE_INST(); + iter = stack_pointer[-1]; materialize(&iter); break; } @@ -1969,8 +1935,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _ITER_NEXT_LIST: { _Py_UopsPESlot iter; _Py_UopsPESlot next; - iter = stack_pointer[-1]; MATERIALIZE_INST(); + iter = stack_pointer[-1]; materialize(&iter); next = sym_new_not_null(ctx); stack_pointer[0] = next; @@ -1981,8 +1947,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _ITER_CHECK_TUPLE: { _Py_UopsPESlot iter; - iter = stack_pointer[-1]; MATERIALIZE_INST(); + iter = stack_pointer[-1]; materialize(&iter); break; } @@ -1991,8 +1957,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _GUARD_NOT_EXHAUSTED_TUPLE: { _Py_UopsPESlot iter; - iter = stack_pointer[-1]; MATERIALIZE_INST(); + iter = stack_pointer[-1]; materialize(&iter); break; } @@ -2000,8 +1966,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _ITER_NEXT_TUPLE: { _Py_UopsPESlot iter; _Py_UopsPESlot next; - iter = stack_pointer[-1]; MATERIALIZE_INST(); + iter = stack_pointer[-1]; materialize(&iter); next = sym_new_not_null(ctx); stack_pointer[0] = next; @@ -2012,8 +1978,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _ITER_CHECK_RANGE: { _Py_UopsPESlot iter; - iter = stack_pointer[-1]; MATERIALIZE_INST(); + iter = stack_pointer[-1]; materialize(&iter); break; } @@ -2022,8 +1988,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _GUARD_NOT_EXHAUSTED_RANGE: { _Py_UopsPESlot iter; - iter = stack_pointer[-1]; MATERIALIZE_INST(); + iter = stack_pointer[-1]; materialize(&iter); break; } @@ -2031,8 +1997,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _ITER_NEXT_RANGE: { _Py_UopsPESlot iter; _Py_UopsPESlot next; - iter = stack_pointer[-1]; MATERIALIZE_INST(); + iter = stack_pointer[-1]; materialize(&iter); next = sym_new_not_null(ctx); stack_pointer[0] = next; @@ -2054,14 +2020,12 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot self_or_null; - owner = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)owner; - (void)attr; - (void)self_or_null; + owner = stack_pointer[-1]; + materialize(&owner); attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = attr; stack_pointer[0] = self_or_null; stack_pointer += 1; @@ -2077,25 +2041,19 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot exit_func; _Py_UopsPESlot unused_1; _Py_UopsPESlot res; + MATERIALIZE_INST(); val = stack_pointer[-1]; + materialize(&val); unused_0 = stack_pointer[-2]; + materialize(&unused_0); lasti = stack_pointer[-3]; + materialize(&lasti); exit_self = stack_pointer[-4]; + materialize(&exit_self); exit_func = stack_pointer[-5]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)val; - (void)unused_0; - (void)lasti; - (void)exit_self; - (void)exit_func; - (void)exit_func; - (void)exit_self; - (void)lasti; - (void)unused_1; - (void)val; - (void)res; + materialize(&exit_func); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2103,11 +2061,12 @@ right = &stack_pointer[(oparg & 0xFF)]; } case _PUSH_EXC_INFO: { - _Py_UopsPESlot new_exc; + _Py_UopsPESlot exc; _Py_UopsPESlot prev_exc; - new_exc = stack_pointer[-1]; + _Py_UopsPESlot new_exc; MATERIALIZE_INST(); - materialize(&new_exc); + exc = stack_pointer[-1]; + materialize(&exc); prev_exc = sym_new_not_null(ctx); new_exc = sym_new_not_null(ctx); stack_pointer[-1] = prev_exc; @@ -2119,16 +2078,16 @@ right = &stack_pointer[(oparg & 0xFF)]; case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { _Py_UopsPESlot owner; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); break; } case _GUARD_KEYS_VERSION: { _Py_UopsPESlot owner; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); break; } @@ -2137,8 +2096,8 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot self = (_Py_UopsPESlot){NULL, 0}; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); attr = sym_new_not_null(ctx); self = sym_new_not_null(ctx); @@ -2153,8 +2112,8 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot self = (_Py_UopsPESlot){NULL, 0}; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); attr = sym_new_not_null(ctx); self = sym_new_not_null(ctx); @@ -2169,8 +2128,8 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot unused_0 = (_Py_UopsPESlot){NULL, 0}; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; @@ -2181,8 +2140,8 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot unused_0 = (_Py_UopsPESlot){NULL, 0}; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; @@ -2191,8 +2150,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CHECK_ATTR_METHOD_LAZY_DICT: { _Py_UopsPESlot owner; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); break; } @@ -2201,8 +2160,8 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot owner; _Py_UopsPESlot attr; _Py_UopsPESlot self = (_Py_UopsPESlot){NULL, 0}; - owner = stack_pointer[-1]; MATERIALIZE_INST(); + owner = stack_pointer[-1]; materialize(&owner); attr = sym_new_not_null(ctx); self = sym_new_not_null(ctx); @@ -2216,22 +2175,23 @@ right = &stack_pointer[(oparg & 0xFF)]; case _MAYBE_EXPAND_METHOD: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; - _Py_UopsPESlot func; + _Py_UopsPESlot *callable; + _Py_UopsPESlot *func; _Py_UopsPESlot *maybe_self; args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + func = &stack_pointer[-2 - oparg]; maybe_self = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-2]; + callable = &stack_pointer[-1]; MATERIALIZE_INST(); - materialize(&callable); + materialize(&callable[0]); materialize(&self_or_null[0]); for (int _i = oparg; --_i >= 0;) { materialize(&args[_i]); } - func = sym_new_not_null(ctx); + func[0] = sym_new_not_null(ctx); maybe_self[0] = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = func; break; } @@ -2242,27 +2202,31 @@ right = &stack_pointer[(oparg & 0xFF)]; case _PY_FRAME_GENERAL: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot new_frame; - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-2]; + callable = &stack_pointer[-1]; MATERIALIZE_INST(); - materialize(&callable); + materialize(&callable[0]); materialize(&self_or_null[0]); for (int _i = oparg; --_i >= 0;) { materialize(&args[_i]); } PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); co = get_code_with_logging((this_instr + 2)); if (co == NULL) { ctx->done = true; break; } - new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL}; - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + _Py_UopsPESlot temp = (_Py_UopsPESlot){ + (_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL}; + new_frame = temp; + stack_pointer[0] = new_frame; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2270,103 +2234,114 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CHECK_FUNCTION_VERSION: { _Py_UopsPESlot *unused_0; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot *unused_1; - unused_0 = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; MATERIALIZE_INST(); + unused_0 = &stack_pointer[-oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&unused_0[_i]); } + self_or_null = &stack_pointer[-1 - oparg]; for (int _i = 1; --_i >= 0;) { materialize(&self_or_null[_i]); } - materialize(&callable); + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } break; } case _CHECK_METHOD_VERSION: { _Py_UopsPESlot *unused_0; _Py_UopsPESlot *null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot *unused_1; - unused_0 = &stack_pointer[-oparg]; - null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; MATERIALIZE_INST(); + unused_0 = &stack_pointer[-oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&unused_0[_i]); } + null = &stack_pointer[-1 - oparg]; for (int _i = 1; --_i >= 0;) { materialize(&null[_i]); } - materialize(&callable); + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } break; } case _EXPAND_METHOD: { _Py_UopsPESlot *unused_0; _Py_UopsPESlot *null; - _Py_UopsPESlot callable; - _Py_UopsPESlot method; + _Py_UopsPESlot *callable; + _Py_UopsPESlot *method; _Py_UopsPESlot *self; _Py_UopsPESlot *unused_1; - unused_0 = &stack_pointer[-oparg]; - null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; MATERIALIZE_INST(); + unused_0 = &stack_pointer[-oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&unused_0[_i]); } + null = &stack_pointer[-1 - oparg]; for (int _i = 1; --_i >= 0;) { materialize(&null[_i]); } - materialize(&callable); - method = sym_new_not_null(ctx); + callable = &stack_pointer[-2 - oparg]; for (int _i = 1; --_i >= 0;) { - self[_i] = sym_new_not_null(ctx); + materialize(&callable[_i]); } - stack_pointer[-2 - oparg] = method; + method = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + method[0] = sym_new_not_null(ctx); + self[0] = sym_new_not_null(ctx); break; } case _CHECK_IS_NOT_PY_CALLABLE: { _Py_UopsPESlot *unused_0; _Py_UopsPESlot *unused_1; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot *unused_2; _Py_UopsPESlot *unused_3; - unused_0 = &stack_pointer[-oparg]; - unused_1 = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; MATERIALIZE_INST(); + unused_0 = &stack_pointer[-oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&unused_0[_i]); } + unused_1 = &stack_pointer[-1 - oparg]; for (int _i = 1; --_i >= 0;) { materialize(&unused_1[_i]); } - materialize(&callable); + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } break; } case _CALL_NON_PY_GENERAL: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); args = &stack_pointer[-oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)args; - (void)self_or_null; - (void)callable; - (void)res; + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2376,46 +2351,48 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { _Py_UopsPESlot *unused_0; _Py_UopsPESlot *null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot *unused_1; - unused_0 = &stack_pointer[-oparg]; - null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; MATERIALIZE_INST(); + unused_0 = &stack_pointer[-oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&unused_0[_i]); } + null = &stack_pointer[-1 - oparg]; for (int _i = 1; --_i >= 0;) { materialize(&null[_i]); } - materialize(&callable); + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } break; } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { _Py_UopsPESlot *unused_0; _Py_UopsPESlot *null; - _Py_UopsPESlot callable; - _Py_UopsPESlot func; + _Py_UopsPESlot *callable; + _Py_UopsPESlot *func; _Py_UopsPESlot *self; _Py_UopsPESlot *unused_1; - unused_0 = &stack_pointer[-oparg]; - null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; MATERIALIZE_INST(); + unused_0 = &stack_pointer[-oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&unused_0[_i]); } + null = &stack_pointer[-1 - oparg]; for (int _i = 1; --_i >= 0;) { materialize(&null[_i]); } - materialize(&callable); - func = sym_new_not_null(ctx); + callable = &stack_pointer[-2 - oparg]; for (int _i = 1; --_i >= 0;) { - self[_i] = sym_new_not_null(ctx); + materialize(&callable[_i]); } - stack_pointer[-2 - oparg] = func; + func = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + func[0] = sym_new_not_null(ctx); + self[0] = sym_new_not_null(ctx); break; } @@ -2427,51 +2404,57 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CHECK_FUNCTION_EXACT_ARGS: { _Py_UopsPESlot *unused_0; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot *unused_1; - unused_0 = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; MATERIALIZE_INST(); + unused_0 = &stack_pointer[-oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&unused_0[_i]); } + self_or_null = &stack_pointer[-1 - oparg]; for (int _i = 1; --_i >= 0;) { materialize(&self_or_null[_i]); } - materialize(&callable); + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } break; } case _CHECK_STACK_SPACE: { _Py_UopsPESlot *unused_0; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot *unused_1; - unused_0 = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; MATERIALIZE_INST(); + unused_0 = &stack_pointer[-oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&unused_0[_i]); } + self_or_null = &stack_pointer[-1 - oparg]; for (int _i = 1; --_i >= 0;) { materialize(&self_or_null[_i]); } - materialize(&callable); + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } break; } case _INIT_CALL_PY_EXACT_ARGS: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot new_frame; args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-2]; + callable = &stack_pointer[-1]; MATERIALIZE_INST(); - materialize(&callable); + materialize(&callable[0]); materialize(&self_or_null[0]); for (int _i = oparg; --_i >= 0;) { materialize(&args[_i]); @@ -2479,6 +2462,8 @@ right = &stack_pointer[(oparg & 0xFF)]; int argcount = oparg; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); co = get_code_with_logging((this_instr + 2)); if (co == NULL) { ctx->done = true; @@ -2491,13 +2476,19 @@ right = &stack_pointer[(oparg & 0xFF)]; args--; argcount++; } + _Py_UopsPESlot temp; if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, args, argcount), NULL}; + temp = (_Py_UopsPESlot){ + (_Py_UopsPESymbol *)frame_new(ctx, co, 0, args, argcount), NULL + }; } else { - new_frame = (_Py_UopsPESlot){(_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL}; + temp = (_Py_UopsPESlot){ + (_Py_UopsPESymbol *)frame_new(ctx, co, 0, NULL, 0), NULL + }; } - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + new_frame = temp; + stack_pointer[0] = new_frame; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2531,9 +2522,11 @@ right = &stack_pointer[(oparg & 0xFF)]; if (first_valid_check_stack == NULL) { first_valid_check_stack = corresponding_check_stack; } - else if (corresponding_check_stack) { - // delete all but the first valid _CHECK_STACK_SPACE - corresponding_check_stack->opcode = _NOP; + else { + if (corresponding_check_stack) { + // delete all but the first valid _CHECK_STACK_SPACE + corresponding_check_stack->opcode = _NOP; + } } corresponding_check_stack = NULL; break; @@ -2544,12 +2537,12 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot null; _Py_UopsPESlot callable; _Py_UopsPESlot res; - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; MATERIALIZE_INST(); + arg = stack_pointer[-1]; materialize(&arg); + null = stack_pointer[-2]; materialize(&null); + callable = stack_pointer[-3]; materialize(&callable); res = sym_new_not_null(ctx); stack_pointer[-3] = res; @@ -2563,16 +2556,15 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot null; _Py_UopsPESlot callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); arg = stack_pointer[-1]; + materialize(&arg); null = stack_pointer[-2]; + materialize(&null); callable = stack_pointer[-3]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)arg; - (void)null; - (void)callable; - (void)res; + materialize(&callable); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2584,16 +2576,15 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot null; _Py_UopsPESlot callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); arg = stack_pointer[-1]; + materialize(&arg); null = stack_pointer[-2]; + materialize(&null); callable = stack_pointer[-3]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)arg; - (void)null; - (void)callable; - (void)res; + materialize(&callable); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2602,39 +2593,40 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CHECK_AND_ALLOCATE_OBJECT: { _Py_UopsPESlot *args; - _Py_UopsPESlot null; - _Py_UopsPESlot callable; - _Py_UopsPESlot self; - _Py_UopsPESlot init; + _Py_UopsPESlot *null; + _Py_UopsPESlot *callable; + _Py_UopsPESlot *init; + _Py_UopsPESlot *self; args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + init = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; uint32_t type_version = (uint32_t)this_instr->operand; + args = &stack_pointer[-2 - oparg]; + null = &stack_pointer[-2]; + callable = &stack_pointer[-1]; (void)type_version; MATERIALIZE_INST(); - materialize(&callable); - materialize(&null); + materialize(&callable[0]); + materialize(&null[0]); for (int _i = oparg; --_i >= 0;) { materialize(&args[_i]); } - self = sym_new_not_null(ctx); - init = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = self; - stack_pointer[-1 - oparg] = init; + self[0] = sym_new_not_null(ctx); + init[0] = sym_new_not_null(ctx); break; } case _CREATE_INIT_FRAME: { _Py_UopsPESlot *args; - _Py_UopsPESlot init; - _Py_UopsPESlot self; + _Py_UopsPESlot *self; + _Py_UopsPESlot *init; _Py_UopsPESlot init_frame; - args = &stack_pointer[-oparg]; - init = stack_pointer[-1 - oparg]; - self = stack_pointer[-2 - oparg]; + args = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-2]; + init = &stack_pointer[-1]; MATERIALIZE_INST(); - materialize(&self); - materialize(&init); + materialize(&init[0]); + materialize(&self[0]); for (int _i = oparg; --_i >= 0;) { materialize(&args[_i]); } @@ -2648,10 +2640,10 @@ right = &stack_pointer[(oparg & 0xFF)]; case _EXIT_INIT_CHECK: { _Py_UopsPESlot should_be_none; - should_be_none = stack_pointer[-1]; MATERIALIZE_INST(); + should_be_none = stack_pointer[-1]; + materialize(&should_be_none); materialize_ctx(ctx); - (void)should_be_none; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2660,18 +2652,23 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CALL_BUILTIN_CLASS: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); args = &stack_pointer[-oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)args; - (void)self_or_null; - (void)callable; - (void)res; + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2681,18 +2678,23 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CALL_BUILTIN_O: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); args = &stack_pointer[-oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)args; - (void)self_or_null; - (void)callable; - (void)res; + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2702,18 +2704,23 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CALL_BUILTIN_FAST: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); args = &stack_pointer[-oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)args; - (void)self_or_null; - (void)callable; - (void)res; + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2723,18 +2730,23 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); args = &stack_pointer[-oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)args; - (void)self_or_null; - (void)callable; - (void)res; + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2744,18 +2756,23 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CALL_LEN: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); args = &stack_pointer[-oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)args; - (void)self_or_null; - (void)callable; - (void)res; + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2765,18 +2782,23 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CALL_ISINSTANCE: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); args = &stack_pointer[-oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)args; - (void)self_or_null; - (void)callable; - (void)res; + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2787,12 +2809,12 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot arg; _Py_UopsPESlot self; _Py_UopsPESlot callable; - arg = stack_pointer[-1]; - self = stack_pointer[-2]; - callable = stack_pointer[-3]; MATERIALIZE_INST(); + arg = stack_pointer[-1]; materialize(&arg); + self = stack_pointer[-2]; materialize(&self); + callable = stack_pointer[-3]; materialize(&callable); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -2802,18 +2824,23 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CALL_METHOD_DESCRIPTOR_O: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); args = &stack_pointer[-oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)args; - (void)self_or_null; - (void)callable; - (void)res; + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2823,18 +2850,23 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); args = &stack_pointer[-oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)args; - (void)self_or_null; - (void)callable; - (void)res; + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2844,18 +2876,23 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CALL_METHOD_DESCRIPTOR_NOARGS: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); args = &stack_pointer[-oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)args; - (void)self_or_null; - (void)callable; - (void)res; + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2865,18 +2902,23 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CALL_METHOD_DESCRIPTOR_FAST: { _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); args = &stack_pointer[-oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)args; - (void)self_or_null; - (void)callable; - (void)res; + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + callable = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2885,20 +2927,55 @@ right = &stack_pointer[(oparg & 0xFF)]; /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ + case _MAYBE_EXPAND_METHOD_KW: { + _Py_UopsPESlot kwnames_in; + _Py_UopsPESlot *args; + _Py_UopsPESlot *self_or_null; + _Py_UopsPESlot *callable; + _Py_UopsPESlot *func; + _Py_UopsPESlot *maybe_self; + _Py_UopsPESlot kwnames_out; + MATERIALIZE_INST(); + kwnames_in = stack_pointer[-1]; + materialize(&kwnames_in); + args = &stack_pointer[-1 - oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } + self_or_null = &stack_pointer[-2 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + callable = &stack_pointer[-3 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - oparg]; + func[0] = sym_new_not_null(ctx); + maybe_self[0] = sym_new_not_null(ctx); + for (int _i = oparg; --_i >= 0;) { + args[_i] = sym_new_not_null(ctx); + } + kwnames_out = sym_new_not_null(ctx); + stack_pointer[-1] = kwnames_out; + break; + } + /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { _Py_UopsPESlot kwnames; _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot new_frame; - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + args = &stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-2]; + callable = &stack_pointer[-1]; MATERIALIZE_INST(); - materialize(&callable); + materialize(&callable[0]); materialize(&self_or_null[0]); for (int _i = oparg; --_i >= 0;) { materialize(&args[_i]); @@ -2916,21 +2993,23 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot kwnames; _Py_UopsPESlot *unused_0; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot *unused_1; - kwnames = stack_pointer[-1]; - unused_0 = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; MATERIALIZE_INST(); + kwnames = stack_pointer[-1]; materialize(&kwnames); + unused_0 = &stack_pointer[-1 - oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&unused_0[_i]); } + self_or_null = &stack_pointer[-2 - oparg]; for (int _i = 1; --_i >= 0;) { materialize(&self_or_null[_i]); } - materialize(&callable); + callable = &stack_pointer[-3 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } break; } @@ -2938,53 +3017,54 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot kwnames; _Py_UopsPESlot *unused_0; _Py_UopsPESlot *null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot *unused_1; - kwnames = stack_pointer[-1]; - unused_0 = &stack_pointer[-1 - oparg]; - null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; MATERIALIZE_INST(); + kwnames = stack_pointer[-1]; materialize(&kwnames); + unused_0 = &stack_pointer[-1 - oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&unused_0[_i]); } + null = &stack_pointer[-2 - oparg]; for (int _i = 1; --_i >= 0;) { materialize(&null[_i]); } - materialize(&callable); + callable = &stack_pointer[-3 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } break; } case _EXPAND_METHOD_KW: { - _Py_UopsPESlot kwnames; - _Py_UopsPESlot *unused_0; + _Py_UopsPESlot unused_0; + _Py_UopsPESlot *unused_1; _Py_UopsPESlot *null; - _Py_UopsPESlot callable; - _Py_UopsPESlot method; + _Py_UopsPESlot *callable; + _Py_UopsPESlot *method; _Py_UopsPESlot *self; - _Py_UopsPESlot *unused_1; - kwnames = stack_pointer[-1]; - unused_0 = &stack_pointer[-1 - oparg]; - null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - self = &stack_pointer[-2 - oparg]; + _Py_UopsPESlot *unused_2; + _Py_UopsPESlot unused_3; MATERIALIZE_INST(); - materialize(&kwnames); + unused_0 = stack_pointer[-1]; + materialize(&unused_0); + unused_1 = &stack_pointer[-1 - oparg]; for (int _i = oparg; --_i >= 0;) { - materialize(&unused_0[_i]); + materialize(&unused_1[_i]); } + null = &stack_pointer[-2 - oparg]; for (int _i = 1; --_i >= 0;) { materialize(&null[_i]); } - materialize(&callable); - method = sym_new_not_null(ctx); + callable = &stack_pointer[-3 - oparg]; for (int _i = 1; --_i >= 0;) { - self[_i] = sym_new_not_null(ctx); + materialize(&callable[_i]); } - kwnames = sym_new_not_null(ctx); - stack_pointer[-3 - oparg] = method; - stack_pointer[-1] = kwnames; + method = &stack_pointer[-3 - oparg]; + self = &stack_pointer[-2 - oparg]; + method[0] = sym_new_not_null(ctx); + self[0] = sym_new_not_null(ctx); break; } @@ -2992,22 +3072,24 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot kwnames; _Py_UopsPESlot *unused_0; _Py_UopsPESlot *unused_1; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot *unused_2; _Py_UopsPESlot *unused_3; - kwnames = stack_pointer[-1]; - unused_0 = &stack_pointer[-1 - oparg]; - unused_1 = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; MATERIALIZE_INST(); + kwnames = stack_pointer[-1]; materialize(&kwnames); + unused_0 = &stack_pointer[-1 - oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&unused_0[_i]); } + unused_1 = &stack_pointer[-2 - oparg]; for (int _i = 1; --_i >= 0;) { materialize(&unused_1[_i]); } - materialize(&callable); + callable = &stack_pointer[-3 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } break; } @@ -3015,20 +3097,25 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot kwnames; _Py_UopsPESlot *args; _Py_UopsPESlot *self_or_null; - _Py_UopsPESlot callable; + _Py_UopsPESlot *callable; _Py_UopsPESlot res; + MATERIALIZE_INST(); kwnames = stack_pointer[-1]; + materialize(&kwnames); args = &stack_pointer[-1 - oparg]; + for (int _i = oparg; --_i >= 0;) { + materialize(&args[_i]); + } self_or_null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)kwnames; - (void)args; - (void)self_or_null; - (void)callable; - (void)res; + for (int _i = 1; --_i >= 0;) { + materialize(&self_or_null[_i]); + } + callable = &stack_pointer[-3 - oparg]; + for (int _i = 1; --_i >= 0;) { + materialize(&callable[_i]); + } res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3037,33 +3124,56 @@ right = &stack_pointer[(oparg & 0xFF)]; /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ - /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + case _MAKE_CALLARGS_A_TUPLE: { + _Py_UopsPESlot kwargs_in = (_Py_UopsPESlot){NULL, 0}; + _Py_UopsPESlot callargs; + _Py_UopsPESlot unused_0; + _Py_UopsPESlot func; + _Py_UopsPESlot unused_1; + _Py_UopsPESlot tuple; + _Py_UopsPESlot kwargs_out = (_Py_UopsPESlot){NULL, 0}; + MATERIALIZE_INST(); + if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } + materialize(&kwargs_in); + callargs = stack_pointer[-1 - (oparg & 1)]; + materialize(&callargs); + unused_0 = stack_pointer[-2 - (oparg & 1)]; + materialize(&unused_0); + func = stack_pointer[-3 - (oparg & 1)]; + materialize(&func); + tuple = sym_new_not_null(ctx); + kwargs_out = sym_new_not_null(ctx); + materialize_ctx(ctx); + stack_pointer[-1 - (oparg & 1)] = tuple; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_out; + break; + } + + /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { _Py_UopsPESlot codeobj_st; _Py_UopsPESlot func; - codeobj_st = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)codeobj_st; - (void)func; + codeobj_st = stack_pointer[-1]; + materialize(&codeobj_st); func = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - _Py_UopsPESlot func_st; + _Py_UopsPESlot func_in; _Py_UopsPESlot attr_st; - func_st = stack_pointer[-1]; - attr_st = stack_pointer[-2]; + _Py_UopsPESlot func_out; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)func_st; - (void)attr_st; - (void)func_st; - func_st = sym_new_not_null(ctx); - stack_pointer[-2] = func_st; + func_in = stack_pointer[-1]; + materialize(&func_in); + attr_st = stack_pointer[-2]; + materialize(&attr_st); + func_out = sym_new_not_null(ctx); + stack_pointer[-2] = func_out; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -3083,14 +3193,14 @@ right = &stack_pointer[(oparg & 0xFF)]; assert(framesize > 0); assert(framesize <= curr_space); curr_space -= framesize; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); co = get_code(this_instr); if (co == NULL) { // might be impossible, but bailing is still safe ctx->done = true; } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -3099,12 +3209,12 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot stop; _Py_UopsPESlot start; _Py_UopsPESlot slice; - if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } - stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; - start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; MATERIALIZE_INST(); + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } materialize(&step); + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; materialize(&stop); + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; materialize(&start); slice = sym_new_not_null(ctx); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; @@ -3116,10 +3226,11 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CONVERT_VALUE: { _Py_UopsPESlot value; _Py_UopsPESlot result; - value = stack_pointer[-1]; MATERIALIZE_INST(); + value = stack_pointer[-1]; materialize(&value); result = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = result; break; } @@ -3127,12 +3238,11 @@ right = &stack_pointer[(oparg & 0xFF)]; case _FORMAT_SIMPLE: { _Py_UopsPESlot value; _Py_UopsPESlot res; - value = stack_pointer[-1]; MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)value; - (void)res; + value = stack_pointer[-1]; + materialize(&value); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-1] = res; break; } @@ -3141,14 +3251,13 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot fmt_spec; _Py_UopsPESlot value; _Py_UopsPESlot res; + MATERIALIZE_INST(); fmt_spec = stack_pointer[-1]; + materialize(&fmt_spec); value = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)fmt_spec; - (void)value; - (void)res; + materialize(&value); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3160,12 +3269,12 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot bottom; _Py_UopsPESlot *unused_1; _Py_UopsPESlot top; - unused_0 = &stack_pointer[-(oparg-1)]; - bottom = stack_pointer[-1 - (oparg-1)]; MATERIALIZE_INST(); + unused_0 = &stack_pointer[-(oparg-1)]; for (int _i = oparg-1; --_i >= 0;) { materialize(&unused_0[_i]); } + bottom = stack_pointer[-1 - (oparg-1)]; materialize(&bottom); top = sym_new_not_null(ctx); stack_pointer[0] = top; @@ -3178,14 +3287,13 @@ right = &stack_pointer[(oparg & 0xFF)]; _Py_UopsPESlot rhs; _Py_UopsPESlot lhs; _Py_UopsPESlot res; + MATERIALIZE_INST(); rhs = stack_pointer[-1]; + materialize(&rhs); lhs = stack_pointer[-2]; - MATERIALIZE_INST(); - materialize_ctx(ctx); - (void)rhs; - (void)lhs; - (void)res; + materialize(&lhs); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3193,23 +3301,25 @@ right = &stack_pointer[(oparg & 0xFF)]; } case _SWAP: { - _Py_UopsPESlot top; + _Py_UopsPESlot top_in; _Py_UopsPESlot *unused_0; - _Py_UopsPESlot bottom; + _Py_UopsPESlot bottom_in; + _Py_UopsPESlot top_out; _Py_UopsPESlot *unused_1; - top = stack_pointer[-1]; - unused_0 = &stack_pointer[-1 - (oparg-2)]; - bottom = stack_pointer[-2 - (oparg-2)]; + _Py_UopsPESlot bottom_out; MATERIALIZE_INST(); - materialize(&top); + top_in = stack_pointer[-1]; + materialize(&top_in); + unused_0 = &stack_pointer[-1 - (oparg-2)]; for (int _i = oparg-2; --_i >= 0;) { materialize(&unused_0[_i]); } - materialize(&bottom); - top = sym_new_not_null(ctx); - bottom = sym_new_not_null(ctx); - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; + bottom_in = stack_pointer[-2 - (oparg-2)]; + materialize(&bottom_in); + top_out = sym_new_not_null(ctx); + bottom_out = sym_new_not_null(ctx); + stack_pointer[-2 - (oparg-2)] = top_out; + stack_pointer[-1] = bottom_out; break; } @@ -3231,8 +3341,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _GUARD_IS_TRUE_POP: { _Py_UopsPESlot flag; - flag = stack_pointer[-1]; MATERIALIZE_INST(); + flag = stack_pointer[-1]; materialize(&flag); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3241,8 +3351,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _GUARD_IS_FALSE_POP: { _Py_UopsPESlot flag; - flag = stack_pointer[-1]; MATERIALIZE_INST(); + flag = stack_pointer[-1]; materialize(&flag); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3251,8 +3361,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _GUARD_IS_NONE_POP: { _Py_UopsPESlot val; - val = stack_pointer[-1]; MATERIALIZE_INST(); + val = stack_pointer[-1]; materialize(&val); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3261,8 +3371,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _GUARD_IS_NOT_NONE_POP: { _Py_UopsPESlot val; - val = stack_pointer[-1]; MATERIALIZE_INST(); + val = stack_pointer[-1]; materialize(&val); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3284,7 +3394,6 @@ right = &stack_pointer[(oparg & 0xFF)]; case _CHECK_STACK_SPACE_OPERAND: { uint32_t framesize = (uint32_t)this_instr->operand; MATERIALIZE_INST(); - (void)framesize; break; } @@ -3334,8 +3443,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _POP_TOP_LOAD_CONST_INLINE_BORROW: { _Py_UopsPESlot pop; _Py_UopsPESlot value; - pop = stack_pointer[-1]; MATERIALIZE_INST(); + pop = stack_pointer[-1]; materialize(&pop); value = sym_new_not_null(ctx); stack_pointer[-1] = value; @@ -3373,10 +3482,36 @@ right = &stack_pointer[(oparg & 0xFF)]; break; } + case _LOAD_GLOBAL_MODULE: { + _Py_UopsPESlot res; + _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; + MATERIALIZE_INST(); + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_GLOBAL_BUILTINS: { + _Py_UopsPESlot res; + _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; + MATERIALIZE_INST(); + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _INTERNAL_INCREMENT_OPT_COUNTER: { _Py_UopsPESlot opt; - opt = stack_pointer[-1]; MATERIALIZE_INST(); + opt = stack_pointer[-1]; materialize(&opt); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3416,8 +3551,8 @@ right = &stack_pointer[(oparg & 0xFF)]; case _ERROR_POP_N: { _Py_UopsPESlot *unused_0; - unused_0 = &stack_pointer[-oparg]; MATERIALIZE_INST(); + unused_0 = &stack_pointer[-oparg]; for (int _i = oparg; --_i >= 0;) { materialize(&unused_0[_i]); } diff --git a/Python/partial_evaluator_cases.c.h.new b/Python/partial_evaluator_cases.c.h.new deleted file mode 100644 index 893ac915a979c2..00000000000000 --- a/Python/partial_evaluator_cases.c.h.new +++ /dev/null @@ -1,70 +0,0 @@ -// This file is generated by Tools/cases_generator/partial_evaluator_generator.py -// from: -// Python/partial_evaluator_bytecodes.c -// Do not edit! - - case _NOP: { - break; - } - - case _CHECK_PERIODIC: { - MATERIALIZE_INST(); - materialize_ctx(ctx); - break; - } - - case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { - MATERIALIZE_INST(); - materialize_ctx(ctx); - break; - } - - /* _QUICKEN_RESUME is not a viable micro-op for tier 2 */ - - case _RESUME_CHECK: { - MATERIALIZE_INST(); - break; - } - - /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ - - case _LOAD_FAST_CHECK: { - _Py_UopsPESlot value; - MATERIALIZE_INST(); - value = GETLOCAL(oparg); - // We guarantee this will error - just bail and don't optimize it. - if (sym_is_null(&value)) { - ctx->done = true; - } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_FAST: { - _Py_UopsPESlot value; - value = GETLOCAL(oparg); - sym_set_origin_inst_override(&value, this_instr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_FAST_AND_CLEAR: { - _Py_UopsPESlot value; - MATERIALIZE_INST(); - value = GETLOCAL(oparg); - GETLOCAL(oparg) = sym_new_null(ctx); - sym_set_origin_inst_override(&value, this_instr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_CONST: { - _Py_UopsPESlot value; - // Should've all been converted by specializer. - Py_UNREACHABLE(); \ No newline at end of file diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py index 57df99f6506a3e..478933af3a03f8 100644 --- a/Tools/cases_generator/partial_evaluator_generator.py +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -81,33 +81,43 @@ def declare_variables(uop: Uop, out: CWriter) -> None: def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: out.emit("MATERIALIZE_INST();\n") unused_count = 0 - if uop.properties.escapes: - out.emit("materialize_ctx(ctx);\n") - for var in reversed(uop.stack.inputs): - name, unused_count = var_name(var, unused_count) - out.emit(f"(void){name};\n") - for var in uop.stack.outputs: - name, unused_count = var_name(var, unused_count) - out.emit(f"(void){name};\n") - else: - for var in reversed(uop.stack.inputs): - name, unused_count = var_name(var, unused_count) - if var.is_array(): - out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"materialize(&{name}[_i]);\n") - out.emit("}\n") - else: - out.emit(f"materialize(&{name});\n") + for var in reversed(uop.stack.inputs): + name, unused_count = var_name(var, unused_count) + old_var_name = var.name + var.name = name + assign, _ = stack.pop(var, assign_unused=True) + var.name = old_var_name + out.emit(assign) + if var.is_array(): + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"materialize(&{name}[_i]);\n") + out.emit("}\n") + else: + out.emit(f"materialize(&{name});\n") + top_offset = stack.top_offset.copy() for var in uop.stack.outputs: + if var.is_array() and not var.peek and not var.name == "unused": + c_offset = top_offset.to_c() + out.emit(f"{var.name} = &stack_pointer[{c_offset}];\n") + top_offset.push(var) + for var in uop.stack.outputs: + local = Local.undefined(var) + stack.push(local) if var.name != "unused" and not var.peek: + local.defined = True if var.is_array(): - out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") - out.emit("}\n") + if var.size == "1": + out.emit(f"{var.name}[0] = sym_new_not_null(ctx);\n") + else: + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") + out.emit("}\n") elif var.name == "null": out.emit(f"{var.name} = sym_new_null(ctx);\n") else: out.emit(f"{var.name} = sym_new_not_null(ctx);\n") + if uop.properties.escapes: + out.emit("materialize_ctx(ctx);\n") class Tier2PEEmitter(Emitter): @@ -143,6 +153,11 @@ def materialize_inputs( else: self.out.emit(f"materialize(&{var.name});\n") + def emit_save(self, storage: Storage) -> None: + storage.flush(self.out) + + def emit_reload(self, storage: Storage) -> None: + pass def write_uop( override: Uop | None, @@ -151,7 +166,6 @@ def write_uop( stack: Stack, debug: bool, ) -> None: - locals: dict[str, Local] = {} prototype = override if override else uop try: out.start_line() @@ -179,6 +193,12 @@ def write_uop( # No reference management of inputs needed. for var in storage.inputs: # type: ignore[possibly-undefined] var.defined = False + base_offset = stack.base_offset.copy() + for var in reversed(uop.stack.inputs): + if var.is_array(): + c_offset = base_offset.to_c() + out.emit(f"{var.name} = &stack_pointer[{c_offset}];\n") + base_offset.push(var) storage = emitter.emit_tokens(override, storage, None) out.start_line() storage.flush(out, cast_type="", extract_bits=False) diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 55cab96d5a0721..670606c14eb4d9 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -299,7 +299,7 @@ def _do_emit( cast_type: str = "uintptr_t", extract_bits: bool = True, ) -> None: - cast = f"({cast_type})" if var.type else "" + cast = f"({cast_type})" if (var.type and cast_type) else "" bits = ".bits" if cast and extract_bits else "" if var.condition == "0": return From 5c577434bc6fdc2efd185bb87f3724fdf5a89d8c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 11 Oct 2024 22:32:23 +0800 Subject: [PATCH 37/46] fix mypy --- .../partial_evaluator_generator.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py index 478933af3a03f8..a373a3d6250d22 100644 --- a/Tools/cases_generator/partial_evaluator_generator.py +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -18,6 +18,7 @@ ROOT, write_header, Emitter, + TokenIterator, ) from cwriter import CWriter from typing import TextIO, Iterator @@ -48,11 +49,11 @@ def type_name(var: StackItem) -> str: def var_name(var: StackItem, unused_count: int) -> tuple[str, int]: if var.name == "unused": - var = f"unused_{unused_count}" + name = f"unused_{unused_count}" unused_count += 1 else: - var = var.name - return var, unused_count + name = var.name + return name, unused_count def declare_variables(uop: Uop, out: CWriter) -> None: @@ -123,14 +124,14 @@ def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: class Tier2PEEmitter(Emitter): def __init__(self, out: CWriter): super().__init__(out) - self._replacers["MATERIALIZE_INPUTS"] = self.materialize_inputs + self._replacers["MATERIALIZE_INPUTS"] = self.materialize_inputs # type: ignore[assignment] def materialize_inputs( self, tkn: Token, - tkn_iter: Iterator[Token], + tkn_iter: TokenIterator, uop: Uop, - stack: Stack, + storage: Storage, inst: Instruction | None, ) -> None: next(tkn_iter) @@ -194,11 +195,11 @@ def write_uop( for var in storage.inputs: # type: ignore[possibly-undefined] var.defined = False base_offset = stack.base_offset.copy() - for var in reversed(uop.stack.inputs): - if var.is_array(): + for input in reversed(uop.stack.inputs): + if input.is_array(): c_offset = base_offset.to_c() - out.emit(f"{var.name} = &stack_pointer[{c_offset}];\n") - base_offset.push(var) + out.emit(f"{input.name} = &stack_pointer[{c_offset}];\n") + base_offset.push(input) storage = emitter.emit_tokens(override, storage, None) out.start_line() storage.flush(out, cast_type="", extract_bits=False) From 3b024787962b06291051440e0800525d7aa490a6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 11 Oct 2024 22:59:31 +0800 Subject: [PATCH 38/46] fix thing --- Python/partial_evaluator_cases.c.h | 16 ++++++++++++++++ .../partial_evaluator_generator.py | 4 +++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index 8a87f4b7e45c3c..b4870b581a477e 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -79,6 +79,7 @@ case _STORE_FAST: { _Py_UopsPESlot value; value = stack_pointer[-1]; + value = stack_pointer[-1]; _PyUOpInstruction *origin = sym_get_origin(&value); // Gets rid of things like x = x. if (sym_is_virtual(&value) && @@ -104,6 +105,7 @@ case _POP_TOP: { _Py_UopsPESlot value; value = stack_pointer[-1]; + value = stack_pointer[-1]; if (!sym_is_virtual(&value)) { MATERIALIZE_INST(); } @@ -584,6 +586,8 @@ _Py_UopsPESlot sub; _Py_UopsPESlot container; _Py_UopsPESlot new_frame; + sub = stack_pointer[-2]; + container = stack_pointer[-1]; MATERIALIZE_INST(); materialize(&container); materialize(&sub); @@ -731,6 +735,7 @@ _Py_UopsPESlot retval; _Py_UopsPESlot res; retval = stack_pointer[-1]; + retval = stack_pointer[-1]; MATERIALIZE_INST(); materialize(&retval); stack_pointer += -1; @@ -801,6 +806,8 @@ _Py_UopsPESlot v; _Py_UopsPESlot receiver; _Py_UopsPESlot gen_frame; + v = stack_pointer[0]; + receiver = stack_pointer[1]; MATERIALIZE_INST(); // We are about to hit the end of the trace: ctx->done = true; @@ -810,6 +817,7 @@ case _YIELD_VALUE: { _Py_UopsPESlot retval; _Py_UopsPESlot value; + retval = stack_pointer[-1]; MATERIALIZE_INST(); materialize(&retval); value = sym_new_unknown(ctx); @@ -870,6 +878,7 @@ _Py_UopsPESlot seq; _Py_UopsPESlot *output; output = &stack_pointer[-1]; + seq = stack_pointer[-1]; /* This has to be done manually */ MATERIALIZE_INST(); materialize(&seq); @@ -934,6 +943,7 @@ _Py_UopsPESlot *right; left = &stack_pointer[-1]; right = &stack_pointer[(oparg & 0xFF)]; + seq = stack_pointer[-1]; /* This has to be done manually */ MATERIALIZE_INST(); materialize(&seq); @@ -1057,6 +1067,7 @@ _Py_UopsPESlot res; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; uint16_t index = (uint16_t)this_instr->operand; + globals_keys = stack_pointer[-1]; (void)index; MATERIALIZE_INST(); materialize(&globals_keys); @@ -1074,6 +1085,7 @@ _Py_UopsPESlot res; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; uint16_t index = (uint16_t)this_instr->operand; + builtins_keys = stack_pointer[-1]; (void)index; MATERIALIZE_INST(); materialize(&builtins_keys); @@ -1526,6 +1538,7 @@ _Py_UopsPESlot owner; _Py_UopsPESlot new_frame; PyObject *fget = (PyObject *)this_instr->operand; + owner = stack_pointer[-1]; MATERIALIZE_INST(); materialize(&owner); new_frame = (_Py_UopsPESlot){NULL, NULL}; @@ -2010,6 +2023,7 @@ case _FOR_ITER_GEN_FRAME: { _Py_UopsPESlot iter; _Py_UopsPESlot gen_frame; + iter = stack_pointer[0]; MATERIALIZE_INST(); /* We are about to hit the end of the trace */ ctx->done = true; @@ -2496,6 +2510,7 @@ case _PUSH_FRAME: { _Py_UopsPESlot new_frame; new_frame = stack_pointer[-1]; + new_frame = stack_pointer[-1]; MATERIALIZE_INST(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2971,6 +2986,7 @@ _Py_UopsPESlot *self_or_null; _Py_UopsPESlot *callable; _Py_UopsPESlot new_frame; + kwnames = stack_pointer[-3 - oparg]; args = &stack_pointer[-2 - oparg]; self_or_null = &stack_pointer[-2]; callable = &stack_pointer[-1]; diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py index a373a3d6250d22..86b9502f88f2e1 100644 --- a/Tools/cases_generator/partial_evaluator_generator.py +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -196,9 +196,11 @@ def write_uop( var.defined = False base_offset = stack.base_offset.copy() for input in reversed(uop.stack.inputs): + c_offset = base_offset.to_c() if input.is_array(): - c_offset = base_offset.to_c() out.emit(f"{input.name} = &stack_pointer[{c_offset}];\n") + else: + out.emit(f"{input.name} = stack_pointer[{c_offset}];\n") base_offset.push(input) storage = emitter.emit_tokens(override, storage, None) out.start_line() From 4985d3f01db6afcd662c1c93cff74ca9783a18ef Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 12 Oct 2024 23:35:47 +0800 Subject: [PATCH 39/46] Fix mypy signature --- Tools/cases_generator/partial_evaluator_generator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py index 86b9502f88f2e1..e1dadd8d9e3a61 100644 --- a/Tools/cases_generator/partial_evaluator_generator.py +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -124,7 +124,7 @@ def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: class Tier2PEEmitter(Emitter): def __init__(self, out: CWriter): super().__init__(out) - self._replacers["MATERIALIZE_INPUTS"] = self.materialize_inputs # type: ignore[assignment] + self._replacers["MATERIALIZE_INPUTS"] = self.materialize_inputs def materialize_inputs( self, @@ -133,7 +133,7 @@ def materialize_inputs( uop: Uop, storage: Storage, inst: Instruction | None, - ) -> None: + ) -> bool: next(tkn_iter) next(tkn_iter) next(tkn_iter) @@ -153,6 +153,7 @@ def materialize_inputs( self.out.emit(f"materialize(&{var.name});\n") else: self.out.emit(f"materialize(&{var.name});\n") + return False def emit_save(self, storage: Storage) -> None: storage.flush(self.out) From 8c45921077999cb4efba705891084ab800e986b8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:06:34 +0800 Subject: [PATCH 40/46] fix upstream merge --- Include/internal/pycore_opcode_metadata.h | 4 ++-- Python/partial_evaluator_cases.c.h | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 74e347cdd34b19..58e583eabbcc46 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1201,7 +1201,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, - [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1237,7 +1237,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, [SETUP_WITH] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, - [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, }; #endif diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index b4870b581a477e..88e9003295abaf 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -76,6 +76,26 @@ break; } + case _LOAD_CONST_IMMORTAL: { + _Py_UopsPESlot value; + MATERIALIZE_INST(); + value = sym_new_not_null(ctx); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_SMALL_INT: { + _Py_UopsPESlot value; + MATERIALIZE_INST(); + value = sym_new_not_null(ctx); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _STORE_FAST: { _Py_UopsPESlot value; value = stack_pointer[-1]; From 67be67aeec0ba8aae3b30c00e1b1a5d3bce70dde Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:21:44 +0800 Subject: [PATCH 41/46] Add tests --- Lib/test/test_generated_cases.py | 171 +++++++++++++++++- .../partial_evaluator_generator.py | 4 + 2 files changed, 174 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 173e405b785ddc..11338970e80a84 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -34,6 +34,7 @@ def skip_if_different_mount_drives(): from stack import Local, Stack import tier1_generator import optimizer_generator + import partial_evaluator_generator def handle_stderr(): @@ -1431,6 +1432,9 @@ def test_instruction_size_macro(self): class TestGeneratedAbstractCases(unittest.TestCase): + + generator = None + def setUp(self) -> None: super().setUp() self.maxDiff = None @@ -1466,7 +1470,8 @@ def run_cases_test(self, input: str, input2: str, expected: str): temp_input.flush() with handle_stderr(): - optimizer_generator.generate_tier2_abstract_from_files( + assert self.generator is not None + self.generator.generate_tier2_abstract_from_files( [self.temp_input_filename, self.temp_input2_filename], self.temp_output_filename ) @@ -1480,6 +1485,9 @@ def run_cases_test(self, input: str, input2: str, expected: str): actual = "".join(lines) self.assertEqual(actual.strip(), expected.strip()) + +class TestGeneratedOptimizerCases(TestGeneratedAbstractCases): + generator = optimizer_generator def test_overridden_abstract(self): input = """ pure op(OP, (--)) { @@ -1580,5 +1588,166 @@ def test_missing_override_failure(self): self.run_cases_test(input, input2, output) +class TestGeneratedPECases(TestGeneratedAbstractCases): + generator = partial_evaluator_generator + + def test_overridden_abstract(self): + input = """ + pure op(OP, (--)) { + SPAM(); + } + """ + input2 = """ + pure op(OP, (--)) { + eggs(); + } + """ + output = """ + case OP: { + eggs(); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_overridden_abstract_args(self): + input = """ + pure op(OP, (arg1 -- out)) { + out = SPAM(arg1); + } + op(OP2, (arg1 -- out)) { + out = EGGS(arg1); + } + """ + input2 = """ + op(OP, (arg1 -- out)) { + out = EGGS(arg1); + } + """ + output = """ + case OP: { + _Py_UopsPESlot arg1; + _Py_UopsPESlot out; + arg1 = stack_pointer[-1]; + arg1 = stack_pointer[-1]; + out = EGGS(arg1); + stack_pointer[-1] = out; + break; + } + + case OP2: { + _Py_UopsPESlot arg1; + _Py_UopsPESlot out; + MATERIALIZE_INST(); + arg1 = stack_pointer[-1]; + materialize(&arg1); + out = sym_new_not_null(ctx); + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_no_overridden_case(self): + input = """ + pure op(OP, (arg1 -- out)) { + out = SPAM(arg1); + } + + pure op(OP2, (arg1 -- out)) { + } + + """ + input2 = """ + pure op(OP2, (arg1 -- out)) { + out = NULL; + } + """ + output = """ + case OP: { + _Py_UopsPESlot arg1; + _Py_UopsPESlot out; + MATERIALIZE_INST(); + arg1 = stack_pointer[-1]; + materialize(&arg1); + out = sym_new_not_null(ctx); + stack_pointer[-1] = out; + break; + } + + case OP2: { + _Py_UopsPESlot arg1; + _Py_UopsPESlot out; + arg1 = stack_pointer[-1]; + out = NULL; + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_missing_override_failure(self): + input = """ + pure op(OP, (arg1 -- out)) { + SPAM(); + } + """ + input2 = """ + pure op(OTHER, (arg1 -- out)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(AssertionError, "All abstract uops"): + self.run_cases_test(input, input2, output) + + + def test_validate_inputs(self): + input = """ + pure op(OP, (arg1 --)) { + SPAM(); + } + """ + input2 = """ + // Non-matching input! + pure op(OP, (arg1, arg2 --)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(AssertionError, "input length don't match"): + self.run_cases_test(input, input2, output) + + def test_materialize_inputs(self): + input = """ + pure op(OP2, (arg1, arg2, arg3[oparg] --)) { + } + """ + input2 = """ + pure op(OP2, (arg1, arg2, arg3[oparg] --)) { + MATERIALIZE_INPUTS(); + } + """ + output = """ + case OP2: { + _Py_UopsPESlot *arg3; + _Py_UopsPESlot arg2; + _Py_UopsPESlot arg1; + arg3 = &stack_pointer[-2 - oparg]; + arg2 = stack_pointer[-2]; + arg1 = stack_pointer[-1]; + materialize(&arg1); + materialize(&arg2); + for (int _i = oparg; --_i >= 0;) { + materialize(&arg3[_i]); + } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + """ + self.run_cases_test(input, input2, output) + + if __name__ == "__main__": unittest.main() diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py index e1dadd8d9e3a61..fdbfb1ceb39a6f 100644 --- a/Tools/cases_generator/partial_evaluator_generator.py +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -29,6 +29,10 @@ DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/partial_evaluator_bytecodes.c").absolute().as_posix() def validate_uop(override: Uop, uop: Uop) -> None: + if len(override.stack.inputs) != len(uop.stack.inputs): + assert False, f"Uop {uop.name} input length don't match." + if len(override.stack.outputs) != len(uop.stack.outputs): + assert False, f"Uop {uop.name} output length don't match." for override_input, uop_input in zip(override.stack.inputs, uop.stack.inputs): if override_input.name != uop_input.name: assert False, f"Uop {uop.name} input names don't match." From 5ee455e6757f96cccf906633f9931364b250e976 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:23:53 +0800 Subject: [PATCH 42/46] Delete 2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst --- .../2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst | 1 - 1 file changed, 1 deletion(-) delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst b/Misc/NEWS.d/next/Core and Builtins/2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst deleted file mode 100644 index 6029db0998ebe9..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst +++ /dev/null @@ -1 +0,0 @@ -Set up a tier 2 partial evaluation pass. Patch by Ken Jin. From 9d8e5746e1beb40197f72ec16a427b24be696876 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 03:25:01 +0000 Subject: [PATCH 43/46] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-10-30-03-24-58.gh-issue-120619.u8o8oB.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-30-03-24-58.gh-issue-120619.u8o8oB.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-30-03-24-58.gh-issue-120619.u8o8oB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-30-03-24-58.gh-issue-120619.u8o8oB.rst new file mode 100644 index 00000000000000..21f7baf43e2c99 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-30-03-24-58.gh-issue-120619.u8o8oB.rst @@ -0,0 +1 @@ +Set up a JIT optimizer partial evaluation pass. Patch by Ken Jin. From aa5f8c2893c741e6aceb51809df7443fb5d40d42 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 4 Nov 2024 08:45:04 +0800 Subject: [PATCH 44/46] fix build problems --- Python/partial_evaluator_bytecodes.c | 8 +++++--- Python/partial_evaluator_cases.c.h | 12 +++++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c index 333bb69260deb3..f231a0364b0d28 100644 --- a/Python/partial_evaluator_bytecodes.c +++ b/Python/partial_evaluator_bytecodes.c @@ -190,19 +190,21 @@ dummy_func(void) { ctx->done = true; } - op(_FOR_ITER_GEN_FRAME, ( -- )) { + op(_FOR_ITER_GEN_FRAME, (iter -- iter, gen_frame)) { MATERIALIZE_INST(); + gen_frame = (_Py_UopsPESlot){NULL, NULL}; /* We are about to hit the end of the trace */ ctx->done = true; } - op(_SEND_GEN_FRAME, ( -- )) { + op(_SEND_GEN_FRAME, (receiver, v -- receiver, gen_frame)) { + gen_frame = (_Py_UopsPESlot){NULL, NULL}; MATERIALIZE_INST(); // We are about to hit the end of the trace: ctx->done = true; } - op(_PUSH_FRAME, (new_frame -- unused if (0))) { + op(_PUSH_FRAME, (new_frame --)) { MATERIALIZE_INST(); SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index 88e9003295abaf..069c15bd1e69b0 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -826,11 +826,13 @@ _Py_UopsPESlot v; _Py_UopsPESlot receiver; _Py_UopsPESlot gen_frame; - v = stack_pointer[0]; - receiver = stack_pointer[1]; + v = stack_pointer[-2]; + receiver = stack_pointer[-1]; + gen_frame = (_Py_UopsPESlot){NULL, NULL}; MATERIALIZE_INST(); // We are about to hit the end of the trace: ctx->done = true; + stack_pointer[-1] = gen_frame; break; } @@ -2043,10 +2045,14 @@ case _FOR_ITER_GEN_FRAME: { _Py_UopsPESlot iter; _Py_UopsPESlot gen_frame; - iter = stack_pointer[0]; + iter = stack_pointer[-1]; MATERIALIZE_INST(); + gen_frame = (_Py_UopsPESlot){NULL, NULL}; /* We are about to hit the end of the trace */ ctx->done = true; + stack_pointer[0] = gen_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } From 93481151dfff2a22393259ec9daacb802f74916b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 4 Dec 2024 23:05:13 +0800 Subject: [PATCH 45/46] Fix changes from upstream --- Python/partial_evaluator.c | 4 +-- Python/partial_evaluator_cases.c.h | 25 +++++++++++++------ .../partial_evaluator_generator.py | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Python/partial_evaluator.c b/Python/partial_evaluator.c index 6c8986040a59a2..89cf2a43ceeaa5 100644 --- a/Python/partial_evaluator.c +++ b/Python/partial_evaluator.c @@ -70,7 +70,7 @@ get_code(_PyUOpInstruction *op) { assert(op->opcode == _PUSH_FRAME || op->opcode == _RETURN_VALUE || op->opcode == _RETURN_GENERATOR); PyCodeObject *co = NULL; - uint64_t operand = op->operand; + uint64_t operand = op->operand0; if (operand == 0) { return NULL; } @@ -90,7 +90,7 @@ static PyCodeObject * get_code_with_logging(_PyUOpInstruction *op) { PyCodeObject *co = NULL; - uint64_t push_operand = op->operand; + uint64_t push_operand = op->operand0; if (push_operand & 1) { co = (PyCodeObject *)(push_operand & ~1); DPRINTF(3, "code=%p ", co); diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h index 069c15bd1e69b0..b7aa63674504cb 100644 --- a/Python/partial_evaluator_cases.c.h +++ b/Python/partial_evaluator_cases.c.h @@ -21,6 +21,8 @@ /* _QUICKEN_RESUME is not a viable micro-op for tier 2 */ + /* _LOAD_BYTECODE is not a viable micro-op for tier 2 */ + case _RESUME_CHECK: { MATERIALIZE_INST(); break; @@ -535,6 +537,7 @@ list_st = stack_pointer[-2]; materialize(&list_st); res = sym_new_not_null(ctx); + materialize_ctx(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1088,7 +1091,7 @@ _Py_UopsPESlot globals_keys; _Py_UopsPESlot res; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; - uint16_t index = (uint16_t)this_instr->operand; + uint16_t index = (uint16_t)this_instr->operand0; globals_keys = stack_pointer[-1]; (void)index; MATERIALIZE_INST(); @@ -1106,7 +1109,7 @@ _Py_UopsPESlot builtins_keys; _Py_UopsPESlot res; _Py_UopsPESlot null = (_Py_UopsPESlot){NULL, 0}; - uint16_t index = (uint16_t)this_instr->operand; + uint16_t index = (uint16_t)this_instr->operand0; builtins_keys = stack_pointer[-1]; (void)index; MATERIALIZE_INST(); @@ -1559,7 +1562,7 @@ case _LOAD_ATTR_PROPERTY_FRAME: { _Py_UopsPESlot owner; _Py_UopsPESlot new_frame; - PyObject *fget = (PyObject *)this_instr->operand; + PyObject *fget = (PyObject *)this_instr->operand0; owner = stack_pointer[-1]; MATERIALIZE_INST(); materialize(&owner); @@ -2292,6 +2295,11 @@ break; } + case _CHECK_FUNCTION_VERSION_INLINE: { + MATERIALIZE_INST(); + break; + } + case _CHECK_METHOD_VERSION: { _Py_UopsPESlot *unused_0; _Py_UopsPESlot *null; @@ -2641,7 +2649,7 @@ args = &stack_pointer[-oparg]; init = &stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; - uint32_t type_version = (uint32_t)this_instr->operand; + uint32_t type_version = (uint32_t)this_instr->operand0; args = &stack_pointer[-2 - oparg]; null = &stack_pointer[-2]; callable = &stack_pointer[-1]; @@ -3434,7 +3442,7 @@ } case _CHECK_STACK_SPACE_OPERAND: { - uint32_t framesize = (uint32_t)this_instr->operand; + uint32_t framesize = (uint32_t)this_instr->operand0; MATERIALIZE_INST(); break; } @@ -3445,7 +3453,7 @@ } case _EXIT_TRACE: { - PyObject *exit_p = (PyObject *)this_instr->operand; + PyObject *exit_p = (PyObject *)this_instr->operand0; MATERIALIZE_INST(); materialize_ctx(ctx); (void)exit_p; @@ -3460,7 +3468,7 @@ case _LOAD_CONST_INLINE: { _Py_UopsPESlot value; - PyObject *ptr = (PyObject *)this_instr->operand; + PyObject *ptr = (PyObject *)this_instr->operand0; MATERIALIZE_INST(); value = sym_new_const(ctx, ptr); sym_set_origin_inst_override(&value, this_instr); @@ -3472,7 +3480,7 @@ case _LOAD_CONST_INLINE_BORROW: { _Py_UopsPESlot value; - PyObject *ptr = (PyObject *)this_instr->operand; + PyObject *ptr = (PyObject *)this_instr->operand0; MATERIALIZE_INST(); value = sym_new_const(ctx, ptr); sym_set_origin_inst_override(&value, this_instr); @@ -3598,6 +3606,7 @@ for (int _i = oparg; --_i >= 0;) { materialize(&unused_0[_i]); } + materialize_ctx(ctx); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); break; diff --git a/Tools/cases_generator/partial_evaluator_generator.py b/Tools/cases_generator/partial_evaluator_generator.py index fdbfb1ceb39a6f..3840b26c5bca70 100644 --- a/Tools/cases_generator/partial_evaluator_generator.py +++ b/Tools/cases_generator/partial_evaluator_generator.py @@ -193,7 +193,7 @@ def write_uop( else: type = f"uint{cache.size*16}_t " cast = f"uint{cache.size*16}_t" - out.emit(f"{type}{cache.name} = ({cast})this_instr->operand;\n") + out.emit(f"{type}{cache.name} = ({cast})this_instr->operand0;\n") if override: emitter = Tier2PEEmitter(out) # No reference management of inputs needed. From 8270defc5b7f694b9d0147cf77857f57410e2804 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 4 Dec 2024 23:05:37 +0800 Subject: [PATCH 46/46] Update test_opt.py --- Lib/test/test_capi/test_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 753527c46290c1..696c318c4d91aa 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1491,7 +1491,7 @@ def thing(a): res, ex = self._run_with_optimizer(thing, 1) - self.assertEqual(res, 19) + self.assertEqual(res, 4095) self.assertIsNotNone(ex) self.assertEqual(list(iter_opnames(ex)).count("_POP_TOP"), 0) self.assertTrue(ex.is_valid()) @@ -1505,7 +1505,7 @@ def thing(a): res, ex = self._run_with_optimizer(thing, 1) - self.assertEqual(res, 19) + self.assertEqual(res, 4095) self.assertIsNotNone(ex) self.assertEqual(list(iter_opnames(ex)).count("_LOAD_FAST_1"), 0) self.assertTrue(ex.is_valid())