Skip to content

Commit

Permalink
Introduce helpers for (un)specializing instructions
Browse files Browse the repository at this point in the history 
Consolidate the code to specialize/unspecialize instructions into
two helper functions and use them in `_Py_Specialize_BinaryOp`.
The resulting code is more concise and keeps all of the logic at
the point where we decide to specialize/unspecialize an instruction.

If folks are amenable to this change, we can incrementally introduce
their use as we work through making each specialization family
thread-safe.
  • Loading branch information
@mpage
mpage committed Nov 4, 2024
1 parent 2e95c5b commit 98883f9
Showing 1 changed file with 88 additions and 48 deletions.
136 changes: 88 additions & 48 deletions Python/specialize.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,25 +24,6 @@ extern const char *_PyUOpName(int index);
* ./adaptive.md
*/

#ifdef Py_GIL_DISABLED
#define SET_OPCODE_OR_RETURN(instr, opcode) \
do { \
uint8_t old_op = _Py_atomic_load_uint8_relaxed(&(instr)->op.code); \
if (old_op >= MIN_INSTRUMENTED_OPCODE) { \
/* Lost race with instrumentation */ \
return; \
} \
if (!_Py_atomic_compare_exchange_uint8(&(instr)->op.code, &old_op, \
(opcode))) { \
/* Lost race with instrumentation */ \
assert(old_op >= MIN_INSTRUMENTED_OPCODE); \
return; \
} \
} while (0)
#else
#define SET_OPCODE_OR_RETURN(instr, opcode) (instr)->op.code = (opcode)
#endif

#ifdef Py_STATS
GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 };
static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats };
Expand Down Expand Up @@ -687,6 +668,77 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts,
#define SPEC_FAIL_CONTAINS_OP_LIST 11
#define SPEC_FAIL_CONTAINS_OP_USER_CLASS 12

static int
set_opcode(_Py_CODEUNIT *instr, uint8_t opcode)
{
#ifdef Py_GIL_DISABLED
uint8_t old_op = _Py_atomic_load_uint8_relaxed(&instr->op.code);
if (old_op >= MIN_INSTRUMENTED_OPCODE) {
/* Lost race with instrumentation */
return 0;
}
if (!_Py_atomic_compare_exchange_uint8(&instr->op.code, &old_op, opcode)) {
/* Lost race with instrumentation */
assert(old_op >= MIN_INSTRUMENTED_OPCODE);
return 0;
}
return 1;
#else
instr->op.code = opcode;
return 1;
#endif
}

static inline void
set_counter(_Py_BackoffCounter *counter, _Py_BackoffCounter value)
{
FT_ATOMIC_STORE_UINT16_RELAXED(counter->value_and_backoff,
value.value_and_backoff);
}

static inline _Py_BackoffCounter
load_counter(_Py_BackoffCounter *counter)
{
_Py_BackoffCounter result = {
.value_and_backoff =
FT_ATOMIC_LOAD_UINT16_RELAXED(counter->value_and_backoff)};
return result;
}

static inline void
specialize(_Py_CODEUNIT *instr, uint8_t specialized_opcode, void *cache)
{
assert(!PyErr_Occurred());
uint8_t generic_opcode = _PyOpcode_Deopt[specialized_opcode];
if (!set_opcode(instr, specialized_opcode)) {
STAT_INC(generic_opcode, failure);
SPECIALIZATION_FAIL(generic_opcode, SPEC_FAIL_OTHER);
return;
}
if (cache != NULL) {
uint8_t num_caches = _PyOpcode_Caches[generic_opcode];
memcpy(instr + 1, cache, sizeof(_Py_CODEUNIT) * num_caches);
}
set_counter((_Py_BackoffCounter *)instr + 1, adaptive_counter_cooldown());
}

static inline void
unspecialize(_Py_CODEUNIT *instr, int reason)
{
assert(!PyErr_Occurred());
uint8_t opcode = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.code);
uint8_t generic_opcode = _PyOpcode_Deopt[opcode];
STAT_INC(generic_opcode, failure);
if (!set_opcode(instr, generic_opcode)) {
SPECIALIZATION_FAIL(generic_opcode, SPEC_FAIL_OTHER);
return;
}
SPECIALIZATION_FAIL(generic_opcode, reason);
_Py_BackoffCounter *counter = (_Py_BackoffCounter *)instr + 1;
_Py_BackoffCounter cur = load_counter(counter);
set_counter(counter, adaptive_counter_backoff(cur));
}

static int function_kind(PyCodeObject *code);
static bool function_check_args(PyObject *o, int expected_argcount, int opcode);
static uint32_t function_get_version(PyObject *o, int opcode);
Expand Down Expand Up @@ -2195,7 +2247,6 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs)
}
}

#ifdef Py_STATS
static int
binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs)
{
Expand Down Expand Up @@ -2263,7 +2314,6 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs)
}
Py_UNREACHABLE();
}
#endif // Py_STATS

void
_Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr,
Expand All @@ -2273,8 +2323,6 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in
PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st);
assert(ENABLE_SPECIALIZATION_FT);
assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP);
_PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1);
uint8_t specialized_op;
switch (oparg) {
case NB_ADD:
case NB_INPLACE_ADD:
Expand All @@ -2285,19 +2333,19 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in
_Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_BINARY_OP + 1];
bool to_store = (next.op.code == STORE_FAST);
if (to_store && PyStackRef_AsPyObjectBorrow(locals[next.op.arg]) == lhs) {
specialized_op = BINARY_OP_INPLACE_ADD_UNICODE;
goto success;
specialize(instr, BINARY_OP_INPLACE_ADD_UNICODE, NULL);
return;
}
specialized_op = BINARY_OP_ADD_UNICODE;
goto success;
specialize(instr, BINARY_OP_ADD_UNICODE, NULL);
return;
}
if (PyLong_CheckExact(lhs)) {
specialized_op = BINARY_OP_ADD_INT;
goto success;
specialize(instr, BINARY_OP_ADD_INT, NULL);
return;
}
if (PyFloat_CheckExact(lhs)) {
specialized_op = BINARY_OP_ADD_FLOAT;
goto success;
specialize(instr, BINARY_OP_ADD_FLOAT, NULL);
return;
}
break;
case NB_MULTIPLY:
Expand All @@ -2306,12 +2354,12 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in
break;
}
if (PyLong_CheckExact(lhs)) {
specialized_op = BINARY_OP_MULTIPLY_INT;
goto success;
specialize(instr, BINARY_OP_MULTIPLY_INT, NULL);
return;
}
if (PyFloat_CheckExact(lhs)) {
specialized_op = BINARY_OP_MULTIPLY_FLOAT;
goto success;
specialize(instr, BINARY_OP_MULTIPLY_FLOAT, NULL);
return;
}
break;
case NB_SUBTRACT:
Expand All @@ -2320,24 +2368,16 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in
break;
}
if (PyLong_CheckExact(lhs)) {
specialized_op = BINARY_OP_SUBTRACT_INT;
goto success;
specialize(instr, BINARY_OP_SUBTRACT_INT, NULL);
return;
}
if (PyFloat_CheckExact(lhs)) {
specialized_op = BINARY_OP_SUBTRACT_FLOAT;
goto success;
specialize(instr, BINARY_OP_SUBTRACT_FLOAT, NULL);
return;
}
break;
}
SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs));
STAT_INC(BINARY_OP, failure);
SET_OPCODE_OR_RETURN(instr, BINARY_OP);
cache->counter = adaptive_counter_backoff(cache->counter);
return;
success:
STAT_INC(BINARY_OP, success);
SET_OPCODE_OR_RETURN(instr, specialized_op);
cache->counter = adaptive_counter_cooldown();
unspecialize(instr, binary_op_fail_kind(oparg, lhs, rhs));
}


Expand Down

0 comments on commit 98883f9

Please sign in to comment.