From f55273b3b7124dc570911724107c2440f37905fc Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 7 Oct 2024 11:46:33 +0100 Subject: [PATCH] GH-116968: Remove branch from advance_backoff_counter (GH-124469) --- Include/internal/pycore_backoff.h | 46 ++++++++++++++----------------- Python/bytecodes.c | 4 +-- Python/executor_cases.c.h | 4 +-- Python/instrumentation.c | 8 +++--- 4 files changed, 29 insertions(+), 33 deletions(-) diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index a9d1bce127e63d..20436a68b69677 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -15,13 +15,7 @@ extern "C" { typedef struct { - union { - struct { - uint16_t backoff : 4; - uint16_t value : 12; - }; - uint16_t as_counter; // For printf("%#x", ...) - }; + uint16_t value_and_backoff; } _Py_BackoffCounter; @@ -38,17 +32,19 @@ typedef struct { and a 4-bit 'backoff' field. When resetting the counter, the backoff field is incremented (until it reaches a limit) and the value is set to a bit mask representing the value 2**backoff - 1. - The maximum backoff is 12 (the number of value bits). + The maximum backoff is 12 (the number of bits in the value). There is an exceptional value which must not be updated, 0xFFFF. */ -#define UNREACHABLE_BACKOFF 0xFFFF +#define BACKOFF_BITS 4 +#define MAX_BACKOFF 12 +#define UNREACHABLE_BACKOFF 15 static inline bool is_unreachable_backoff_counter(_Py_BackoffCounter counter) { - return counter.as_counter == UNREACHABLE_BACKOFF; + return counter.value_and_backoff == UNREACHABLE_BACKOFF; } static inline _Py_BackoffCounter @@ -57,8 +53,7 @@ make_backoff_counter(uint16_t value, uint16_t backoff) assert(backoff <= 15); assert(value <= 0xFFF); _Py_BackoffCounter result; - result.value = value; - result.backoff = backoff; + result.value_and_backoff = (value << BACKOFF_BITS) | backoff; return result; } @@ -66,7 +61,7 @@ static inline _Py_BackoffCounter forge_backoff_counter(uint16_t counter) { _Py_BackoffCounter result; - result.as_counter = counter; + result.value_and_backoff = counter; return result; } @@ -74,35 +69,36 @@ static inline _Py_BackoffCounter restart_backoff_counter(_Py_BackoffCounter counter) { assert(!is_unreachable_backoff_counter(counter)); - if (counter.backoff < 12) { - return make_backoff_counter((1 << (counter.backoff + 1)) - 1, counter.backoff + 1); + int backoff = counter.value_and_backoff & 15; + if (backoff < MAX_BACKOFF) { + return make_backoff_counter((1 << (backoff + 1)) - 1, backoff + 1); } else { - return make_backoff_counter((1 << 12) - 1, 12); + return make_backoff_counter((1 << MAX_BACKOFF) - 1, MAX_BACKOFF); } } static inline _Py_BackoffCounter pause_backoff_counter(_Py_BackoffCounter counter) { - return make_backoff_counter(counter.value | 1, counter.backoff); + _Py_BackoffCounter result; + result.value_and_backoff = counter.value_and_backoff | (1 << BACKOFF_BITS); + return result; } static inline _Py_BackoffCounter advance_backoff_counter(_Py_BackoffCounter counter) { - if (!is_unreachable_backoff_counter(counter)) { - return make_backoff_counter((counter.value - 1) & 0xFFF, counter.backoff); - } - else { - return counter; - } + _Py_BackoffCounter result; + result.value_and_backoff = counter.value_and_backoff - (1 << BACKOFF_BITS); + return result; } static inline bool backoff_counter_triggers(_Py_BackoffCounter counter) { - return counter.value == 0; + /* Test whether the value is zero and the backoff is not UNREACHABLE_BACKOFF */ + return counter.value_and_backoff < UNREACHABLE_BACKOFF; } /* Initial JUMP_BACKWARD counter. @@ -136,7 +132,7 @@ initial_temperature_backoff_counter(void) static inline _Py_BackoffCounter initial_unreachable_backoff_counter(void) { - return forge_backoff_counter(UNREACHABLE_BACKOFF); + return make_backoff_counter(0, UNREACHABLE_BACKOFF); } #ifdef __cplusplus diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c712c772201e10..f251b79e00ebe7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4705,7 +4705,7 @@ dummy_func( printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); } @@ -4794,7 +4794,7 @@ dummy_func( printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index fdfec66b73c730..1716d5d8e68827 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -5252,7 +5252,7 @@ printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); } @@ -5390,7 +5390,7 @@ printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 8fd7c08beac92a..e1e494c31a1120 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -643,8 +643,8 @@ de_instrument(PyCodeObject *code, int i, int event) CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, deinstrumented); if (_PyOpcode_Caches[deinstrumented]) { - FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter, - adaptive_counter_warmup().as_counter); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff, + adaptive_counter_warmup().value_and_backoff); } } @@ -719,8 +719,8 @@ instrument(PyCodeObject *code, int i) assert(instrumented); FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented); if (_PyOpcode_Caches[deopt]) { - FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter, - adaptive_counter_warmup().as_counter); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff, + adaptive_counter_warmup().value_and_backoff); instr[1].counter = adaptive_counter_warmup(); } }