1berry/cpython
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-132554: Fix tier2 FOR_ITER implementation and optimizations (GH-135137)

Browse Source
This commit is contained in:
Mark Shannon 2025-06-05 18:53:57 +01:00 committed by GitHub
parent d9cad074d5
commit b90ecea9e6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 155 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Include/internal/pycore_ceval.h
View File

@ -353,7 +353,8 @@ PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyS
extern int _PyRunRemoteDebugger(PyThreadState *tstate);
#endif
_PyStackRef _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index);
PyAPI_FUNC(_PyStackRef)
_PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame* frame, _PyStackRef iter, _PyStackRef *index_ptr);
#ifdef __cplusplus
}

42
Include/internal/pycore_stackref.h
View File

@ -62,14 +62,15 @@ PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filenam
extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref);
static const _PyStackRef PyStackRef_NULL = { .index = 0 };
static const _PyStackRef PyStackRef_ERROR = { .index = 2 };
// Use the first 3 even numbers for None, True and False.
// Odd numbers are reserved for (tagged) integers
#define PyStackRef_None ((_PyStackRef){ .index = 2 } )
#define PyStackRef_False ((_PyStackRef){ .index = 4 })
#define PyStackRef_True ((_PyStackRef){ .index = 6 })
#define PyStackRef_None ((_PyStackRef){ .index = 4 } )
#define PyStackRef_False ((_PyStackRef){ .index = 6 })
#define PyStackRef_True ((_PyStackRef){ .index = 8 })
#define INITIAL_STACKREF_INDEX 8
#define INITIAL_STACKREF_INDEX 10
static inline int
PyStackRef_IsNull(_PyStackRef ref)
@ -77,6 +78,19 @@ PyStackRef_IsNull(_PyStackRef ref)
return ref.index == 0;
}
static inline bool
PyStackRef_IsError(_PyStackRef ref)
{
return ref.index == 2;
}
static inline bool
PyStackRef_IsValid(_PyStackRef ref)
{
/* Invalid values are ERROR and NULL */
return !PyStackRef_IsError(ref) && !PyStackRef_IsNull(ref);
}
static inline int
PyStackRef_IsTrue(_PyStackRef ref)
{
@ -104,6 +118,7 @@ PyStackRef_IsTaggedInt(_PyStackRef ref)
static inline PyObject *
_PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber)
{
assert(!PyStackRef_IsError(ref));
assert(!PyStackRef_IsTaggedInt(ref));
_Py_stackref_record_borrow(ref, filename, linenumber);
return _Py_stackref_get_object(ref);
@ -155,6 +170,7 @@ _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber)
static inline void
_PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber)
{
assert(!PyStackRef_IsError(ref));
if (PyStackRef_IsNull(ref)) {
return;
}
@ -165,6 +181,7 @@ _PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber)
static inline _PyStackRef
_PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
{
assert(!PyStackRef_IsError(ref));
if (PyStackRef_IsTaggedInt(ref)) {
return ref;
}
@ -241,9 +258,25 @@ PyStackRef_IsNullOrInt(_PyStackRef ref);
#else
#define Py_INT_TAG 3
#define Py_TAG_INVALID 2
#define Py_TAG_REFCNT 1
#define Py_TAG_BITS 3
static const _PyStackRef PyStackRef_ERROR = { .bits = Py_TAG_INVALID };
static inline bool
PyStackRef_IsError(_PyStackRef ref)
{
return ref.bits == Py_TAG_INVALID;
}
static inline bool
PyStackRef_IsValid(_PyStackRef ref)
{
/* Invalid values are ERROR and NULL */
return ref.bits >= Py_INT_TAG;
}
static inline bool
PyStackRef_IsTaggedInt(_PyStackRef i)
{
@ -284,6 +317,7 @@ PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref)
static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED};
#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits)
#define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED })
#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED })

11
Lib/test/test_capi/test_opt.py
View File

@ -1183,6 +1183,17 @@ class TestUopsOptimization(unittest.TestCase):
self.assertIsNotNone(ex)
self.assertIn("_RETURN_GENERATOR", get_opnames(ex))
def test_for_iter(self):
def testfunc(n):
t = 0
for i in set(range(n)):
t += i
return t
res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD)
self.assertEqual(res, TIER2_THRESHOLD * (TIER2_THRESHOLD - 1) // 2)
self.assertIsNotNone(ex)
self.assertIn("_FOR_ITER_TIER_TWO", get_opnames(ex))
@unittest.skip("Tracing into generators currently isn't supported.")
def test_for_iter_gen(self):
def gen(n):

97
Python/bytecodes.c
View File

@ -3125,100 +3125,49 @@ dummy_func(
}
replaced op(_FOR_ITER, (iter, null_or_index -- iter, null_or_index, next)) {
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (PyStackRef_IsTaggedInt(null_or_index)) {
next = _PyForIter_NextWithIndex(iter_o, null_or_index);
if (PyStackRef_IsNull(next)) {
JUMPBY(oparg + 1);
DISPATCH();
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
if (!PyStackRef_IsValid(item)) {
if (PyStackRef_IsError(item)) {
ERROR_NO_POP();
}
null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
}
else {
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
if (!matches) {
ERROR_NO_POP();
}
_PyEval_MonitorRaise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
}
/* iterator ended normally */
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
/* Jump forward oparg, then skip following END_FOR */
JUMPBY(oparg + 1);
DISPATCH();
}
next = PyStackRef_FromPyObjectSteal(next_o);
// Jump forward by oparg and skip the following END_FOR
JUMPBY(oparg + 1);
DISPATCH();
}
next = item;
}
op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) {
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
EXIT_IF(!PyStackRef_IsNull(null_or_index));
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
if (!matches) {
ERROR_NO_POP();
}
_PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
_PyErr_Clear(tstate);
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
if (!PyStackRef_IsValid(item)) {
if (PyStackRef_IsError(item)) {
ERROR_NO_POP();
}
/* iterator ended normally */
/* The translator sets the deopt target just past the matching END_FOR */
EXIT_IF(true);
}
next = PyStackRef_FromPyObjectSteal(next_o);
// Common case: no jump, leave it to the code generator
next = item;
}
macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER;
inst(INSTRUMENTED_FOR_ITER, (unused/1, iter, null_or_index -- iter, null_or_index, next)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (PyStackRef_IsTaggedInt(null_or_index)) {
next = _PyForIter_NextWithIndex(iter_o, null_or_index);
if (PyStackRef_IsNull(next)) {
JUMPBY(oparg + 1);
DISPATCH();
}
null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
}
else {
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
if (next_o != NULL) {
next = PyStackRef_FromPyObjectSteal(next_o);
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
}
else {
if (_PyErr_Occurred(tstate)) {
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
if (!matches) {
ERROR_NO_POP();
}
_PyEval_MonitorRaise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
}
/* iterator ended normally */
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
/* Skip END_FOR */
JUMPBY(oparg + 1);
DISPATCH();
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
if (!PyStackRef_IsValid(item)) {
if (PyStackRef_IsError(item)) {
ERROR_NO_POP();
}
// Jump forward by oparg and skip the following END_FOR
JUMPBY(oparg + 1);
DISPATCH();
}
next = item;
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
}
op(_ITER_CHECK_LIST, (iter, null_or_index -- iter, null_or_index)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
EXIT_IF(Py_TYPE(iter_o) != &PyList_Type);

28
Python/ceval.c
View File

@ -3439,8 +3439,8 @@ _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *na
return value;
}
_PyStackRef
_PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index)
static _PyStackRef
foriter_next(PyObject *seq, _PyStackRef index)
{
assert(PyStackRef_IsTaggedInt(index));
assert(PyTuple_CheckExact(seq) || PyList_CheckExact(seq));
@ -3459,6 +3459,30 @@ _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index)
return PyStackRef_FromPyObjectSteal(item);
}
_PyStackRef _PyForIter_VirtualIteratorNext(PyThreadState* tstate, _PyInterpreterFrame* frame, _PyStackRef iter, _PyStackRef* index_ptr)
{
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyStackRef index = *index_ptr;
if (PyStackRef_IsTaggedInt(index)) {
*index_ptr = PyStackRef_IncrementTaggedIntNoOverflow(index);
return foriter_next(iter_o, index);
}
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
_PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
_PyErr_Clear(tstate);
}
else {
return PyStackRef_ERROR;
}
}
return PyStackRef_NULL;
}
return PyStackRef_FromPyObjectSteal(next_o);
}
/* Check if a 'cls' provides the given special method. */
static inline int
type_has_special_method(PyTypeObject *cls, PyObject *name)

25
Python/executor_cases.c.h generated
View File

@ -4268,33 +4268,20 @@
_PyStackRef next;
null_or_index = stack_pointer[-1];
iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (!PyStackRef_IsNull(null_or_index)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!matches) {
JUMP_TO_ERROR();
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
_PyErr_Clear(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!PyStackRef_IsValid(item)) {
if (PyStackRef_IsError(item)) {
JUMP_TO_ERROR();
}
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
next = PyStackRef_FromPyObjectSteal(next_o);
next = item;
stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());

91
Python/generated_cases.c.h generated
View File

@ -5753,41 +5753,18 @@
}
// _FOR_ITER
{
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (PyStackRef_IsTaggedInt(null_or_index)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
next = _PyForIter_NextWithIndex(iter_o, null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (PyStackRef_IsNull(next)) {
JUMPBY(oparg + 1);
DISPATCH();
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!PyStackRef_IsValid(item)) {
if (PyStackRef_IsError(item)) {
JUMP_TO_LABEL(error);
}
null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
}
else {
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!matches) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_MonitorRaise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
JUMPBY(oparg + 1);
DISPATCH();
}
next = PyStackRef_FromPyObjectSteal(next_o);
JUMPBY(oparg + 1);
stack_pointer[-1] = null_or_index;
DISPATCH();
}
next = item;
}
stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
@ -7059,45 +7036,19 @@
/* Skip 1 cache entry */
null_or_index = stack_pointer[-1];
iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (PyStackRef_IsTaggedInt(null_or_index)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
next = _PyForIter_NextWithIndex(iter_o, null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (PyStackRef_IsNull(next)) {
JUMPBY(oparg + 1);
DISPATCH();
}
null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
}
else {
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (next_o != NULL) {
next = PyStackRef_FromPyObjectSteal(next_o);
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
}
else {
if (_PyErr_Occurred(tstate)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!matches) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_MonitorRaise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
JUMPBY(oparg + 1);
DISPATCH();
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!PyStackRef_IsValid(item)) {
if (PyStackRef_IsError(item)) {
JUMP_TO_LABEL(error);
}
JUMPBY(oparg + 1);
stack_pointer[-1] = null_or_index;
DISPATCH();
}
next = item;
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;

11
Python/optimizer_bytecodes.c
View File

@ -840,6 +840,17 @@ dummy_func(void) {
value = sym_new_unknown(ctx);
}
op(_GET_ITER, (iterable -- iter, index_or_null)) {
if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) {
iter = iterable;
index_or_null = sym_new_not_null(ctx);
}
else {
iter = sym_new_not_null(ctx);
index_or_null = sym_new_unknown(ctx);
}
}
op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame: _Py_UOpsAbstractFrame*)) {
gen_frame = NULL;
/* We are about to hit the end of the trace */

12
Python/optimizer_cases.c.h generated
View File

@ -1562,10 +1562,18 @@
}
case _GET_ITER: {
JitOptSymbol *iterable;
JitOptSymbol *iter;
JitOptSymbol *index_or_null;
iter = sym_new_not_null(ctx);
index_or_null = sym_new_not_null(ctx);
iterable = stack_pointer[-1];
if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) {
iter = iterable;
index_or_null = sym_new_not_null(ctx);
}
else {
iter = sym_new_not_null(ctx);
index_or_null = sym_new_unknown(ctx);
}
stack_pointer[-1] = iter;
stack_pointer[0] = index_or_null;
stack_pointer += 1;

4
Python/stackrefs.c
View File

@ -40,6 +40,7 @@ make_table_entry(PyObject *obj, const char *filename, int linenumber)
PyObject *
_Py_stackref_get_object(_PyStackRef ref)
{
assert(!PyStackRef_IsError(ref));
if (ref.index == 0) {
return NULL;
}
@ -64,6 +65,7 @@ PyStackRef_Is(_PyStackRef a, _PyStackRef b)
PyObject *
_Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber)
{
assert(!PyStackRef_IsError(ref));
PyInterpreterState *interp = PyInterpreterState_Get();
if (ref.index >= interp->next_stackref) {
_Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", (void *)ref.index, filename, linenumber);
@ -128,6 +130,7 @@ _Py_stackref_create(PyObject *obj, const char *filename, int linenumber)
void
_Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber)
{
assert(!PyStackRef_IsError(ref));
if (ref.index < INITIAL_STACKREF_INDEX) {
return;
}
@ -152,6 +155,7 @@ _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber
void
_Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref)
{
assert(!PyStackRef_IsError(ref));
assert(ref.index < INITIAL_STACKREF_INDEX);
TableEntry *entry = make_table_entry(obj, "builtin-object", 0);
if (entry == NULL) {

2
Tools/cases_generator/analyzer.py
View File

@ -681,6 +681,8 @@ NON_ESCAPING_FUNCTIONS = (
"PyStackRef_UntagInt",
"PyStackRef_IncrementTaggedIntNoOverflow",
"PyStackRef_IsNullOrInt",
"PyStackRef_IsError",
"PyStackRef_IsValid",
)

2
Tools/cases_generator/stack.py
View File

@ -496,7 +496,7 @@ class Storage:
f"Expected '{undefined}' to be defined before '{out.name}'"
else:
undefined = out.name
while len(self.outputs) > self.peeks and not self.needs_defining(self.outputs[0]):
while len(self.outputs) > self.peeks and not self.needs_defining(self.outputs[self.peeks]):
out = self.outputs.pop(self.peeks)
self.stack.push(out)