1berry/cpython
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
cpython/Python/generated_cases.c.h
Ken Jin 22b0de2755
gh-117139: Convert the evaluation stack to stack refs (#118450) 
This PR sets up tagged pointers for CPython.

The general idea is to create a separate struct _PyStackRef for everything on the evaluation stack to store the bits. This forces the C compiler to warn us if we try to cast things or pull things out of the struct directly.

Only for free threading: We tag the low bit if something is deferred - that means we skip incref and decref operations on it. This behavior may change in the future if Mark's plans to defer all objects in the interpreter loop pans out.

This implies a strict stack reference discipline is required. ALL incref and decref operations on stackrefs must use the stackref variants. It is unsafe to untag something then do normal incref/decref ops on it.

The new incref and decref variants are called dup and close. They mimic a "handle" API operating on these stackrefs.

Please read Include/internal/pycore_stackref.h for more information!

---------

Co-authored-by: Mark Shannon <9448417+markshannon@users.noreply.github.com>
2024-06-27 03:10:43 +08:00

6799 lines
282 KiB
C
Generated
Raw Blame History

// This file is generated by Tools/cases_generator/tier1_generator.py
// from:
// Python/bytecodes.c
// Do not edit!
#ifdef TIER_TWO
#error "This file is for Tier 1 only"
#endif
#define TIER_ONE 1
TARGET(BINARY_OP) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP);
PREDICTED(BINARY_OP);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
_PyStackRef rhs;
_PyStackRef lhs;
_PyStackRef res;
// _SPECIALIZE_BINARY_OP
rhs = stack_pointer[-1];
lhs = stack_pointer[-2];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
DISPATCH_SAME_OPARG();
}
STAT_INC(BINARY_OP, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
assert(NB_ADD <= oparg);
assert(oparg <= NB_INPLACE_XOR);
}
// _BINARY_OP
{
PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs);
PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs);
assert(_PyEval_BinaryOps[oparg]);
PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o);
PyStackRef_CLOSE(lhs);
PyStackRef_CLOSE(rhs);
if (res_o == NULL) goto pop_2_error;
res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_OP_ADD_FLOAT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
// _GUARD_BOTH_FLOAT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_ADD_FLOAT
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
double dres =
((PyFloatObject *)left_o)->ob_fval +
((PyFloatObject *)right_o)->ob_fval;
PyObject *res_o;
DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_OP_ADD_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_ADD_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
// _GUARD_BOTH_INT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_ADD_INT
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
_Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
if (res_o == NULL) goto pop_2_error;
res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_OP_ADD_UNICODE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
// _GUARD_BOTH_UNICODE
right = stack_pointer[-1];
left = stack_pointer[-2];
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_ADD_UNICODE
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
PyObject *res_o = PyUnicode_Concat(left_o, right_o);
_Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc);
if (res_o == NULL) goto pop_2_error;
res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_OP_INPLACE_ADD_UNICODE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
_PyStackRef right;
_PyStackRef left;
// _GUARD_BOTH_UNICODE
right = stack_pointer[-1];
left = stack_pointer[-2];
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_INPLACE_ADD_UNICODE
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(next_instr->op.code == STORE_FAST);
_PyStackRef *target_local = &GETLOCAL(next_instr->op.arg);
DEOPT_IF(!PyStackRef_Is(*target_local, left), BINARY_OP);
STAT_INC(BINARY_OP, hit);
/* Handle `left = left + right` or `left += right` for str.
*
* When possible, extend `left` in place rather than
* allocating a new PyUnicodeObject. This attempts to avoid
* quadratic behavior when one neglects to use str.join().
*
* If `left` has only two references remaining (one from
* the stack, one in the locals), DECREFing `left` leaves
* only the locals reference, so PyUnicode_Append knows
* that the string is safe to mutate.
*/
assert(Py_REFCNT(left_o) >= 2);
_Py_DECREF_NO_DEALLOC(left_o);
PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local);
PyUnicode_Append(&temp, right_o);
*target_local = PyStackRef_FromPyObjectSteal(temp);
_Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc);
if (PyStackRef_IsNull(*target_local)) goto pop_2_error;
// The STORE_FAST is already done.
assert(next_instr->op.code == STORE_FAST);
SKIP_OVER(1);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_OP_MULTIPLY_FLOAT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
// _GUARD_BOTH_FLOAT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_MULTIPLY_FLOAT
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
double dres =
((PyFloatObject *)left_o)->ob_fval *
((PyFloatObject *)right_o)->ob_fval;
PyObject *res_o;
DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_OP_MULTIPLY_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
// _GUARD_BOTH_INT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_MULTIPLY_INT
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
_Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
if (res_o == NULL) goto pop_2_error;
res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_OP_SUBTRACT_FLOAT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
// _GUARD_BOTH_FLOAT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_SUBTRACT_FLOAT
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
double dres =
((PyFloatObject *)left_o)->ob_fval -
((PyFloatObject *)right_o)->ob_fval;
PyObject *res_o;
DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_OP_SUBTRACT_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
// _GUARD_BOTH_INT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_SUBTRACT_INT
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
_Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);;
if (res_o == NULL) goto pop_2_error;
res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_SLICE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BINARY_SLICE);
_PyStackRef stop;
_PyStackRef start;
_PyStackRef container;
_PyStackRef res;
stop = stack_pointer[-1];
start = stack_pointer[-2];
container = stack_pointer[-3];
PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start),
PyStackRef_AsPyObjectSteal(stop));
PyObject *res_o;
// Can't use ERROR_IF() here, because we haven't
// DECREF'ed container yet, and we still own slice.
if (slice == NULL) {
res_o = NULL;
}
else {
res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice);
Py_DECREF(slice);
}
PyStackRef_CLOSE(container);
if (res_o == NULL) goto pop_3_error;
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_SUBSCR) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR);
PREDICTED(BINARY_SUBSCR);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
_PyStackRef sub;
_PyStackRef container;
_PyStackRef res;
// _SPECIALIZE_BINARY_SUBSCR
sub = stack_pointer[-1];
container = stack_pointer[-2];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_BinarySubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(BINARY_SUBSCR, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _BINARY_SUBSCR
{
PyObject *container_o = PyStackRef_AsPyObjectBorrow(container);
PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub);
PyObject *res_o = PyObject_GetItem(container_o, sub_o);
PyStackRef_CLOSE(container);
PyStackRef_CLOSE(sub);
if (res_o == NULL) goto pop_2_error;
res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_SUBSCR_DICT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_DICT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
_PyStackRef sub_st;
_PyStackRef dict_st;
_PyStackRef res;
/* Skip 1 cache entry */
sub_st = stack_pointer[-1];
dict_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o;
int rc = PyDict_GetItemRef(dict, sub, &res_o);
if (rc == 0) {
_PyErr_SetKeyError(sub);
}
PyStackRef_CLOSE(dict_st);
PyStackRef_CLOSE(sub_st);
if (rc <= 0) goto pop_2_error;
// not found or error
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_SUBSCR_GETITEM) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
_PyStackRef sub_st;
_PyStackRef container_st;
/* Skip 1 cache entry */
sub_st = stack_pointer[-1];
container_st = stack_pointer[-2];
PyObject *container = PyStackRef_AsPyObjectBorrow(container_st);
DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR);
PyTypeObject *tp = Py_TYPE(container);
DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR);
PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;
PyObject *cached = ht->_spec_cache.getitem;
DEOPT_IF(cached == NULL, BINARY_SUBSCR);
assert(PyFunction_Check(cached));
PyFunctionObject *getitem = (PyFunctionObject *)cached;
uint32_t cached_version = ht->_spec_cache.getitem_version;
DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR);
PyCodeObject *code = (PyCodeObject *)getitem->func_code;
assert(code->co_argcount == 2);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
Py_INCREF(getitem);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2);
STACK_SHRINK(2);
new_frame->localsplus[0] = container_st;
new_frame->localsplus[1] = sub_st;
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
}
TARGET(BINARY_SUBSCR_LIST_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
_PyStackRef sub_st;
_PyStackRef list_st;
_PyStackRef res;
/* Skip 1 cache entry */
sub_st = stack_pointer[-1];
list_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);
// Deopt unless 0 <= sub < PyList_Size(list)
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o = PyList_GET_ITEM(list, index);
assert(res_o != NULL);
Py_INCREF(res_o);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
PyStackRef_CLOSE(list_st);
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_SUBSCR_STR_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
_PyStackRef sub_st;
_PyStackRef str_st;
_PyStackRef res;
/* Skip 1 cache entry */
sub_st = stack_pointer[-1];
str_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR);
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR);
// Specialize for reading an ASCII character from any string:
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
PyStackRef_CLOSE(str_st);
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BINARY_SUBSCR_TUPLE_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
_PyStackRef sub_st;
_PyStackRef tuple_st;
_PyStackRef res;
/* Skip 1 cache entry */
sub_st = stack_pointer[-1];
tuple_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);
// Deopt unless 0 <= sub < PyTuple_Size(list)
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
assert(res_o != NULL);
Py_INCREF(res_o);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
PyStackRef_CLOSE(tuple_st);
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BUILD_CONST_KEY_MAP) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_CONST_KEY_MAP);
_PyStackRef keys;
_PyStackRef *values;
_PyStackRef map;
keys = stack_pointer[-1];
values = &stack_pointer[-1 - oparg];
PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys);
assert(PyTuple_CheckExact(keys_o));
assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg);
STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
if (CONVERSION_FAILED(values_o)) {
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(values[_i]);
}
PyStackRef_CLOSE(keys);
if (true) { stack_pointer += -1 - oparg; goto error; }
}
PyObject *map_o = _PyDict_FromItems(
&PyTuple_GET_ITEM(keys_o, 0), 1,
values_o, 1, oparg);
STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(values[_i]);
}
PyStackRef_CLOSE(keys);
if (map_o == NULL) { stack_pointer += -1 - oparg; goto error; }
map = PyStackRef_FromPyObjectSteal(map_o);
stack_pointer[-1 - oparg] = map;
stack_pointer += -oparg;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BUILD_LIST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_LIST);
_PyStackRef *values;
_PyStackRef list;
values = &stack_pointer[-oparg];
STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
if (CONVERSION_FAILED(values_o)) {
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(values[_i]);
}
if (true) { stack_pointer += -oparg; goto error; }
}
PyObject *list_o = _PyList_FromArraySteal(values_o, oparg);
STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
if (list_o == NULL) { stack_pointer += -oparg; goto error; }
list = PyStackRef_FromPyObjectSteal(list_o);
stack_pointer[-oparg] = list;
stack_pointer += 1 - oparg;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BUILD_MAP) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_MAP);
_PyStackRef *values;
_PyStackRef map;
values = &stack_pointer[-oparg*2];
STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o);
if (CONVERSION_FAILED(values_o)) {
for (int _i = oparg*2; --_i >= 0;) {
PyStackRef_CLOSE(values[_i]);
}
if (true) { stack_pointer += -oparg*2; goto error; }
}
PyObject *map_o = _PyDict_FromItems(
values_o, 2,
values_o+1, 2,
oparg);
STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
for (int _i = oparg*2; --_i >= 0;) {
PyStackRef_CLOSE(values[_i]);
}
if (map_o == NULL) { stack_pointer += -oparg*2; goto error; }
map = PyStackRef_FromPyObjectSteal(map_o);
stack_pointer[-oparg*2] = map;
stack_pointer += 1 - oparg*2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BUILD_SET) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_SET);
_PyStackRef *values;
_PyStackRef set;
values = &stack_pointer[-oparg];
PyObject *set_o = PySet_New(NULL);
if (set_o == NULL) {
goto error;
}
int err = 0;
for (int i = 0; i < oparg; i++) {
PyObject *item = PyStackRef_AsPyObjectSteal(values[i]);
if (err == 0) {
err = PySet_Add(set_o, item);
}
Py_DECREF(item);
}
if (err != 0) {
Py_DECREF(set_o);
if (true) { stack_pointer += -oparg; goto error; }
}
set = PyStackRef_FromPyObjectSteal(set_o);
stack_pointer[-oparg] = set;
stack_pointer += 1 - oparg;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BUILD_SLICE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_SLICE);
_PyStackRef step = PyStackRef_NULL;
_PyStackRef stop;
_PyStackRef start;
_PyStackRef slice;
if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; }
stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)];
start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)];
PyObject *start_o = PyStackRef_AsPyObjectBorrow(start);
PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop);
PyObject *step_o = PyStackRef_AsPyObjectBorrow(step);
PyObject *slice_o = PySlice_New(start_o, stop_o, step_o);
PyStackRef_CLOSE(start);
PyStackRef_CLOSE(stop);
PyStackRef_XCLOSE(step);
if (slice_o == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; }
slice = PyStackRef_FromPyObjectSteal(slice_o);
stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice;
stack_pointer += -1 - ((oparg == 3) ? 1 : 0);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BUILD_STRING) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_STRING);
_PyStackRef *pieces;
_PyStackRef str;
pieces = &stack_pointer[-oparg];
STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o);
if (CONVERSION_FAILED(pieces_o)) {
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(pieces[_i]);
}
if (true) { stack_pointer += -oparg; goto error; }
}
PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg);
STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o);
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(pieces[_i]);
}
if (str_o == NULL) { stack_pointer += -oparg; goto error; }
str = PyStackRef_FromPyObjectSteal(str_o);
stack_pointer[-oparg] = str;
stack_pointer += 1 - oparg;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(BUILD_TUPLE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_TUPLE);
_PyStackRef *values;
_PyStackRef tup;
values = &stack_pointer[-oparg];
STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
if (CONVERSION_FAILED(values_o)) {
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(values[_i]);
}
if (true) { stack_pointer += -oparg; goto error; }
}
PyObject *tup_o = _PyTuple_FromArraySteal(values_o, oparg);
STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
if (tup_o == NULL) { stack_pointer += -oparg; goto error; }
tup = PyStackRef_FromPyObjectSteal(tup_o);
stack_pointer[-oparg] = tup;
stack_pointer += 1 - oparg;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(CACHE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CACHE);
assert(0 && "Executing a cache.");
Py_FatalError("Executing a cache.");
DISPATCH();
}
TARGET(CALL) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL);
PREDICTED(CALL);
_Py_CODEUNIT *this_instr = next_instr - 4;
(void)this_instr;
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
// _SPECIALIZE_CALL
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null));
DISPATCH_SAME_OPARG();
}
STAT_INC(CALL, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
/* Skip 2 cache entries */
// _CALL
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
// oparg counts all of the args, but *not* self:
int total_args = oparg;
if (self_or_null_o != NULL) {
args--;
total_args++;
}
else if (Py_TYPE(callable_o) == &PyMethod_Type) {
args--;
total_args++;
PyObject *self = ((PyMethodObject *)callable_o)->im_self;
args[0] = PyStackRef_FromPyObjectNew(self);
PyObject *method = ((PyMethodObject *)callable_o)->im_func;
args[-1] = PyStackRef_FromPyObjectNew(method);
PyStackRef_CLOSE(callable);
callable_o = method;
callable = args[-1];
}
// Check if the call can be inlined or not
if (Py_TYPE(callable_o) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall)
{
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
args, total_args, NULL
);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
goto error;
}
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
}
/* Callable is not a normal Python function */
STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
PyStackRef_CLOSE(callable);
PyStackRef_CLOSE(self_or_null);
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(args[_i]);
}
if (true) { stack_pointer += -2 - oparg; goto error; }
}
PyObject *res_o = PyObject_Vectorcall(
callable_o, args_o,
total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
NULL);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
if (opcode == INSTRUMENTED_CALL) {
PyObject *arg = total_args == 0 ?
&_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]);
if (res_o == NULL) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
frame, this_instr, callable_o, arg);
}
else {
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_C_RETURN,
frame, this_instr, callable_o, arg);
if (err < 0) {
Py_CLEAR(res_o);
}
}
}
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
PyStackRef_CLOSE(callable);
for (int i = 0; i < total_args; i++) {
PyStackRef_CLOSE(args[i]);
}
if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_ALLOC_AND_ENTER_INIT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef *args;
_PyStackRef null;
_PyStackRef callable;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
args = &stack_pointer[-oparg];
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
/* This instruction does the following:
* 1. Creates the object (by calling ``object.__new__``)
* 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``)
* 3. Pushes the frame for ``__init__`` to the frame stack
* */
_PyCallCache *cache = (_PyCallCache *)&this_instr[1];
DEOPT_IF(!PyStackRef_IsNull(null), CALL);
DEOPT_IF(!PyType_Check(callable_o), CALL);
PyTypeObject *tp = (PyTypeObject *)callable_o;
DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL);
assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o;
PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
PyCodeObject *code = (PyCodeObject *)init->func_code;
DEOPT_IF(code->co_argcount != oparg+1, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL);
STAT_INC(CALL, hit);
PyObject *self = _PyType_NewManagedObject(tp);
if (self == NULL) {
goto error;
}
PyStackRef_CLOSE(callable);
_PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
tstate, (PyCodeObject *)&_Py_InitCleanup, 1);
assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK);
/* Push self onto stack of shim */
Py_INCREF(self);
shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
Py_INCREF(init);
_PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1);
/* Copy self followed by args to __init__ frame */
init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
for (int i = 0; i < oparg; i++) {
init_frame->localsplus[i+1] = args[i];
}
frame->return_offset = (uint16_t)(next_instr - this_instr);
STACK_SHRINK(oparg+2);
_PyFrame_SetStackPointer(frame, stack_pointer);
/* Link frames */
init_frame->previous = shim;
shim->previous = frame;
frame = tstate->current_frame = init_frame;
CALL_STAT_INC(inlined_py_calls);
/* Account for pushing the extra frame.
* We don't check recursion depth here,
* as it will be checked after start_frame */
tstate->py_recursion_remaining--;
goto start_frame;
}
TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef null;
_PyStackRef callable;
_PyStackRef func;
_PyStackRef self;
_PyStackRef self_or_null;
_PyStackRef *args;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, CALL);
}
// _CHECK_CALL_BOUND_METHOD_EXACT_ARGS
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
DEOPT_IF(!PyStackRef_IsNull(null), CALL);
DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type, CALL);
}
// _INIT_CALL_BOUND_METHOD_EXACT_ARGS
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
STAT_INC(CALL, hit);
stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS
stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization
self = stack_pointer[-1 - oparg];
func = stack_pointer[-2 - oparg];
PyStackRef_CLOSE(callable);
// self may be unused in tier 1, so silence warnings.
(void)self;
}
// _CHECK_FUNCTION_VERSION
callable = func;
{
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
DEOPT_IF(!PyFunction_Check(callable_o), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
DEOPT_IF(func->func_version != func_version, CALL);
}
// _CHECK_FUNCTION_EXACT_ARGS
self_or_null = stack_pointer[-1 - oparg];
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
assert(PyFunction_Check(callable_o));
PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null)), CALL);
}
// _CHECK_STACK_SPACE
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
}
// _INIT_CALL_PY_EXACT_ARGS
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
_PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
}
// _SAVE_RETURN_OFFSET
{
#if TIER_ONE
frame->return_offset = (uint16_t)(next_instr - this_instr);
#endif
#if TIER_TWO
frame->return_offset = oparg;
#endif
}
// _PUSH_FRAME
{
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(CALL_BOUND_METHOD_GENERAL) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef null;
_PyStackRef callable;
_PyStackRef method;
_PyStackRef self;
_PyStackRef *args;
_PyStackRef self_or_null;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, CALL);
}
// _CHECK_METHOD_VERSION
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL);
PyObject *func = ((PyMethodObject *)callable_o)->im_func;
DEOPT_IF(!PyFunction_Check(func), CALL);
DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL);
DEOPT_IF(!PyStackRef_IsNull(null), CALL);
}
// _EXPAND_METHOD
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
assert(PyStackRef_IsNull(null));
assert(Py_TYPE(callable_o) == &PyMethod_Type);
self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self);
stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL
method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func);
assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method)));
PyStackRef_CLOSE(callable);
}
// _PY_FRAME_GENERAL
args = &stack_pointer[-oparg];
self_or_null = self;
callable = method;
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
// oparg counts all of the args, but *not* self:
int total_args = oparg;
if (self_or_null_o != NULL) {
args--;
total_args++;
}
assert(Py_TYPE(callable_o) == &PyFunction_Type);
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
args, total_args, NULL
);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
if (new_frame == NULL) {
goto error;
}
}
// _SAVE_RETURN_OFFSET
{
#if TIER_ONE
frame->return_offset = (uint16_t)(next_instr - this_instr);
#endif
#if TIER_TWO
frame->return_offset = oparg;
#endif
}
// _PUSH_FRAME
{
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(CALL_BUILTIN_CLASS) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_CLASS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_BUILTIN_CLASS
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(!PyType_Check(callable_o), CALL);
PyTypeObject *tp = (PyTypeObject *)callable_o;
DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
STAT_INC(CALL, hit);
STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
PyStackRef_CLOSE(callable);
PyStackRef_CLOSE(self_or_null);
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(args[_i]);
}
if (true) { stack_pointer += -2 - oparg; goto error; }
}
PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
PyStackRef_CLOSE(args[i]);
}
PyStackRef_CLOSE(callable);
if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_BUILTIN_FAST) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_FAST);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_BUILTIN_FAST
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
/* Builtin METH_FASTCALL functions, without keywords */
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
/* res = func(self, args, nargs) */
STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
PyStackRef_CLOSE(callable);
PyStackRef_CLOSE(self_or_null);
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(args[_i]);
}
if (true) { stack_pointer += -2 - oparg; goto error; }
}
PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)(
PyCFunction_GET_SELF(callable_o),
args_o,
total_args);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
PyStackRef_CLOSE(args[i]);
}
PyStackRef_CLOSE(callable);
if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_BUILTIN_FAST_WITH_KEYWORDS
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL);
STAT_INC(CALL, hit);
/* res = func(self, args, nargs, kwnames) */
PyCFunctionFastWithKeywords cfunc =
(PyCFunctionFastWithKeywords)(void(*)(void))
PyCFunction_GET_FUNCTION(callable_o);
STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
PyStackRef_CLOSE(callable);
PyStackRef_CLOSE(self_or_null);
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(args[_i]);
}
if (true) { stack_pointer += -2 - oparg; goto error; }
}
PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
PyStackRef_CLOSE(args[i]);
}
PyStackRef_CLOSE(callable);
if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_BUILTIN_O) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_O);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_BUILTIN_O
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
/* Builtin METH_O functions */
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL);
// CPython promises to check all non-vectorcall function calls.
DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
_PyStackRef arg = args[0];
_Py_EnterRecursiveCallTstateUnchecked(tstate);
PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg));
_Py_LeaveRecursiveCallTstate(tstate);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
PyStackRef_CLOSE(arg);
PyStackRef_CLOSE(callable);
if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_FUNCTION_EX) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_FUNCTION_EX);
PREDICTED(CALL_FUNCTION_EX);
_Py_CODEUNIT *this_instr = next_instr - 1;
(void)this_instr;
_PyStackRef kwargs_st = PyStackRef_NULL;
_PyStackRef callargs_st;
_PyStackRef func_st;
_PyStackRef result;
if (oparg & 1) { kwargs_st = stack_pointer[-(oparg & 1)]; }
callargs_st = stack_pointer[-1 - (oparg & 1)];
func_st = stack_pointer[-3 - (oparg & 1)];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
// DICT_MERGE is called before this opcode if there are kwargs.
// It converts all dict subtypes in kwargs into regular dicts.
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
if (!PyTuple_CheckExact(callargs)) {
if (check_args_iterable(tstate, func, callargs) < 0) {
goto error;
}
PyObject *tuple = PySequence_Tuple(callargs);
if (tuple == NULL) {
goto error;
}
PyStackRef_CLOSE(callargs_st);
callargs_st = PyStackRef_FromPyObjectSteal(tuple);
callargs = tuple;
}
assert(PyTuple_CheckExact(callargs));
EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func);
if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) {
PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ?
PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING;
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, func, arg);
if (err) goto error;
result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs));
if (!PyFunction_Check(func) && !PyMethod_Check(func)) {
if (PyStackRef_IsNull(result)) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
frame, this_instr, func, arg);
}
else {
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_C_RETURN,
frame, this_instr, func, arg);
if (err < 0) {
PyStackRef_CLEAR(result);
}
}
}
}
else {
if (Py_TYPE(func) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
assert(PyTuple_CheckExact(callargs));
Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate,
(PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals,
nargs, callargs, kwargs);
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
if (new_frame == NULL) {
goto error;
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
DISPATCH_INLINED(new_frame);
}
result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs));
}
PyStackRef_CLOSE(func_st);
PyStackRef_CLOSE(callargs_st);
PyStackRef_XCLOSE(kwargs_st);
assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL);
if (PyStackRef_IsNull(result)) { stack_pointer += -3 - (oparg & 1); goto error; }
stack_pointer[-3 - (oparg & 1)] = result;
stack_pointer += -2 - (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_INTRINSIC_1) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_INTRINSIC_1);
_PyStackRef value;
_PyStackRef res;
value = stack_pointer[-1];
assert(oparg <= MAX_INTRINSIC_1);
PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value));
PyStackRef_CLOSE(value);
if (res_o == NULL) goto pop_1_error;
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(CALL_INTRINSIC_2) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_INTRINSIC_2);
_PyStackRef value1_st;
_PyStackRef value2_st;
_PyStackRef res;
value1_st = stack_pointer[-1];
value2_st = stack_pointer[-2];
assert(oparg <= MAX_INTRINSIC_2);
PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st);
PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st);
PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1);
PyStackRef_CLOSE(value2_st);
PyStackRef_CLOSE(value1_st);
if (res_o == NULL) goto pop_2_error;
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(CALL_ISINSTANCE) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_ISINSTANCE);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* isinstance(o, o2) */
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(total_args != 2, CALL);
PyInterpreterState *interp = tstate->interp;
DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL);
STAT_INC(CALL, hit);
_PyStackRef cls_stackref = args[1];
_PyStackRef inst_stackref = args[0];
int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref));
if (retval < 0) {
goto error;
}
res = retval ? PyStackRef_True : PyStackRef_False;
assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL));
PyStackRef_CLOSE(inst_stackref);
PyStackRef_CLOSE(cls_stackref);
PyStackRef_CLOSE(callable);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(CALL_KW) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_KW);
PREDICTED(CALL_KW);
_Py_CODEUNIT *this_instr = next_instr - 1;
(void)this_instr;
_PyStackRef kwnames;
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
kwnames = stack_pointer[-1];
args = &stack_pointer[-1 - oparg];
self_or_null = stack_pointer[-2 - oparg];
callable = stack_pointer[-3 - oparg];
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames);
// oparg counts all of the args, but *not* self:
int total_args = oparg;
if (self_or_null_o != NULL) {
args--;
total_args++;
}
if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) {
args--;
total_args++;
PyObject *self = ((PyMethodObject *)callable_o)->im_self;
args[0] = PyStackRef_FromPyObjectNew(self);
PyObject *method = ((PyMethodObject *)callable_o)->im_func;
args[-1] = PyStackRef_FromPyObjectNew(method);
PyStackRef_CLOSE(callable);
callable_o = method;
callable = args[-1];
}
int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o);
// Check if the call can be inlined or not
if (Py_TYPE(callable_o) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall)
{
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
args, positional_args, kwnames_o
);
PyStackRef_CLOSE(kwnames);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 3);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
goto error;
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
DISPATCH_INLINED(new_frame);
}
/* Callable is not a normal Python function */
STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
PyStackRef_CLOSE(callable);
PyStackRef_CLOSE(self_or_null);
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(args[_i]);
}
PyStackRef_CLOSE(kwnames);
if (true) { stack_pointer += -3 - oparg; goto error; }
}
PyObject *res_o = PyObject_Vectorcall(
callable_o, args_o,
positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
kwnames_o);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
if (opcode == INSTRUMENTED_CALL_KW) {
PyObject *arg = total_args == 0 ?
&_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]);
if (res_o == NULL) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
frame, this_instr, callable_o, arg);
}
else {
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_C_RETURN,
frame, this_instr, callable_o, arg);
if (err < 0) {
Py_CLEAR(res_o);
}
}
}
PyStackRef_CLOSE(kwnames);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
PyStackRef_CLOSE(callable);
for (int i = 0; i < total_args; i++) {
PyStackRef_CLOSE(args[i]);
}
if (res_o == NULL) { stack_pointer += -3 - oparg; goto error; }
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-3 - oparg] = res;
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_LEN) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_LEN);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* len(o) */
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
PyInterpreterState *interp = tstate->interp;
DEOPT_IF(callable_o != interp->callable_cache.len, CALL);
STAT_INC(CALL, hit);
_PyStackRef arg_stackref = args[0];
PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref);
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
goto error;
}
PyObject *res_o = PyLong_FromSsize_t(len_i);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
if (res_o == NULL) {
GOTO_ERROR(error);
}
PyStackRef_CLOSE(callable);
PyStackRef_CLOSE(arg_stackref);
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(CALL_LIST_APPEND) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_LIST_APPEND);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef arg;
_PyStackRef self;
_PyStackRef callable;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
arg = stack_pointer[-1];
self = stack_pointer[-2];
callable = stack_pointer[-3];
assert(oparg == 1);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *self_o = PyStackRef_AsPyObjectBorrow(self);
PyInterpreterState *interp = tstate->interp;
DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL);
assert(self_o != NULL);
DEOPT_IF(!PyList_Check(self_o), CALL);
STAT_INC(CALL, hit);
if (_PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)) < 0) {
goto pop_1_error; // Since arg is DECREF'ed already
}
PyStackRef_CLOSE(self);
PyStackRef_CLOSE(callable);
STACK_SHRINK(3);
// Skip POP_TOP
assert(next_instr->op.code == POP_TOP);
SKIP_OVER(1);
DISPATCH();
}
TARGET(CALL_METHOD_DESCRIPTOR_FAST) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_METHOD_DESCRIPTOR_FAST
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
/* Builtin METH_FASTCALL methods, without keywords */
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]);
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
STAT_INC(CALL, hit);
PyCFunctionFast cfunc =
(PyCFunctionFast)(void(*)(void))meth->ml_meth;
int nargs = total_args - 1;
STACKREFS_TO_PYOBJECTS(args, nargs, args_o);
if (CONVERSION_FAILED(args_o)) {
PyStackRef_CLOSE(callable);
PyStackRef_CLOSE(self_or_null);
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(args[_i]);
}
if (true) { stack_pointer += -2 - oparg; goto error; }
}
PyObject *res_o = cfunc(self, (args_o + 1), nargs);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the arguments. */
for (int i = 0; i < total_args; i++) {
PyStackRef_CLOSE(args[i]);
}
PyStackRef_CLOSE(callable);
if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL);
PyTypeObject *d_type = method->d_common.d_type;
PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]);
DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL);
STAT_INC(CALL, hit);
int nargs = total_args - 1;
PyCFunctionFastWithKeywords cfunc =
(PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
STACKREFS_TO_PYOBJECTS(args, nargs, args_o);
if (CONVERSION_FAILED(args_o)) {
PyStackRef_CLOSE(callable);
PyStackRef_CLOSE(self_or_null);
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(args[_i]);
}
if (true) { stack_pointer += -2 - oparg; goto error; }
}
PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
PyStackRef_CLOSE(args[i]);
}
PyStackRef_CLOSE(callable);
if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_METHOD_DESCRIPTOR_NOARGS
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
assert(oparg == 0 || oparg == 1);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = method->d_method;
_PyStackRef self_stackref = args[0];
PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref);
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
// CPython promises to check all non-vectorcall function calls.
DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
PyStackRef_CLOSE(self_stackref);
PyStackRef_CLOSE(callable);
if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_METHOD_DESCRIPTOR_O) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_METHOD_DESCRIPTOR_O
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
DEOPT_IF(total_args != 2, CALL);
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != METH_O, CALL);
// CPython promises to check all non-vectorcall function calls.
DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
_PyStackRef arg_stackref = args[1];
_PyStackRef self_stackref = args[0];
DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref),
method->d_common.d_type), CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
PyObject *res_o = _PyCFunction_TrampolineCall(cfunc,
PyStackRef_AsPyObjectBorrow(self_stackref),
PyStackRef_AsPyObjectBorrow(arg_stackref));
_Py_LeaveRecursiveCallTstate(tstate);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
PyStackRef_CLOSE(self_stackref);
PyStackRef_CLOSE(arg_stackref);
PyStackRef_CLOSE(callable);
if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NON_PY_GENERAL) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_NON_PY_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef callable;
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CHECK_IS_NOT_PY_CALLABLE
callable = stack_pointer[-2 - oparg];
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
DEOPT_IF(PyFunction_Check(callable_o), CALL);
DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL);
}
// _CALL_NON_PY_GENERAL
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
{
#if TIER_ONE
assert(opcode != INSTRUMENTED_CALL);
#endif
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
int total_args = oparg;
if (self_or_null_o != NULL) {
args--;
total_args++;
}
/* Callable is not a normal Python function */
STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
PyStackRef_CLOSE(callable);
PyStackRef_CLOSE(self_or_null);
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(args[_i]);
}
if (true) { stack_pointer += -2 - oparg; goto error; }
}
PyObject *res_o = PyObject_Vectorcall(
callable_o, args_o,
total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
NULL);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
PyStackRef_CLOSE(callable);
for (int i = 0; i < total_args; i++) {
PyStackRef_CLOSE(args[i]);
}
if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_PY_EXACT_ARGS) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_PY_EXACT_ARGS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef callable;
_PyStackRef self_or_null;
_PyStackRef *args;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, CALL);
}
// _CHECK_FUNCTION_VERSION
callable = stack_pointer[-2 - oparg];
{
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
DEOPT_IF(!PyFunction_Check(callable_o), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
DEOPT_IF(func->func_version != func_version, CALL);
}
// _CHECK_FUNCTION_EXACT_ARGS
self_or_null = stack_pointer[-1 - oparg];
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
assert(PyFunction_Check(callable_o));
PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null)), CALL);
}
// _CHECK_STACK_SPACE
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
}
// _INIT_CALL_PY_EXACT_ARGS
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
_PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
}
// _SAVE_RETURN_OFFSET
{
#if TIER_ONE
frame->return_offset = (uint16_t)(next_instr - this_instr);
#endif
#if TIER_TWO
frame->return_offset = oparg;
#endif
}
// _PUSH_FRAME
{
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(CALL_PY_GENERAL) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_PY_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef callable;
_PyStackRef *args;
_PyStackRef self_or_null;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, CALL);
}
// _CHECK_FUNCTION_VERSION
callable = stack_pointer[-2 - oparg];
{
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
DEOPT_IF(!PyFunction_Check(callable_o), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
DEOPT_IF(func->func_version != func_version, CALL);
}
// _PY_FRAME_GENERAL
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
// oparg counts all of the args, but *not* self:
int total_args = oparg;
if (self_or_null_o != NULL) {
args--;
total_args++;
}
assert(Py_TYPE(callable_o) == &PyFunction_Type);
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
args, total_args, NULL
);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
if (new_frame == NULL) {
goto error;
}
}
// _SAVE_RETURN_OFFSET
{
#if TIER_ONE
frame->return_offset = (uint16_t)(next_instr - this_instr);
#endif
#if TIER_TWO
frame->return_offset = oparg;
#endif
}
// _PUSH_FRAME
{
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(CALL_STR_1) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_STR_1);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef arg;
_PyStackRef null;
_PyStackRef callable;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_STR_1
arg = stack_pointer[-1];
null = stack_pointer[-2];
callable = stack_pointer[-3];
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
DEOPT_IF(!PyStackRef_IsNull(null), CALL);
DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL);
STAT_INC(CALL, hit);
res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o));
PyStackRef_CLOSE(arg);
if (PyStackRef_IsNull(res)) goto pop_3_error;
}
// _CHECK_PERIODIC
{
}
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_TUPLE_1) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_TUPLE_1);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef arg;
_PyStackRef null;
_PyStackRef callable;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_TUPLE_1
arg = stack_pointer[-1];
null = stack_pointer[-2];
callable = stack_pointer[-3];
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
DEOPT_IF(!PyStackRef_IsNull(null), CALL);
DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL);
STAT_INC(CALL, hit);
res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o));
PyStackRef_CLOSE(arg);
if (PyStackRef_IsNull(res)) goto pop_3_error;
}
// _CHECK_PERIODIC
{
}
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_TYPE_1) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_TYPE_1);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef arg;
_PyStackRef null;
_PyStackRef callable;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
arg = stack_pointer[-1];
null = stack_pointer[-2];
callable = stack_pointer[-3];
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
DEOPT_IF(!PyStackRef_IsNull(null), CALL);
DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL);
STAT_INC(CALL, hit);
res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o)));
PyStackRef_CLOSE(arg);
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(CHECK_EG_MATCH) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CHECK_EG_MATCH);
_PyStackRef match_type_st;
_PyStackRef exc_value_st;
_PyStackRef rest;
_PyStackRef match;
match_type_st = stack_pointer[-1];
exc_value_st = stack_pointer[-2];
PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st);
PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st);
if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) {
PyStackRef_CLOSE(exc_value_st);
PyStackRef_CLOSE(match_type_st);
if (true) goto pop_2_error;
}
PyObject *match_o = NULL;
PyObject *rest_o = NULL;
int res = _PyEval_ExceptionGroupMatch(exc_value, match_type,
&match_o, &rest_o);
PyStackRef_CLOSE(exc_value_st);
PyStackRef_CLOSE(match_type_st);
if (res < 0) goto pop_2_error;
assert((match_o == NULL) == (rest_o == NULL));
if (match_o == NULL) goto pop_2_error;
if (!Py_IsNone(match_o)) {
PyErr_SetHandledException(match_o);
}
rest = PyStackRef_FromPyObjectSteal(rest_o);
match = PyStackRef_FromPyObjectSteal(match_o);
stack_pointer[-2] = rest;
stack_pointer[-1] = match;
DISPATCH();
}
TARGET(CHECK_EXC_MATCH) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CHECK_EXC_MATCH);
_PyStackRef right;
_PyStackRef left;
_PyStackRef b;
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyExceptionInstance_Check(left_o));
if (_PyEval_CheckExceptTypeValid(tstate, right_o) < 0) {
PyStackRef_CLOSE(right);
if (true) goto pop_1_error;
}
int res = PyErr_GivenExceptionMatches(left_o, right_o);
PyStackRef_CLOSE(right);
b = res ? PyStackRef_True : PyStackRef_False;
stack_pointer[-1] = b;
DISPATCH();
}
TARGET(CLEANUP_THROW) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(CLEANUP_THROW);
_PyStackRef exc_value_st;
_PyStackRef last_sent_val_st;
_PyStackRef sub_iter_st;
_PyStackRef none;
_PyStackRef value;
exc_value_st = stack_pointer[-1];
last_sent_val_st = stack_pointer[-2];
sub_iter_st = stack_pointer[-3];
PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st);
assert(throwflag);
assert(exc_value && PyExceptionInstance_Check(exc_value));
if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) {
value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value);
PyStackRef_CLOSE(sub_iter_st);
PyStackRef_CLOSE(last_sent_val_st);
PyStackRef_CLOSE(exc_value_st);
none = PyStackRef_None;
}
else {
_PyErr_SetRaisedException(tstate, Py_NewRef(exc_value));
monitor_reraise(tstate, frame, this_instr);
goto exception_unwind;
}
stack_pointer[-3] = none;
stack_pointer[-2] = value;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(COMPARE_OP) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP);
PREDICTED(COMPARE_OP);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
// _SPECIALIZE_COMPARE_OP
right = stack_pointer[-1];
left = stack_pointer[-2];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_CompareOp(left, right, next_instr, oparg);
DISPATCH_SAME_OPARG();
}
STAT_INC(COMPARE_OP, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _COMPARE_OP
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert((oparg >> 5) <= Py_GE);
PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5);
PyStackRef_CLOSE(left);
PyStackRef_CLOSE(right);
if (res_o == NULL) goto pop_2_error;
if (oparg & 16) {
int res_bool = PyObject_IsTrue(res_o);
Py_DECREF(res_o);
if (res_bool < 0) goto pop_2_error;
res = res_bool ? PyStackRef_True : PyStackRef_False;
}
else {
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(COMPARE_OP_FLOAT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
// _GUARD_BOTH_FLOAT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP);
DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP);
}
/* Skip 1 cache entry */
// _COMPARE_OP_FLOAT
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(COMPARE_OP, hit);
double dleft = PyFloat_AS_DOUBLE(left_o);
double dright = PyFloat_AS_DOUBLE(right_o);
// 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg
int sign_ish = COMPARISON_BIT(dleft, dright);
_Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc);
_Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc);
res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(COMPARE_OP_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_INT);
static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
// _GUARD_BOTH_INT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP);
DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP);
}
/* Skip 1 cache entry */
// _COMPARE_OP_INT
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP);
DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 &&
_PyLong_DigitCount((PyLongObject *)right_o) <= 1);
Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o);
Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o);
// 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg
int sign_ish = COMPARISON_BIT(ileft, iright);
_Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(COMPARE_OP_STR) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_STR);
static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
// _GUARD_BOTH_UNICODE
right = stack_pointer[-1];
left = stack_pointer[-2];
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP);
DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP);
}
/* Skip 1 cache entry */
// _COMPARE_OP_STR
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(COMPARE_OP, hit);
int eq = _PyUnicode_Equal(left_o, right_o);
assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE);
_Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc);
assert(eq == 0 || eq == 1);
assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS);
assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS);
res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(CONTAINS_OP) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP);
PREDICTED(CONTAINS_OP);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
_PyStackRef right;
_PyStackRef left;
_PyStackRef b;
// _SPECIALIZE_CONTAINS_OP
right = stack_pointer[-1];
left = stack_pointer[-2];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_ContainsOp(right, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(CONTAINS_OP, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _CONTAINS_OP
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
int res = PySequence_Contains(right_o, left_o);
PyStackRef_CLOSE(left);
PyStackRef_CLOSE(right);
if (res < 0) goto pop_2_error;
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
}
stack_pointer[-2] = b;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(CONTAINS_OP_DICT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP_DICT);
static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size");
_PyStackRef right;
_PyStackRef left;
_PyStackRef b;
/* Skip 1 cache entry */
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP);
STAT_INC(CONTAINS_OP, hit);
int res = PyDict_Contains(right_o, left_o);
PyStackRef_CLOSE(left);
PyStackRef_CLOSE(right);
if (res < 0) goto pop_2_error;
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(CONTAINS_OP_SET) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP_SET);
static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size");
_PyStackRef right;
_PyStackRef left;
_PyStackRef b;
/* Skip 1 cache entry */
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP);
STAT_INC(CONTAINS_OP, hit);
// Note: both set and frozenset use the same seq_contains method!
int res = _PySet_Contains((PySetObject *)right_o, left_o);
PyStackRef_CLOSE(left);
PyStackRef_CLOSE(right);
if (res < 0) goto pop_2_error;
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(CONVERT_VALUE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CONVERT_VALUE);
_PyStackRef value;
_PyStackRef result;
value = stack_pointer[-1];
conversion_func conv_fn;
assert(oparg >= FVC_STR && oparg <= FVC_ASCII);
conv_fn = _PyEval_ConversionFuncs[oparg];
PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value));
PyStackRef_CLOSE(value);
if (result_o == NULL) goto pop_1_error;
result = PyStackRef_FromPyObjectSteal(result_o);
stack_pointer[-1] = result;
DISPATCH();
}
TARGET(COPY) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(COPY);
_PyStackRef bottom;
_PyStackRef top;
bottom = stack_pointer[-1 - (oparg-1)];
assert(oparg > 0);
top = PyStackRef_DUP(bottom);
stack_pointer[0] = top;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(COPY_FREE_VARS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(COPY_FREE_VARS);
/* Copy closure variables to free variables */
PyCodeObject *co = _PyFrame_GetCode(frame);
assert(PyFunction_Check(frame->f_funcobj));
PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure;
assert(oparg == co->co_nfreevars);
int offset = co->co_nlocalsplus - oparg;
for (int i = 0; i < oparg; ++i) {
PyObject *o = PyTuple_GET_ITEM(closure, i);
frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o);
}
DISPATCH();
}
TARGET(DELETE_ATTR) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_ATTR);
_PyStackRef owner;
owner = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name);
PyStackRef_CLOSE(owner);
if (err) goto pop_1_error;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(DELETE_DEREF) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_DEREF);
PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
// Can't use ERROR_IF here.
// Fortunately we don't need its superpower.
PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL);
if (oldobj == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
goto error;
}
Py_DECREF(oldobj);
DISPATCH();
}
TARGET(DELETE_FAST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_FAST);
_PyStackRef v = GETLOCAL(oparg);
if (PyStackRef_IsNull(v)) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
if (1) goto error;
}
SETLOCAL(oparg, PyStackRef_NULL);
DISPATCH();
}
TARGET(DELETE_GLOBAL) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_GLOBAL);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyDict_Pop(GLOBALS(), name, NULL);
// Can't use ERROR_IF here.
if (err < 0) {
goto error;
}
if (err == 0) {
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
goto error;
}
DISPATCH();
}
TARGET(DELETE_NAME) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_NAME);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *ns = LOCALS();
int err;
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals when deleting %R", name);
goto error;
}
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
if (err != 0) {
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG,
name);
goto error;
}
DISPATCH();
}
TARGET(DELETE_SUBSCR) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_SUBSCR);
_PyStackRef sub;
_PyStackRef container;
sub = stack_pointer[-1];
container = stack_pointer[-2];
/* del container[sub] */
int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container),
PyStackRef_AsPyObjectBorrow(sub));
PyStackRef_CLOSE(container);
PyStackRef_CLOSE(sub);
if (err) goto pop_2_error;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(DICT_MERGE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DICT_MERGE);
_PyStackRef update;
_PyStackRef dict;
_PyStackRef callable;
update = stack_pointer[-1];
dict = stack_pointer[-2 - (oparg - 1)];
callable = stack_pointer[-5 - (oparg - 1)];
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict);
PyObject *update_o = PyStackRef_AsPyObjectBorrow(update);
if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) {
_PyEval_FormatKwargsError(tstate, callable_o, update_o);
PyStackRef_CLOSE(update);
if (true) goto pop_1_error;
}
PyStackRef_CLOSE(update);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(DICT_UPDATE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DICT_UPDATE);
_PyStackRef update;
_PyStackRef dict;
update = stack_pointer[-1];
dict = stack_pointer[-2 - (oparg - 1)];
PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict);
PyObject *update_o = PyStackRef_AsPyObjectBorrow(update);
if (PyDict_Update(dict_o, update_o) < 0) {
if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
_PyErr_Format(tstate, PyExc_TypeError,
"'%.200s' object is not a mapping",
Py_TYPE(update_o)->tp_name);
}
PyStackRef_CLOSE(update);
if (true) goto pop_1_error;
}
PyStackRef_CLOSE(update);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(END_ASYNC_FOR) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(END_ASYNC_FOR);
_PyStackRef exc_st;
_PyStackRef awaitable_st;
exc_st = stack_pointer[-1];
awaitable_st = stack_pointer[-2];
PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st);
assert(exc && PyExceptionInstance_Check(exc));
if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) {
PyStackRef_CLOSE(awaitable_st);
PyStackRef_CLOSE(exc_st);
}
else {
Py_INCREF(exc);
_PyErr_SetRaisedException(tstate, exc);
monitor_reraise(tstate, frame, this_instr);
goto exception_unwind;
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(END_FOR) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(END_FOR);
_PyStackRef value;
value = stack_pointer[-1];
PyStackRef_CLOSE(value);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(END_SEND) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(END_SEND);
_PyStackRef value;
_PyStackRef receiver;
value = stack_pointer[-1];
receiver = stack_pointer[-2];
(void)receiver;
PyStackRef_CLOSE(receiver);
stack_pointer[-2] = value;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(ENTER_EXECUTOR) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(ENTER_EXECUTOR);
#ifdef _Py_TIER2
PyCodeObject *code = _PyFrame_GetCode(frame);
_PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
assert(executor->vm_data.index == INSTR_OFFSET() - 1);
assert(executor->vm_data.code == code);
assert(executor->vm_data.valid);
assert(tstate->previous_executor == NULL);
/* If the eval breaker is set then stay in tier 1.
* This avoids any potentially infinite loops
* involving _RESUME_CHECK */
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
opcode = executor->vm_data.opcode;
oparg = (oparg & ~255) | executor->vm_data.oparg;
next_instr = this_instr;
if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) {
PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
}
DISPATCH_GOTO();
}
tstate->previous_executor = Py_None;
Py_INCREF(executor);
GOTO_TIER_TWO(executor);
#else
Py_FatalError("ENTER_EXECUTOR is not supported in this build");
#endif /* _Py_TIER2 */
DISPATCH();
}
TARGET(EXIT_INIT_CHECK) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(EXIT_INIT_CHECK);
_PyStackRef should_be_none;
should_be_none = stack_pointer[-1];
assert(STACK_LEVEL() == 2);
if (!PyStackRef_Is(should_be_none, PyStackRef_None)) {
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name);
goto error;
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(EXTENDED_ARG) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(EXTENDED_ARG);
assert(oparg);
opcode = next_instr->op.code;
oparg = oparg << 8 | next_instr->op.arg;
PRE_DISPATCH_GOTO();
DISPATCH_GOTO();
}
TARGET(FORMAT_SIMPLE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(FORMAT_SIMPLE);
_PyStackRef value;
_PyStackRef res;
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
/* If value is a unicode object, then we know the result
* of format(value) is value itself. */
if (!PyUnicode_CheckExact(value_o)) {
res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL));
PyStackRef_CLOSE(value);
if (PyStackRef_IsNull(res)) goto pop_1_error;
}
else {
res = value;
}
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(FORMAT_WITH_SPEC) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(FORMAT_WITH_SPEC);
_PyStackRef fmt_spec;
_PyStackRef value;
_PyStackRef res;
fmt_spec = stack_pointer[-1];
value = stack_pointer[-2];
PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec));
PyStackRef_CLOSE(value);
PyStackRef_CLOSE(fmt_spec);
if (res_o == NULL) goto pop_2_error;
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(FOR_ITER) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER);
PREDICTED(FOR_ITER);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
_PyStackRef iter;
_PyStackRef next;
// _SPECIALIZE_FOR_ITER
iter = stack_pointer[-1];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_ForIter(iter, next_instr, oparg);
DISPATCH_SAME_OPARG();
}
STAT_INC(FOR_ITER, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _FOR_ITER
{
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
if (next_o == NULL) {
next = PyStackRef_NULL;
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
goto error;
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
}
/* iterator ended normally */
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
PyStackRef_CLOSE(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */
JUMPBY(oparg + 2);
DISPATCH();
}
next = PyStackRef_FromPyObjectSteal(next_o);
// Common case: no jump, leave it to the code generator
}
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(FOR_ITER_GEN) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_GEN);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
_PyStackRef iter;
_PyInterpreterFrame *gen_frame;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, FOR_ITER);
}
// _FOR_ITER_GEN_FRAME
iter = stack_pointer[-1];
{
PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter);
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER);
STAT_INC(FOR_ITER, hit);
gen_frame = &gen->gi_iframe;
_PyFrame_StackPush(gen_frame, PyStackRef_None);
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
// oparg is the return offset from the next instruction.
frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg);
}
// _PUSH_FRAME
new_frame = gen_frame;
{
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(FOR_ITER_LIST) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_LIST);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
_PyStackRef iter;
_PyStackRef next;
/* Skip 1 cache entry */
// _ITER_CHECK_LIST
iter = stack_pointer[-1];
{
DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER);
}
// _ITER_JUMP_LIST
{
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyListIterObject *it = (_PyListIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyListIter_Type);
STAT_INC(FOR_ITER, hit);
PyListObject *seq = it->it_seq;
if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) {
it->it_index = -1;
#ifndef Py_GIL_DISABLED
if (seq != NULL) {
it->it_seq = NULL;
Py_DECREF(seq);
}
#endif
PyStackRef_CLOSE(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */
JUMPBY(oparg + 2);
DISPATCH();
}
}
// _ITER_NEXT_LIST
{
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyListIterObject *it = (_PyListIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
assert(seq);
assert(it->it_index < PyList_GET_SIZE(seq));
next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
}
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(FOR_ITER_RANGE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_RANGE);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
_PyStackRef iter;
_PyStackRef next;
/* Skip 1 cache entry */
// _ITER_CHECK_RANGE
iter = stack_pointer[-1];
{
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER);
}
// _ITER_JUMP_RANGE
{
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
STAT_INC(FOR_ITER, hit);
if (r->len <= 0) {
STACK_SHRINK(1);
PyStackRef_CLOSE(iter);
// Jump over END_FOR and POP_TOP instructions.
JUMPBY(oparg + 2);
DISPATCH();
}
}
// _ITER_NEXT_RANGE
{
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
assert(r->len > 0);
long value = r->start;
r->start = value + r->step;
r->len--;
PyObject *res = PyLong_FromLong(value);
if (res == NULL) goto error;
next = PyStackRef_FromPyObjectSteal(res);
}
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(FOR_ITER_TUPLE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_TUPLE);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
_PyStackRef iter;
_PyStackRef next;
/* Skip 1 cache entry */
// _ITER_CHECK_TUPLE
iter = stack_pointer[-1];
{
DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER);
}
// _ITER_JUMP_TUPLE
{
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
STAT_INC(FOR_ITER, hit);
PyTupleObject *seq = it->it_seq;
if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) {
if (seq != NULL) {
it->it_seq = NULL;
Py_DECREF(seq);
}
PyStackRef_CLOSE(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */
JUMPBY(oparg + 2);
DISPATCH();
}
}
// _ITER_NEXT_TUPLE
{
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
PyTupleObject *seq = it->it_seq;
assert(seq);
assert(it->it_index < PyTuple_GET_SIZE(seq));
next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++));
}
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(GET_AITER) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_AITER);
_PyStackRef obj;
_PyStackRef iter;
obj = stack_pointer[-1];
unaryfunc getter = NULL;
PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj);
PyObject *iter_o;
PyTypeObject *type = Py_TYPE(obj_o);
if (type->tp_as_async != NULL) {
getter = type->tp_as_async->am_aiter;
}
if (getter == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' requires an object with "
"__aiter__ method, got %.100s",
type->tp_name);
PyStackRef_CLOSE(obj);
if (true) goto pop_1_error;
}
iter_o = (*getter)(obj_o);
PyStackRef_CLOSE(obj);
if (iter_o == NULL) goto pop_1_error;
if (Py_TYPE(iter_o)->tp_as_async == NULL ||
Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' received an object from __aiter__ "
"that does not implement __anext__: %.100s",
Py_TYPE(iter_o)->tp_name);
Py_DECREF(iter_o);
if (true) goto pop_1_error;
}
iter = PyStackRef_FromPyObjectSteal(iter_o);
stack_pointer[-1] = iter;
DISPATCH();
}
TARGET(GET_ANEXT) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_ANEXT);
_PyStackRef aiter;
_PyStackRef awaitable;
aiter = stack_pointer[-1];
unaryfunc getter = NULL;
PyObject *next_iter = NULL;
PyObject *awaitable_o;
PyObject *aiter_o = PyStackRef_AsPyObjectBorrow(aiter);
PyTypeObject *type = Py_TYPE(aiter_o);
if (PyAsyncGen_CheckExact(aiter_o)) {
awaitable_o = type->tp_as_async->am_anext(aiter_o);
if (awaitable_o == NULL) {
goto error;
}
} else {
if (type->tp_as_async != NULL){
getter = type->tp_as_async->am_anext;
}
if (getter != NULL) {
next_iter = (*getter)(aiter_o);
if (next_iter == NULL) {
goto error;
}
}
else {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
goto error;
}
awaitable_o = _PyCoro_GetAwaitableIter(next_iter);
if (awaitable_o == NULL) {
_PyErr_FormatFromCause(
PyExc_TypeError,
"'async for' received an invalid object "
"from __anext__: %.100s",
Py_TYPE(next_iter)->tp_name);
Py_DECREF(next_iter);
goto error;
} else {
Py_DECREF(next_iter);
}
}
awaitable = PyStackRef_FromPyObjectSteal(awaitable_o);
stack_pointer[0] = awaitable;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(GET_AWAITABLE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_AWAITABLE);
_PyStackRef iterable;
_PyStackRef iter;
iterable = stack_pointer[-1];
PyObject *iter_o = _PyCoro_GetAwaitableIter(PyStackRef_AsPyObjectBorrow(iterable));
if (iter_o == NULL) {
_PyEval_FormatAwaitableError(tstate,
Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)), oparg);
}
PyStackRef_CLOSE(iterable);
if (iter_o != NULL && PyCoro_CheckExact(iter_o)) {
PyObject *yf = _PyGen_yf((PyGenObject*)iter_o);
if (yf != NULL) {
/* `iter` is a coroutine object that is being
awaited, `yf` is a pointer to the current awaitable
being awaited on. */
Py_DECREF(yf);
Py_CLEAR(iter_o);
_PyErr_SetString(tstate, PyExc_RuntimeError,
"coroutine is being awaited already");
/* The code below jumps to `error` if `iter` is NULL. */
}
}
if (iter_o == NULL) goto pop_1_error;
iter = PyStackRef_FromPyObjectSteal(iter_o);
stack_pointer[-1] = iter;
DISPATCH();
}
TARGET(GET_ITER) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_ITER);
_PyStackRef iterable;
_PyStackRef iter;
iterable = stack_pointer[-1];
/* before: [obj]; after [getiter(obj)] */
iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)));
PyStackRef_CLOSE(iterable);
if (PyStackRef_IsNull(iter)) goto pop_1_error;
stack_pointer[-1] = iter;
DISPATCH();
}
TARGET(GET_LEN) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_LEN);
_PyStackRef obj;
_PyStackRef len;
obj = stack_pointer[-1];
// PUSH(len(TOS))
Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj));
if (len_i < 0) goto error;
PyObject *len_o = PyLong_FromSsize_t(len_i);
if (len_o == NULL) goto error;
len = PyStackRef_FromPyObjectSteal(len_o);
stack_pointer[0] = len;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(GET_YIELD_FROM_ITER) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_YIELD_FROM_ITER);
_PyStackRef iterable;
_PyStackRef iter;
iterable = stack_pointer[-1];
/* before: [obj]; after [getiter(obj)] */
PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable);
if (PyCoro_CheckExact(iterable_o)) {
/* `iterable` is a coroutine */
if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
/* and it is used in a 'yield from' expression of a
regular generator. */
_PyErr_SetString(tstate, PyExc_TypeError,
"cannot 'yield from' a coroutine object "
"in a non-coroutine generator");
goto error;
}
iter = iterable;
}
else if (PyGen_CheckExact(iterable_o)) {
iter = iterable;
}
else {
/* `iterable` is not a generator. */
iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o));
if (PyStackRef_IsNull(iter)) {
goto error;
}
PyStackRef_CLOSE(iterable);
}
stack_pointer[-1] = iter;
DISPATCH();
}
TARGET(IMPORT_FROM) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IMPORT_FROM);
_PyStackRef from;
_PyStackRef res;
from = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *res_o = import_from(tstate, PyStackRef_AsPyObjectBorrow(from), name);
if (res_o == NULL) goto error;
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(IMPORT_NAME) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IMPORT_NAME);
_PyStackRef fromlist;
_PyStackRef level;
_PyStackRef res;
fromlist = stack_pointer[-1];
level = stack_pointer[-2];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *res_o = import_name(tstate, frame, name,
PyStackRef_AsPyObjectBorrow(fromlist),
PyStackRef_AsPyObjectBorrow(level));
PyStackRef_CLOSE(level);
PyStackRef_CLOSE(fromlist);
if (res_o == NULL) goto pop_2_error;
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(INSTRUMENTED_CALL) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 4;
INSTRUCTION_STATS(INSTRUMENTED_CALL);
/* Skip 3 cache entries */
int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL;
int total_args = oparg + is_meth;
PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2));
PyObject *arg = total_args == 0 ?
&_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args));
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, function, arg);
if (err) goto error;
PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
GO_TO_INSTRUCTION(CALL);
}
TARGET(INSTRUMENTED_CALL_FUNCTION_EX) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX);
GO_TO_INSTRUCTION(CALL_FUNCTION_EX);
}
TARGET(INSTRUMENTED_CALL_KW) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_CALL_KW);
int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2));
int total_args = oparg + is_meth;
PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3));
PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING
: PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1));
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, function, arg);
if (err) goto error;
GO_TO_INSTRUCTION(CALL_KW);
}
TARGET(INSTRUMENTED_END_FOR) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_END_FOR);
_PyStackRef value;
_PyStackRef receiver;
value = stack_pointer[-1];
receiver = stack_pointer[-2];
/* Need to create a fake StopIteration error here,
* to conform to PEP 380 */
if (PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver))) {
if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) {
goto error;
}
}
PyStackRef_CLOSE(value);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(INSTRUMENTED_END_SEND) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_END_SEND);
_PyStackRef value;
_PyStackRef receiver;
value = stack_pointer[-1];
receiver = stack_pointer[-2];
PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver);
if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) {
if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) {
goto error;
}
}
PyStackRef_CLOSE(receiver);
stack_pointer[-2] = value;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(INSTRUMENTED_FOR_ITER) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER);
/* Skip 1 cache entry */
_Py_CODEUNIT *target;
_PyStackRef iter_stackref = TOP();
PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref);
PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter);
if (next != NULL) {
PUSH(PyStackRef_FromPyObjectSteal(next));
target = next_instr;
}
else {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
goto error;
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
}
/* iterator ended normally */
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
STACK_SHRINK(1);
PyStackRef_CLOSE(iter_stackref);
/* Skip END_FOR and POP_TOP */
target = next_instr + oparg + 2;
}
INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH);
DISPATCH();
}
TARGET(INSTRUMENTED_INSTRUCTION) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION);
int next_opcode = _Py_call_instrumentation_instruction(
tstate, frame, this_instr);
if (next_opcode < 0) goto error;
next_instr = this_instr;
if (_PyOpcode_Caches[next_opcode]) {
PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter);
}
assert(next_opcode > 0 && next_opcode < 256);
opcode = next_opcode;
DISPATCH_GOTO();
}
TARGET(INSTRUMENTED_JUMP_BACKWARD) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD);
/* Skip 1 cache entry */
CHECK_EVAL_BREAKER();
INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP);
DISPATCH();
}
TARGET(INSTRUMENTED_JUMP_FORWARD) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD);
INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP);
DISPATCH();
}
TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR);
/* Skip 1 cache entry */
// cancel out the decrement that will happen in LOAD_SUPER_ATTR; we
// don't want to specialize instrumented instructions
PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
GO_TO_INSTRUCTION(LOAD_SUPER_ATTR);
}
TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE);
/* Skip 1 cache entry */
_PyStackRef cond = POP();
assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
int flag = PyStackRef_Is(cond, PyStackRef_False);
int offset = flag * oparg;
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
DISPATCH();
}
TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE);
/* Skip 1 cache entry */
_PyStackRef value_stackref = POP();
int flag = PyStackRef_Is(value_stackref, PyStackRef_None);
int offset;
if (flag) {
offset = oparg;
}
else {
PyStackRef_CLOSE(value_stackref);
offset = 0;
}
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
DISPATCH();
}
TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE);
/* Skip 1 cache entry */
_PyStackRef value_stackref = POP();
int offset;
int nflag = PyStackRef_Is(value_stackref, PyStackRef_None);
if (nflag) {
offset = 0;
}
else {
PyStackRef_CLOSE(value_stackref);
offset = oparg;
}
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | !nflag;
#endif
INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
DISPATCH();
}
TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE);
/* Skip 1 cache entry */
_PyStackRef cond = POP();
assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
int flag = PyStackRef_Is(cond, PyStackRef_True);
int offset = flag * oparg;
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
DISPATCH();
}
TARGET(INSTRUMENTED_RESUME) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_RESUME);
uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK;
uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
if (code_version != global_version && tstate->tracing == 0) {
if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) {
goto error;
}
next_instr = this_instr;
}
else {
if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
CHECK_EVAL_BREAKER();
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_call_instrumentation(
tstate, oparg > 0, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) goto error;
if (frame->instr_ptr != this_instr) {
/* Instrumentation has jumped */
next_instr = frame->instr_ptr;
DISPATCH();
}
}
DISPATCH();
}
TARGET(INSTRUMENTED_RETURN_CONST) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST);
PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
if (err) goto error;
Py_INCREF(retval);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_LeaveRecursiveCallPy(tstate);
assert(frame != &entry_frame);
// GH-99729: We need to unlink the frame *before* clearing it:
_PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
_PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(retval));
LOAD_IP(frame->return_offset);
goto resume_frame;
}
TARGET(INSTRUMENTED_RETURN_VALUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE);
_PyStackRef retval;
retval = stack_pointer[-1];
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, PyStackRef_AsPyObjectBorrow(retval));
if (err) goto error;
STACK_SHRINK(1);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_LeaveRecursiveCallPy(tstate);
assert(frame != &entry_frame);
// GH-99729: We need to unlink the frame *before* clearing it:
_PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
_PyFrame_StackPush(frame, retval);
LOAD_IP(frame->return_offset);
goto resume_frame;
}
TARGET(INSTRUMENTED_YIELD_VALUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE);
_PyStackRef retval;
retval = stack_pointer[-1];
assert(frame != &entry_frame);
frame->instr_ptr = next_instr;
PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame);
assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1);
assert(oparg == 0 || oparg == 1);
gen->gi_frame_state = FRAME_SUSPENDED + oparg;
_PyFrame_SetStackPointer(frame, stack_pointer - 1);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_YIELD,
frame, this_instr, PyStackRef_AsPyObjectBorrow(retval));
if (err) goto error;
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
_PyInterpreterFrame *gen_frame = frame;
frame = tstate->current_frame = frame->previous;
gen_frame->previous = NULL;
_PyFrame_StackPush(frame, retval);
/* We don't know which of these is relevant here, so keep them equal */
assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER);
LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND);
goto resume_frame;
}
TARGET(INTERPRETER_EXIT) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INTERPRETER_EXIT);
_PyStackRef retval;
retval = stack_pointer[-1];
assert(frame == &entry_frame);
assert(_PyFrame_IsIncomplete(frame));
/* Restore previous frame and return. */
tstate->current_frame = frame->previous;
assert(!_PyErr_Occurred(tstate));
tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS;
return PyStackRef_AsPyObjectSteal(retval);
}
TARGET(IS_OP) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IS_OP);
_PyStackRef right;
_PyStackRef left;
_PyStackRef b;
right = stack_pointer[-1];
left = stack_pointer[-2];
#ifdef Py_GIL_DISABLED
// On free-threaded builds, objects are conditionally immortalized.
// So their bits don't always compare equally.
int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg;
#else
int res = PyStackRef_Is(left, right) ^ oparg;
#endif
PyStackRef_CLOSE(left);
PyStackRef_CLOSE(right);
b = res ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(JUMP_BACKWARD) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(JUMP_BACKWARD);
/* Skip 1 cache entry */
CHECK_EVAL_BREAKER();
assert(oparg <= INSTR_OFFSET());
JUMPBY(-oparg);
#ifdef _Py_TIER2
#if ENABLE_SPECIALIZATION
_Py_BackoffCounter counter = this_instr[1].counter;
if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) {
_Py_CODEUNIT *start = this_instr;
/* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */
while (oparg > 255) {
oparg >>= 8;
start--;
}
_PyExecutorObject *executor;
int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor);
if (optimized < 0) goto error;
if (optimized) {
assert(tstate->previous_executor == NULL);
tstate->previous_executor = Py_None;
GOTO_TIER_TWO(executor);
}
else {
this_instr[1].counter = restart_backoff_counter(counter);
}
}
else {
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
}
#endif /* ENABLE_SPECIALIZATION */
#endif /* _Py_TIER2 */
DISPATCH();
}
TARGET(JUMP_BACKWARD_NO_INTERRUPT) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT);
/* This bytecode is used in the `yield from` or `await` loop.
* If there is an interrupt, we want it handled in the innermost
* generator or coroutine, so we deliberately do not check it here.
* (see bpo-30039).
*/
JUMPBY(-oparg);
DISPATCH();
}
TARGET(JUMP_FORWARD) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(JUMP_FORWARD);
JUMPBY(oparg);
DISPATCH();
}
TARGET(LIST_APPEND) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LIST_APPEND);
_PyStackRef v;
_PyStackRef list;
v = stack_pointer[-1];
list = stack_pointer[-2 - (oparg-1)];
if (_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list),
PyStackRef_AsPyObjectSteal(v)) < 0) goto pop_1_error;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LIST_EXTEND) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LIST_EXTEND);
_PyStackRef iterable_st;
_PyStackRef list_st;
iterable_st = stack_pointer[-1];
list_st = stack_pointer[-2 - (oparg-1)];
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st);
PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
if (none_val == NULL) {
if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
(Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable)))
{
_PyErr_Clear(tstate);
_PyErr_Format(tstate, PyExc_TypeError,
"Value after * must be an iterable, not %.200s",
Py_TYPE(iterable)->tp_name);
}
PyStackRef_CLOSE(iterable_st);
if (true) goto pop_1_error;
}
assert(Py_IsNone(none_val));
PyStackRef_CLOSE(iterable_st);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_ATTR) {
frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR);
PREDICTED(LOAD_ATTR);
_Py_CODEUNIT *this_instr = next_instr - 10;
(void)this_instr;
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self_or_null = PyStackRef_NULL;
// _SPECIALIZE_LOAD_ATTR
owner = stack_pointer[-1];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr;
_Py_Specialize_LoadAttr(owner, next_instr, name);
DISPATCH_SAME_OPARG();
}
STAT_INC(LOAD_ATTR, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
/* Skip 8 cache entries */
// _LOAD_ATTR
{
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
PyObject *attr_o;
if (oparg & 1) {
/* Designed to work in tandem with CALL, pushes two values. */
attr_o = NULL;
if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) {
/* We can bypass temporary bound method object.
meth is unbound method and obj is self.
meth | self | arg1 | ... | argN
*/
assert(attr_o != NULL); // No errors on this branch
self_or_null = owner; // Transfer ownership
}
else {
/* meth is not an unbound method (but a regular attr, or
something was returned by a descriptor protocol). Set
the second element of the stack to NULL, to signal
CALL that it's not a method call.
meth | NULL | arg1 | ... | argN
*/
PyStackRef_CLOSE(owner);
if (attr_o == NULL) goto pop_1_error;
self_or_null = PyStackRef_NULL;
}
}
else {
/* Classic, pushes one value. */
attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
PyStackRef_CLOSE(owner);
if (attr_o == NULL) goto pop_1_error;
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
}
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = self_or_null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_ATTR_CLASS) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_CLASS);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _CHECK_ATTR_CLASS
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR);
assert(type_version != 0);
DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_CLASS
{
PyObject *descr = read_obj(&this_instr[6].cache);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = PyStackRef_FromPyObjectNew(descr);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
/* Skip 1 cache entry */
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
uint32_t func_version = read_u32(&this_instr[4].cache);
PyObject *getattribute = read_obj(&this_instr[6].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)getattribute;
assert(func_version != 0);
DEOPT_IF(f->func_version != func_version, LOAD_ATTR);
PyCodeObject *code = (PyCodeObject *)f->func_code;
assert(code->co_argcount == 2);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
Py_INCREF(f);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2);
// Manipulate stack directly because we exit with DISPATCH_INLINED().
STACK_SHRINK(1);
new_frame->localsplus[0] = owner;
new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name);
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
}
TARGET(LOAD_ATTR_INSTANCE_VALUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _CHECK_MANAGED_OBJECT_HAS_VALUES
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
}
// _LOAD_ATTR_INSTANCE_VALUE
{
uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index];
DEOPT_IF(attr_o == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr_o);
null = PyStackRef_NULL;
attr = PyStackRef_FromPyObjectSteal(attr_o);
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_ATTR_METHOD_LAZY_DICT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _CHECK_ATTR_METHOD_LAZY_DICT
{
uint16_t dictoffset = read_u16(&this_instr[4].cache);
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = *(PyObject **)ptr;
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL, LOAD_ATTR);
}
/* Skip 1 cache entry */
// _LOAD_ATTR_METHOD_LAZY_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert(oparg & 1);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_ATTR_METHOD_NO_DICT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_METHOD_NO_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert(oparg & 1);
assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_ATTR_METHOD_WITH_VALUES) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
}
// _LOAD_ATTR_METHOD_WITH_VALUES
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert(oparg & 1);
/* Cached method object */
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_ATTR_MODULE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_MODULE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _CHECK_ATTR_MODULE
owner = stack_pointer[-1];
{
uint32_t dict_version = read_u32(&this_instr[2].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(!PyModule_CheckExact(owner_o), LOAD_ATTR);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict != NULL);
DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR);
}
// _LOAD_ATTR_MODULE
{
uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < dict->ma_keys->dk_nentries);
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
PyObject *attr_o = ep->me_value;
DEOPT_IF(attr_o == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr_o);
attr = PyStackRef_FromPyObjectSteal(attr_o);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_NONDESCRIPTOR_NO_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert((oparg & 1) == 0);
assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
PyStackRef_CLOSE(owner);
attr = PyStackRef_FromPyObjectNew(descr);
}
stack_pointer[-1] = attr;
DISPATCH();
}
TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
}
// _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert((oparg & 1) == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
PyStackRef_CLOSE(owner);
attr = PyStackRef_FromPyObjectNew(descr);
}
stack_pointer[-1] = attr;
DISPATCH();
}
TARGET(LOAD_ATTR_PROPERTY) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_PROPERTY);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
/* Skip 1 cache entry */
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
uint32_t func_version = read_u32(&this_instr[4].cache);
PyObject *fget = read_obj(&this_instr[6].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
assert(Py_IS_TYPE(fget, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)fget;
assert(func_version != 0);
DEOPT_IF(f->func_version != func_version, LOAD_ATTR);
PyCodeObject *code = (PyCodeObject *)f->func_code;
assert(code->co_argcount == 1);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(fget);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1);
// Manipulate stack directly because we exit with DISPATCH_INLINED().
STACK_SHRINK(1);
new_frame->localsplus[0] = owner;
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
}
TARGET(LOAD_ATTR_SLOT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_SLOT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _LOAD_ATTR_SLOT
{
uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
char *addr = (char *)owner_o + index;
PyObject *attr_o = *(PyObject **)addr;
DEOPT_IF(attr_o == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
attr = PyStackRef_FromPyObjectNew(attr_o);
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_ATTR_WITH_HINT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _CHECK_ATTR_WITH_HINT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
DEOPT_IF(dict == NULL, LOAD_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
}
// _LOAD_ATTR_WITH_HINT
{
uint16_t hint = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject *attr_o;
PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, LOAD_ATTR);
attr_o = ep->me_value;
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, LOAD_ATTR);
attr_o = ep->me_value;
}
DEOPT_IF(attr_o == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr_o);
attr = PyStackRef_FromPyObjectSteal(attr_o);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_BUILD_CLASS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_BUILD_CLASS);
_PyStackRef bc;
PyObject *bc_o;
if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0) goto error;
if (bc_o == NULL) {
_PyErr_SetString(tstate, PyExc_NameError,
"__build_class__ not found");
if (true) goto error;
}
bc = PyStackRef_FromPyObjectSteal(bc_o);
stack_pointer[0] = bc;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_COMMON_CONSTANT) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_COMMON_CONSTANT);
_PyStackRef value;
// Keep in sync with _common_constants in opcode.py
switch(oparg) {
case CONSTANT_ASSERTIONERROR:
value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError);
break;
case CONSTANT_NOTIMPLEMENTEDERROR:
value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError);
break;
default:
Py_FatalError("bad LOAD_COMMON_CONSTANT oparg");
}
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_CONST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_CONST);
_PyStackRef value;
value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_DEREF) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_DEREF);
_PyStackRef value;
PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
PyObject *value_o = PyCell_GetRef(cell);
if (value_o == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
if (true) goto error;
}
value = PyStackRef_FromPyObjectSteal(value_o);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_FAST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST);
_PyStackRef value;
assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
value = PyStackRef_DUP(GETLOCAL(oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_FAST_AND_CLEAR) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR);
_PyStackRef value;
value = GETLOCAL(oparg);
// do not use SETLOCAL here, it decrefs the old value
GETLOCAL(oparg) = PyStackRef_NULL;
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_FAST_CHECK) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_CHECK);
_PyStackRef value;
_PyStackRef value_s = GETLOCAL(oparg);
if (PyStackRef_IsNull(value_s)) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
if (1) goto error;
}
value = PyStackRef_DUP(value_s);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_FAST_LOAD_FAST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST);
_PyStackRef value1;
_PyStackRef value2;
uint32_t oparg1 = oparg >> 4;
uint32_t oparg2 = oparg & 15;
value1 = PyStackRef_DUP(GETLOCAL(oparg1));
value2 = PyStackRef_DUP(GETLOCAL(oparg2));
stack_pointer[0] = value1;
stack_pointer[1] = value2;
stack_pointer += 2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_FROM_DICT_OR_DEREF) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF);
_PyStackRef class_dict_st;
_PyStackRef value;
class_dict_st = stack_pointer[-1];
PyObject *value_o;
PyObject *name;
PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st);
assert(class_dict);
assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) {
goto error;
}
if (!value_o) {
PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
value_o = PyCell_GetRef(cell);
if (value_o == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
goto error;
}
}
PyStackRef_CLOSE(class_dict_st);
value = PyStackRef_FromPyObjectSteal(value_o);
stack_pointer[-1] = value;
DISPATCH();
}
TARGET(LOAD_FROM_DICT_OR_GLOBALS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS);
_PyStackRef mod_or_class_dict;
_PyStackRef v;
mod_or_class_dict = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *v_o;
if (PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o) < 0) {
goto error;
}
if (v_o == NULL) {
if (PyDict_CheckExact(GLOBALS())
&& PyDict_CheckExact(BUILTINS()))
{
v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
(PyDictObject *)BUILTINS(),
name);
if (v_o == NULL) {
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
goto error;
}
}
else {
/* Slow-path if globals or builtins is not a dict */
/* namespace 1: globals */
if (PyMapping_GetOptionalItem(GLOBALS(), name, &v_o) < 0) goto pop_1_error;
if (v_o == NULL) {
/* namespace 2: builtins */
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) goto pop_1_error;
if (v_o == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
if (true) goto pop_1_error;
}
}
}
}
PyStackRef_CLOSE(mod_or_class_dict);
v = PyStackRef_FromPyObjectSteal(v_o);
stack_pointer[-1] = v;
DISPATCH();
}
TARGET(LOAD_GLOBAL) {
frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL);
PREDICTED(LOAD_GLOBAL);
_Py_CODEUNIT *this_instr = next_instr - 5;
(void)this_instr;
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
// _SPECIALIZE_LOAD_GLOBAL
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr;
_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
DISPATCH_SAME_OPARG();
}
STAT_INC(LOAD_GLOBAL, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
/* Skip 1 cache entry */
/* Skip 1 cache entry */
/* Skip 1 cache entry */
// _LOAD_GLOBAL
{
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
PyObject *res_o;
if (PyDict_CheckExact(GLOBALS())
&& PyDict_CheckExact(BUILTINS()))
{
res_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
(PyDictObject *)BUILTINS(),
name);
if (res_o == NULL) {
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
if (true) goto error;
}
}
else {
/* Slow-path if globals or builtins is not a dict */
/* namespace 1: globals */
if (PyMapping_GetOptionalItem(GLOBALS(), name, &res_o) < 0) goto error;
if (res_o == NULL) {
/* namespace 2: builtins */
if (PyMapping_GetOptionalItem(BUILTINS(), name, &res_o) < 0) goto error;
if (res_o == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
if (true) goto error;
}
}
}
null = PyStackRef_NULL;
res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_GLOBAL_BUILTIN) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN);
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_GLOBALS_VERSION
{
uint16_t version = read_u16(&this_instr[2].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
assert(DK_IS_UNICODE(dict->ma_keys));
}
// _GUARD_BUILTINS_VERSION
{
uint16_t version = read_u16(&this_instr[3].cache);
PyDictObject *dict = (PyDictObject *)BUILTINS();
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
assert(DK_IS_UNICODE(dict->ma_keys));
}
// _LOAD_GLOBAL_BUILTINS
{
uint16_t index = read_u16(&this_instr[4].cache);
PyDictObject *bdict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
PyObject *res_o = entries[index].me_value;
DEOPT_IF(res_o == NULL, LOAD_GLOBAL);
Py_INCREF(res_o);
STAT_INC(LOAD_GLOBAL, hit);
null = PyStackRef_NULL;
res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_GLOBAL_MODULE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_MODULE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_GLOBALS_VERSION
{
uint16_t version = read_u16(&this_instr[2].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
assert(DK_IS_UNICODE(dict->ma_keys));
}
/* Skip 1 cache entry */
// _LOAD_GLOBAL_MODULE
{
uint16_t index = read_u16(&this_instr[4].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
PyObject *res_o = entries[index].me_value;
DEOPT_IF(res_o == NULL, LOAD_GLOBAL);
Py_INCREF(res_o);
STAT_INC(LOAD_GLOBAL, hit);
null = PyStackRef_NULL;
res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_LOCALS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_LOCALS);
_PyStackRef locals;
PyObject *l = LOCALS();
if (l == NULL) {
_PyErr_SetString(tstate, PyExc_SystemError,
"no locals found");
if (true) goto error;
}
locals = PyStackRef_FromPyObjectNew(l);;
stack_pointer[0] = locals;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_NAME) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_NAME);
_PyStackRef v;
PyObject *v_o;
PyObject *mod_or_class_dict = LOCALS();
if (mod_or_class_dict == NULL) {
_PyErr_SetString(tstate, PyExc_SystemError,
"no locals found");
if (true) goto error;
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) {
goto error;
}
if (v_o == NULL) {
if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) {
goto error;
}
if (v_o == NULL) {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) {
goto error;
}
if (v_o == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
goto error;
}
}
}
v = PyStackRef_FromPyObjectSteal(v_o);
stack_pointer[0] = v;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_SPECIAL) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_SPECIAL);
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self_or_null;
owner = stack_pointer[-1];
assert(oparg <= SPECIAL_MAX);
PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner);
PyObject *name = _Py_SpecialMethods[oparg].name;
PyObject *self_or_null_o;
attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o));
if (PyStackRef_IsNull(attr)) {
if (!_PyErr_Occurred(tstate)) {
_PyErr_Format(tstate, PyExc_TypeError,
_Py_SpecialMethods[oparg].error,
Py_TYPE(owner_o)->tp_name);
}
}
if (PyStackRef_IsNull(attr)) goto pop_1_error;
self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o);
stack_pointer[-1] = attr;
stack_pointer[0] = self_or_null;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_SUPER_ATTR) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR);
PREDICTED(LOAD_SUPER_ATTR);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
_PyStackRef class_st;
_PyStackRef global_super_st;
_PyStackRef self_st;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
// _SPECIALIZE_LOAD_SUPER_ATTR
class_st = stack_pointer[-2];
global_super_st = stack_pointer[-3];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
int load_method = oparg & 1;
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method);
DISPATCH_SAME_OPARG();
}
STAT_INC(LOAD_SUPER_ATTR, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _LOAD_SUPER_ATTR
self_st = stack_pointer[-1];
{
PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, global_super, arg);
if (err) goto pop_3_error;
}
// we make no attempt to optimize here; specializations should
// handle any case whose performance we care about
PyObject *stack[] = {class, self};
PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL);
if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
if (super == NULL) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
frame, this_instr, global_super, arg);
}
else {
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_C_RETURN,
frame, this_instr, global_super, arg);
if (err < 0) {
Py_CLEAR(super);
}
}
}
PyStackRef_CLOSE(global_super_st);
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
if (super == NULL) goto pop_3_error;
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
attr = PyStackRef_FromPyObjectSteal(PyObject_GetAttr(super, name));
Py_DECREF(super);
if (PyStackRef_IsNull(attr)) goto pop_3_error;
null = PyStackRef_NULL;
}
stack_pointer[-3] = attr;
if (oparg & 1) stack_pointer[-2] = null;
stack_pointer += -2 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_SUPER_ATTR_ATTR) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR);
static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size");
_PyStackRef self_st;
_PyStackRef class_st;
_PyStackRef global_super_st;
_PyStackRef attr_st;
/* Skip 1 cache entry */
self_st = stack_pointer[-1];
class_st = stack_pointer[-2];
global_super_st = stack_pointer[-3];
PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
assert(!(oparg & 1));
DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
PyStackRef_CLOSE(global_super_st);
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
if (attr == NULL) goto pop_3_error;
attr_st = PyStackRef_FromPyObjectSteal(attr);
stack_pointer[-3] = attr_st;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_SUPER_ATTR_METHOD) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD);
static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size");
_PyStackRef self_st;
_PyStackRef class_st;
_PyStackRef global_super_st;
_PyStackRef attr;
_PyStackRef self_or_null;
/* Skip 1 cache entry */
self_st = stack_pointer[-1];
class_st = stack_pointer[-2];
global_super_st = stack_pointer[-3];
PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
assert(oparg & 1);
DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
PyTypeObject *cls = (PyTypeObject *)class;
int method_found = 0;
PyObject *attr_o = _PySuper_Lookup(cls, self, name,
Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL);
PyStackRef_CLOSE(global_super_st);
PyStackRef_CLOSE(class_st);
if (attr_o == NULL) {
PyStackRef_CLOSE(self_st);
if (true) goto pop_3_error;
}
if (method_found) {
self_or_null = self_st; // transfer ownership
} else {
PyStackRef_CLOSE(self_st);
self_or_null = PyStackRef_NULL;
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
stack_pointer[-3] = attr;
stack_pointer[-2] = self_or_null;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(MAKE_CELL) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAKE_CELL);
// "initial" is probably NULL but not if it's an arg (or set
// via the f_locals proxy before MAKE_CELL has run).
PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
goto error;
}
SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell));
DISPATCH();
}
TARGET(MAKE_FUNCTION) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAKE_FUNCTION);
_PyStackRef codeobj_st;
_PyStackRef func;
codeobj_st = stack_pointer[-1];
PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st);
PyFunctionObject *func_obj = (PyFunctionObject *)
PyFunction_New(codeobj, GLOBALS());
PyStackRef_CLOSE(codeobj_st);
if (func_obj == NULL) {
goto error;
}
_PyFunction_SetVersion(
func_obj, ((PyCodeObject *)codeobj)->co_version);
func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj);
stack_pointer[-1] = func;
DISPATCH();
}
TARGET(MAP_ADD) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAP_ADD);
_PyStackRef value;
_PyStackRef key;
_PyStackRef dict_st;
value = stack_pointer[-1];
key = stack_pointer[-2];
dict_st = stack_pointer[-3 - (oparg - 1)];
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
assert(PyDict_CheckExact(dict));
/* dict[key] = value */
// Do not DECREF INPUTS because the function steals the references
if (_PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value)) != 0) goto pop_2_error;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(MATCH_CLASS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_CLASS);
_PyStackRef names;
_PyStackRef type;
_PyStackRef subject;
_PyStackRef attrs;
names = stack_pointer[-1];
type = stack_pointer[-2];
subject = stack_pointer[-3];
// Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
// None on failure.
assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names)));
PyObject *attrs_o = _PyEval_MatchClass(tstate,
PyStackRef_AsPyObjectBorrow(subject),
PyStackRef_AsPyObjectBorrow(type), oparg,
PyStackRef_AsPyObjectBorrow(names));
PyStackRef_CLOSE(subject);
PyStackRef_CLOSE(type);
PyStackRef_CLOSE(names);
if (attrs_o) {
assert(PyTuple_CheckExact(attrs_o)); // Success!
attrs = PyStackRef_FromPyObjectSteal(attrs_o);
}
else {
if (_PyErr_Occurred(tstate)) goto pop_3_error;
// Error!
attrs = PyStackRef_None; // Failure!
}
stack_pointer[-3] = attrs;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(MATCH_KEYS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_KEYS);
_PyStackRef keys;
_PyStackRef subject;
_PyStackRef values_or_none;
keys = stack_pointer[-1];
subject = stack_pointer[-2];
// On successful match, PUSH(values). Otherwise, PUSH(None).
PyObject *values_or_none_o = _PyEval_MatchKeys(tstate,
PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys));
if (values_or_none_o == NULL) goto error;
values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o);
stack_pointer[0] = values_or_none;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(MATCH_MAPPING) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_MAPPING);
_PyStackRef subject;
_PyStackRef res;
subject = stack_pointer[-1];
int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
res = match ? PyStackRef_True : PyStackRef_False;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(MATCH_SEQUENCE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_SEQUENCE);
_PyStackRef subject;
_PyStackRef res;
subject = stack_pointer[-1];
int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
res = match ? PyStackRef_True : PyStackRef_False;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(NOP) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(NOP);
DISPATCH();
}
TARGET(POP_EXCEPT) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(POP_EXCEPT);
_PyStackRef exc_value;
exc_value = stack_pointer[-1];
_PyErr_StackItem *exc_info = tstate->exc_info;
Py_XSETREF(exc_info->exc_value,
PyStackRef_AsPyObjectBorrow(exc_value) == Py_None
? NULL : PyStackRef_AsPyObjectSteal(exc_value));
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(POP_JUMP_IF_FALSE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_FALSE);
_PyStackRef cond;
/* Skip 1 cache entry */
cond = stack_pointer[-1];
assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
int flag = PyStackRef_Is(cond, PyStackRef_False);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
JUMPBY(oparg * flag);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(POP_JUMP_IF_NONE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_NONE);
_PyStackRef value;
_PyStackRef b;
_PyStackRef cond;
/* Skip 1 cache entry */
// _IS_NONE
value = stack_pointer[-1];
{
if (PyStackRef_Is(value, PyStackRef_None)) {
b = PyStackRef_True;
}
else {
b = PyStackRef_False;
PyStackRef_CLOSE(value);
}
}
// _POP_JUMP_IF_TRUE
cond = b;
{
assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
int flag = PyStackRef_Is(cond, PyStackRef_True);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
JUMPBY(oparg * flag);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(POP_JUMP_IF_NOT_NONE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE);
_PyStackRef value;
_PyStackRef b;
_PyStackRef cond;
/* Skip 1 cache entry */
// _IS_NONE
value = stack_pointer[-1];
{
if (PyStackRef_Is(value, PyStackRef_None)) {
b = PyStackRef_True;
}
else {
b = PyStackRef_False;
PyStackRef_CLOSE(value);
}
}
// _POP_JUMP_IF_FALSE
cond = b;
{
assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
int flag = PyStackRef_Is(cond, PyStackRef_False);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
JUMPBY(oparg * flag);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(POP_JUMP_IF_TRUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_TRUE);
_PyStackRef cond;
/* Skip 1 cache entry */
cond = stack_pointer[-1];
assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
int flag = PyStackRef_Is(cond, PyStackRef_True);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
JUMPBY(oparg * flag);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(POP_TOP) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(POP_TOP);
_PyStackRef value;
value = stack_pointer[-1];
PyStackRef_CLOSE(value);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(PUSH_EXC_INFO) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(PUSH_EXC_INFO);
_PyStackRef new_exc;
_PyStackRef prev_exc;
new_exc = stack_pointer[-1];
_PyErr_StackItem *exc_info = tstate->exc_info;
if (exc_info->exc_value != NULL) {
prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value);
}
else {
prev_exc = PyStackRef_None;
}
assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc)));
exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc);
stack_pointer[-1] = prev_exc;
stack_pointer[0] = new_exc;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(PUSH_NULL) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(PUSH_NULL);
_PyStackRef res;
res = PyStackRef_NULL;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(RAISE_VARARGS) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(RAISE_VARARGS);
_PyStackRef *args;
args = &stack_pointer[-oparg];
PyObject *cause = NULL, *exc = NULL;
switch (oparg) {
case 2:
cause = PyStackRef_AsPyObjectSteal(args[1]);
/* fall through */
case 1:
exc = PyStackRef_AsPyObjectSteal(args[0]);
/* fall through */
case 0:
if (do_raise(tstate, exc, cause)) {
assert(oparg == 0);
monitor_reraise(tstate, frame, this_instr);
goto exception_unwind;
}
break;
default:
_PyErr_SetString(tstate, PyExc_SystemError,
"bad RAISE_VARARGS oparg");
break;
}
if (true) { stack_pointer += -oparg; goto error; }
}
TARGET(RERAISE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(RERAISE);
_PyStackRef exc_st;
_PyStackRef *values;
exc_st = stack_pointer[-1];
values = &stack_pointer[-1 - oparg];
PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st);
assert(oparg >= 0 && oparg <= 2);
if (oparg) {
PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]);
if (PyLong_Check(lasti)) {
frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti);
assert(!_PyErr_Occurred(tstate));
}
else {
assert(PyLong_Check(lasti));
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
goto error;
}
}
assert(exc && PyExceptionInstance_Check(exc));
Py_INCREF(exc);
_PyErr_SetRaisedException(tstate, exc);
monitor_reraise(tstate, frame, this_instr);
goto exception_unwind;
}
TARGET(RESERVED) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESERVED);
assert(0 && "Executing RESERVED instruction.");
Py_FatalError("Executing RESERVED instruction.");
DISPATCH();
}
TARGET(RESUME) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESUME);
PREDICTED(RESUME);
_Py_CODEUNIT *this_instr = next_instr - 1;
(void)this_instr;
assert(frame == tstate->current_frame);
if (tstate->tracing == 0) {
uintptr_t global_version =
_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &
~_PY_EVAL_EVENTS_MASK;
PyCodeObject* code = _PyFrame_GetCode(frame);
uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version);
assert((code_version & 255) == 0);
if (code_version != global_version) {
int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
if (err) goto error;
next_instr = this_instr;
DISPATCH();
}
assert(this_instr->op.code == RESUME ||
this_instr->op.code == RESUME_CHECK ||
this_instr->op.code == INSTRUMENTED_RESUME ||
this_instr->op.code == ENTER_EXECUTOR);
if (this_instr->op.code == RESUME) {
#if ENABLE_SPECIALIZATION
FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);
#endif /* ENABLE_SPECIALIZATION */
}
}
if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
CHECK_EVAL_BREAKER();
}
DISPATCH();
}
TARGET(RESUME_CHECK) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESUME_CHECK);
static_assert(0 == 0, "incorrect cache size");
#if defined(__EMSCRIPTEN__)
DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME);
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
DEOPT_IF(eval_breaker != version, RESUME);
DISPATCH();
}
TARGET(RETURN_CONST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RETURN_CONST);
_PyStackRef value;
_PyStackRef retval;
_PyStackRef res;
// _LOAD_CONST
{
value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg));
}
// _RETURN_VALUE
retval = value;
{
#if TIER_ONE
assert(frame != &entry_frame);
#endif
_PyFrame_SetStackPointer(frame, stack_pointer);
assert(EMPTY());
_Py_LeaveRecursiveCallPy(tstate);
// GH-99729: We need to unlink the frame *before* clearing it:
_PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
LOAD_SP();
LOAD_IP(frame->return_offset);
res = retval;
LLTRACE_RESUME_FRAME();
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(RETURN_GENERATOR) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RETURN_GENERATOR);
_PyStackRef res;
assert(PyFunction_Check(frame->f_funcobj));
PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj;
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
if (gen == NULL) {
goto error;
}
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyInterpreterFrame *gen_frame = &gen->gi_iframe;
frame->instr_ptr++;
_PyFrame_Copy(frame, gen_frame);
assert(frame->frame_obj == NULL);
gen->gi_frame_state = FRAME_CREATED;
gen_frame->owner = FRAME_OWNED_BY_GENERATOR;
_Py_LeaveRecursiveCallPy(tstate);
res = PyStackRef_FromPyObjectSteal((PyObject *)gen);
_PyInterpreterFrame *prev = frame->previous;
_PyThreadState_PopFrame(tstate, frame);
frame = tstate->current_frame = prev;
LOAD_IP(frame->return_offset);
LOAD_SP();
LLTRACE_RESUME_FRAME();
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(RETURN_VALUE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RETURN_VALUE);
_PyStackRef retval;
_PyStackRef res;
retval = stack_pointer[-1];
#if TIER_ONE
assert(frame != &entry_frame);
#endif
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
assert(EMPTY());
_Py_LeaveRecursiveCallPy(tstate);
// GH-99729: We need to unlink the frame *before* clearing it:
_PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
LOAD_SP();
LOAD_IP(frame->return_offset);
res = retval;
LLTRACE_RESUME_FRAME();
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(SEND) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(SEND);
PREDICTED(SEND);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
_PyStackRef receiver;
_PyStackRef v;
_PyStackRef retval;
// _SPECIALIZE_SEND
receiver = stack_pointer[-2];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_Send(receiver, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(SEND, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _SEND
v = stack_pointer[-1];
{
PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver);
PyObject *retval_o;
assert(frame != &entry_frame);
if ((tstate->interp->eval_frame == NULL) &&
(Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) &&
((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING)
{
PyGenObject *gen = (PyGenObject *)receiver_o;
_PyInterpreterFrame *gen_frame = &gen->gi_iframe;
STACK_SHRINK(1);
_PyFrame_StackPush(gen_frame, v);
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
assert(next_instr - this_instr + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)(next_instr - this_instr + oparg);
DISPATCH_INLINED(gen_frame);
}
if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) {
retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o);
}
else {
retval_o = PyObject_CallMethodOneArg(receiver_o,
&_Py_ID(send),
PyStackRef_AsPyObjectBorrow(v));
}
if (retval_o == NULL) {
if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)
) {
monitor_raise(tstate, frame, this_instr);
}
if (_PyGen_FetchStopIterationValue(&retval_o) == 0) {
assert(retval_o != NULL);
JUMPBY(oparg);
}
else {
goto error;
}
}
PyStackRef_CLOSE(v);
retval = PyStackRef_FromPyObjectSteal(retval_o);
}
stack_pointer[-1] = retval;
DISPATCH();
}
TARGET(SEND_GEN) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(SEND_GEN);
static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size");
_PyStackRef v;
_PyStackRef receiver;
/* Skip 1 cache entry */
v = stack_pointer[-1];
receiver = stack_pointer[-2];
DEOPT_IF(tstate->interp->eval_frame, SEND);
PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver);
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND);
STAT_INC(SEND, hit);
_PyInterpreterFrame *gen_frame = &gen->gi_iframe;
STACK_SHRINK(1);
_PyFrame_StackPush(gen_frame, v);
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
assert(next_instr - this_instr + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)(next_instr - this_instr + oparg);
DISPATCH_INLINED(gen_frame);
}
TARGET(SETUP_ANNOTATIONS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SETUP_ANNOTATIONS);
int err;
PyObject *ann_dict;
if (LOCALS() == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when setting up annotations");
if (true) goto error;
}
/* check if __annotations__ in locals()... */
if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error;
if (ann_dict == NULL) {
ann_dict = PyDict_New();
if (ann_dict == NULL) goto error;
err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__),
ann_dict);
Py_DECREF(ann_dict);
if (err) goto error;
}
else {
Py_DECREF(ann_dict);
}
DISPATCH();
}
TARGET(SET_ADD) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_ADD);
_PyStackRef v;
_PyStackRef set;
v = stack_pointer[-1];
set = stack_pointer[-2 - (oparg-1)];
int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set),
PyStackRef_AsPyObjectSteal(v));
PyStackRef_CLOSE(v);
if (err) goto pop_1_error;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(SET_FUNCTION_ATTRIBUTE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE);
_PyStackRef func_st;
_PyStackRef attr_st;
func_st = stack_pointer[-1];
attr_st = stack_pointer[-2];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st);
assert(PyFunction_Check(func));
PyFunctionObject *func_obj = (PyFunctionObject *)func;
switch(oparg) {
case MAKE_FUNCTION_CLOSURE:
assert(func_obj->func_closure == NULL);
func_obj->func_closure = attr;
break;
case MAKE_FUNCTION_ANNOTATIONS:
assert(func_obj->func_annotations == NULL);
func_obj->func_annotations = attr;
break;
case MAKE_FUNCTION_KWDEFAULTS:
assert(PyDict_CheckExact(attr));
assert(func_obj->func_kwdefaults == NULL);
func_obj->func_kwdefaults = attr;
break;
case MAKE_FUNCTION_DEFAULTS:
assert(PyTuple_CheckExact(attr));
assert(func_obj->func_defaults == NULL);
func_obj->func_defaults = attr;
break;
case MAKE_FUNCTION_ANNOTATE:
assert(PyCallable_Check(attr));
assert(func_obj->func_annotate == NULL);
func_obj->func_annotate = attr;
break;
default:
Py_UNREACHABLE();
}
stack_pointer[-2] = func_st;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(SET_UPDATE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_UPDATE);
_PyStackRef iterable;
_PyStackRef set;
iterable = stack_pointer[-1];
set = stack_pointer[-2 - (oparg-1)];
int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set),
PyStackRef_AsPyObjectBorrow(iterable));
PyStackRef_CLOSE(iterable);
if (err < 0) goto pop_1_error;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(STORE_ATTR) {
frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR);
PREDICTED(STORE_ATTR);
_Py_CODEUNIT *this_instr = next_instr - 5;
(void)this_instr;
_PyStackRef owner;
_PyStackRef v;
// _SPECIALIZE_STORE_ATTR
owner = stack_pointer[-1];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
next_instr = this_instr;
_Py_Specialize_StoreAttr(owner, next_instr, name);
DISPATCH_SAME_OPARG();
}
STAT_INC(STORE_ATTR, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
/* Skip 3 cache entries */
// _STORE_ATTR
v = stack_pointer[-2];
{
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner),
name, PyStackRef_AsPyObjectSteal(v));
PyStackRef_CLOSE(v);
PyStackRef_CLOSE(owner);
if (err) goto pop_2_error;
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(STORE_ATTR_INSTANCE_VALUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
_PyStackRef owner;
_PyStackRef value;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
}
// _GUARD_DORV_NO_DICT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(_PyObject_GetManagedDict(owner_o), STORE_ATTR);
DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0, STORE_ATTR);
}
// _STORE_ATTR_INSTANCE_VALUE
value = stack_pointer[-2];
{
uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
STAT_INC(STORE_ATTR, hit);
assert(_PyObject_GetManagedDict(owner_o) == NULL);
PyDictValues *values = _PyObject_InlineValues(owner_o);
PyObject *old_value = values->values[index];
values->values[index] = PyStackRef_AsPyObjectSteal(value);
if (old_value == NULL) {
_PyDictValues_AddToInsertionOrder(values, index);
}
else {
Py_DECREF(old_value);
}
PyStackRef_CLOSE(owner);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(STORE_ATTR_SLOT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_SLOT);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
_PyStackRef owner;
_PyStackRef value;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
}
// _STORE_ATTR_SLOT
value = stack_pointer[-2];
{
uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
char *addr = (char *)owner_o + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
*(PyObject **)addr = PyStackRef_AsPyObjectSteal(value);
Py_XDECREF(old_value);
PyStackRef_CLOSE(owner);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(STORE_ATTR_WITH_HINT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
_PyStackRef owner;
_PyStackRef value;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
}
// _STORE_ATTR_WITH_HINT
value = stack_pointer[-2];
{
uint16_t hint = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
DEOPT_IF(dict == NULL, STORE_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR);
PyObject *old_value;
uint64_t new_version;
if (DK_IS_UNICODE(dict->ma_keys)) {
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, STORE_ATTR);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
ep->me_value = PyStackRef_AsPyObjectSteal(value);
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, STORE_ATTR);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
ep->me_value = PyStackRef_AsPyObjectSteal(value);
}
Py_DECREF(old_value);
STAT_INC(STORE_ATTR, hit);
/* Ensure dict is GC tracked if it needs to be */
if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) {
_PyObject_GC_TRACK(dict);
}
/* PEP 509 */
dict->ma_version_tag = new_version;
PyStackRef_CLOSE(owner);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(STORE_DEREF) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_DEREF);
_PyStackRef v;
v = stack_pointer[-1];
PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v));
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(STORE_FAST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST);
_PyStackRef value;
value = stack_pointer[-1];
SETLOCAL(oparg, value);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(STORE_FAST_LOAD_FAST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST_LOAD_FAST);
_PyStackRef value1;
_PyStackRef value2;
value1 = stack_pointer[-1];
uint32_t oparg1 = oparg >> 4;
uint32_t oparg2 = oparg & 15;
SETLOCAL(oparg1, value1);
value2 = PyStackRef_DUP(GETLOCAL(oparg2));
stack_pointer[-1] = value2;
DISPATCH();
}
TARGET(STORE_FAST_STORE_FAST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST_STORE_FAST);
_PyStackRef value1;
_PyStackRef value2;
value1 = stack_pointer[-1];
value2 = stack_pointer[-2];
uint32_t oparg1 = oparg >> 4;
uint32_t oparg2 = oparg & 15;
SETLOCAL(oparg1, value1);
SETLOCAL(oparg2, value2);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(STORE_GLOBAL) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_GLOBAL);
_PyStackRef v;
v = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v));
PyStackRef_CLOSE(v);
if (err) goto pop_1_error;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(STORE_NAME) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_NAME);
_PyStackRef v;
v = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *ns = LOCALS();
int err;
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when storing %R", name);
PyStackRef_CLOSE(v);
if (true) goto pop_1_error;
}
if (PyDict_CheckExact(ns))
err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
else
err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
PyStackRef_CLOSE(v);
if (err) goto pop_1_error;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(STORE_SLICE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_SLICE);
_PyStackRef stop;
_PyStackRef start;
_PyStackRef container;
_PyStackRef v;
stop = stack_pointer[-1];
start = stack_pointer[-2];
container = stack_pointer[-3];
v = stack_pointer[-4];
PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start),
PyStackRef_AsPyObjectSteal(stop));
int err;
if (slice == NULL) {
err = 1;
}
else {
err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v));
Py_DECREF(slice);
}
PyStackRef_CLOSE(v);
PyStackRef_CLOSE(container);
if (err) goto pop_4_error;
stack_pointer += -4;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(STORE_SUBSCR) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR);
PREDICTED(STORE_SUBSCR);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
_PyStackRef sub;
_PyStackRef container;
_PyStackRef v;
// _SPECIALIZE_STORE_SUBSCR
sub = stack_pointer[-1];
container = stack_pointer[-2];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_StoreSubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(STORE_SUBSCR, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _STORE_SUBSCR
v = stack_pointer[-3];
{
/* container[sub] = v */
int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v));
PyStackRef_CLOSE(v);
PyStackRef_CLOSE(container);
PyStackRef_CLOSE(sub);
if (err) goto pop_3_error;
}
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(STORE_SUBSCR_DICT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR_DICT);
static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size");
_PyStackRef sub_st;
_PyStackRef dict_st;
_PyStackRef value;
/* Skip 1 cache entry */
sub_st = stack_pointer[-1];
dict_st = stack_pointer[-2];
value = stack_pointer[-3];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
STAT_INC(STORE_SUBSCR, hit);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value));
PyStackRef_CLOSE(dict_st);
if (err) goto pop_3_error;
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(STORE_SUBSCR_LIST_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT);
static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size");
_PyStackRef sub_st;
_PyStackRef list_st;
_PyStackRef value;
/* Skip 1 cache entry */
sub_st = stack_pointer[-1];
list_st = stack_pointer[-2];
value = stack_pointer[-3];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);
// Ensure nonnegative, zero-or-one-digit ints.
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR);
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
// Ensure index < len(list)
DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR);
STAT_INC(STORE_SUBSCR, hit);
PyObject *old_value = PyList_GET_ITEM(list, index);
PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value));
assert(old_value != NULL);
Py_DECREF(old_value);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
PyStackRef_CLOSE(list_st);
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(SWAP) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SWAP);
_PyStackRef top;
_PyStackRef bottom;
top = stack_pointer[-1];
bottom = stack_pointer[-2 - (oparg-2)];
assert(oparg >= 2);
stack_pointer[-2 - (oparg-2)] = top;
stack_pointer[-1] = bottom;
DISPATCH();
}
TARGET(TO_BOOL) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL);
PREDICTED(TO_BOOL);
_Py_CODEUNIT *this_instr = next_instr - 4;
(void)this_instr;
_PyStackRef value;
_PyStackRef res;
// _SPECIALIZE_TO_BOOL
value = stack_pointer[-1];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_ToBool(value, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(TO_BOOL, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
/* Skip 2 cache entries */
// _TO_BOOL
{
int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value));
PyStackRef_CLOSE(value);
if (err < 0) goto pop_1_error;
res = err ? PyStackRef_True : PyStackRef_False;
}
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(TO_BOOL_ALWAYS_TRUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
_PyStackRef owner;
_PyStackRef value;
_PyStackRef res;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL);
}
// _REPLACE_WITH_TRUE
value = owner;
{
PyStackRef_CLOSE(value);
res = PyStackRef_True;
}
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(TO_BOOL_BOOL) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_BOOL);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
_PyStackRef value;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
DEOPT_IF(!PyBool_Check(PyStackRef_AsPyObjectBorrow(value)), TO_BOOL);
STAT_INC(TO_BOOL, hit);
DISPATCH();
}
TARGET(TO_BOOL_INT) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_INT);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
_PyStackRef value;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL);
STAT_INC(TO_BOOL, hit);
if (_PyLong_IsZero((PyLongObject *)value_o)) {
assert(_Py_IsImmortal(value_o));
res = PyStackRef_False;
}
else {
PyStackRef_CLOSE(value);
res = PyStackRef_True;
}
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(TO_BOOL_LIST) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_LIST);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
_PyStackRef value;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL);
STAT_INC(TO_BOOL, hit);
res = Py_SIZE(value_o) ? PyStackRef_True : PyStackRef_False;
PyStackRef_CLOSE(value);
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(TO_BOOL_NONE) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_NONE);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
_PyStackRef value;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
// This one is a bit weird, because we expect *some* failures:
DEOPT_IF(!PyStackRef_Is(value, PyStackRef_None), TO_BOOL);
STAT_INC(TO_BOOL, hit);
res = PyStackRef_False;
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(TO_BOOL_STR) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_STR);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
_PyStackRef value;
_PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL);
STAT_INC(TO_BOOL, hit);
if (value_o == &_Py_STR(empty)) {
assert(_Py_IsImmortal(value_o));
res = PyStackRef_False;
}
else {
assert(Py_SIZE(value_o));
PyStackRef_CLOSE(value);
res = PyStackRef_True;
}
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(UNARY_INVERT) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_INVERT);
_PyStackRef value;
_PyStackRef res;
value = stack_pointer[-1];
PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value));
PyStackRef_CLOSE(value);
if (res_o == NULL) goto pop_1_error;
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(UNARY_NEGATIVE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_NEGATIVE);
_PyStackRef value;
_PyStackRef res;
value = stack_pointer[-1];
PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value));
PyStackRef_CLOSE(value);
if (res_o == NULL) goto pop_1_error;
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(UNARY_NOT) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_NOT);
_PyStackRef value;
_PyStackRef res;
value = stack_pointer[-1];
assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value)));
res = PyStackRef_Is(value, PyStackRef_False)
? PyStackRef_True : PyStackRef_False;
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(UNPACK_EX) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNPACK_EX);
_PyStackRef seq;
seq = stack_pointer[-1];
int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
_PyStackRef *top = stack_pointer + totalargs - 1;
int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top);
PyStackRef_CLOSE(seq);
if (res == 0) goto pop_1_error;
stack_pointer += (oparg >> 8) + (oparg & 0xFF);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(UNPACK_SEQUENCE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE);
PREDICTED(UNPACK_SEQUENCE);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
_PyStackRef seq;
// _SPECIALIZE_UNPACK_SEQUENCE
seq = stack_pointer[-1];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_UnpackSequence(seq, next_instr, oparg);
DISPATCH_SAME_OPARG();
}
STAT_INC(UNPACK_SEQUENCE, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
(void)seq;
(void)counter;
}
// _UNPACK_SEQUENCE
{
_PyStackRef *top = stack_pointer + oparg - 1;
int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top);
PyStackRef_CLOSE(seq);
if (res == 0) goto pop_1_error;
}
stack_pointer += -1 + oparg;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(UNPACK_SEQUENCE_LIST) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST);
static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
_PyStackRef seq;
_PyStackRef *values;
/* Skip 1 cache entry */
seq = stack_pointer[-1];
values = &stack_pointer[-1];
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE);
DEOPT_IF(PyList_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE);
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyList_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
*values++ = PyStackRef_FromPyObjectNew(items[i]);
}
PyStackRef_CLOSE(seq);
stack_pointer += -1 + oparg;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(UNPACK_SEQUENCE_TUPLE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE);
static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
_PyStackRef seq;
_PyStackRef *values;
/* Skip 1 cache entry */
seq = stack_pointer[-1];
values = &stack_pointer[-1];
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE);
DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE);
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyTuple_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
*values++ = PyStackRef_FromPyObjectNew(items[i]);
}
PyStackRef_CLOSE(seq);
stack_pointer += -1 + oparg;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(UNPACK_SEQUENCE_TWO_TUPLE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE);
static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
_PyStackRef seq;
_PyStackRef val1;
_PyStackRef val0;
/* Skip 1 cache entry */
seq = stack_pointer[-1];
assert(oparg == 2);
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE);
DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE);
STAT_INC(UNPACK_SEQUENCE, hit);
val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0));
val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1));
PyStackRef_CLOSE(seq);
stack_pointer[-1] = val1;
stack_pointer[0] = val0;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(WITH_EXCEPT_START) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(WITH_EXCEPT_START);
_PyStackRef val;
_PyStackRef lasti;
_PyStackRef exit_self;
_PyStackRef exit_func;
_PyStackRef res;
val = stack_pointer[-1];
lasti = stack_pointer[-3];
exit_self = stack_pointer[-4];
exit_func = stack_pointer[-5];
/* At the top of the stack are 4 values:
- val: TOP = exc_info()
- unused: SECOND = previous exception
- lasti: THIRD = lasti of exception in exc_info()
- exit_self: FOURTH = the context or NULL
- exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method
We call FOURTH(type(TOP), TOP, GetTraceback(TOP)).
Then we push the __exit__ return value.
*/
PyObject *exc, *tb;
PyObject *val_o = PyStackRef_AsPyObjectBorrow(val);
PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func);
assert(val_o && PyExceptionInstance_Check(val_o));
exc = PyExceptionInstance_Class(val_o);
tb = PyException_GetTraceback(val_o);
if (tb == NULL) {
tb = Py_None;
}
else {
Py_DECREF(tb);
}
assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti)));
(void)lasti; // Shut up compiler warning if asserts are off
PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb};
int has_self = !PyStackRef_IsNull(exit_self);
res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self,
(3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL));
if (PyStackRef_IsNull(res)) goto error;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(YIELD_VALUE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(YIELD_VALUE);
_PyStackRef retval;
_PyStackRef value;
retval = stack_pointer[-1];
// NOTE: It's important that YIELD_VALUE never raises an exception!
// The compiler treats any exception raised here as a failed close()
// or throw() call.
#if TIER_ONE
assert(frame != &entry_frame);
#endif
frame->instr_ptr++;
PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame);
assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1);
assert(oparg == 0 || oparg == 1);
gen->gi_frame_state = FRAME_SUSPENDED + oparg;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
_PyInterpreterFrame *gen_frame = frame;
frame = tstate->current_frame = frame->previous;
gen_frame->previous = NULL;
/* We don't know which of these is relevant here, so keep them equal */
assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER);
#if TIER_ONE
assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE ||
frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR);
#endif
LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND);
LOAD_SP();
value = retval;
LLTRACE_RESUME_FRAME();
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
#undef TIER_ONE
Reference in New Issue View Git Blame Copy Permalink