diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index f13bc2178b1e7e..4c8ad11447aa59 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -1001,7 +1001,7 @@ struct _is { struct ast_state ast; struct types_state types; struct callable_cache callable_cache; - PyObject *common_consts[NUM_COMMON_CONSTANTS]; + _PyStackRef common_consts[NUM_COMMON_CONSTANTS]; bool jit; bool compiling; diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 238e17bea303d3..b2456c6c18541b 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -9313,7 +9313,7 @@ INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DUP(tstate->interp->common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3bd489122da9d4..b6149ef3e5ac52 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1972,7 +1972,7 @@ dummy_func( inst(LOAD_COMMON_CONSTANT, ( -- value)) { // Keep in sync with _common_constants in opcode.py assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DUP(tstate->interp->common_consts[oparg]); } inst(LOAD_BUILD_CLASS, ( -- bc)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b6a2821db3007e..f64ca48fcc7d0f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -9394,7 +9394,7 @@ _PyStackRef value; oparg = CURRENT_OPARG(); assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DUP(tstate->interp->common_consts[oparg]); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -9408,7 +9408,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DUP(tstate->interp->common_consts[oparg]); _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); @@ -9424,7 +9424,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DUP(tstate->interp->common_consts[oparg]); _tos_cache2 = value; _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 224426b7aa44bc..f114c44a832f84 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -11,6 +11,7 @@ #include "pycore_opcode_utils.h" #include "pycore_opcode_metadata.h" // OPCODE_HAS_ARG, etc #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_stackref.h" // PyStackRef_AsPyObjectBorrow() #include @@ -1330,7 +1331,8 @@ get_const_value(int opcode, int oparg, PyObject *co_consts) } if (opcode == LOAD_COMMON_CONSTANT) { assert(oparg < NUM_COMMON_CONSTANTS); - return Py_NewRef(_PyInterpreterState_GET()->common_consts[oparg]); + return Py_NewRef(PyStackRef_AsPyObjectBorrow( + _PyInterpreterState_GET()->common_consts[oparg])); } if (constant == NULL) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2623105656c90c..8307e805330a74 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9312,7 +9312,7 @@ INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DUP(tstate->interp->common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 96dbaea5a5797e..3aded576caa432 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -2,6 +2,7 @@ #include "pycore_long.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" +#include "pycore_stackref.h" #include "pycore_typeobject.h" #include "pycore_uops.h" #include "pycore_uop_ids.h" @@ -870,15 +871,11 @@ dummy_func(void) { op(_LOAD_COMMON_CONSTANT, (-- value)) { assert(oparg < NUM_COMMON_CONSTANTS); - PyObject *val = _PyInterpreterState_GET()->common_consts[oparg]; - if (_Py_IsImmortal(val)) { - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(sym_new_const(ctx, val)); - } - else { - ADD_OP(_LOAD_CONST_INLINE, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); - } + PyObject *val = PyStackRef_AsPyObjectBorrow( + _PyInterpreterState_GET()->common_consts[oparg]); + assert(_Py_IsImmortal(val)); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_SMALL_INT, (-- value)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index f336549d2ed244..55bd92299d3d5c 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1991,15 +1991,11 @@ case _LOAD_COMMON_CONSTANT: { JitOptRef value; assert(oparg < NUM_COMMON_CONSTANTS); - PyObject *val = _PyInterpreterState_GET()->common_consts[oparg]; - if (_Py_IsImmortal(val)) { - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(sym_new_const(ctx, val)); - } - else { - ADD_OP(_LOAD_CONST_INLINE, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); - } + PyObject *val = PyStackRef_AsPyObjectBorrow( + _PyInterpreterState_GET()->common_consts[oparg]); + assert(_Py_IsImmortal(val)); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 46579a45f4cc39..e017269a5ac5b2 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -28,6 +28,7 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_stackref.h" // PyStackRef_FromPyObjectBorrow() #include "pycore_stats.h" // _PyStats_InterpInit() #include "pycore_sysmodule.h" // _PySys_ClearAttrString() #include "pycore_traceback.h" // PyUnstable_TracebackThreads() @@ -878,22 +879,25 @@ pycore_init_builtins(PyThreadState *tstate) goto error; } - interp->common_consts[CONSTANT_ASSERTIONERROR] = PyExc_AssertionError; - interp->common_consts[CONSTANT_NOTIMPLEMENTEDERROR] = PyExc_NotImplementedError; - interp->common_consts[CONSTANT_BUILTIN_TUPLE] = (PyObject *)&PyTuple_Type; - interp->common_consts[CONSTANT_BUILTIN_ALL] = all; - interp->common_consts[CONSTANT_BUILTIN_ANY] = any; - interp->common_consts[CONSTANT_BUILTIN_LIST] = (PyObject *)&PyList_Type; - interp->common_consts[CONSTANT_BUILTIN_SET] = (PyObject *)&PySet_Type; - interp->common_consts[CONSTANT_NONE] = Py_None; - interp->common_consts[CONSTANT_EMPTY_STR] = + PyObject *common_objs[NUM_COMMON_CONSTANTS] = {NULL}; + common_objs[CONSTANT_ASSERTIONERROR] = PyExc_AssertionError; + common_objs[CONSTANT_NOTIMPLEMENTEDERROR] = PyExc_NotImplementedError; + common_objs[CONSTANT_BUILTIN_TUPLE] = (PyObject *)&PyTuple_Type; + common_objs[CONSTANT_BUILTIN_ALL] = all; + common_objs[CONSTANT_BUILTIN_ANY] = any; + common_objs[CONSTANT_BUILTIN_LIST] = (PyObject *)&PyList_Type; + common_objs[CONSTANT_BUILTIN_SET] = (PyObject *)&PySet_Type; + common_objs[CONSTANT_NONE] = Py_None; + common_objs[CONSTANT_EMPTY_STR] = Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_STR); - interp->common_consts[CONSTANT_TRUE] = Py_True; - interp->common_consts[CONSTANT_FALSE] = Py_False; - interp->common_consts[CONSTANT_MINUS_ONE] = + common_objs[CONSTANT_TRUE] = Py_True; + common_objs[CONSTANT_FALSE] = Py_False; + common_objs[CONSTANT_MINUS_ONE] = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS - 1]; for (int i = 0; i < NUM_COMMON_CONSTANTS; i++) { - assert(interp->common_consts[i] != NULL); + assert(common_objs[i] != NULL); + _Py_SetImmortal(common_objs[i]); + interp->common_consts[i] = PyStackRef_FromPyObjectBorrow(common_objs[i]); } PyObject *list_append = _PyType_Lookup(&PyList_Type, &_Py_ID(append)); diff --git a/Python/pystate.c b/Python/pystate.c index bf2616a49148a7..199b369facd59c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -12,8 +12,9 @@ #include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interpframe.h" // _PyThreadState_HasStackSpace() -#include "pycore_object.h" // _PyType_InitCache() +#include "pycore_object.h" // _PyType_InitCache(), _Py_ClearImmortal() #include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() +#include "pycore_opcode_utils.h" // NUM_COMMON_CONSTANTS #include "pycore_optimizer.h" // JIT_CLEANUP_THRESHOLD #include "pycore_parking_lot.h" // _PyParkingLot_AfterFork() #include "pycore_pyerrors.h" // _PyErr_Clear() @@ -21,7 +22,7 @@ #include "pycore_pymem.h" // _PyMem_DebugEnabled() #include "pycore_runtime.h" // _PyRuntime #include "pycore_runtime_init.h" // _PyRuntimeState_INIT -#include "pycore_stackref.h" // Py_STACKREF_DEBUG +#include "pycore_stackref.h" // PyStackRef_AsPyObjectBorrow() #include "pycore_stats.h" // FT_STAT_WORLD_STOP_INC() #include "pycore_time.h" // _PyTime_Init() #include "pycore_uniqueid.h" // _PyObject_FinalizePerThreadRefcounts() @@ -777,6 +778,36 @@ extern void _Py_stackref_report_leaks(PyInterpreterState *interp); #endif +static int +common_const_is_initialized(_PyStackRef ref) +{ +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + return !PyStackRef_IsNull(ref); +#else + return ref.bits != 0 && !PyStackRef_IsNull(ref); +#endif +} + + +static void +common_constants_clear(PyInterpreterState *interp) +{ + for (int i = 0; i < NUM_COMMON_CONSTANTS; i++) { + _PyStackRef ref = interp->common_consts[i]; + if (!common_const_is_initialized(ref)) { + continue; + } + PyObject *obj = PyStackRef_AsPyObjectBorrow(ref); + PyStackRef_XCLOSE(ref); + interp->common_consts[i] = PyStackRef_NULL; + // Refcount reclamation skips heap immortals; release manually. + if (_Py_IsImmortal(obj) && !_Py_IsStaticImmortal(obj)) { + _Py_ClearImmortal(obj); + } + } +} + + static void interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) { @@ -903,6 +934,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) PyDict_Clear(interp->builtins); Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); + common_constants_clear(interp); #if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) # ifdef Py_STACKREF_CLOSE_DEBUG