From c9173ae74b28548ff388e01e49ad6d4d487ad820 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 19 Jul 2026 20:04:41 +0100 Subject: [PATCH 1/6] gh-153946: Defer automatic GC in object graph builders --- Include/internal/pycore_gc.h | 8 ++ Include/internal/pycore_interp_structs.h | 1 + Lib/test/test_gc.py | 74 +++++++++++++++++++ .../2026-07-19-12-00-00.gh-issue-153946.rst | 4 + Modules/_testinternalcapi.c | 16 ++++ Parser/asdl_c.py | 5 ++ Python/Python-ast.c | 5 ++ Python/gc.c | 26 ++++++- Python/gc_free_threading.c | 57 ++++++++++++-- Python/marshal.c | 10 ++- 10 files changed, 197 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-00-00.gh-issue-153946.rst diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 84cbb56a9192156..9c9f1091f34c1d1 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -330,6 +330,14 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); + +// These calls nest across all threads in an interpreter. Allocation counters +// continue advancing, explicit collections remain enabled, and a collection +// that becomes due remains eligible at the next normal scheduling opportunity. +// An automatic collection already starting concurrently may emit callbacks, +// but the free-threaded collector will not traverse the protected object graph. +PyAPI_FUNC(void) _PyGC_DeferAutomaticCollection(PyThreadState *tstate); +PyAPI_FUNC(void) _PyGC_ResumeAutomaticCollection(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); union _PyStackRef; diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 0623adce693d465..90b89b4b9b6c95b 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -224,6 +224,7 @@ struct gc_stats { struct _gc_runtime_state { /* Is automatic collection enabled? */ int enabled; + int automatic_collection_pause_count; int debug; /* linked lists of container objects */ #ifndef Py_GIL_DISABLED diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 4c721c01a34f070..204b170879275f5 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -91,6 +91,80 @@ def __tp_del__(self): ############################################################################### class GCTests(unittest.TestCase): + @staticmethod + def total_collections(): + return sum(stats["collections"] for stats in gc.get_stats()) + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_defer_automatic_collection(self): + was_enabled = gc.isenabled() + gc.enable() + try: + with gc_threshold(1, 0, 0): + _testinternalcapi.defer_automatic_gc() + try: + before = self.total_collections() + objects = [[] for _ in range(10_000)] + self.assertEqual(self.total_collections(), before) + self.assertTrue(gc.isenabled()) + + gc.collect() + self.assertEqual(self.total_collections(), before + 1) + finally: + _testinternalcapi.resume_automatic_gc() + finally: + if not was_enabled: + gc.disable() + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_defer_automatic_collection_nested(self): + was_enabled = gc.isenabled() + gc.enable() + try: + with gc_threshold(1, 0, 0): + _testinternalcapi.defer_automatic_gc() + try: + _testinternalcapi.defer_automatic_gc() + try: + before = self.total_collections() + objects = [[] for _ in range(10_000)] + self.assertEqual(self.total_collections(), before) + finally: + _testinternalcapi.resume_automatic_gc() + + objects.extend([] for _ in range(10_000)) + self.assertEqual(self.total_collections(), before) + finally: + _testinternalcapi.resume_automatic_gc() + + objects.extend([] for _ in range(10_000)) + self.assertGreater(self.total_collections(), before) + finally: + if not was_enabled: + gc.disable() + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_defer_automatic_collection_across_threads(self): + was_enabled = gc.isenabled() + gc.enable() + try: + with gc_threshold(1, 0, 0): + _testinternalcapi.defer_automatic_gc() + try: + before = self.total_collections() + objects = [] + thread = threading.Thread( + target=lambda: objects.extend( + [] for _ in range(10_000))) + thread.start() + thread.join() + self.assertEqual(self.total_collections(), before) + finally: + _testinternalcapi.resume_automatic_gc() + finally: + if not was_enabled: + gc.disable() + def test_list(self): l = [] l.append(l) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-00-00.gh-issue-153946.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-00-00.gh-issue-153946.rst new file mode 100644 index 000000000000000..cbe8ceefc0ef61a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-00-00.gh-issue-153946.rst @@ -0,0 +1,4 @@ +Add a private, nestable API for temporarily deferring automatic garbage +collection without changing the interpreter-visible garbage collector state. +Use it to avoid unproductive collections while converting ASTs to Python +objects and unmarshalling objects from memory. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index e9950bb232431c6..e08139d77b3e662 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2927,6 +2927,20 @@ get_tracked_heap_size(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyLong_FromInt64(PyInterpreterState_Get()->gc.heap_size); } +static PyObject * +defer_automatic_gc(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + _PyGC_DeferAutomaticCollection(_PyThreadState_GET()); + Py_RETURN_NONE; +} + +static PyObject * +resume_automatic_gc(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + _PyGC_ResumeAutomaticCollection(_PyThreadState_GET()); + Py_RETURN_NONE; +} + static PyObject * is_static_immortal(PyObject *self, PyObject *op) { @@ -3378,6 +3392,8 @@ static PyMethodDef module_functions[] = { {"identify_type_slot_wrappers", identify_type_slot_wrappers, METH_NOARGS}, {"has_deferred_refcount", has_deferred_refcount, METH_O}, {"get_tracked_heap_size", get_tracked_heap_size, METH_NOARGS}, + {"defer_automatic_gc", defer_automatic_gc, METH_NOARGS}, + {"resume_automatic_gc", resume_automatic_gc, METH_NOARGS}, {"is_static_immortal", is_static_immortal, METH_O}, {"incref_decref_delayed", incref_decref_delayed, METH_O}, GET_NEXT_DICT_KEYS_VERSION_METHODDEF diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index d53886866f54f86..8a1bf5d31071b0d 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -2117,7 +2117,11 @@ class PartingShots(StaticVisitor): if (state == NULL) { return NULL; } + PyThreadState *tstate = _PyThreadState_GET(); + // The new objects cannot be cyclic until the completed tree is returned. + _PyGC_DeferAutomaticCollection(tstate); PyObject *result = ast2obj_mod(state, t); + _PyGC_ResumeAutomaticCollection(tstate); return result; } @@ -2254,6 +2258,7 @@ def generate_module_def(mod, metadata, f, internal_h): #include "pycore_ast.h" #include "pycore_ast_state.h" // struct ast_state #include "pycore_ceval.h" // _Py_EnterRecursiveCall() + #include "pycore_gc.h" // _PyGC_DeferAutomaticCollection() #include "pycore_lock.h" // _PyOnceFlag #include "pycore_modsupport.h" // _PyArg_NoPositional() #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Python/Python-ast.c b/Python/Python-ast.c index f36072dfce098c6..2d8760e26f746fc 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -4,6 +4,7 @@ #include "pycore_ast.h" #include "pycore_ast_state.h" // struct ast_state #include "pycore_ceval.h" // _Py_EnterRecursiveCall() +#include "pycore_gc.h" // _PyGC_DeferAutomaticCollection() #include "pycore_lock.h" // _PyOnceFlag #include "pycore_modsupport.h" // _PyArg_NoPositional() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -18544,7 +18545,11 @@ PyObject* PyAST_mod2obj(mod_ty t) if (state == NULL) { return NULL; } + PyThreadState *tstate = _PyThreadState_GET(); + // The new objects cannot be cyclic until the completed tree is returned. + _PyGC_DeferAutomaticCollection(tstate); PyObject *result = ast2obj_mod(state, t); + _PyGC_ResumeAutomaticCollection(tstate); return result; } diff --git a/Python/gc.c b/Python/gc.c index 201c621bcc3cb9b..04eb692f45c28b4 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1783,6 +1783,26 @@ PyGC_IsEnabled(void) return gcstate->enabled; } +void +_PyGC_DeferAutomaticCollection(PyThreadState *tstate) +{ + GCState *gcstate = &tstate->interp->gc; + int previous = _Py_atomic_add_int( + &gcstate->automatic_collection_pause_count, 1); + (void)previous; + assert(previous >= 0); +} + +void +_PyGC_ResumeAutomaticCollection(PyThreadState *tstate) +{ + GCState *gcstate = &tstate->interp->gc; + int previous = _Py_atomic_add_int( + &gcstate->automatic_collection_pause_count, -1); + (void)previous; + assert(previous > 0); +} + /* Public API to invoke gc.collect() from C */ Py_ssize_t PyGC_Collect(void) @@ -1984,6 +2004,8 @@ _PyObject_GC_Link(PyObject *op) gcstate->generations[0].count++; /* number of allocated GC objects */ if (gcstate->generations[0].count > gcstate->generations[0].threshold && gcstate->enabled && + !_Py_atomic_load_int_relaxed( + &gcstate->automatic_collection_pause_count) && gcstate->generations[0].threshold && !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) @@ -1996,7 +2018,9 @@ void _Py_RunGC(PyThreadState *tstate) { GCState *gcstate = get_gc_state(); - if (!gcstate->enabled) { + if (!gcstate->enabled || + _Py_atomic_load_int_relaxed( + &gcstate->automatic_collection_pause_count)) { return; } gc_collect_main(tstate, GENERATION_AUTO, _Py_GC_REASON_HEAP); diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index fbd13d1e4d87f25..c12904b72d3a15d 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2002,7 +2002,9 @@ gc_should_collect(GCState *gcstate) int count = _Py_atomic_load_int_relaxed(&gcstate->young.count); int threshold = gcstate->young.threshold; int gc_enabled = _Py_atomic_load_int_relaxed(&gcstate->enabled); - if (count <= threshold || threshold == 0 || !gc_enabled) { + int pause_count = _Py_atomic_load_int_relaxed( + &gcstate->automatic_collection_pause_count); + if (count <= threshold || threshold == 0 || !gc_enabled || pause_count) { return false; } if (gcstate->old[0].threshold == 0) { @@ -2065,11 +2067,21 @@ record_deallocation(PyThreadState *tstate) } } -static void -gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, int generation) +static bool +gc_collect_internal(PyInterpreterState *interp, + struct collection_state *state, int generation) { _PyEval_StopTheWorld(interp); + // A concurrent deferral may begin after this collection has emitted its + // start notification, but it must take effect before any heap traversal. + if (state->reason == _Py_GC_REASON_HEAP && + _Py_atomic_load_int( + &state->gcstate->automatic_collection_pause_count)) { + _PyEval_StartTheWorld(interp); + return false; + } + // update collection and allocation counters if (generation+1 < NUM_GENERATIONS) { state->gcstate->old[generation].count += 1; @@ -2114,7 +2126,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, if (err < 0) { _PyEval_StartTheWorld(interp); PyErr_NoMemory(); - return; + return true; } } #endif @@ -2124,7 +2136,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, if (err < 0) { _PyEval_StartTheWorld(interp); PyErr_NoMemory(); - return; + return true; } #ifdef GC_DEBUG @@ -2171,7 +2183,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, cleanup_worklist(&state->wrcb_to_call); cleanup_worklist(&state->objs_to_decref); PyErr_NoMemory(); - return; + return true; } // Call tp_clear on objects in the unreachable set. This will cause @@ -2181,6 +2193,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, // Append objects with legacy finalizers to the "gc.garbage" list. handle_legacy_finalizers(state); + return true; } static struct gc_generation_stats * @@ -2258,7 +2271,17 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) .reason = reason, }; - gc_collect_internal(interp, &state, generation); + if (!gc_collect_internal(interp, &state, generation)) { + if (PyDTrace_GC_DONE_ENABLED()) { + PyDTrace_GC_DONE(0); + } + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "stop", generation, 0, 0, 0, 0.0); + } + gcstate->frame = NULL; + _Py_atomic_store_int(&gcstate->collecting, 0); + return 0; + } m = state.collected; n = state.uncollectable; @@ -2549,6 +2572,26 @@ PyGC_IsEnabled(void) return _Py_atomic_load_int_relaxed(&gcstate->enabled); } +void +_PyGC_DeferAutomaticCollection(PyThreadState *tstate) +{ + GCState *gcstate = &tstate->interp->gc; + int previous = _Py_atomic_add_int( + &gcstate->automatic_collection_pause_count, 1); + (void)previous; + assert(previous >= 0); +} + +void +_PyGC_ResumeAutomaticCollection(PyThreadState *tstate) +{ + GCState *gcstate = &tstate->interp->gc; + int previous = _Py_atomic_add_int( + &gcstate->automatic_collection_pause_count, -1); + (void)previous; + assert(previous > 0); +} + /* Public API to invoke gc.collect() from C */ Py_ssize_t PyGC_Collect(void) diff --git a/Python/marshal.c b/Python/marshal.c index b11f2dca57a226c..a426541217c8ae5 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -9,6 +9,7 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_code.h" // _PyCode_New() +#include "pycore_gc.h" // _PyGC_DeferAutomaticCollection() #include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_long.h" // _PyLong_IsZero() #include "pycore_object.h" // _PyObject_IsUniquelyReferenced @@ -1772,12 +1773,19 @@ read_object(RFILE *p) if (PySys_Audit("marshal.loads", "y#", p->ptr, (Py_ssize_t)(p->end - p->ptr)) < 0) { return NULL; } + PyThreadState *tstate = _PyThreadState_GET(); + _PyGC_DeferAutomaticCollection(tstate); + v = r_object(p); + _PyGC_ResumeAutomaticCollection(tstate); } else if (p->fp || p->readable) { if (PySys_Audit("marshal.load", NULL) < 0) { return NULL; } + v = r_object(p); + } + else { + v = r_object(p); } - v = r_object(p); if (v == NULL && !PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object"); return v; From 9319acdf1a2c0e43d6530b58d06172937a2cd79f Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 19 Jul 2026 20:26:53 +0100 Subject: [PATCH 2/6] gh-153946: Refine automatic GC deferral --- Include/internal/pycore_gc.h | 8 +++----- Lib/test/test_gc.py | 2 ++ Parser/asdl_c.py | 1 - Python/Python-ast.c | 1 - Python/gc.c | 20 +++++++------------- Python/gc_free_threading.c | 16 ++++++++-------- Python/marshal.c | 18 ++++++++++-------- 7 files changed, 30 insertions(+), 36 deletions(-) diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 9c9f1091f34c1d1..6776990423a5bad 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -331,11 +331,9 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); -// These calls nest across all threads in an interpreter. Allocation counters -// continue advancing, explicit collections remain enabled, and a collection -// that becomes due remains eligible at the next normal scheduling opportunity. -// An automatic collection already starting concurrently may emit callbacks, -// but the free-threaded collector will not traverse the protected object graph. +// Nesting is interpreter-wide. Explicit collections and allocation counting +// continue, and resuming does not schedule a collection immediately. +// An in-flight free-threaded collection may still emit start and stop callbacks. PyAPI_FUNC(void) _PyGC_DeferAutomaticCollection(PyThreadState *tstate); PyAPI_FUNC(void) _PyGC_ResumeAutomaticCollection(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 204b170879275f5..212cb23cbf67503 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -144,6 +144,7 @@ def test_defer_automatic_collection_nested(self): gc.disable() @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + @threading_helper.requires_working_threading() def test_defer_automatic_collection_across_threads(self): was_enabled = gc.isenabled() gc.enable() @@ -158,6 +159,7 @@ def test_defer_automatic_collection_across_threads(self): [] for _ in range(10_000))) thread.start() thread.join() + self.assertEqual(len(objects), 10_000) self.assertEqual(self.total_collections(), before) finally: _testinternalcapi.resume_automatic_gc() diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 8a1bf5d31071b0d..bcfdc2df408cac2 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -2118,7 +2118,6 @@ class PartingShots(StaticVisitor): return NULL; } PyThreadState *tstate = _PyThreadState_GET(); - // The new objects cannot be cyclic until the completed tree is returned. _PyGC_DeferAutomaticCollection(tstate); PyObject *result = ast2obj_mod(state, t); _PyGC_ResumeAutomaticCollection(tstate); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 2d8760e26f746fc..14492935710fbc3 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -18546,7 +18546,6 @@ PyObject* PyAST_mod2obj(mod_ty t) return NULL; } PyThreadState *tstate = _PyThreadState_GET(); - // The new objects cannot be cyclic until the completed tree is returned. _PyGC_DeferAutomaticCollection(tstate); PyObject *result = ast2obj_mod(state, t); _PyGC_ResumeAutomaticCollection(tstate); diff --git a/Python/gc.c b/Python/gc.c index 04eb692f45c28b4..591d0bd62d6efa4 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1787,20 +1787,16 @@ void _PyGC_DeferAutomaticCollection(PyThreadState *tstate) { GCState *gcstate = &tstate->interp->gc; - int previous = _Py_atomic_add_int( - &gcstate->automatic_collection_pause_count, 1); - (void)previous; - assert(previous >= 0); + assert(gcstate->automatic_collection_pause_count >= 0); + gcstate->automatic_collection_pause_count++; } void _PyGC_ResumeAutomaticCollection(PyThreadState *tstate) { GCState *gcstate = &tstate->interp->gc; - int previous = _Py_atomic_add_int( - &gcstate->automatic_collection_pause_count, -1); - (void)previous; - assert(previous > 0); + assert(gcstate->automatic_collection_pause_count > 0); + gcstate->automatic_collection_pause_count--; } /* Public API to invoke gc.collect() from C */ @@ -2003,10 +1999,9 @@ _PyObject_GC_Link(PyObject *op) gc->_gc_prev = 0; gcstate->generations[0].count++; /* number of allocated GC objects */ if (gcstate->generations[0].count > gcstate->generations[0].threshold && - gcstate->enabled && - !_Py_atomic_load_int_relaxed( - &gcstate->automatic_collection_pause_count) && gcstate->generations[0].threshold && + gcstate->enabled && + !gcstate->automatic_collection_pause_count && !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) { @@ -2019,8 +2014,7 @@ _Py_RunGC(PyThreadState *tstate) { GCState *gcstate = get_gc_state(); if (!gcstate->enabled || - _Py_atomic_load_int_relaxed( - &gcstate->automatic_collection_pause_count)) { + gcstate->automatic_collection_pause_count) { return; } gc_collect_main(tstate, GENERATION_AUTO, _Py_GC_REASON_HEAP); diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index c12904b72d3a15d..41e18147d914ef9 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2001,10 +2001,12 @@ gc_should_collect(GCState *gcstate) { int count = _Py_atomic_load_int_relaxed(&gcstate->young.count); int threshold = gcstate->young.threshold; - int gc_enabled = _Py_atomic_load_int_relaxed(&gcstate->enabled); - int pause_count = _Py_atomic_load_int_relaxed( - &gcstate->automatic_collection_pause_count); - if (count <= threshold || threshold == 0 || !gc_enabled || pause_count) { + if (count <= threshold || threshold == 0) { + return false; + } + if (!_Py_atomic_load_int_relaxed(&gcstate->enabled) || + _Py_atomic_load_int_relaxed( + &gcstate->automatic_collection_pause_count)) { return false; } if (gcstate->old[0].threshold == 0) { @@ -2073,8 +2075,7 @@ gc_collect_internal(PyInterpreterState *interp, { _PyEval_StopTheWorld(interp); - // A concurrent deferral may begin after this collection has emitted its - // start notification, but it must take effect before any heap traversal. + // Close the race with a deferral that started before the world stopped. if (state->reason == _Py_GC_REASON_HEAP && _Py_atomic_load_int( &state->gcstate->automatic_collection_pause_count)) { @@ -2246,8 +2247,6 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) s->object_stats.object_visits = 0; } #endif - GC_STAT_ADD(generation, collections, 1); - if (reason != _Py_GC_REASON_SHUTDOWN) { invoke_gc_callback(tstate, "start", generation, 0, 0, 0, 0.0); } @@ -2282,6 +2281,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) _Py_atomic_store_int(&gcstate->collecting, 0); return 0; } + GC_STAT_ADD(generation, collections, 1); m = state.collected; n = state.uncollectable; diff --git a/Python/marshal.c b/Python/marshal.c index a426541217c8ae5..28fbeeac1c1bbcb 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1765,26 +1765,28 @@ static PyObject * read_object(RFILE *p) { PyObject *v; + int from_memory = p->ptr && p->end; if (PyErr_Occurred()) { fprintf(stderr, "XXX readobject called with exception set\n"); return NULL; } - if (p->ptr && p->end) { + if (from_memory) { if (PySys_Audit("marshal.loads", "y#", p->ptr, (Py_ssize_t)(p->end - p->ptr)) < 0) { return NULL; } - PyThreadState *tstate = _PyThreadState_GET(); - _PyGC_DeferAutomaticCollection(tstate); - v = r_object(p); - _PyGC_ResumeAutomaticCollection(tstate); } else if (p->fp || p->readable) { if (PySys_Audit("marshal.load", NULL) < 0) { return NULL; } - v = r_object(p); } - else { - v = r_object(p); + PyThreadState *tstate; + if (from_memory) { + tstate = _PyThreadState_GET(); + _PyGC_DeferAutomaticCollection(tstate); + } + v = r_object(p); + if (from_memory) { + _PyGC_ResumeAutomaticCollection(tstate); } if (v == NULL && !PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object"); From 920df6a23179763e9f10bc9545904310b5399c72 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 19 Jul 2026 20:28:35 +0100 Subject: [PATCH 3/6] gh-153946: Fix NEWS entry filename --- ...-153946.rst => 2026-07-19-12-00-00.gh-issue-153946.r6GkP2.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Misc/NEWS.d/next/Core_and_Builtins/{2026-07-19-12-00-00.gh-issue-153946.rst => 2026-07-19-12-00-00.gh-issue-153946.r6GkP2.rst} (100%) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-00-00.gh-issue-153946.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-00-00.gh-issue-153946.r6GkP2.rst similarity index 100% rename from Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-00-00.gh-issue-153946.rst rename to Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-00-00.gh-issue-153946.r6GkP2.rst From 35cfb621f8d391d3912ae275e62f324006071029 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 26 Aug 2026 01:06:11 +0100 Subject: [PATCH 4/6] gh-153946: Reset automatic GC deferral after fork --- Include/internal/pycore_gc.h | 1 + Lib/test/test_gc.py | 22 ++++++++++++++++++++++ Modules/posixmodule.c | 3 +++ Python/gc.c | 6 ++++++ Python/gc_free_threading.c | 7 +++++++ 5 files changed, 39 insertions(+) diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 6776990423a5bad..f8a521f97a3815c 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -314,6 +314,7 @@ static inline void _PyObject_GC_UNTRACK( */ extern void _PyGC_InitState(struct _gc_runtime_state *); +extern void _PyGC_AfterFork(PyInterpreterState *interp); extern Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason); extern void _PyGC_CollectNoFail(PyThreadState *tstate); diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 212cb23cbf67503..53e603099bd1619 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -11,6 +11,7 @@ from test.support import threading_helper, gc_threshold import gc +import os import sys import sysconfig import textwrap @@ -167,6 +168,27 @@ def test_defer_automatic_collection_across_threads(self): if not was_enabled: gc.disable() + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + @support.requires_fork() + def test_defer_automatic_collection_reset_after_fork(self): + was_enabled = gc.isenabled() + gc.enable() + try: + with gc_threshold(1, 0, 0): + _testinternalcapi.defer_automatic_gc() + try: + pid = os.fork() + if pid == 0: + before = self.total_collections() + objects = [[] for _ in range(10_000)] + os._exit(self.total_collections() <= before) + support.wait_process(pid, exitcode=0) + finally: + _testinternalcapi.resume_automatic_gc() + finally: + if not was_enabled: + gc.disable() + def test_list(self): l = [] l.append(l) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a114a617918f7ce..ddf8988f6456d55 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -18,6 +18,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_ReInitThreads() #include "pycore_fileutils.h" // _Py_closerange() +#include "pycore_gc.h" // _PyGC_AfterFork() #include "pycore_import.h" // _PyImport_AcquireLock() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_jit_unwind.h" // _Py_jit_debug_mutex @@ -776,6 +777,8 @@ PyOS_AfterFork_Child(void) goto fatal_error; } + _PyGC_AfterFork(tstate->interp); + #if defined(PY_HAVE_JIT_GDB_UNWIND) // The child can inherit this mutex locked if another thread held it at // fork(), but the child itself cannot be inside gdb_jit_register_code(). diff --git a/Python/gc.c b/Python/gc.c index 591d0bd62d6efa4..9320a9ed67df835 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1799,6 +1799,12 @@ _PyGC_ResumeAutomaticCollection(PyThreadState *tstate) gcstate->automatic_collection_pause_count--; } +void +_PyGC_AfterFork(PyInterpreterState *interp) +{ + interp->gc.automatic_collection_pause_count = 0; +} + /* Public API to invoke gc.collect() from C */ Py_ssize_t PyGC_Collect(void) diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 41e18147d914ef9..bcd7fb339e4a8ef 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2592,6 +2592,13 @@ _PyGC_ResumeAutomaticCollection(PyThreadState *tstate) assert(previous > 0); } +void +_PyGC_AfterFork(PyInterpreterState *interp) +{ + _Py_atomic_store_int( + &interp->gc.automatic_collection_pause_count, 0); +} + /* Public API to invoke gc.collect() from C */ Py_ssize_t PyGC_Collect(void) From 488c75b3c5d0cc05ee904f3218f7216f83cb1872 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 26 Aug 2026 01:20:23 +0100 Subject: [PATCH 5/6] gh-153946: Suppress expected fork warning in test --- Lib/test/test_gc.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 53e603099bd1619..83a7913942a3d05 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -17,6 +17,7 @@ import textwrap import threading import time +import warnings import weakref try: @@ -177,7 +178,13 @@ def test_defer_automatic_collection_reset_after_fork(self): with gc_threshold(1, 0, 0): _testinternalcapi.defer_automatic_gc() try: - pid = os.fork() + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="This process .* use of fork.*", + category=DeprecationWarning, + ) + pid = os.fork() if pid == 0: before = self.total_collections() objects = [[] for _ in range(10_000)] From c723f523005070cfaa184c7916242434fd40ff5b Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 26 Aug 2026 20:27:28 +0100 Subject: [PATCH 6/6] gh-153946: Preserve active GC deferrals after fork --- Include/internal/pycore_gc.h | 2 +- Include/internal/pycore_tstate.h | 8 +++-- Lib/test/test_gc.py | 50 +++++++++++++++++++++----------- Modules/posixmodule.c | 2 +- Python/gc.c | 12 ++++++-- Python/gc_free_threading.c | 12 ++++++-- 6 files changed, 60 insertions(+), 26 deletions(-) diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index f8a521f97a3815c..acb86a6e13fe46d 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -314,7 +314,7 @@ static inline void _PyObject_GC_UNTRACK( */ extern void _PyGC_InitState(struct _gc_runtime_state *); -extern void _PyGC_AfterFork(PyInterpreterState *interp); +extern void _PyGC_AfterFork(PyThreadState *tstate); extern Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason); extern void _PyGC_CollectNoFail(PyThreadState *tstate); diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index eb2b0c84acdc7c8..d4290ef795cecf3 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -16,12 +16,14 @@ extern "C" { #include "pycore_uop.h" // struct _PyUOpInstruction #include "pycore_structs.h" -#ifdef Py_GIL_DISABLED struct _gc_thread_state { + /* Number of active automatic collection deferrals owned by this thread. */ + int automatic_collection_pause_count; +#ifdef Py_GIL_DISABLED /* Thread-local allocation count. */ Py_ssize_t alloc_count; -}; #endif +}; // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The @@ -64,11 +66,11 @@ typedef struct _PyThreadStateImpl { struct llist_node asyncio_tasks_head; struct _qsbr_thread_state *qsbr; // only used by free-threaded build struct llist_node mem_free_queue; // delayed free queue + struct _gc_thread_state gc; #ifdef Py_GIL_DISABLED // Stack references for the current thread that exist on the C stack struct _PyCStackRef *c_stack_refs; - struct _gc_thread_state gc; struct _mimalloc_thread_state mimalloc; struct _Py_freelists freelists; struct _brc_thread_state brc; diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 83a7913942a3d05..aace978f28a6cc5 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -171,27 +171,43 @@ def test_defer_automatic_collection_across_threads(self): @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") @support.requires_fork() - def test_defer_automatic_collection_reset_after_fork(self): + @threading_helper.requires_working_threading() + def test_defer_automatic_collection_after_fork(self): was_enabled = gc.isenabled() gc.enable() try: with gc_threshold(1, 0, 0): - _testinternalcapi.defer_automatic_gc() - try: - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - message="This process .* use of fork.*", - category=DeprecationWarning, - ) - pid = os.fork() - if pid == 0: - before = self.total_collections() - objects = [[] for _ in range(10_000)] - os._exit(self.total_collections() <= before) - support.wait_process(pid, exitcode=0) - finally: - _testinternalcapi.resume_automatic_gc() + ready = threading.Event() + release = threading.Event() + + def defer_in_thread(): + _testinternalcapi.defer_automatic_gc() + try: + ready.set() + release.wait() + finally: + _testinternalcapi.resume_automatic_gc() + + thread = threading.Thread(target=defer_in_thread) + with threading_helper.start_threads([thread], release.set): + self.assertTrue(ready.wait(support.SHORT_TIMEOUT)) + _testinternalcapi.defer_automatic_gc() + try: + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="This process .* use of fork.*", + category=DeprecationWarning, + ) + pid = os.fork() + if pid == 0: + _testinternalcapi.resume_automatic_gc() + before = self.total_collections() + objects = [[] for _ in range(10_000)] + os._exit(self.total_collections() <= before) + support.wait_process(pid, exitcode=0) + finally: + _testinternalcapi.resume_automatic_gc() finally: if not was_enabled: gc.disable() diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index ddf8988f6456d55..f118d8c24243371 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -777,7 +777,7 @@ PyOS_AfterFork_Child(void) goto fatal_error; } - _PyGC_AfterFork(tstate->interp); + _PyGC_AfterFork(tstate); #if defined(PY_HAVE_JIT_GDB_UNWIND) // The child can inherit this mutex locked if another thread held it at diff --git a/Python/gc.c b/Python/gc.c index 9320a9ed67df835..91ae84ecd0914db 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1787,22 +1787,30 @@ void _PyGC_DeferAutomaticCollection(PyThreadState *tstate) { GCState *gcstate = &tstate->interp->gc; + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; assert(gcstate->automatic_collection_pause_count >= 0); + assert(tstate_impl->gc.automatic_collection_pause_count >= 0); gcstate->automatic_collection_pause_count++; + tstate_impl->gc.automatic_collection_pause_count++; } void _PyGC_ResumeAutomaticCollection(PyThreadState *tstate) { GCState *gcstate = &tstate->interp->gc; + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; assert(gcstate->automatic_collection_pause_count > 0); + assert(tstate_impl->gc.automatic_collection_pause_count > 0); gcstate->automatic_collection_pause_count--; + tstate_impl->gc.automatic_collection_pause_count--; } void -_PyGC_AfterFork(PyInterpreterState *interp) +_PyGC_AfterFork(PyThreadState *tstate) { - interp->gc.automatic_collection_pause_count = 0; + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + tstate->interp->gc.automatic_collection_pause_count = + tstate_impl->gc.automatic_collection_pause_count; } /* Public API to invoke gc.collect() from C */ diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index bcd7fb339e4a8ef..554416b2ae953e3 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2576,27 +2576,35 @@ void _PyGC_DeferAutomaticCollection(PyThreadState *tstate) { GCState *gcstate = &tstate->interp->gc; + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; int previous = _Py_atomic_add_int( &gcstate->automatic_collection_pause_count, 1); (void)previous; assert(previous >= 0); + assert(tstate_impl->gc.automatic_collection_pause_count >= 0); + tstate_impl->gc.automatic_collection_pause_count++; } void _PyGC_ResumeAutomaticCollection(PyThreadState *tstate) { GCState *gcstate = &tstate->interp->gc; + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; int previous = _Py_atomic_add_int( &gcstate->automatic_collection_pause_count, -1); (void)previous; assert(previous > 0); + assert(tstate_impl->gc.automatic_collection_pause_count > 0); + tstate_impl->gc.automatic_collection_pause_count--; } void -_PyGC_AfterFork(PyInterpreterState *interp) +_PyGC_AfterFork(PyThreadState *tstate) { + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; _Py_atomic_store_int( - &interp->gc.automatic_collection_pause_count, 0); + &tstate->interp->gc.automatic_collection_pause_count, + tstate_impl->gc.automatic_collection_pause_count); } /* Public API to invoke gc.collect() from C */