theap: mi_on_thread_idle_pending_own, for a thread that comes back only for its own large pages

include/mimalloc.h

@@ -220,8 +220,18 @@ mi_decl_export void mi_collect(bool force)      mi_attr_noexcept;
 // RETURNED, and three more times at the most: two will do as a rule, and what is left after
 // those are pages that other threads are using, for which it can stay true as long as they
 // do. The `_start`/`_end` pair below does all of that by itself.
+// A thread that is one of many that go idle around the threads that work (a compiler or a
+// collector helper thread of a runtime) asks `mi_on_thread_idle_pending_own` instead, which is
+// true only when one of those pages is its own. Each such thread passes the abandoned pages of
+// all of them, which is where the buffers of the threads that run are: if every helper called
+// again while any of those was young, their sweeps would end an epoch per interval and a thread
+// that takes its buffers again every few intervals would find them given back each time.
+// (The `_start`/`_end` pair still comes back on the first answer, for any page: it was made for
+// the threads that run, whose buffers those pages hold. A helper thread that parks with it comes
+// back for the pages of the others as before.)
 mi_decl_export void mi_on_thread_idle(void)     mi_attr_noexcept;
 mi_decl_export bool mi_on_thread_idle_pending(void) mi_attr_noexcept;  // did the last sweep of this thread's heaps (its `mi_on_thread_idle`, or the scavenger's during a park) leave free blocks of a large page that was just in use? Not to be called between `_start` and `_end`.
+mi_decl_export bool mi_on_thread_idle_pending_own(void) mi_attr_noexcept;  // ..of a large page of this thread itself (not only of abandoned pages of its heaps, which other threads may be using)? Same rules.
 mi_decl_export bool mi_on_thread_idle_start(void) mi_attr_noexcept;  // about to block: hand the theaps to the scavenger. false = nothing handed off, no _end needed
 mi_decl_export void mi_on_thread_idle_end(void)   mi_attr_noexcept;  // awake again: take them back (pairs with a true from _start)
 mi_decl_export void mi_scavenger_stop(void)      mi_attr_noexcept;  // stop and join the background scavenger thread; a no-op if it is not running

include/mimalloc/types.h

@@ -869,6 +869,7 @@ struct mi_tld_s {
   size_t                holes_sweep_skipped;  // per-sweep counters, folded into the process-wide ones in `_mi_page_purge_holes_end`
   size_t                holes_sweep_visited;
   bool                  holes_sweep_deferred; // this sweep left the free blocks of a large page that was just in use (see `_mi_page_purge_holes`)
+  bool                  holes_sweep_deferred_own;   // ..and one such page is a page of this thread, not an abandoned one of its heaps (`mi_on_thread_idle_pending_own`)
   uint32_t              holes_sweep_epoch;    // the epoch of the sweep (`page.c`) when the current sweep began: its pages were compared with that one or a later one
   uint32_t              holes_park_epoch;     // ..and that of the first sweep of the current park: what the thread left when it parked is from that epoch or an earlier one
   _Atomic(size_t)       holes_floor_kept;     // bytes that the last sweep of this tld left under `purge_holes_large_floor` (its share of `mi_holes_floor_kept` in `page.c`)

src/init.c

@@ -132,7 +132,7 @@ static mi_decl_cache_align mi_tld_t mi_tld_detached = {
   NULL,                   // subproc_next
   0, 0,                   // holes_sweep_seq / _last
   false, false, 0, 0,     // holes_sweeping / _full / _skipped / _visited
-  false, 0, 0,            // holes_sweep_deferred / holes_sweep_epoch / holes_park_epoch
+  false, false, 0, 0,     // holes_sweep_deferred / _own / holes_sweep_epoch / holes_park_epoch
   MI_ATOMIC_VAR_INIT(0),  // holes_floor_kept
   NULL,                   // holes_floor_list
   NULL,                   // theap_spare

src/page.c

@@ -1240,7 +1240,7 @@ bool _mi_page_purge_holes_free_page_stays(mi_page_t* page, mi_tld_t* tld) {
   if (!_mi_page_purge_holes_large_page_waits(page, tld)) return false;
   uint32_t stamp;
   bool stays = false;
-  if (mi_page_sweep_state_is_alloc(page, &stamp) && mi_page_sweep_state_alloc_age(stamp) < 2) { tld->holes_sweep_deferred = true; stays = true; }
+  if (mi_page_sweep_state_is_alloc(page, &stamp) && mi_page_sweep_state_alloc_age(stamp) < 2) { tld->holes_sweep_deferred = true; tld->holes_sweep_deferred_own = true; stays = true; }   // (only a page of the thread comes here)
   else { stays = mi_page_holes_floor_keep(page, tld); }
   if (stays) { page->retire_expire = 0; }   // (not `_mi_theap_collect_retired`: the sweeps decide from here on)
   return stays;
@@ -1359,7 +1359,12 @@ void _mi_page_purge_holes(mi_page_t* page, mi_tld_t* tld) {
       // is full, so that is where most of a thread's own buffers are (and those of others; hence a fixed number of
       // sweeps, not "until nothing is left", in `_mi_theap_sweep_parked`). With nothing to take now, the epoch stays for
       // what is freed until the next sweep.
-      if (page->free != NULL || mi_page_unformed_tail_pending(page)) { tld->holes_sweep_deferred = true; }
+      // Which of the two it was is kept apart: a thread that comes back only for abandoned pages takes the buffers of
+      // the threads that use them, an epoch sooner with every sweep it adds (`mi_on_thread_idle_pending_own`).
+      if (page->free != NULL || mi_page_unformed_tail_pending(page)) {
+        tld->holes_sweep_deferred = true;
+        if (!mi_page_is_abandoned(page)) { tld->holes_sweep_deferred_own = true; }
+      }
       return;
     }
   }

src/subproc.c

@@ -463,6 +463,7 @@ void _mi_process_fork_child(void) {
       mi_atomic_store_relaxed(&t->park_swept, (uint32_t)0);
       t->holes_sweeping = false;   // a sweep that was running at the fork (by the scavenger, or by a thread that is gone) does not continue here
       t->holes_sweep_deferred = false;
+      t->holes_sweep_deferred_own = false;
       mi_atomic_store_relaxed(&t->holes_floor_kept, (size_t)0);   // (`_mi_page_purge_holes_forked_child` zeroes the sum)
       t->holes_floor_list = NULL;
     }

src/theap.c

@@ -349,6 +349,7 @@ void mi_on_thread_idle(void) mi_attr_noexcept {
   if (theap0->tld->thread_id != _mi_thread_id()) return;
   mi_tld_t* const tld = theap0->tld;
   tld->holes_sweep_deferred = false;
+  tld->holes_sweep_deferred_own = false;
   _mi_thread_idle_work(tld, theap0);
 }
 
@@ -363,6 +364,20 @@ bool mi_on_thread_idle_pending(void) mi_attr_noexcept {
   return theap0->tld->holes_sweep_deferred;
 }
 
+// ..and was one of those a page of this thread itself, not an abandoned page of its heaps? The answer for a thread
+// that may be one of many that go idle around the threads that work (the compiler and collector helper threads of a
+// runtime): an abandoned large page is one that became full, which is where the buffers of the threads that run are,
+// and every one of those helpers passes all of them. If each came back while any was young, their sweeps would end an
+// epoch per interval, and a thread that takes its buffers again every few intervals would find them given back each
+// time. A thread that holds nothing of its own has nothing to come back for: what it leaves in the abandoned pages
+// goes at the next sweep of any thread that finds it old. Same rules as for `mi_on_thread_idle_pending`.
+bool mi_on_thread_idle_pending_own(void) mi_attr_noexcept {
+  mi_theap_t* const theap0 = _mi_theap_default();
+  if (theap0 == NULL || !mi_theap_is_initialized(theap0) || theap0->tld == NULL) return false;
+  if (theap0->tld->thread_id != _mi_thread_id()) return false;
+  return theap0->tld->holes_sweep_deferred_own;
+}
+
 // Declare that this thread will not allocate or free until `mi_on_thread_idle_end` -- the sweep's
 // precondition -- so the scavenger can do it while we block.
 //
@@ -467,6 +482,7 @@ mi_msecs_t _mi_theap_sweep_parked(mi_subproc_t* subproc) {
     // is not due, so the next one in the list is claimed.
     const uint32_t swept = mi_atomic_load_relaxed(&claimed->park_swept);
     claimed->holes_sweep_deferred = false;
+    claimed->holes_sweep_deferred_own = false;
     _mi_thread_idle_work(claimed, theap0);
     // After the work: the sweep that comes back is to find the epoch that this one began (if it did) as old as the interval.
     claimed->holes_sweep_last = _mi_clock_now();