8247845: Shenandoah: refactor TLAB/GCLAB retirement code

Reviewed-by: rkennke

Author: shipilev

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -1044,28 +1044,79 @@ void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
   }
 }
 
+class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
+public:
+  ShenandoahCheckCleanGCLABClosure() {}
+  void do_thread(Thread* thread) {
+    PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
+    assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
+    assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");
+  }
+};
+
 class ShenandoahRetireGCLABClosure : public ThreadClosure {
+private:
+  bool const _resize;
 public:
+  ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
   void do_thread(Thread* thread) {
     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
     assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
     gclab->retire();
+    if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
+      ShenandoahThreadLocalData::set_gclab_size(thread, 0);
+    }
   }
 };
 
-void ShenandoahHeap::make_parsable(bool retire_tlabs) {
-  if (UseTLAB) {
-    CollectedHeap::ensure_parsability(retire_tlabs);
+void ShenandoahHeap::labs_make_parsable() {
+  assert(UseTLAB, "Only call with UseTLAB");
+
+  ShenandoahRetireGCLABClosure cl(false);
+
+  for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
+    ThreadLocalAllocBuffer& tlab = t->tlab();
+    tlab.make_parsable();
+    cl.do_thread(t);
   }
-  ShenandoahRetireGCLABClosure cl;
+
+  workers()->threads_do(&cl);
+}
+
+void ShenandoahHeap::tlabs_retire(bool resize) {
+  assert(UseTLAB, "Only call with UseTLAB");
+  assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");
+
+  ThreadLocalAllocStats stats;
+
+  for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
+    ThreadLocalAllocBuffer& tlab = t->tlab();
+    tlab.retire(&stats);
+    if (resize) {
+      tlab.resize();
+    }
+  }
+
+  stats.publish();
+
+#ifdef ASSERT
+  ShenandoahCheckCleanGCLABClosure cl;
   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
     cl.do_thread(t);
   }
   workers()->threads_do(&cl);
+#endif
 }
 
-void ShenandoahHeap::resize_tlabs() {
-  CollectedHeap::resize_all_tlabs();
+void ShenandoahHeap::gclabs_retire(bool resize) {
+  assert(UseTLAB, "Only call with UseTLAB");
+  assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");
+
+  ShenandoahRetireGCLABClosure cl(resize);
+  for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
+    cl.do_thread(t);
+  }
+  workers()->threads_do(&cl);
 }
 
 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
@@ -1122,25 +1173,6 @@ size_t ShenandoahHeap::max_tlab_size() const {
   return ShenandoahHeapRegion::max_tlab_size_words();
 }
 
-class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure {
-public:
-  void do_thread(Thread* thread) {
-    PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
-    gclab->retire();
-    if (ShenandoahThreadLocalData::gclab_size(thread) > 0) {
-      ShenandoahThreadLocalData::set_gclab_size(thread, 0);
-    }
-  }
-};
-
-void ShenandoahHeap::retire_and_reset_gclabs() {
-  ShenandoahRetireAndResetGCLABClosure cl;
-  for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
-    cl.do_thread(t);
-  }
-  workers()->threads_do(&cl);
-}
-
 void ShenandoahHeap::collect(GCCause::Cause cause) {
   control_thread()->request_gc(cause);
 }
@@ -1173,8 +1205,8 @@ jlong ShenandoahHeap::millis_since_last_gc() {
 }
 
 void ShenandoahHeap::prepare_for_verify() {
-  if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
-    make_parsable(false);
+  if (SafepointSynchronize::is_at_safepoint() && UseTLAB) {
+    labs_make_parsable();
   }
 }
 
@@ -1264,7 +1296,7 @@ class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
  * This is public API, used in preparation of object_iterate().
  * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
  * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
- * control, we call SH::make_tlabs_parsable().
+ * control, we call SH::tlabs_retire, SH::gclabs_retire.
  */
 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
   // No-op.
@@ -1420,10 +1452,13 @@ void ShenandoahHeap::op_init_mark() {
   }
 
   set_concurrent_mark_in_progress(true);
-  // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
-  {
-    ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
-    make_parsable(true);
+
+  // We need to reset all TLABs because they might be below the TAMS, and we need to mark
+  // the objects in them. Do not let mutators allocate any new objects in their current TLABs.
+  // It is also a good place to resize the TLAB sizes for future allocations.
+  if (UseTLAB) {
+    ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_manage_tlabs);
+    tlabs_retire(ResizeTLAB);
   }
 
   {
@@ -1437,11 +1472,6 @@ void ShenandoahHeap::op_init_mark() {
 
   concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
 
-  if (UseTLAB) {
-    ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
-    resize_tlabs();
-  }
-
   if (ShenandoahPacing) {
     pacer()->setup_for_mark();
   }
@@ -1539,9 +1569,9 @@ void ShenandoahHeap::op_final_mark() {
     // which would be outside the collection set, so no cset writes would happen there.
     // Weaker one: new allocations would happen past update watermark, and so less work would
     // be needed for reference updates (would update the large filler instead).
-    {
-      ShenandoahGCPhase phase(ShenandoahPhaseTimings::retire_tlabs);
-      make_parsable(true);
+    if (UseTLAB) {
+      ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_manage_labs);
+      tlabs_retire(false);
     }
 
     {
@@ -1917,10 +1947,6 @@ void ShenandoahHeap::op_full(GCCause::Cause cause) {
   metrics.snap_before();
 
   full_gc()->do_it(cause);
-  if (UseTLAB) {
-    ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
-    resize_all_tlabs();
-  }
 
   metrics.snap_after();
 
@@ -2511,9 +2537,12 @@ void ShenandoahHeap::op_init_updaterefs() {
 
   set_evacuation_in_progress(false);
 
-  {
-    ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_retire_gclabs);
-    retire_and_reset_gclabs();
+  // Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to
+  // make them parsable for update code to work correctly. Plus, we can compute new sizes
+  // for future GCLABs here.
+  if (UseTLAB) {
+    ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_manage_gclabs);
+    gclabs_retire(true);
   }
 
   if (ShenandoahVerify) {

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -595,7 +595,6 @@ class ShenandoahHeap : public CollectedHeap {
   inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
   HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
   HeapWord* allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size);
-  void retire_and_reset_gclabs();
 
 public:
   HeapWord* allocate_memory(ShenandoahAllocRequest& request);
@@ -615,10 +614,11 @@ class ShenandoahHeap : public CollectedHeap {
   size_t max_tlab_size() const;
   size_t tlab_used(Thread* ignored) const;
 
-  void resize_tlabs();
+  void ensure_parsability(bool retire_labs);
 
-  void ensure_parsability(bool retire_tlabs);
-  void make_parsable(bool retire_tlabs);
+  void labs_make_parsable();
+  void tlabs_retire(bool resize);
+  void gclabs_retire(bool resize);
 
 // ---------- Marking support
 //

src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp

@@ -145,7 +145,10 @@ void ShenandoahMarkCompact::do_it(GCCause::Cause gc_cause) {
     _preserved_marks->init(heap->workers()->active_workers());
   }
 
-  heap->make_parsable(true);
+  if (UseTLAB) {
+    heap->gclabs_retire(true);
+    heap->tlabs_retire(true);
+  }
 
   OrderAccess::fence();
 

src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp

@@ -60,11 +60,10 @@ class outputStream;
                                                                                        \
   f(init_mark_gross,                                "Pause Init Mark (G)")             \
   f(init_mark,                                      "Pause Init Mark (N)")             \
-  f(make_parsable,                                  "  Make Parsable")                 \
+  f(init_manage_tlabs,                              "  Manage TLABs")                  \
   f(init_update_region_states,                      "  Update Region States")          \
   f(scan_roots,                                     "  Scan Roots")                    \
   SHENANDOAH_PAR_PHASE_DO(scan_,                    "    S: ", f)                      \
-  f(resize_tlabs,                                   "  Resize TLABs")                  \
                                                                                        \
   f(conc_mark,                                      "Concurrent Marking")              \
   f(conc_mark_roots,                                "  Roots ")                        \
@@ -86,7 +85,7 @@ class outputStream;
   SHENANDOAH_PAR_PHASE_DO(purge_weak_par_,          "      WR: ", f)                   \
   f(purge_cldg,                                     "    CLDG")                        \
   f(final_update_region_states,                     "  Update Region States")          \
-  f(retire_tlabs,                                   "  Retire TLABs")                  \
+  f(final_manage_labs,                              "  Manage GC/TLABs")               \
   f(choose_cset,                                    "  Choose Collection Set")         \
   f(final_rebuild_freeset,                          "  Rebuild Free Set")              \
   f(init_evac,                                      "  Initial Evacuation")            \
@@ -113,7 +112,7 @@ class outputStream;
                                                                                        \
   f(init_update_refs_gross,                         "Pause Init  Update Refs (G)")     \
   f(init_update_refs,                               "Pause Init  Update Refs (N)")     \
-  f(init_update_refs_retire_gclabs,                 "  Retire GCLABs")                 \
+  f(init_update_refs_manage_gclabs,                 "  Manage GCLABs")                 \
                                                                                        \
   f(conc_update_refs,                               "Concurrent Update Refs")          \
                                                                                        \
@@ -166,7 +165,6 @@ class outputStream;
   f(full_gc_copy_objects_humong,                    "    Humongous Objects")           \
   f(full_gc_copy_objects_reset_complete,            "    Reset Complete Bitmap")       \
   f(full_gc_copy_objects_rebuild,                   "    Rebuild Region Sets")         \
-  f(full_gc_resize_tlabs,                           "  Resize TLABs")                  \
   f(full_gc_heapdump_post,                          "  Post Heap Dump")                \
                                                                                        \
   f(conc_uncommit,                                  "Concurrent Uncommit")             \

src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp

@@ -690,7 +690,10 @@ void ShenandoahVerifier::verify_at_safepoint(const char *label,
   }
 
   OrderAccess::fence();
-  _heap->make_parsable(false);
+
+  if (UseTLAB) {
+    _heap->labs_make_parsable();
+  }
 
   // Allocate temporary bitmap for storing marking wavefront:
   _verification_bit_map->clear();