8240872: Shenandoah: Avoid updating new regions from start of evacuation

Reviewed-by: shade

Author: rkennke

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp

@@ -184,13 +184,16 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah
     // Record actual allocation size
     req.set_actual_size(size);
 
-    if (req.is_gc_alloc() && _heap->is_concurrent_traversal_in_progress()) {
-      // Traversal needs to traverse through GC allocs. Adjust TAMS to the new top
-      // so that these allocations appear below TAMS, and thus get traversed.
-      // See top of shenandoahTraversal.cpp for an explanation.
-      _heap->marking_context()->capture_top_at_mark_start(r);
-      _heap->traversal_gc()->traversal_set()->add_region_check_for_duplicates(r);
-      OrderAccess::fence();
+    if (req.is_gc_alloc()) {
+      r->set_update_watermark(r->top());
+      if (_heap->is_concurrent_traversal_in_progress()) {
+        // Traversal needs to traverse through GC allocs. Adjust TAMS to the new top
+        // so that these allocations appear below TAMS, and thus get traversed.
+        // See top of shenandoahTraversal.cpp for an explanation.
+        _heap->marking_context()->capture_top_at_mark_start(r);
+        _heap->traversal_gc()->traversal_set()->add_region_check_for_duplicates(r);
+        OrderAccess::fence();
+      }
     }
   }
 

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -1531,6 +1531,13 @@ void ShenandoahHeap::op_final_mark() {
         verifier()->verify_before_evacuation();
       }
 
+      // Remember limit for updating refs. It's guaranteed that we get no from-space-refs written
+      // from here on.
+      for (uint i = 0; i < num_regions(); i++) {
+        ShenandoahHeapRegion* r = get_region(i);
+        r->set_update_watermark(r->top());
+      }
+
       set_evacuation_in_progress(true);
       // From here on, we need to update references.
       set_has_forwarded_objects(true);
@@ -2391,13 +2398,13 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
     ShenandoahHeapRegion* r = _regions->next();
     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
     while (r != NULL) {
-      HeapWord* top_at_start_ur = r->concurrent_iteration_safe_limit();
-      assert (top_at_start_ur >= r->bottom(), "sanity");
+      HeapWord* update_watermark = r->get_update_watermark();
+      assert (update_watermark >= r->bottom(), "sanity");
       if (r->is_active() && !r->is_cset()) {
-        _heap->marked_object_oop_iterate(r, &cl, top_at_start_ur);
+        _heap->marked_object_oop_iterate(r, &cl, update_watermark);
       }
       if (ShenandoahPacing) {
-        _heap->pacer()->report_updaterefs(pointer_delta(top_at_start_ur, r->bottom()));
+        _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
       }
       if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
         return;
@@ -2435,10 +2442,6 @@ void ShenandoahHeap::op_init_updaterefs() {
     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_prepare);
 
     make_parsable(true);
-    for (uint i = 0; i < num_regions(); i++) {
-      ShenandoahHeapRegion* r = get_region(i);
-      r->set_concurrent_iteration_safe_limit(r->top());
-    }
 
     // Reset iterator.
     _update_refs_iterator.reset();

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp

@@ -71,7 +71,8 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start
   _seqnum_last_alloc_mutator(0),
   _seqnum_last_alloc_gc(0),
   _live_data(0),
-  _critical_pins(0) {
+  _critical_pins(0),
+  _update_watermark(start) {
 
   ContiguousSpace::initialize(_reserved, true, committed);
 }
@@ -479,6 +480,7 @@ void ShenandoahHeapRegion::recycle() {
   reset_alloc_metadata();
 
   _heap->marking_context()->reset_top_at_mark_start(this);
+  set_update_watermark(bottom());
 
   make_empty();
 }

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@@ -261,6 +261,8 @@ class ShenandoahHeapRegion : public ContiguousSpace {
   volatile size_t _live_data;
   volatile size_t _critical_pins;
 
+  HeapWord* _update_watermark;
+
   // Claim some space at the end to protect next region
   DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);
 
@@ -428,6 +430,16 @@ class ShenandoahHeapRegion : public ContiguousSpace {
     return _seqnum_last_alloc_gc;
   }
 
+  HeapWord* get_update_watermark() const {
+    assert(bottom() <= _update_watermark && _update_watermark <= top(), "within bounds");
+    return _update_watermark;
+  }
+
+  void set_update_watermark(HeapWord* w) {
+    assert(bottom() <= w && w <= top(), "within bounds");
+    _update_watermark = w;
+  }
+
 private:
   void do_commit();
   void do_uncommit();

src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp

@@ -224,7 +224,6 @@ class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure {
   void heap_region_do(ShenandoahHeapRegion *r) {
     _ctx->capture_top_at_mark_start(r);
     r->clear_live_data();
-    r->set_concurrent_iteration_safe_limit(r->top());
   }
 };