Improve handling of promotion and old generation evacuation failures

Reviewed-by: kdnilsen

Author: William Kemper

src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp

@@ -294,18 +294,6 @@ void ShenandoahOldHeuristics::get_coalesce_and_fill_candidates(ShenandoahHeapReg
   }
 }
 
-bool ShenandoahOldHeuristics::should_defer_gc() {
-  if (unprocessed_old_collection_candidates() > 0) {
-    // Cannot start a new old-gen GC until previous one has finished.
-    //
-    // Future refinement: under certain circumstances, we might be more sophisticated about this choice.
-    // For example, we could choose to abandon the previous old collection before it has completed evacuations,
-    // but this would require that we coalesce and fill all garbage within unevacuated collection-set regions.
-    return true;
-  }
-  return false;
-}
-
 void ShenandoahOldHeuristics::abandon_collection_candidates() {
   _old_collection_candidates = 0;
   _next_old_collection_candidate = 0;
@@ -315,7 +303,12 @@ void ShenandoahOldHeuristics::abandon_collection_candidates() {
   _first_coalesce_and_fill_candidate = 0;
 }
 
+void ShenandoahOldHeuristics::handle_promotion_failure() {
+  _promotion_failed = true;
+}
+
 void ShenandoahOldHeuristics::record_cycle_start() {
+  _promotion_failed = false;
   _trigger_heuristic->record_cycle_start();
 }
 
@@ -324,9 +317,23 @@ void ShenandoahOldHeuristics::record_cycle_end() {
 }
 
 bool ShenandoahOldHeuristics::should_start_gc() {
-  if (should_defer_gc()) {
+  // Cannot start a new old-gen GC until previous one has finished.
+  //
+  // Future refinement: under certain circumstances, we might be more sophisticated about this choice.
+  // For example, we could choose to abandon the previous old collection before it has completed evacuations,
+  // but this would require that we coalesce and fill all garbage within unevacuated collection-set regions.
+  if (unprocessed_old_collection_candidates() > 0) {
     return false;
   }
+
+  // If there's been a promotion failure (and we don't have regions already scheduled for evacuation),
+  // start a new old generation collection.
+  if (_promotion_failed) {
+    log_info(gc)("Trigger: Promotion Failure");
+    return true;
+  }
+
+  // Otherwise, defer to configured heuristic for gc trigger.
   return _trigger_heuristic->should_start_gc();
 }
 

src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp

@@ -66,10 +66,12 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics {
   // This can be the 'static' or 'adaptive' heuristic.
   ShenandoahHeuristics* _trigger_heuristic;
 
+  // Flag is set when promotion failure is detected (by gc thread), cleared when old generation collection begins (by control thread)
+  volatile bool _promotion_failed;
+
   // Prepare for evacuation of old-gen regions by capturing the mark results of a recently completed concurrent mark pass.
   void prepare_for_old_collections();
 
-
  protected:
   virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, RegionData* data, size_t data_size,
                                                      size_t free) override;
@@ -108,12 +110,15 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics {
   // end of the array.
   void get_coalesce_and_fill_candidates(ShenandoahHeapRegion** buffer);
 
-  bool should_defer_gc();
-
   // If a GLOBAL gc occurs, it will collect the entire heap which invalidates any collection candidates being
   // held by this heuristic for supplying mixed collections.
   void abandon_collection_candidates();
 
+  // Notify the heuristic of promotion failures. The promotion attempt will be skipped and the object will
+  // be evacuated into the young generation. The collection should complete normally, but we want to schedule
+  // an old collection as soon as possible.
+  void handle_promotion_failure();
+
   virtual void record_cycle_start() override;
 
   virtual void record_cycle_end() override;

src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp

@@ -142,8 +142,10 @@ void ShenandoahControlThread::run_service() {
 
       ShenandoahHeuristics* heuristics = _degen_generation->heuristics();
       generation = _degen_generation->generation_mode();
+      bool old_gen_evacuation_failed = heap->clear_old_evacuation_failure();
 
-      if (ShenandoahDegeneratedGC && heuristics->should_degenerate_cycle()) {
+      // Do not bother with degenerated cycle if old generation evacuation failed.
+      if (ShenandoahDegeneratedGC && heuristics->should_degenerate_cycle() && !old_gen_evacuation_failed) {
         heuristics->record_allocation_failure_gc();
         policy->record_alloc_failure_to_degenerated(degen_point);
         set_gc_mode(stw_degenerated);
@@ -370,7 +372,8 @@ void ShenandoahControlThread::run_service() {
       last_shrink_time = current;
     }
 
-    {
+    // Don't wait around if there was an allocation failure - start the next cycle immediately.
+    if (!is_alloc_failure_gc()) {
       // The timed wait is necessary because this thread has a responsibility to send
       // 'alloc_words' to the pacer when it does not perform a GC.
       MonitorLocker lock(&_control_lock, Mutex::_no_safepoint_check_flag);

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -824,6 +824,16 @@ void ShenandoahHeap::handle_old_evacuation(HeapWord* obj, size_t words, bool pro
   }
 }
 
+void ShenandoahHeap::handle_old_evacuation_failure() {
+  if (_old_gen_oom_evac.try_set()) {
+    log_info(gc)("Old gen evac failure.");
+  }
+}
+
+void ShenandoahHeap::handle_promotion_failure() {
+  old_heuristics()->handle_promotion_failure();
+}
+
 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
   // New object should fit the GCLAB size
   size_t min_size = MAX2(size, PLAB::min_size());

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -668,9 +668,12 @@ class ShenandoahHeap : public CollectedHeap {
 private:
   ShenandoahCollectionSet* _collection_set;
   ShenandoahEvacOOMHandler _oom_evac_handler;
+  ShenandoahSharedFlag _old_gen_oom_evac;
 
   inline oop try_evacuate_object(oop src, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahRegionAffiliation target_gen);
   void handle_old_evacuation(HeapWord* obj, size_t words, bool promotion);
+  void handle_old_evacuation_failure();
+  void handle_promotion_failure();
 
 public:
   static address in_cset_fast_test_addr();
@@ -691,6 +694,8 @@ class ShenandoahHeap : public CollectedHeap {
   inline void enter_evacuation(Thread* t);
   inline void leave_evacuation(Thread* t);
 
+  inline bool clear_old_evacuation_failure();
+
 // ---------- Generational support
 //
 private:

src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp

@@ -315,7 +315,8 @@ inline HeapWord* ShenandoahHeap::allocate_from_plab(Thread* thread, size_t size)
 }
 
 inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
-  if (ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) {
+  assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
+  if (ShenandoahThreadLocalData::is_oom_during_evac(thread)) {
     // This thread went through the OOM during evac protocol and it is safe to return
     // the forward pointer. It must not attempt to evacuate any more.
     return ShenandoahBarrierSet::resolve_forwarded(p);
@@ -388,9 +389,17 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah
 #endif
 
   if (copy == NULL) {
-    if (target_gen == OLD_GENERATION && from_region->affiliation() == YOUNG_GENERATION) {
-      // TODO: Inform old generation heuristic of promotion failure
-      return NULL;
+    if (target_gen == OLD_GENERATION) {
+      assert(mode()->is_generational(), "Should only be here in generational mode.");
+      if (from_region->is_young()) {
+        // Signal that promotion failed. Will evacuate this old object somewhere in young gen.
+        handle_promotion_failure();
+        return NULL;
+      } else {
+        // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
+        // after the evacuation threads have finished.
+        handle_old_evacuation_failure();
+      }
     }
 
     control_thread()->handle_alloc_failure_evac(size);
@@ -466,6 +475,10 @@ void ShenandoahHeap::increase_object_age(oop obj, uint additional_age) {
   }
 }
 
+inline bool ShenandoahHeap::clear_old_evacuation_failure() {
+  return _old_gen_oom_evac.try_unset();
+}
+
 inline bool ShenandoahHeap::is_old(oop obj) const {
   return is_gc_generation_young() && is_in_old(obj);
 }