8254739: G1: Optimize evacuation failure for regions with few failed objects

src/hotspot/share/gc/g1/g1EvacFailure.cpp

@@ -68,6 +68,7 @@ class RemoveSelfForwardPtrObjClosure: public ObjectClosure {
   // dead too) already.
   void do_object(oop obj) {
     HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);
+    assert(_last_forwarded_object_end <= obj_addr, "should iterate in ascending address order");
     assert(_hr->is_in(obj_addr), "sanity");
 
     // The object failed to move.

src/hotspot/share/gc/g1/g1EvacFailureObjectsSet.cpp

@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2021, Huawei and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/g1EvacFailureObjectsSet.hpp"
+#include "gc/g1/g1CollectedHeap.hpp"
+#include "gc/g1/g1SegmentedArray.inline.hpp"
+#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/heapRegion.inline.hpp"
+#include "utilities/quickSort.hpp"
+
+
+const G1SegmentedArrayAllocOptions G1EvacFailureObjectsSet::_alloc_options =
+  G1SegmentedArrayAllocOptions((uint)sizeof(OffsetInRegion), BufferLength, UINT_MAX, Alignment);
+
+G1SegmentedArrayBufferList<mtGC> G1EvacFailureObjectsSet::_free_buffer_list;
+
+#ifdef ASSERT
+void G1EvacFailureObjectsSet::assert_is_valid_offset(size_t offset) const {
+  const uint max_offset = 1u << (HeapRegion::LogOfHRGrainBytes - LogHeapWordSize);
+  assert(offset < max_offset, "must be, but is " SIZE_FORMAT, offset);
+}
+#endif
+
+oop G1EvacFailureObjectsSet::from_offset(OffsetInRegion offset) const {
+  assert_is_valid_offset(offset);
+  return cast_to_oop(_bottom + offset);
+}
+
+G1EvacFailureObjectsSet::OffsetInRegion G1EvacFailureObjectsSet::cast_to_offset(oop obj) const {
+  const HeapWord* o = cast_from_oop<const HeapWord*>(obj);
+  size_t offset = pointer_delta(o, _bottom);
+  assert_is_valid_offset(offset);
+  assert(obj == from_offset(offset), "must be");
+  return static_cast<OffsetInRegion>(offset);
+}
+
+G1EvacFailureObjectsSet::G1EvacFailureObjectsSet(uint region_idx, HeapWord* bottom) :
+  DEBUG_ONLY(_region_idx(region_idx) COMMA)
+  _bottom(bottom),
+  _offsets("", &_alloc_options, &_free_buffer_list)  {
+  assert(HeapRegion::LogOfHRGrainBytes < 32, "must be");
+}
+
+void G1EvacFailureObjectsSet::record(oop obj) {
+  assert(obj != NULL, "must be");
+  assert(_region_idx == G1CollectedHeap::heap()->heap_region_containing(obj)->hrm_index(), "must be");
+  OffsetInRegion* e = _offsets.allocate();
+  *e = cast_to_offset(obj);
+}
+
+// Helper class to join, sort and iterate over the previously collected segmented
+// array of objects that failed evacuation.
+class G1EvacFailureObjectsIterator {
+  typedef G1EvacFailureObjectsSet::OffsetInRegion OffsetInRegion;
+  friend class G1SegmentedArray<OffsetInRegion, mtGC>;
+  friend class G1SegmentedArrayBuffer<mtGC>;
+
+  G1EvacFailureObjectsSet* _collector;
+  const G1SegmentedArray<OffsetInRegion, mtGC>* _segments;
+  OffsetInRegion* _offset_array;
+  uint _array_length;
+
+  static int order_oop(OffsetInRegion a, OffsetInRegion b) {
+    return static_cast<int>(a-b);
+  }
+
+  void join_and_sort() {
+    uint num = _segments->num_allocated_nodes();
+    _offset_array = NEW_C_HEAP_ARRAY(OffsetInRegion, num, mtGC);
+
+    _segments->iterate_nodes(*this);
+    assert(_array_length == num, "must be %u, %u", _array_length, num);
+
+    QuickSort::sort(_offset_array, _array_length, order_oop, true);
+  }
+
+  void iterate_internal(ObjectClosure* closure) {
+    for (uint i = 0; i < _array_length; i++) {
+      _collector->assert_is_valid_offset(_offset_array[i]);
+      oop cur = _collector->from_offset(_offset_array[i]);
+      closure->do_object(cur);
+    }
+
+    FREE_C_HEAP_ARRAY(OffsetInRegion, _offset_array);
+  }
+
+  // Callback of G1SegmentedArray::iterate_nodes
+  void visit_buffer(G1SegmentedArrayBuffer<mtGC>* node, uint length) {
+    node->copy_to(&_offset_array[_array_length]);
+    _array_length += length;
+
+    // Verify elements in the node
+    DEBUG_ONLY(node->iterate_elems(*this));
+  }
+
+#ifdef ASSERT
+  // Callback of G1SegmentedArrayBuffer::iterate_elems
+  // Verify a single element in a segment node
+  void visit_elem(void* elem) {
+    uint* ptr = (uint*)elem;
+    _collector->assert_is_valid_offset(*ptr);
+  }
+#endif
+
+public:
+  G1EvacFailureObjectsIterator(G1EvacFailureObjectsSet* collector) :
+    _collector(collector),
+    _segments(&_collector->_offsets),
+    _offset_array(nullptr),
+    _array_length(0) { }
+
+  ~G1EvacFailureObjectsIterator() { }
+
+  void iterate(ObjectClosure* closure) {
+    join_and_sort();
+    iterate_internal(closure);
+  }
+};
+
+void G1EvacFailureObjectsSet::iterate(ObjectClosure* closure) {
+  assert_at_safepoint();
+
+  G1EvacFailureObjectsIterator iterator(this);
+  iterator.iterate(closure);
+
+  _offsets.drop_all();
+}

src/hotspot/share/gc/g1/g1EvacFailureObjsInHR.hpp → src/hotspot/share/gc/g1/g1EvacFailureObjectsSet.hpp

@@ -29,65 +29,53 @@
 #include "memory/iterator.hpp"
 #include "oops/oop.hpp"
 
-// This class
-//   1. records the objects per region which have failed to evacuate.
-//   2. speeds up removing self forwarded ptrs in post evacuation phase.
-//
-class G1EvacFailureObjsInHR {
+class G1EvacFailureObjectsIterator;
+
+// This class collects addresses of objects that failed evacuation in a specific
+// heap region.
+// Provides sorted iteration of these elements for processing during the remove
+// self forwards phase.
+class G1EvacFailureObjectsSet {
+  friend class G1EvacFailureObjectsIterator;
 
 public:
-  typedef uint Elem;
+  // Storage type of an object that failed evacuation within a region. Given
+  // heap region size and possible object locations within a region, it is
+  // sufficient to use an uint here to save some space instead of full pointers.
+  typedef uint OffsetInRegion;
 
 private:
   static const uint BufferLength = 256;
-  static const uint MaxBufferLength;
   static const uint Alignment = 4;
 
   static const G1SegmentedArrayAllocOptions _alloc_options;
+
   // This free list is shared among evacuation failure process in all regions.
   static G1SegmentedArrayBufferList<mtGC> _free_buffer_list;
 
-  const Elem _max_offset;
-  const uint _region_idx;
+  DEBUG_ONLY(const uint _region_idx;)
+
+  // Region bottom
   const HeapWord* _bottom;
 
-  // To improve space efficiency, elements are offset rather than raw addr
-  G1SegmentedArray<Elem, mtGC> _nodes_array;
-  // Local array contains the _nodes_array data in flat layout
-  Elem* _offset_array;
-  uint _objs_num;
+  // Offsets within region containing objects that failed evacuation.
+  G1SegmentedArray<OffsetInRegion, mtGC> _offsets;
 
-private:
-  oop cast_from_offset(Elem offset) {
-    return cast_to_oop(_bottom + offset);
-  }
-  Elem cast_from_oop_addr(oop obj) {
-    const HeapWord* o = cast_from_oop<const HeapWord*>(obj);
-    size_t offset = pointer_delta(o, _bottom);
-    return static_cast<Elem>(offset);
-  }
-
-  // Copy buffers' data to local array, must be called at safepoint
-  void compact();
-  void sort();
-  void clear_array();
-  // Iterate through evac failure objects in local array
-  void iterate_internal(ObjectClosure* closure);
+  void assert_is_valid_offset(size_t offset) const NOT_DEBUG_RETURN;
+  // Converts between an offset within a region and an oop address.
+  oop from_offset(OffsetInRegion offset) const;
+  OffsetInRegion cast_to_offset(oop obj) const;
 
 public:
-  G1EvacFailureObjsInHR(uint region_idx, HeapWord* bottom);
-  ~G1EvacFailureObjsInHR();
+  G1EvacFailureObjectsSet(uint region_idx, HeapWord* bottom);
+  ~G1EvacFailureObjectsSet() { }
 
-  // Record an evac failure object
+  // Record an object that failed evacuation.
   void record(oop obj);
-  // Iterate through evac failure objects
-  void iterate(ObjectClosure* closure);
-
-  // Copy a buffer data to local array
-  void visit_buffer(G1SegmentedArrayBuffer<mtGC>* node, uint limit);
 
-  // Verify elements in the buffer
-  void visit_elem(void* elem);
+  // Apply the given ObjectClosure to all objects that failed evacuation. Objects
+  // are passed in increasing address order.
+  void iterate(ObjectClosure* closure);
 };
 
 

src/hotspot/share/gc/g1/g1SegmentedArray.hpp

@@ -74,11 +74,12 @@ class G1SegmentedArrayBuffer : public CHeapObj<flag> {
   size_t mem_size() const { return sizeof(*this) + (size_t)_num_elems * _elem_size; }
 
   uint length() const {
-    // _next_allocate might grow greater than _num_elems in multi-thread env,
-    // so, here we need to return the adjusted real length value.
-    return _next_allocate > _num_elems ? _num_elems : _next_allocate;
+    // _next_allocate might grow larger than _num_elems in multi-thread environments
+    // due to races.
+    return MIN2(_next_allocate, _num_elems);
   }
 
+  // Copies the (valid) contents of this buffer into the destination.
   void copy_to(void* dest) const {
     ::memcpy(dest, _buffer, length() * _elem_size);
   }

src/hotspot/share/gc/g1/g1SegmentedArray.inline.hpp

@@ -269,15 +269,12 @@ template <typename Visitor>
 void G1SegmentedArray<Elem, flag>::iterate_nodes(Visitor& v) const {
   G1SegmentedArrayBuffer<flag>* cur = Atomic::load_acquire(&_first);
 
-  if (cur != nullptr) {
-    assert(_last != nullptr, "If there is at least one element, there must be a last one.");
+  assert((cur != nullptr) == (_last != nullptr),
+         "If there is at least one element, there must be a last one");
 
-    while (cur != nullptr) {
-      uint limit = cur->length();
-      v.visit_buffer(cur, limit);
-
-      cur = cur->next();
-    }
+  while (cur != nullptr) {
+    v.visit_buffer(cur, cur->length());
+    cur = cur->next();
   }
 }
 

src/hotspot/share/gc/g1/heapRegion.hpp

@@ -26,7 +26,7 @@
 #define SHARE_GC_G1_HEAPREGION_HPP
 
 #include "gc/g1/g1BlockOffsetTable.hpp"
-#include "gc/g1/g1EvacFailureObjsInHR.hpp"
+#include "gc/g1/g1EvacFailureObjectsSet.hpp"
 #include "gc/g1/g1HeapRegionTraceType.hpp"
 #include "gc/g1/g1SurvRateGroup.hpp"
 #include "gc/g1/heapRegionTracer.hpp"
@@ -259,7 +259,7 @@ class HeapRegion : public CHeapObj<mtGC> {
 
   uint _node_index;
 
-  G1EvacFailureObjsInHR _evac_failure_objs;
+  G1EvacFailureObjectsSet _evac_failure_objs;
 
   void report_region_type_change(G1HeapRegionTraceType::Type to);