8283566: G1: Improve G1BarrierSet::enqueue performance

Reviewed-by: tschatzl, ayang

Author: shipilev

src/hotspot/share/gc/g1/g1BarrierSet.cpp

@@ -61,13 +61,6 @@ G1BarrierSet::G1BarrierSet(G1CardTable* card_table) :
   _dirty_card_queue_set(&_dirty_card_queue_buffer_allocator)
 {}
 
-void G1BarrierSet::enqueue(oop pre_val) {
-  // Nulls should have been already filtered.
-  assert(oopDesc::is_oop(pre_val, true), "Error");
-  SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(Thread::current());
-  G1BarrierSet::satb_mark_queue_set().enqueue(queue, pre_val);
-}
-
 template <class T> void
 G1BarrierSet::write_ref_array_pre_work(T* dst, size_t count) {
   G1SATBMarkQueueSet& queue_set = G1BarrierSet::satb_mark_queue_set();

src/hotspot/share/gc/g1/g1BarrierSet.hpp

@@ -56,10 +56,12 @@ class G1BarrierSet: public CardTableBarrierSet {
   }
 
   // Add "pre_val" to a set of objects that may have been disconnected from the
-  // pre-marking object graph.
-  static void enqueue(oop pre_val);
+  // pre-marking object graph. Prefer the version that takes location, as it
+  // can avoid touching the heap unnecessarily.
+  template <class T> static void enqueue(T* dst);
+  static void enqueue_preloaded(oop pre_val);
 
-  static void enqueue_if_weak(DecoratorSet decorators, oop value);
+  static void enqueue_preloaded_if_weak(DecoratorSet decorators, oop value);
 
   template <class T> void write_ref_array_pre_work(T* dst, size_t count);
   virtual void write_ref_array_pre(oop* dst, size_t count, bool dest_uninitialized);

src/hotspot/share/gc/g1/g1BarrierSet.inline.hpp

@@ -28,22 +28,43 @@
 #include "gc/g1/g1BarrierSet.hpp"
 
 #include "gc/g1/g1CardTable.hpp"
+#include "gc/g1/g1ThreadLocalData.hpp"
 #include "gc/shared/accessBarrierSupport.inline.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
 #include "oops/oop.hpp"
 
+inline void G1BarrierSet::enqueue_preloaded(oop pre_val) {
+  // Nulls should have been already filtered.
+  assert(oopDesc::is_oop(pre_val, true), "Error");
+
+  G1SATBMarkQueueSet& queue_set = G1BarrierSet::satb_mark_queue_set();
+  if (!queue_set.is_active()) return;
+
+  SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(Thread::current());
+  queue_set.enqueue_known_active(queue, pre_val);
+}
+
+template <class T>
+inline void G1BarrierSet::enqueue(T* dst) {
+  G1SATBMarkQueueSet& queue_set = G1BarrierSet::satb_mark_queue_set();
+  if (!queue_set.is_active()) return;
+
+  T heap_oop = RawAccess<MO_RELAXED>::oop_load(dst);
+  if (!CompressedOops::is_null(heap_oop)) {
+    SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(Thread::current());
+    queue_set.enqueue_known_active(queue, CompressedOops::decode_not_null(heap_oop));
+  }
+}
+
 template <DecoratorSet decorators, typename T>
 inline void G1BarrierSet::write_ref_field_pre(T* field) {
   if (HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value ||
       HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
     return;
   }
 
-  T heap_oop = RawAccess<MO_RELAXED>::oop_load(field);
-  if (!CompressedOops::is_null(heap_oop)) {
-    enqueue(CompressedOops::decode_not_null(heap_oop));
-  }
+  enqueue(field);
 }
 
 template <DecoratorSet decorators, typename T>
@@ -55,7 +76,7 @@ inline void G1BarrierSet::write_ref_field_post(T* field, oop new_val) {
   }
 }
 
-inline void G1BarrierSet::enqueue_if_weak(DecoratorSet decorators, oop value) {
+inline void G1BarrierSet::enqueue_preloaded_if_weak(DecoratorSet decorators, oop value) {
   assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known");
   // Loading from a weak or phantom reference needs enqueueing, as
   // the object may not have been reachable (part of the snapshot)
@@ -65,7 +86,7 @@ inline void G1BarrierSet::enqueue_if_weak(DecoratorSet decorators, oop value) {
   const bool needs_enqueue     = (!peek && !on_strong_oop_ref);
 
   if (needs_enqueue && value != NULL) {
-    enqueue(value);
+    enqueue_preloaded(value);
   }
 }
 
@@ -74,7 +95,7 @@ template <typename T>
 inline oop G1BarrierSet::AccessBarrier<decorators, BarrierSetT>::
 oop_load_not_in_heap(T* addr) {
   oop value = ModRef::oop_load_not_in_heap(addr);
-  enqueue_if_weak(decorators, value);
+  enqueue_preloaded_if_weak(decorators, value);
   return value;
 }
 
@@ -83,15 +104,15 @@ template <typename T>
 inline oop G1BarrierSet::AccessBarrier<decorators, BarrierSetT>::
 oop_load_in_heap(T* addr) {
   oop value = ModRef::oop_load_in_heap(addr);
-  enqueue_if_weak(decorators, value);
+  enqueue_preloaded_if_weak(decorators, value);
   return value;
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
 inline oop G1BarrierSet::AccessBarrier<decorators, BarrierSetT>::
 oop_load_in_heap_at(oop base, ptrdiff_t offset) {
   oop value = ModRef::oop_load_in_heap_at(base, offset);
-  enqueue_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value);
+  enqueue_preloaded_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value);
   return value;
 }
 

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -2279,7 +2279,7 @@ void G1CollectedHeap::object_iterate_parallel(ObjectClosure* cl, uint worker_id,
 }
 
 void G1CollectedHeap::keep_alive(oop obj) {
-  G1BarrierSet::enqueue(obj);
+  G1BarrierSet::enqueue_preloaded(obj);
 }
 
 void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) const {