8274054: Add custom enqueue calls during reference processing

Reviewed-by: ayang, kbarrett, sjohanss

Author: Thomas Schatzl

src/hotspot/share/gc/g1/g1ConcurrentMark.cpp

@@ -1581,8 +1581,9 @@ class G1CMRefProcProxyTask : public RefProcProxyTask {
     G1CMIsAliveClosure is_alive(&_g1h);
     uint index = (_tm == RefProcThreadModel::Single) ? 0 : worker_id;
     G1CMKeepAliveAndDrainClosure keep_alive(&_cm, _cm.task(index), _tm == RefProcThreadModel::Single);
+    BarrierEnqueueDiscoveredFieldClosure enqueue;
     G1CMDrainMarkingStackClosure complete_gc(&_cm, _cm.task(index), _tm == RefProcThreadModel::Single);
-    _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &complete_gc);
+    _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &enqueue, &complete_gc);
   }
 
   void prepare_run_task_hook() override {
@@ -1695,6 +1696,7 @@ void G1ConcurrentMark::preclean() {
   SuspendibleThreadSetJoiner joiner;
 
   G1CMKeepAliveAndDrainClosure keep_alive(this, task(0), true /* is_serial */);
+  BarrierEnqueueDiscoveredFieldClosure enqueue;
   G1CMDrainMarkingStackClosure drain_mark_stack(this, task(0), true /* is_serial */);
 
   set_concurrency_and_phase(1, true);
@@ -1706,6 +1708,7 @@ void G1ConcurrentMark::preclean() {
   ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(rp, false);
   rp->preclean_discovered_references(rp->is_alive_non_header(),
                                      &keep_alive,
+                                     &enqueue,
                                      &drain_mark_stack,
                                      &yield_cl,
                                      _gc_timer_cm);

src/hotspot/share/gc/g1/g1FullCollector.cpp

@@ -248,8 +248,9 @@ class G1FullGCRefProcProxyTask : public RefProcProxyTask {
     G1IsAliveClosure is_alive(&_collector);
     uint index = (_tm == RefProcThreadModel::Single) ? 0 : worker_id;
     G1FullKeepAliveClosure keep_alive(_collector.marker(index));
+    BarrierEnqueueDiscoveredFieldClosure enqueue;
     G1FollowStackClosure* complete_gc = _collector.marker(index)->stack_closure();
-    _rp_task->rp_work(worker_id, &is_alive, &keep_alive, complete_gc);
+    _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &enqueue, complete_gc);
   }
 };
 

src/hotspot/share/gc/g1/g1YoungCollector.cpp

@@ -912,8 +912,9 @@ class G1STWRefProcProxyTask : public RefProcProxyTask {
 
     G1STWIsAliveClosure is_alive(&_g1h);
     G1CopyingKeepAliveClosure keep_alive(&_g1h, pss);
+    BarrierEnqueueDiscoveredFieldClosure enqueue;
     G1ParEvacuateFollowersClosure complete_gc(&_g1h, pss, &_task_queues, _tm == RefProcThreadModel::Single ? nullptr : &_terminator, G1GCPhaseTimes::ObjCopy);
-    _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &complete_gc);
+    _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &enqueue, &complete_gc);
 
     // We have completed copying any necessary live referent objects.
     assert(pss->queue_is_empty(), "both queue and overflow should be empty");

src/hotspot/share/gc/parallel/psParallelCompact.cpp

@@ -2067,8 +2067,9 @@ class ParallelCompactRefProcProxyTask : public RefProcProxyTask {
     assert(worker_id < _max_workers, "sanity");
     ParCompactionManager* cm = (_tm == RefProcThreadModel::Single) ? ParCompactionManager::get_vmthread_cm() : ParCompactionManager::gc_thread_compaction_manager(worker_id);
     PCMarkAndPushClosure keep_alive(cm);
+    BarrierEnqueueDiscoveredFieldClosure enqueue;
     ParCompactionManager::FollowStackClosure complete_gc(cm, (_tm == RefProcThreadModel::Single) ? nullptr : &_terminator, worker_id);
-    _rp_task->rp_work(worker_id, PSParallelCompact::is_alive_closure(), &keep_alive, &complete_gc);
+    _rp_task->rp_work(worker_id, PSParallelCompact::is_alive_closure(), &keep_alive, &enqueue, &complete_gc);
   }
 
   void prepare_run_task_hook() override {

src/hotspot/share/gc/parallel/psScavenge.cpp

@@ -209,9 +209,10 @@ class ParallelScavengeRefProcProxyTask : public RefProcProxyTask {
     assert(worker_id < _max_workers, "sanity");
     PSPromotionManager* promotion_manager = (_tm == RefProcThreadModel::Single) ? PSPromotionManager::vm_thread_promotion_manager() : PSPromotionManager::gc_thread_promotion_manager(worker_id);
     PSIsAliveClosure is_alive;
-    PSKeepAliveClosure keep_alive(promotion_manager);;
+    PSKeepAliveClosure keep_alive(promotion_manager);
+    BarrierEnqueueDiscoveredFieldClosure enqueue;
     PSEvacuateFollowersClosure complete_gc(promotion_manager, (_marks_oops_alive && _tm == RefProcThreadModel::Multi) ? &_terminator : nullptr, worker_id);;
-    _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &complete_gc);
+    _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &enqueue, &complete_gc);
   }
 
   void prepare_run_task_hook() override {

src/hotspot/share/gc/serial/serialGcRefProcProxyTask.hpp

@@ -41,7 +41,8 @@ class SerialGCRefProcProxyTask : public RefProcProxyTask {
 
   void work(uint worker_id) override {
     assert(worker_id < _max_workers, "sanity");
-    _rp_task->rp_work(worker_id, &_is_alive, &_keep_alive, &_complete_gc);
+    BarrierEnqueueDiscoveredFieldClosure enqueue;
+    _rp_task->rp_work(worker_id, &_is_alive, &_keep_alive, &enqueue, &_complete_gc);
   }
 };
 

src/hotspot/share/gc/shared/referenceProcessor.cpp

@@ -218,6 +218,12 @@ ReferenceProcessorStats ReferenceProcessor::process_discovered_references(RefPro
   return stats;
 }
 
+void BarrierEnqueueDiscoveredFieldClosure::enqueue(oop reference, oop value) {
+  HeapAccess<AS_NO_KEEPALIVE>::oop_store_at(reference,
+                                            java_lang_ref_Reference::discovered_offset(),
+                                            value);
+}
+
 void DiscoveredListIterator::load_ptrs(DEBUG_ONLY(bool allow_null_referent)) {
   _current_discovered_addr = java_lang_ref_Reference::discovered_addr_raw(_current_discovered);
   oop discovered = java_lang_ref_Reference::discovered(_current_discovered);
@@ -271,18 +277,16 @@ void DiscoveredListIterator::clear_referent() {
 }
 
 void DiscoveredListIterator::enqueue() {
-  HeapAccess<AS_NO_KEEPALIVE>::oop_store_at(_current_discovered,
-                                            java_lang_ref_Reference::discovered_offset(),
-                                            _next_discovered);
+  _enqueue->enqueue(_current_discovered, _next_discovered);
 }
 
 void DiscoveredListIterator::complete_enqueue() {
-  if (_prev_discovered != NULL) {
+  if (_prev_discovered != nullptr) {
     // This is the last object.
     // Swap refs_list into pending list and set obj's
     // discovered to what we read from the pending list.
     oop old = Universe::swap_reference_pending_list(_refs_list.head());
-    HeapAccess<AS_NO_KEEPALIVE>::oop_store_at(_prev_discovered, java_lang_ref_Reference::discovered_offset(), old);
+    _enqueue->enqueue(_prev_discovered, old);
   }
 }
 
@@ -307,8 +311,9 @@ inline void log_enqueued_ref(const DiscoveredListIterator& iter, const char* rea
 size_t ReferenceProcessor::process_discovered_list_work(DiscoveredList&    refs_list,
                                                         BoolObjectClosure* is_alive,
                                                         OopClosure*        keep_alive,
+                                                        EnqueueDiscoveredFieldClosure* enqueue,
                                                         bool               do_enqueue_and_clear) {
-  DiscoveredListIterator iter(refs_list, keep_alive, is_alive);
+  DiscoveredListIterator iter(refs_list, keep_alive, is_alive, enqueue);
   while (iter.has_next()) {
     iter.load_ptrs(DEBUG_ONLY(discovery_is_concurrent() /* allow_null_referent */));
     if (iter.referent() == NULL) {
@@ -350,8 +355,9 @@ size_t ReferenceProcessor::process_discovered_list_work(DiscoveredList&    refs_
 }
 
 size_t ReferenceProcessor::process_final_keep_alive_work(DiscoveredList& refs_list,
-                                                         OopClosure*     keep_alive) {
-  DiscoveredListIterator iter(refs_list, keep_alive, NULL);
+                                                         OopClosure*     keep_alive,
+                                                         EnqueueDiscoveredFieldClosure* enqueue) {
+  DiscoveredListIterator iter(refs_list, keep_alive, NULL, enqueue);
   while (iter.has_next()) {
     iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */));
     // keep the referent and followers around
@@ -421,7 +427,8 @@ size_t ReferenceProcessor::total_reference_count(ReferenceType type) const {
 void RefProcTask::process_discovered_list(uint worker_id,
                                           ReferenceType ref_type,
                                           BoolObjectClosure* is_alive,
-                                          OopClosure* keep_alive) {
+                                          OopClosure* keep_alive,
+                                          EnqueueDiscoveredFieldClosure* enqueue) {
   ReferenceProcessor::RefProcSubPhases subphase;
   DiscoveredList* dl;
   switch (ref_type) {
@@ -453,6 +460,7 @@ void RefProcTask::process_discovered_list(uint worker_id,
     size_t const removed = _ref_processor.process_discovered_list_work(dl[worker_id],
                                                                        is_alive,
                                                                        keep_alive,
+                                                                       enqueue,
                                                                        do_enqueue_and_clear);
     _phase_times->add_ref_cleared(ref_type, removed);
   }
@@ -468,14 +476,15 @@ class RefProcSoftWeakFinalPhaseTask: public RefProcTask {
   void rp_work(uint worker_id,
                BoolObjectClosure* is_alive,
                OopClosure* keep_alive,
+               EnqueueDiscoveredFieldClosure* enqueue,
                VoidClosure* complete_gc) override {
     RefProcWorkerTimeTracker t(_phase_times->soft_weak_final_refs_phase_worker_time_sec(), tracker_id(worker_id));
 
-    process_discovered_list(worker_id, REF_SOFT, is_alive, keep_alive);
+    process_discovered_list(worker_id, REF_SOFT, is_alive, keep_alive, enqueue);
 
-    process_discovered_list(worker_id, REF_WEAK, is_alive, keep_alive);
+    process_discovered_list(worker_id, REF_WEAK, is_alive, keep_alive, enqueue);
 
-    process_discovered_list(worker_id, REF_FINAL, is_alive, keep_alive);
+    process_discovered_list(worker_id, REF_FINAL, is_alive, keep_alive, enqueue);
 
     // Close the reachable set; needed for collectors which keep_alive_closure do
     // not immediately complete their work.
@@ -493,9 +502,10 @@ class RefProcKeepAliveFinalPhaseTask: public RefProcTask {
   void rp_work(uint worker_id,
                BoolObjectClosure* is_alive,
                OopClosure* keep_alive,
+               EnqueueDiscoveredFieldClosure* enqueue,
                VoidClosure* complete_gc) override {
     RefProcSubPhasesWorkerTimeTracker tt(ReferenceProcessor::KeepAliveFinalRefsSubPhase, _phase_times, tracker_id(worker_id));
-    _ref_processor.process_final_keep_alive_work(_ref_processor._discoveredFinalRefs[worker_id], keep_alive);
+    _ref_processor.process_final_keep_alive_work(_ref_processor._discoveredFinalRefs[worker_id], keep_alive, enqueue);
     // Close the reachable set
     complete_gc->do_void();
   }
@@ -511,8 +521,9 @@ class RefProcPhantomPhaseTask: public RefProcTask {
   void rp_work(uint worker_id,
                BoolObjectClosure* is_alive,
                OopClosure* keep_alive,
+               EnqueueDiscoveredFieldClosure* enqueue,
                VoidClosure* complete_gc) override {
-    process_discovered_list(worker_id, REF_PHANTOM, is_alive, keep_alive);
+    process_discovered_list(worker_id, REF_PHANTOM, is_alive, keep_alive, enqueue);
 
     // Close the reachable set; needed for collectors which keep_alive_closure do
     // not immediately complete their work.
@@ -1039,6 +1050,7 @@ bool ReferenceProcessor::has_discovered_references() {
 
 void ReferenceProcessor::preclean_discovered_references(BoolObjectClosure* is_alive,
                                                         OopClosure* keep_alive,
+                                                        EnqueueDiscoveredFieldClosure* enqueue,
                                                         VoidClosure* complete_gc,
                                                         YieldClosure* yield,
                                                         GCTimer* gc_timer) {
@@ -1053,7 +1065,7 @@ void ReferenceProcessor::preclean_discovered_references(BoolObjectClosure* is_al
         return;
       }
       if (preclean_discovered_reflist(_discoveredSoftRefs[i], is_alive,
-                                      keep_alive, complete_gc, yield)) {
+                                      keep_alive, enqueue, complete_gc, yield)) {
         log_reflist("SoftRef abort: ", _discoveredSoftRefs, _max_num_queues);
         return;
       }
@@ -1070,7 +1082,7 @@ void ReferenceProcessor::preclean_discovered_references(BoolObjectClosure* is_al
         return;
       }
       if (preclean_discovered_reflist(_discoveredWeakRefs[i], is_alive,
-                                      keep_alive, complete_gc, yield)) {
+                                      keep_alive, enqueue, complete_gc, yield)) {
         log_reflist("WeakRef abort: ", _discoveredWeakRefs, _max_num_queues);
         return;
       }
@@ -1087,7 +1099,7 @@ void ReferenceProcessor::preclean_discovered_references(BoolObjectClosure* is_al
         return;
       }
       if (preclean_discovered_reflist(_discoveredFinalRefs[i], is_alive,
-                                      keep_alive, complete_gc, yield)) {
+                                      keep_alive, enqueue, complete_gc, yield)) {
         log_reflist("FinalRef abort: ", _discoveredFinalRefs, _max_num_queues);
         return;
       }
@@ -1104,7 +1116,7 @@ void ReferenceProcessor::preclean_discovered_references(BoolObjectClosure* is_al
         return;
       }
       if (preclean_discovered_reflist(_discoveredPhantomRefs[i], is_alive,
-                                      keep_alive, complete_gc, yield)) {
+                                      keep_alive, enqueue, complete_gc, yield)) {
         log_reflist("PhantomRef abort: ", _discoveredPhantomRefs, _max_num_queues);
         return;
       }
@@ -1124,9 +1136,10 @@ void ReferenceProcessor::preclean_discovered_references(BoolObjectClosure* is_al
 bool ReferenceProcessor::preclean_discovered_reflist(DiscoveredList&    refs_list,
                                                      BoolObjectClosure* is_alive,
                                                      OopClosure*        keep_alive,
+                                                     EnqueueDiscoveredFieldClosure* enqueue,
                                                      VoidClosure*       complete_gc,
                                                      YieldClosure*      yield) {
-  DiscoveredListIterator iter(refs_list, keep_alive, is_alive);
+  DiscoveredListIterator iter(refs_list, keep_alive, is_alive, enqueue);
   while (iter.has_next()) {
     if (yield->should_return_fine_grain()) {
       return true;

src/hotspot/share/gc/shared/referenceProcessor.hpp

@@ -38,6 +38,26 @@ class ReferenceProcessorPhaseTimes;
 class RefProcTask;
 class RefProcProxyTask;
 
+// Provides a callback to the garbage collector to set the given value to the
+// discovered field of the j.l.ref.Reference instance. This is called during STW
+// reference processing when iterating over the discovered lists for all
+// discovered references.
+// Typically garbage collectors may just call the barrier, but for some garbage
+// collectors the barrier environment (e.g. card table) may not be set up correctly
+// at the point of invocation.
+class EnqueueDiscoveredFieldClosure {
+public:
+  // For the given j.l.ref.Reference reference, set the discovered field to value.
+  virtual void enqueue(oop reference, oop value) = 0;
+};
+
+// EnqueueDiscoveredFieldClosure that executes the default barrier on the discovered
+// field of the j.l.ref.Reference reference with the given value.
+class BarrierEnqueueDiscoveredFieldClosure : public EnqueueDiscoveredFieldClosure {
+public:
+  void enqueue(oop reference, oop value) override;
+};
+
 // List of discovered references.
 class DiscoveredList {
 public:
@@ -66,7 +86,6 @@ class DiscoveredList {
 
 // Iterator for the list of discovered references.
 class DiscoveredListIterator {
-private:
   DiscoveredList&    _refs_list;
   HeapWord*          _prev_discovered_addr;
   oop                _prev_discovered;
@@ -78,6 +97,7 @@ class DiscoveredListIterator {
 
   OopClosure*        _keep_alive;
   BoolObjectClosure* _is_alive;
+  EnqueueDiscoveredFieldClosure* _enqueue;
 
   DEBUG_ONLY(
   oop                _first_seen; // cyclic linked list check
@@ -89,7 +109,8 @@ class DiscoveredListIterator {
 public:
   inline DiscoveredListIterator(DiscoveredList&    refs_list,
                                 OopClosure*        keep_alive,
-                                BoolObjectClosure* is_alive);
+                                BoolObjectClosure* is_alive,
+                                EnqueueDiscoveredFieldClosure* enqueue);
 
   // End Of List.
   inline bool has_next() const { return _current_discovered != NULL; }
@@ -255,12 +276,14 @@ class ReferenceProcessor : public ReferenceDiscoverer {
   size_t process_discovered_list_work(DiscoveredList&    refs_list,
                                       BoolObjectClosure* is_alive,
                                       OopClosure*        keep_alive,
+                                      EnqueueDiscoveredFieldClosure* enqueue,
                                       bool               do_enqueue_and_clear);
 
   // Keep alive followers of referents for FinalReferences. Must only be called for
   // those.
   size_t process_final_keep_alive_work(DiscoveredList& refs_list,
-                                       OopClosure* keep_alive);
+                                       OopClosure* keep_alive,
+                                       EnqueueDiscoveredFieldClosure* enqueue);
 
 
   void setup_policy(bool always_clear) {
@@ -291,6 +314,7 @@ class ReferenceProcessor : public ReferenceDiscoverer {
   // (or predicates involved) by other threads.
   void preclean_discovered_references(BoolObjectClosure* is_alive,
                                       OopClosure*        keep_alive,
+                                      EnqueueDiscoveredFieldClosure* enqueue,
                                       VoidClosure*       complete_gc,
                                       YieldClosure*      yield,
                                       GCTimer*           gc_timer);
@@ -307,6 +331,7 @@ class ReferenceProcessor : public ReferenceDiscoverer {
   bool preclean_discovered_reflist(DiscoveredList&    refs_list,
                                    BoolObjectClosure* is_alive,
                                    OopClosure*        keep_alive,
+                                   EnqueueDiscoveredFieldClosure* enqueue,
                                    VoidClosure*       complete_gc,
                                    YieldClosure*      yield);
 
@@ -542,7 +567,8 @@ class RefProcTask : StackObj {
   void process_discovered_list(uint worker_id,
                                ReferenceType ref_type,
                                BoolObjectClosure* is_alive,
-                               OopClosure* keep_alive);
+                               OopClosure* keep_alive,
+                               EnqueueDiscoveredFieldClosure* enqueue);
 public:
   RefProcTask(ReferenceProcessor& ref_processor,
               ReferenceProcessorPhaseTimes* phase_times)
@@ -552,6 +578,7 @@ class RefProcTask : StackObj {
   virtual void rp_work(uint worker_id,
                        BoolObjectClosure* is_alive,
                        OopClosure* keep_alive,
+                       EnqueueDiscoveredFieldClosure* enqueue,
                        VoidClosure* complete_gc) = 0;
 };
 

src/hotspot/share/gc/shared/referenceProcessor.inline.hpp

@@ -60,7 +60,8 @@ void DiscoveredList::clear() {
 
 DiscoveredListIterator::DiscoveredListIterator(DiscoveredList&    refs_list,
                                                OopClosure*        keep_alive,
-                                               BoolObjectClosure* is_alive):
+                                               BoolObjectClosure* is_alive,
+                                               EnqueueDiscoveredFieldClosure* enqueue):
   _refs_list(refs_list),
   _prev_discovered_addr(refs_list.adr_head()),
   _prev_discovered(NULL),
@@ -70,6 +71,7 @@ DiscoveredListIterator::DiscoveredListIterator(DiscoveredList&    refs_list,
   _referent(NULL),
   _keep_alive(keep_alive),
   _is_alive(is_alive),
+  _enqueue(enqueue),
 #ifdef ASSERT
   _first_seen(refs_list.head()),
 #endif