8258254: Move PtrQueue flush to PtrQueueSet subclasses

Reviewed-by: tschatzl, shade

Author: Kim Barrett

src/hotspot/share/gc/g1/g1BarrierSet.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -158,6 +158,14 @@ void G1BarrierSet::on_thread_attach(Thread* thread) {
 void G1BarrierSet::on_thread_detach(Thread* thread) {
   // Flush any deferred card marks.
   CardTableBarrierSet::on_thread_detach(thread);
-  G1ThreadLocalData::satb_mark_queue(thread).flush();
-  G1ThreadLocalData::dirty_card_queue(thread).on_thread_detach();
+  {
+    SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(thread);
+    G1BarrierSet::satb_mark_queue_set().flush_queue(queue);
+  }
+  {
+    G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(thread);
+    G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set();
+    qset.flush_queue(queue);
+    qset.record_detached_refinement_stats(queue.refinement_stats());
+  }
 }

src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -55,21 +55,10 @@ G1DirtyCardQueue::G1DirtyCardQueue(G1DirtyCardQueueSet* qset) :
 { }
 
 G1DirtyCardQueue::~G1DirtyCardQueue() {
-  flush();
+  G1BarrierSet::dirty_card_queue_set().flush_queue(*this);
   delete _refinement_stats;
 }
 
-void G1DirtyCardQueue::flush() {
-  _refinement_stats->inc_dirtied_cards(size());
-  flush_impl();
-}
-
-void G1DirtyCardQueue::on_thread_detach() {
-  assert(this == &G1ThreadLocalData::dirty_card_queue(Thread::current()), "precondition");
-  flush();
-  dirty_card_qset()->record_detached_refinement_stats(_refinement_stats);
-}
-
 // Assumed to be zero by concurrent threads.
 static uint par_ids_start() { return 0; }
 
@@ -95,6 +84,14 @@ uint G1DirtyCardQueueSet::num_par_ids() {
   return (uint)os::initial_active_processor_count();
 }
 
+void G1DirtyCardQueueSet::flush_queue(G1DirtyCardQueue& queue) {
+  if (queue.buffer() != nullptr) {
+    G1ConcurrentRefineStats* stats = queue.refinement_stats();
+    stats->inc_dirtied_cards(buffer_size() - queue.index());
+  }
+  PtrQueueSet::flush_queue(queue);
+}
+
 void G1DirtyCardQueueSet::enqueue(G1DirtyCardQueue& queue,
                                   volatile CardValue* card_ptr) {
   CardValue* value = const_cast<CardValue*>(card_ptr);
@@ -645,13 +642,16 @@ void G1DirtyCardQueueSet::concatenate_logs() {
   set_max_cards(MaxCardsUnlimited);
 
   struct ConcatenateThreadLogClosure : public ThreadClosure {
+    G1DirtyCardQueueSet& _qset;
+    ConcatenateThreadLogClosure(G1DirtyCardQueueSet& qset) : _qset(qset) {}
     virtual void do_thread(Thread* t) {
-      G1DirtyCardQueue& dcq = G1ThreadLocalData::dirty_card_queue(t);
-      if (!dcq.is_empty()) {
-        dcq.flush();
+      G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(t);
+      if ((queue.buffer() != nullptr) &&
+          (queue.index() != _qset.buffer_size())) {
+        _qset.flush_queue(queue);
       }
     }
-  } closure;
+  } closure(*this);
   Threads::threads_do(&closure);
 
   G1BarrierSet::shared_dirty_card_queue().flush();

src/hotspot/share/gc/g1/g1DirtyCardQueue.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -49,19 +49,12 @@ class G1DirtyCardQueue: public PtrQueue {
   // doing something else, with auto-flush on completion.
   ~G1DirtyCardQueue();
 
-  // Process queue entries and release resources.
-  void flush();
-
   inline G1DirtyCardQueueSet* dirty_card_qset() const;
 
   G1ConcurrentRefineStats* refinement_stats() const {
     return _refinement_stats;
   }
 
-  // To be called by the barrier set's on_thread_detach, to notify this
-  // object of the corresponding state change of its owning thread.
-  void on_thread_detach();
-
   // Compiler support.
   static ByteSize byte_offset_of_index() {
     return PtrQueue::byte_offset_of_index<G1DirtyCardQueue>();
@@ -313,6 +306,8 @@ class G1DirtyCardQueueSet: public PtrQueueSet {
 
   G1BufferNodeList take_all_completed_buffers();
 
+  void flush_queue(G1DirtyCardQueue& queue);
+
   using CardValue = G1CardTable::CardValue;
   void enqueue(G1DirtyCardQueue& queue, volatile CardValue* card_ptr);
 

src/hotspot/share/gc/g1/g1EvacFailure.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -219,7 +219,7 @@ class RemoveSelfForwardPtrHRClosure: public HeapRegionClosure {
   }
 
   ~RemoveSelfForwardPtrHRClosure() {
-    _rdcq.flush();
+    _rdc_local_qset.flush_queue(_rdcq);
     _rdc_local_qset.flush();
   }
 

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -114,7 +114,7 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
 }
 
 size_t G1ParScanThreadState::flush(size_t* surviving_young_words) {
-  _rdcq.flush();
+  _rdc_local_qset.flush_queue(_rdcq);
   _rdc_local_qset.flush();
   flush_numa_stats();
   // Update allocation statistics.

src/hotspot/share/gc/g1/g1RedirtyCardsQueue.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -68,6 +68,10 @@ void G1RedirtyCardsLocalQueueSet::flush() {
   _buffers = G1BufferNodeList();
 }
 
+void G1RedirtyCardsLocalQueueSet::flush_queue(G1RedirtyCardsQueue& queue) {
+  PtrQueueSet::flush_queue(queue);
+}
+
 // G1RedirtyCardsQueue
 
 G1RedirtyCardsQueue::G1RedirtyCardsQueue(G1RedirtyCardsLocalQueueSet* qset) :
@@ -76,14 +80,10 @@ G1RedirtyCardsQueue::G1RedirtyCardsQueue(G1RedirtyCardsLocalQueueSet* qset) :
 
 #ifdef ASSERT
 G1RedirtyCardsQueue::~G1RedirtyCardsQueue() {
-  assert(is_empty(), "unflushed queue");
+  assert(buffer() == nullptr, "unflushed queue");
 }
 #endif // ASSERT
 
-void G1RedirtyCardsQueue::flush() {
-  flush_impl();
-}
-
 // G1RedirtyCardsQueueSet
 
 G1RedirtyCardsQueueSet::G1RedirtyCardsQueueSet(BufferNode::Allocator* allocator) :

src/hotspot/share/gc/g1/g1RedirtyCardsQueue.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -50,16 +50,15 @@ class G1RedirtyCardsLocalQueueSet : public PtrQueueSet {
 
   // Transfer all completed buffers to the shared qset.
   void flush();
+
+  void flush_queue(G1RedirtyCardsQueue& queue);
 };
 
 // Worker-local queues of card table entries.
 class G1RedirtyCardsQueue : public PtrQueue {
 public:
   G1RedirtyCardsQueue(G1RedirtyCardsLocalQueueSet* qset);
   ~G1RedirtyCardsQueue() NOT_DEBUG(= default);
-
-  // Flushes all enqueued cards to qset.
-  void flush();
 };
 
 // Card table entries to be redirtied and the cards reprocessed later.

src/hotspot/share/gc/shared/ptrQueue.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -46,32 +46,6 @@ PtrQueue::~PtrQueue() {
   assert(_buf == NULL, "queue must be flushed before delete");
 }
 
-void PtrQueue::flush_impl() {
-  if (_buf != NULL) {
-    BufferNode* node = BufferNode::make_node_from_buffer(_buf, index());
-    if (is_empty()) {
-      // No work to do.
-      qset()->deallocate_buffer(node);
-    } else {
-      qset()->enqueue_completed_buffer(node);
-    }
-    _buf = NULL;
-    set_index(0);
-  }
-}
-
-void PtrQueue::allocate_buffer() {
-  _buf = qset()->allocate_buffer();
-  reset();
-}
-
-void PtrQueue::enqueue_completed_buffer() {
-  assert(_buf != NULL, "precondition");
-  BufferNode* node = BufferNode::make_node_from_buffer(_buf, index());
-  qset()->enqueue_completed_buffer(node);
-  allocate_buffer();
-}
-
 BufferNode* BufferNode::allocate(size_t size) {
   size_t byte_size = size * sizeof(void*);
   void* data = NEW_C_HEAP_ARRAY(char, buffer_offset() + byte_size, mtGC);
@@ -225,6 +199,21 @@ PtrQueueSet::PtrQueueSet(BufferNode::Allocator* allocator) :
 
 PtrQueueSet::~PtrQueueSet() {}
 
+void PtrQueueSet::flush_queue(PtrQueue& queue) {
+  void** buffer = queue.buffer();
+  if (buffer != nullptr) {
+    size_t index = queue.index();
+    queue.set_buffer(nullptr);
+    queue.set_index(0);
+    BufferNode* node = BufferNode::make_node_from_buffer(buffer, index);
+    if (index == buffer_size()) {
+      deallocate_buffer(node);
+    } else {
+      enqueue_completed_buffer(node);
+    }
+  }
+}
+
 bool PtrQueueSet::try_enqueue(PtrQueue& queue, void* value) {
   size_t index = queue.index();
   if (index == 0) return false;

src/hotspot/share/gc/shared/ptrQueue.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -79,14 +79,6 @@ class PtrQueue {
 
   PtrQueueSet* qset() const { return _qset; }
 
-  // Process queue entries and release resources.
-  void flush_impl();
-
-  void allocate_buffer();
-
-  // Enqueue the current buffer in the qset and allocate a new buffer.
-  void enqueue_completed_buffer();
-
   // Initialize this queue to contain a null buffer, and be part of the
   // given PtrQueueSet.
   PtrQueue(PtrQueueSet* qset);
@@ -268,6 +260,10 @@ class PtrQueueSet {
   PtrQueueSet(BufferNode::Allocator* allocator);
   ~PtrQueueSet();
 
+  // If queue has any buffered enqueued data, transfer it to this qset.
+  // Otherwise, deallocate queue's buffer.
+  void flush_queue(PtrQueue& queue);
+
   // Add value to queue's buffer, returning true.  If buffer is full
   // or if queue doesn't have a buffer, does nothing and returns false.
   bool try_enqueue(PtrQueue& queue, void* value);

src/hotspot/share/gc/shared/satbMarkQueue.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -47,13 +47,6 @@ SATBMarkQueue::SATBMarkQueue(SATBMarkQueueSet* qset) :
   _active(false)
 { }
 
-void SATBMarkQueue::flush() {
-  // Filter now to possibly save work later.  If filtering empties the
-  // buffer then flush_impl can deallocate the buffer.
-  filter();
-  flush_impl();
-}
-
 void SATBMarkQueue::apply_closure_and_empty(SATBBufferClosure* cl) {
   assert(SafepointSynchronize::is_at_safepoint(),
          "SATB queues must only be processed at safepoints");
@@ -235,6 +228,13 @@ bool SATBMarkQueueSet::apply_closure_to_completed_buffer(SATBBufferClosure* cl)
   }
 }
 
+void SATBMarkQueueSet::flush_queue(SATBMarkQueue& queue) {
+  // Filter now to possibly save work later.  If filtering empties the
+  // buffer then flush_queue can deallocate the buffer.
+  filter(queue);
+  PtrQueueSet::flush_queue(queue);
+}
+
 void SATBMarkQueueSet::enqueue_known_active(SATBMarkQueue& queue, oop obj) {
   assert(queue.is_active(), "precondition");
   void* value = cast_from_oop<void*>(obj);

src/hotspot/share/gc/shared/satbMarkQueue.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -63,9 +63,6 @@ class SATBMarkQueue: public PtrQueue {
   bool is_active() const { return _active; }
   void set_active(bool value) { _active = value; }
 
-  // Process queue entries and free resources.
-  void flush();
-
   inline SATBMarkQueueSet* satb_qset() const;
 
   // Apply cl to the active part of the buffer.
@@ -150,6 +147,8 @@ class SATBMarkQueueSet: public PtrQueueSet {
   // buffer; the leading entries may be excluded due to filtering.
   bool apply_closure_to_completed_buffer(SATBBufferClosure* cl);
 
+  void flush_queue(SATBMarkQueue& queue);
+
   // When active, add obj to queue by calling enqueue_known_active.
   void enqueue(SATBMarkQueue& queue, oop obj) {
     if (queue.is_active()) enqueue_known_active(queue, obj);

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -116,7 +116,7 @@ void ShenandoahBarrierSet::on_thread_attach(Thread *thread) {
 
 void ShenandoahBarrierSet::on_thread_detach(Thread *thread) {
   SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread);
-  queue.flush();
+  _satb_mark_queue_set.flush_queue(queue);
   if (thread->is_Java_thread()) {
     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
     if (gclab != NULL) {