8280396: G1: Full gc mark stack draining should prefer to make work available to other threads

Reviewed-by: sjohanss, ayang

Author: Thomas Schatzl

Diff for: src/hotspot/share/gc/g1/g1FullGCMarker.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -71,15 +71,21 @@ class G1FullGCMarker : public CHeapObj<mtGC> {
   G1RegionMarkStatsCache _mark_stats_cache;
 
   inline bool is_empty();
-  inline bool pop_object(oop& obj);
-  inline bool pop_objarray(ObjArrayTask& array);
   inline void push_objarray(oop obj, size_t index);
   inline bool mark_object(oop obj);
 
   // Marking helpers
   inline void follow_object(oop obj);
   inline void follow_array(objArrayOop array);
   inline void follow_array_chunk(objArrayOop array, int index);
+
+  inline void drain_oop_stack();
+  // Transfer contents from the objArray task queue overflow stack to the shared
+  // objArray stack.
+  // Returns true and a valid task if there has not been enough space in the shared
+  // objArray stack, otherwise the task is invalid.
+  inline bool transfer_objArray_overflow_stack(ObjArrayTask& task);
+
 public:
   G1FullGCMarker(G1FullCollector* collector,
                  uint worker_id,

Diff for: src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -91,20 +91,12 @@ inline bool G1FullGCMarker::is_empty() {
   return _oop_stack.is_empty() && _objarray_stack.is_empty();
 }
 
-inline bool G1FullGCMarker::pop_object(oop& oop) {
-  return _oop_stack.pop_overflow(oop) || _oop_stack.pop_local(oop);
-}
-
 inline void G1FullGCMarker::push_objarray(oop obj, size_t index) {
   ObjArrayTask task(obj, index);
   assert(task.is_valid(), "bad ObjArrayTask");
   _objarray_stack.push(task);
 }
 
-inline bool G1FullGCMarker::pop_objarray(ObjArrayTask& arr) {
-  return _objarray_stack.pop_overflow(arr) || _objarray_stack.pop_local(arr);
-}
-
 inline void G1FullGCMarker::follow_array(objArrayOop array) {
   follow_klass(array->klass());
   // Don't push empty arrays to avoid unnecessary work.
@@ -159,16 +151,40 @@ inline void G1FullGCMarker::follow_object(oop obj) {
   }
 }
 
-void G1FullGCMarker::drain_stack() {
-  do {
-    oop obj;
-    while (pop_object(obj)) {
+inline void G1FullGCMarker::drain_oop_stack() {
+  oop obj;
+  while (_oop_stack.pop_overflow(obj)) {
+    if (!_oop_stack.try_push_to_taskqueue(obj)) {
       assert(_bitmap->is_marked(obj), "must be marked");
       follow_object(obj);
     }
-    // Process ObjArrays one at a time to avoid marking stack bloat.
+  }
+  while (_oop_stack.pop_local(obj)) {
+    assert(_bitmap->is_marked(obj), "must be marked");
+    follow_object(obj);
+  }
+}
+
+inline bool G1FullGCMarker::transfer_objArray_overflow_stack(ObjArrayTask& task) {
+  // It is desirable to move as much as possible work from the overflow queue to
+  // the shared queue as quickly as possible.
+  while (_objarray_stack.pop_overflow(task)) {
+    if (!_objarray_stack.try_push_to_taskqueue(task)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+void G1FullGCMarker::drain_stack() {
+  do {
+    // First, drain regular oop stack.
+    drain_oop_stack();
+
+    // Then process ObjArrays one at a time to avoid marking stack bloat.
     ObjArrayTask task;
-    if (pop_objarray(task)) {
+    if (transfer_objArray_overflow_stack(task) ||
+      _objarray_stack.pop_local(task)) {
       follow_array_chunk(objArrayOop(task.obj()), task.index());
     }
   } while (!is_empty());