8286829: Shenandoah: fix Shenandoah Loom support

Reviewed-by: shade

Author: zhengyu123

src/hotspot/share/gc/shenandoah/shenandoahBarrierSetNMethod.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2019, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -60,8 +60,13 @@ bool ShenandoahBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) {
     return false;
   }
 
-  // Heal oops and disarm
+  // Heal oops
   ShenandoahNMethod::heal_nmethod(nm);
+
+  // CodeCache sweeper support
+  nm->mark_as_maybe_on_continuation();
+
+  // Disarm
   ShenandoahNMethod::disarm_nmethod(nm);
   return true;
 }

src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2019, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_HPP
 #define SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_HPP
 
+#include "code/nmethod.hpp"
 #include "memory/iterator.hpp"
 #include "oops/accessDecorators.hpp"
 #include "runtime/handshake.hpp"
@@ -71,7 +72,12 @@ class ShenandoahKeepAliveClosure : public OopClosure {
   void do_oop_work(T* p);
 };
 
-class ShenandoahUpdateRefsClosure: public OopClosure {
+class ShenandoahOopClosureBase : public MetadataVisitingOopIterateClosure {
+public:
+  inline void do_nmethod(nmethod* nm);
+};
+
+class ShenandoahUpdateRefsClosure: public ShenandoahOopClosureBase {
 private:
   ShenandoahHeap* _heap;
 public:
@@ -84,7 +90,7 @@ class ShenandoahUpdateRefsClosure: public OopClosure {
 };
 
 template <DecoratorSet MO = MO_UNORDERED>
-class ShenandoahEvacuateUpdateMetadataClosure: public BasicOopIterateClosure {
+class ShenandoahEvacuateUpdateMetadataClosure: public ShenandoahOopClosureBase {
 private:
   ShenandoahHeap* const _heap;
   Thread* const         _thread;
@@ -99,7 +105,7 @@ class ShenandoahEvacuateUpdateMetadataClosure: public BasicOopIterateClosure {
 };
 
 // Context free version, cannot cache calling thread
-class ShenandoahEvacuateUpdateRootsClosure : public BasicOopIterateClosure {
+class ShenandoahEvacuateUpdateRootsClosure : public ShenandoahOopClosureBase {
 private:
   ShenandoahHeap* const _heap;
 public:

src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp

@@ -30,6 +30,7 @@
 #include "gc/shenandoah/shenandoahAsserts.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
+#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
@@ -67,8 +68,12 @@ BoolObjectClosure* ShenandoahIsAliveSelector::is_alive_closure() {
          reinterpret_cast<BoolObjectClosure*>(&_alive_cl);
 }
 
+void ShenandoahOopClosureBase::do_nmethod(nmethod* nm) {
+  nm->run_nmethod_entry_barrier();
+}
+
 ShenandoahKeepAliveClosure::ShenandoahKeepAliveClosure() :
-  _bs(static_cast<ShenandoahBarrierSet*>(BarrierSet::barrier_set())) {
+  _bs(ShenandoahBarrierSet::barrier_set()) {
 }
 
 void ShenandoahKeepAliveClosure::do_oop(oop* p) {

src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2017, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -244,7 +244,13 @@ class ShenandoahNMethodUnlinkClosure : public NMethodClosure {
     if (_bs->is_armed(nm)) {
       ShenandoahEvacOOMScope oom_evac_scope;
       ShenandoahNMethod::heal_nmethod_metadata(nm_data);
-      _bs->disarm(nm);
+      if (Continuations::enabled()) {
+        // Loom needs to know about visited nmethods. Arm the nmethods to get
+        // mark_as_maybe_on_continuation() callbacks when they are used again.
+        _bs->arm(nm, 0);
+      } else {
+        _bs->disarm(nm);
+      }
     }
 
     // Clear compiled ICs and exception caches

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
 
 #include "gc/shared/barrierSetNMethod.hpp"
 #include "gc/shared/collectorCounters.hpp"
+#include "gc/shared/continuationGCSupport.inline.hpp"
 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
@@ -506,6 +507,10 @@ class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionCl
   bool is_thread_safe() { return true; }
 };
 
+void ShenandoahConcurrentGC::start_mark() {
+  _mark.start_mark();
+}
+
 void ShenandoahConcurrentGC::op_init_mark() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
@@ -525,6 +530,8 @@ void ShenandoahConcurrentGC::op_init_mark() {
 
   heap->set_concurrent_mark_in_progress(true);
 
+  start_mark();
+
   {
     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
     ShenandoahInitMarkUpdateRegionStateClosure cl;

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp

@@ -106,6 +106,8 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
   void op_final_roots();
   void op_cleanup_complete();
 
+  void start_mark();
+
   // Messages for GC trace events, they have to be immortal for
   // passing around the logging/tracing systems
   const char* init_mark_event_message() const;

src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp

@@ -41,6 +41,7 @@
 #include "gc/shenandoah/shenandoahUtils.hpp"
 #include "memory/iterator.inline.hpp"
 #include "memory/resourceArea.hpp"
+#include "runtime/continuation.hpp"
 
 class ShenandoahConcurrentMarkingTask : public WorkerTask {
 private:
@@ -238,6 +239,8 @@ void ShenandoahConcurrentMark::finish_mark() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   heap->set_concurrent_mark_in_progress(false);
   heap->mark_complete_marking_context();
+
+  end_mark();
 }
 
 void ShenandoahConcurrentMark::finish_mark_work() {

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2312,16 +2312,15 @@ bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
 
   // Objects allocated after marking start are implicitly alive, don't need any barriers during
   // marking phase.
-  if (is_concurrent_mark_in_progress() && marking_context()->allocated_after_mark_start(obj)) {
-    return false;
+  if (is_concurrent_mark_in_progress() &&
+     !marking_context()->allocated_after_mark_start(obj)) {
+    return true;
   }
 
-  // Objects allocated after evacuation start are guaranteed in to-space, don't need any barriers
-  // during evacuation/update references phases.
-  if (has_forwarded_objects() &&
-      cast_from_oop<HeapWord*>(obj) >= heap_region_containing(obj)->get_update_watermark()) {
-    return false;
+  // Can not guarantee obj is deeply good.
+  if (has_forwarded_objects()) {
+    return true;
   }
 
-  return true;
+  return false;
 }

src/hotspot/share/gc/shenandoah/shenandoahMark.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -33,6 +33,7 @@
 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
 #include "gc/shenandoah/shenandoahUtils.hpp"
 #include "gc/shenandoah/shenandoahVerifier.hpp"
+#include "runtime/continuation.hpp"
 
 ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q,  ShenandoahReferenceProcessor* rp) :
   MetadataVisitingOopIterateClosure(rp),
@@ -45,6 +46,20 @@ ShenandoahMark::ShenandoahMark() :
   _task_queues(ShenandoahHeap::heap()->marking_context()->task_queues()) {
 }
 
+void ShenandoahMark::start_mark() {
+  // Tell the sweeper that we start a marking cycle.
+  if (!Continuations::is_gc_marking_cycle_active()) {
+    Continuations::on_gc_marking_cycle_start();
+  }
+}
+
+void ShenandoahMark::end_mark() {
+  // Tell the sweeper that we finished a marking cycle.
+  // Unlike other GCs, we do not arm the nmethods
+  // when marking terminates.
+  Continuations::on_gc_marking_cycle_finish();
+}
+
 void ShenandoahMark::clear() {
   // Clean up marking stacks.
   ShenandoahObjToScanQueueSet* queues = ShenandoahHeap::heap()->marking_context()->task_queues();

src/hotspot/share/gc/shenandoah/shenandoahMark.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -50,6 +50,10 @@ class ShenandoahMark: public StackObj {
 
   static void clear();
 
+  // Loom support
+  void start_mark();
+  void end_mark();
+
   // Helpers
   inline ShenandoahObjToScanQueueSet* task_queues() const;
   inline ShenandoahObjToScanQueue* get_queue(uint index) const;

src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2015, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@
 
 #include "gc/shenandoah/shenandoahMark.hpp"
 
+#include "gc/shared/continuationGCSupport.inline.hpp"
 #include "gc/shenandoah/shenandoahAsserts.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
@@ -69,6 +70,12 @@ void ShenandoahMark::do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveD
   if (task->is_not_chunked()) {
     if (obj->is_instance()) {
       // Case 1: Normal oop, process as usual.
+      if (ContinuationGCSupport::relativize_stack_chunk(obj)) {
+          // Loom doesn't support mixing of weak marking and strong marking of
+          // stack chunks.
+          cl->set_weak(false);
+      }
+
       obj->oop_iterate(cl);
       dedup_string<STRING_DEDUP>(obj, req);
     } else if (obj->is_objArray()) {

src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2019, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,7 +27,9 @@
 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
+#include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
 #include "memory/resourceArea.hpp"
+#include "runtime/continuation.hpp"
 
 ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>& oops, bool non_immediate_oops) :
   _nm(nm), _oops(NULL), _oops_count(0), _unregistered(false) {
@@ -159,14 +161,14 @@ void ShenandoahNMethod::heal_nmethod(nmethod* nm) {
     ShenandoahKeepAliveClosure cl;
     data->oops_do(&cl);
   } else if (heap->is_concurrent_weak_root_in_progress() ||
-             heap->is_concurrent_strong_root_in_progress()) {
+             heap->is_concurrent_strong_root_in_progress() ) {
     ShenandoahEvacOOMScope evac_scope;
     heal_nmethod_metadata(data);
   } else {
     // There is possibility that GC is cancelled when it arrives final mark.
     // In this case, concurrent root phase is skipped and degenerated GC should be
     // followed, where nmethods are disarmed.
-    assert(heap->cancelled_gc(), "What else?");
+    assert(heap->cancelled_gc() || Continuations::enabled(), "What else?");
   }
 }
 

src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2015, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
 #define SHARE_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_HPP
 
 #include "gc/shared/stringdedup/stringDedup.hpp"
+#include "gc/shenandoah/shenandoahClosures.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
 #include "gc/shenandoah/shenandoahUtils.hpp"
@@ -58,6 +59,11 @@ class ShenandoahMarkRefsSuperClosure : public MetadataVisitingOopIterateClosure
   void set_weak(bool weak) {
     _weak = weak;
   }
+
+  virtual void do_nmethod(nmethod* nm) {
+    assert(!is_weak(), "Can't handle weak marking of nmethods");
+    nm->run_nmethod_entry_barrier();
+  }
 };
 
 class ShenandoahMarkUpdateRefsSuperClosure : public ShenandoahMarkRefsSuperClosure {
@@ -86,7 +92,6 @@ class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkUpdateRefsSuperClos
 
   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   virtual void do_oop(oop* p)       { do_oop_work(p); }
-  virtual bool do_metadata()        { return false; }
 };
 
 class ShenandoahMarkUpdateRefsMetadataClosure : public ShenandoahMarkUpdateRefsSuperClosure {
@@ -100,7 +105,6 @@ class ShenandoahMarkUpdateRefsMetadataClosure : public ShenandoahMarkUpdateRefsS
 
   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   virtual void do_oop(oop* p)       { do_oop_work(p); }
-  virtual bool do_metadata()        { return true; }
 };
 
 
@@ -115,7 +119,6 @@ class ShenandoahMarkRefsClosure : public ShenandoahMarkRefsSuperClosure {
 
   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   virtual void do_oop(oop* p)       { do_oop_work(p); }
-  virtual bool do_metadata()        { return false; }
 };
 
 
@@ -130,10 +133,9 @@ class ShenandoahMarkRefsMetadataClosure : public ShenandoahMarkRefsSuperClosure
 
   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   virtual void do_oop(oop* p)       { do_oop_work(p); }
-  virtual bool do_metadata()        { return true; }
 };
 
-class ShenandoahUpdateRefsSuperClosure : public BasicOopIterateClosure {
+class ShenandoahUpdateRefsSuperClosure : public ShenandoahOopClosureBase {
 protected:
   ShenandoahHeap* _heap;
 

src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -101,6 +101,8 @@ void ShenandoahSTWMark::mark() {
     heap->verifier()->verify_roots_no_forwarded();
   }
 
+  start_mark();
+
   uint nworkers = heap->workers()->active_workers();
   task_queues()->reserve(nworkers);
 
@@ -116,6 +118,7 @@ void ShenandoahSTWMark::mark() {
   }
 
   heap->mark_complete_marking_context();
+  end_mark();
 
   assert(task_queues()->is_empty(), "Should be empty");
   TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());

src/hotspot/share/runtime/continuationFreezeThaw.cpp

@@ -1199,7 +1199,6 @@ NOINLINE void FreezeBase::finish_freeze(const frame& f, const frame& top) {
     // old chunks are all in GC mode.
     assert(!UseG1GC, "G1 can not deal with allocating outside of eden");
     assert(!UseZGC, "ZGC can not deal with allocating chunks visible to marking");
-    assert(!UseShenandoahGC, "Shenandoah can not deal with allocating chunks visible to marking");
     ContinuationGCSupport::transform_stack_chunk(_cont.tail());
     // For objects in the old generation we must maintain the remembered set
     _cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>();
@@ -1282,11 +1281,16 @@ stackChunkOop Freeze<ConfigT>::allocate_chunk(size_t stack_size) {
 
   assert(chunk->parent() == nullptr || chunk->parent()->is_stackChunk(), "");
 
+  // Shenandoah: even continuation is good, it does not mean it is deeply good.
+  if (UseShenandoahGC && chunk->requires_barriers()) {
+    fast_oop = nullptr;
+  }
+
   if (fast_oop != nullptr) {
     assert(!chunk->requires_barriers(), "Unfamiliar GC requires barriers on TLAB allocation");
   } else {
-    assert(!UseZGC || !UseShenandoahGC || !chunk->requires_barriers(), "Allocated ZGC/ShenandoahGC object requires barriers");
-    _barriers = !UseZGC && !UseShenandoahGC && chunk->requires_barriers();
+    assert(!UseZGC || !chunk->requires_barriers(), "Allocated ZGC object requires barriers");
+    _barriers = !UseZGC && chunk->requires_barriers();
 
     if (_barriers) {
       log_develop_trace(continuations)("allocation requires barriers");