Merge

Author: AlanBateman

src/hotspot/share/code/codeCache.cpp

@@ -829,7 +829,6 @@ void CodeCache::purge_exception_caches() {
 }
 
 uint8_t CodeCache::_unloading_cycle = 1;
-uint64_t CodeCache::_marking_cycle = 0;
 
 void CodeCache::increment_unloading_cycle() {
   // 2-bit value (see IsUnloadingState in nmethod.cpp for details)
@@ -840,28 +839,6 @@ void CodeCache::increment_unloading_cycle() {
   }
 }
 
-void CodeCache::increment_marking_cycle() {
-  ++_marking_cycle;
-  BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
-  if (bs_nm != NULL) {
-    bs_nm->arm_all_nmethods();
-  }
-}
-
-bool CodeCache::is_marking_cycle_active() {
-  return (_marking_cycle % 2) == 1;
-}
-
-void CodeCache::start_marking_cycle() {
-  assert(!is_marking_cycle_active(), "Previous marking cycle never ended");
-  increment_marking_cycle();
-}
-
-void CodeCache::finish_marking_cycle() {
-  assert(is_marking_cycle_active(), "Marking cycle started before last one finished");
-  increment_marking_cycle();
-}
-
 CodeCache::UnloadingScope::UnloadingScope(BoolObjectClosure* is_alive)
   : _is_unloading_behaviour(is_alive)
 {

src/hotspot/share/code/codeCache.hpp

@@ -96,7 +96,6 @@ class CodeCache : AllStatic {
   static address _high_bound;                           // Upper bound of CodeHeap addresses
   static int _number_of_nmethods_with_dependencies;     // Total number of nmethods with dependencies
   static uint8_t _unloading_cycle;                      // Global state for recognizing old nmethods that need to be unloaded
-  static uint64_t _marking_cycle;
 
   static ExceptionCache* volatile _exception_cache_purge_list;
 
@@ -203,14 +202,6 @@ class CodeCache : AllStatic {
 
   static void increment_unloading_cycle();
 
- private:
-  static void increment_marking_cycle();
-
- public:
-  static void start_marking_cycle();
-  static void finish_marking_cycle();
-  static bool is_marking_cycle_active();
-  static uint64_t marking_cycle() { return _marking_cycle; }
   static void release_exception_cache(ExceptionCache* entry);
   static void purge_exception_caches();
 

src/hotspot/share/code/nmethod.cpp

@@ -631,7 +631,7 @@ nmethod::nmethod(
     // values something that will never match a pc like the nmethod vtable entry
     _exception_offset        = 0;
     _orig_pc_offset          = 0;
-    _marking_cycle           = 0;
+    _gc_epoch                = Continuations::gc_epoch();
 
     _consts_offset           = data_offset();
     _stub_offset             = content_offset()      + code_buffer->total_offset_of(code_buffer->stubs());
@@ -768,7 +768,7 @@ nmethod::nmethod(
     _comp_level              = comp_level;
     _orig_pc_offset          = orig_pc_offset;
     _hotness_counter         = NMethodSweeper::hotness_counter_reset_val();
-    _marking_cycle           = CodeCache::marking_cycle();
+    _gc_epoch                = Continuations::gc_epoch();
 
     // Section offsets
     _consts_offset           = content_offset()      + code_buffer->total_offset_of(code_buffer->consts());
@@ -1232,22 +1232,17 @@ void nmethod::mark_as_seen_on_stack() {
 
 void nmethod::mark_as_maybe_on_continuation() {
   assert(is_alive(), "Must be an alive method");
-  _marking_cycle = CodeCache::marking_cycle();
+  _gc_epoch = Continuations::gc_epoch();
 }
 
-bool nmethod::is_not_on_continuation_stack() {
+bool nmethod::is_maybe_on_continuation_stack() {
   if (!Continuations::enabled()) {
-    return true;
+    return false;
   }
 
-  // Odd marking cycles are found during concurrent marking. Even numbers are found
-  // in nmethods that are marked when GC is inactive (e.g. nmethod entry barriers during
-  // normal execution). Therefore we align up by 2 so that nmethods encountered during
-  // concurrent marking are treated as if they were encountered in the inactive phase
-  // after that concurrent GC. Each GC increments the marking cycle twice - once when
-  // it starts and once when it ends. So we can only be sure there are no new continuations
-  // when they have not been encountered from before a GC to after a GC.
-  return CodeCache::marking_cycle() >= align_up(_marking_cycle, 2) + 2;
+  // If the condition below is true, it means that the nmethod was found to
+  // be alive the previous completed marking cycle.
+  return _gc_epoch >= Continuations::previous_completed_gc_marking_cycle();
 }
 
 // Tell if a non-entrant method can be converted to a zombie (i.e.,
@@ -1268,7 +1263,7 @@ bool nmethod::can_convert_to_zombie() {
   // If an is_unloading() nmethod is still not_entrant, then it is not safe to
   // convert it to zombie due to GC unloading interactions. However, if it
   // has become unloaded, then it is okay to convert such nmethods to zombie.
-  return stack_traversal_mark()+1 < NMethodSweeper::traversal_count() && is_not_on_continuation_stack() &&
+  return stack_traversal_mark()+1 < NMethodSweeper::traversal_count() && !is_maybe_on_continuation_stack() &&
          !is_locked_by_vm() && (!is_unloading() || is_unloaded());
 }
 

src/hotspot/share/code/nmethod.hpp

@@ -74,7 +74,7 @@ class nmethod : public CompiledMethod {
   // Shared fields for all nmethod's
   int       _entry_bci;        // != InvocationEntryBci if this nmethod is an on-stack replacement method
 
-  uint64_t  _marking_cycle;
+  uint64_t  _gc_epoch;
 
   // To support simple linked-list chaining of nmethods:
   nmethod*  _osr_link;         // from InstanceKlass::osr_nmethods_head
@@ -571,7 +571,7 @@ class nmethod : public CompiledMethod {
   // See comment at definition of _last_seen_on_stack
   void mark_as_seen_on_stack();
   void mark_as_maybe_on_continuation();
-  bool is_not_on_continuation_stack();
+  bool is_maybe_on_continuation_stack();
   bool can_convert_to_zombie();
 
   // Evolution support. We make old (discarded) compiled methods point to new Method*s.

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -3457,13 +3457,14 @@ void G1CollectedHeap::fill_with_dummy_object(HeapWord* start, HeapWord* end, boo
 }
 
 void G1CollectedHeap::start_codecache_marking_cycle_if_inactive() {
-  if (!CodeCache::is_marking_cycle_active()) {
+  if (!Continuations::is_gc_marking_cycle_active()) {
     // This is the normal case when we do not call collect when a
     // concurrent mark is ongoing. We then start a new code marking
     // cycle. If, on the other hand, a concurrent mark is ongoing, we
     // will be conservative and use the last code marking cycle. Code
     // caches marked between the two concurrent marks will live a bit
     // longer than needed.
-    CodeCache::start_marking_cycle();
+    Continuations::on_gc_marking_cycle_start();
+    Continuations::arm_all_nmethods();
   }
 }

src/hotspot/share/gc/g1/g1ConcurrentMark.cpp

@@ -66,6 +66,7 @@
 #include "oops/access.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/continuation.hpp"
 #include "runtime/globals_extension.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/java.hpp"
@@ -1300,7 +1301,8 @@ void G1ConcurrentMark::remark() {
     report_object_count(mark_finished);
   }
 
-  CodeCache::finish_marking_cycle();
+  Continuations::on_gc_marking_cycle_finish();
+  Continuations::arm_all_nmethods();
 
   // Statistics
   double now = os::elapsedTime();

src/hotspot/share/gc/g1/g1FullCollector.cpp

@@ -44,6 +44,7 @@
 #include "gc/shared/weakProcessor.inline.hpp"
 #include "gc/shared/workerPolicy.hpp"
 #include "logging/log.hpp"
+#include "runtime/continuation.hpp"
 #include "runtime/handles.inline.hpp"
 #include "utilities/debug.hpp"
 
@@ -201,7 +202,8 @@ void G1FullCollector::collect() {
 
   phase4_do_compaction();
 
-  CodeCache::finish_marking_cycle();
+  Continuations::on_gc_marking_cycle_finish();
+  Continuations::arm_all_nmethods();
 }
 
 void G1FullCollector::complete_collection() {

src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp

@@ -64,7 +64,7 @@ inline void G1MarkAndPushClosure::do_cld(ClassLoaderData* cld) {
 }
 
 inline void G1MarkAndPushClosure::do_method(Method* m) {
-  m->record_marking_cycle();
+  m->record_gc_epoch();
 }
 
 inline void G1MarkAndPushClosure::do_nmethod(nmethod* nm) {

src/hotspot/share/gc/parallel/psParallelCompact.cpp

@@ -75,6 +75,7 @@
 #include "oops/objArrayKlass.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/continuation.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/java.hpp"
 #include "runtime/safepoint.hpp"
@@ -960,7 +961,8 @@ void PSParallelCompact::pre_compact()
   // Increment the invocation count
   heap->increment_total_collections(true);
 
-  CodeCache::start_marking_cycle();
+  Continuations::on_gc_marking_cycle_start();
+  Continuations::arm_all_nmethods();
 
   // We need to track unique mark sweep invocations as well.
   _total_invocations++;
@@ -992,7 +994,8 @@ void PSParallelCompact::post_compact()
   GCTraceTime(Info, gc, phases) tm("Post Compact", &_gc_timer);
   ParCompactionManager::remove_all_shadow_regions();
 
-  CodeCache::finish_marking_cycle();
+  Continuations::on_gc_marking_cycle_finish();
+  Continuations::arm_all_nmethods();
 
   for (unsigned int id = old_space_id; id < last_space_id; ++id) {
     // Clear the marking bitmap, summary data and split info.

src/hotspot/share/gc/serial/markSweep.inline.hpp

@@ -86,7 +86,7 @@ inline void MarkAndPushClosure::do_oop(oop* p)               { do_oop_work(p); }
 inline void MarkAndPushClosure::do_oop(narrowOop* p)         { do_oop_work(p); }
 inline void MarkAndPushClosure::do_klass(Klass* k)           { MarkSweep::follow_klass(k); }
 inline void MarkAndPushClosure::do_cld(ClassLoaderData* cld) { MarkSweep::follow_cld(cld); }
-inline void MarkAndPushClosure::do_method(Method* m)         { m->record_marking_cycle(); }
+inline void MarkAndPushClosure::do_method(Method* m)         { m->record_gc_epoch(); }
 inline void MarkAndPushClosure::do_nmethod(nmethod* nm)      { nm->follow_nmethod(this); }
 
 template <class T> inline void MarkSweep::adjust_pointer(T* p) {

src/hotspot/share/gc/shared/barrierSetNMethod.cpp

@@ -96,6 +96,8 @@ class BarrierSetNMethodArmClosure : public ThreadClosure {
 };
 
 void BarrierSetNMethod::arm_all_nmethods() {
+  // Change to a new global GC phase. Doing this requires changing the thread-local
+  // disarm value for all threads, to reflect the new GC phase.
   ++_current_phase;
   if (_current_phase == 4) {
     _current_phase = 1;

src/hotspot/share/gc/shared/genCollectedHeap.cpp

@@ -63,6 +63,7 @@
 #include "memory/resourceArea.hpp"
 #include "memory/universe.hpp"
 #include "oops/oop.inline.hpp"
+#include "runtime/continuation.hpp"
 #include "runtime/handles.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/java.hpp"
@@ -607,7 +608,8 @@ void GenCollectedHeap::do_collection(bool           full,
       increment_total_full_collections();
     }
 
-    CodeCache::start_marking_cycle();
+    Continuations::on_gc_marking_cycle_start();
+    Continuations::arm_all_nmethods();
 
     collect_generation(_old_gen,
                        full,
@@ -616,7 +618,8 @@ void GenCollectedHeap::do_collection(bool           full,
                        run_verification && VerifyGCLevel <= 1,
                        do_clear_all_soft_refs);
 
-    CodeCache::finish_marking_cycle();
+    Continuations::on_gc_marking_cycle_finish();
+    Continuations::arm_all_nmethods();
 
     // Adjust generation sizes.
     _old_gen->compute_new_size();

src/hotspot/share/gc/z/zMark.cpp

@@ -56,6 +56,7 @@
 #include "oops/objArrayOop.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/continuation.hpp"
 #include "runtime/handshake.hpp"
 #include "runtime/prefetch.inline.hpp"
 #include "runtime/safepointMechanism.hpp"
@@ -109,7 +110,11 @@ void ZMark::start() {
   // marking information for all pages.
   ZGlobalSeqNum++;
 
-  CodeCache::start_marking_cycle();
+  // Tell the sweeper that we start a marking cycle.
+  // Unlike other GCs, the color switch implicitly changes the nmethods
+  // to be armed, and the thread-local disarm values are lazily updated
+  // when JavaThreads wake up from safepoints.
+  Continuations::on_gc_marking_cycle_start();
 
   // Reset flush/continue counters
   _nproactiveflush = 0;
@@ -817,7 +822,10 @@ bool ZMark::end() {
   // Update statistics
   ZStatMark::set_at_mark_end(_nproactiveflush, _nterminateflush, _ntrycomplete, _ncontinue);
 
-  CodeCache::finish_marking_cycle();
+  // Tell the sweeper that we finished a marking cycle.
+  // Unlike other GCs, we do not arm the nmethods
+  // when marking terminates.
+  Continuations::on_gc_marking_cycle_finish();
 
   // Mark completed
   return true;

src/hotspot/share/memory/iterator.inline.hpp

@@ -60,7 +60,7 @@ inline void ClaimMetadataVisitingOopIterateClosure::do_nmethod(nmethod* nm) {
 
 inline void ClaimMetadataVisitingOopIterateClosure::do_method(Method* m) {
   // Mark interpreted frames for marking_cycle
-  m->record_marking_cycle();
+  m->record_gc_epoch();
 }
 
 // Implementation of the non-virtual do_oop dispatch.

src/hotspot/share/oops/constantPool.cpp

@@ -2213,17 +2213,30 @@ int ConstantPool::copy_cpool_bytes(int cpool_size,
 
 #undef DBG
 
-// For redefinition, if any methods found in InstanceStackChunks, the marking_cycle is
+bool ConstantPool::is_maybe_on_continuation_stack() const {
+  // This method uses the similar logic as nmethod::is_maybe_on_continuation_stack()
+  if (!Continuations::enabled()) {
+    return false;
+  }
+
+  // If the condition below is true, it means that the nmethod was found to
+  // be alive the previous completed marking cycle.
+  return cache()->gc_epoch() >= Continuations::previous_completed_gc_marking_cycle();
+}
+
+// For redefinition, if any methods found in loom stack chunks, the gc_epoch is
 // recorded in their constant pool cache. The on_stack-ness of the constant pool controls whether
 // memory for the method is reclaimed.
 bool ConstantPool::on_stack() const {
-  // See nmethod::is_not_on_continuation_stack for explanation of what this means.
-  // cpCache is null when doing default method processing during class loading, so
-  // won't be on a continuation stack.
-  bool not_on_vthread_stack = !Continuations::enabled()
-       || _cache == nullptr
-       || CodeCache::marking_cycle() >= align_up(cache()->marking_cycle(), 2) + 2;
-  return (_flags &_on_stack) != 0 || !not_on_vthread_stack;
+  if ((_flags &_on_stack) != 0) {
+    return true;
+  }
+
+  if (_cache == nullptr) {
+    return false;
+  }
+
+  return is_maybe_on_continuation_stack();
 }
 
 void ConstantPool::set_on_stack(const bool value) {

src/hotspot/share/oops/constantPool.hpp

@@ -206,6 +206,7 @@ class ConstantPool : public Metadata {
   // can't be removed from the set of previous versions saved in the instance
   // class.
   bool on_stack() const;
+  bool is_maybe_on_continuation_stack() const;
   void set_on_stack(const bool value);
 
   // Faster than MetaspaceObj::is_shared() - used by set_on_stack()

src/hotspot/share/oops/cpCache.cpp

@@ -27,7 +27,6 @@
 #include "classfile/resolutionErrors.hpp"
 #include "classfile/systemDictionary.hpp"
 #include "classfile/vmClasses.hpp"
-#include "code/codeCache.hpp"
 #include "interpreter/bytecodeStream.hpp"
 #include "interpreter/bytecodes.hpp"
 #include "interpreter/interpreter.hpp"
@@ -47,6 +46,7 @@
 #include "prims/methodHandles.hpp"
 #include "runtime/arguments.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/continuation.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/vm_version.hpp"
 #include "utilities/macros.hpp"
@@ -706,9 +706,9 @@ void ConstantPoolCache::initialize(const intArray& inverse_index_map,
   }
 }
 
-// Record the GC marking cycle when redefined vs. when found in the InstanceStackChunks.
-void ConstantPoolCache::record_marking_cycle() {
-  _marking_cycle = CodeCache::marking_cycle();
+// Record the GC marking cycle when redefined vs. when found in the loom stack chunks.
+void ConstantPoolCache::record_gc_epoch() {
+  _gc_epoch = Continuations::gc_epoch();
 }
 
 void ConstantPoolCache::verify_just_initialized() {