8242088: Replace mutually exclusive lists with concurrent alternatives

Reviewed-by: egahlin

Author: Markus Grönlund

src/hotspot/share/jfr/jfr.cpp

@@ -25,14 +25,14 @@
 #include "precompiled.hpp"
 #include "jfr/jfr.hpp"
 #include "jfr/leakprofiler/leakProfiler.hpp"
-#include "jfr/periodic/sampling/jfrThreadSampler.hpp"
 #include "jfr/recorder/jfrRecorder.hpp"
 #include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp"
 #include "jfr/recorder/repository/jfrEmergencyDump.hpp"
 #include "jfr/recorder/service/jfrOptionSet.hpp"
 #include "jfr/recorder/repository/jfrRepository.hpp"
 #include "jfr/support/jfrThreadLocal.hpp"
 #include "runtime/java.hpp"
+#include "runtime/thread.hpp"
 
 bool Jfr::is_enabled() {
   return JfrRecorder::is_enabled();

src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp

@@ -38,6 +38,7 @@
 #include "jfr/recorder/storage/jfrStorageUtils.inline.hpp"
 #include "jfr/utilities/jfrBigEndian.hpp"
 #include "jfr/utilities/jfrIterator.hpp"
+#include "jfr/utilities/jfrLinkedList.inline.hpp"
 #include "jfr/utilities/jfrThreadIterator.hpp"
 #include "jfr/utilities/jfrTypes.hpp"
 #include "jfr/writers/jfrJavaEventWriter.hpp"
@@ -50,9 +51,7 @@
 #include "runtime/os.inline.hpp"
 #include "runtime/safepoint.hpp"
 
-typedef JfrCheckpointManager::Buffer* BufferPtr;
-
-static JfrCheckpointManager* _instance = NULL;
+typedef JfrCheckpointManager::BufferPtr BufferPtr;
 
 static volatile bool constant_pending = false;
 
@@ -70,6 +69,8 @@ static void set_constant_pending() {
   }
 }
 
+static JfrCheckpointManager* _instance = NULL;
+
 JfrCheckpointManager& JfrCheckpointManager::instance() {
   return *_instance;
 }
@@ -89,7 +90,6 @@ void JfrCheckpointManager::destroy() {
 JfrCheckpointManager::JfrCheckpointManager(JfrChunkWriter& cw) :
   _free_list_mspace(NULL),
   _epoch_transition_mspace(NULL),
-  _lock(NULL),
   _service_thread(NULL),
   _chunkwriter(cw),
   _checkpoint_epoch_state(JfrTraceIdEpoch::epoch()) {}
@@ -101,9 +101,6 @@ JfrCheckpointManager::~JfrCheckpointManager() {
   if (_epoch_transition_mspace != NULL) {
     delete _epoch_transition_mspace;
   }
-  if (_lock != NULL) {
-    delete _lock;
-  }
   JfrTypeManager::destroy();
 }
 
@@ -126,40 +123,22 @@ bool JfrCheckpointManager::initialize() {
   if (_epoch_transition_mspace == NULL) {
     return false;
   }
-  assert(_lock == NULL, "invariant");
-  _lock = new Mutex(Monitor::leaf - 1, "Checkpoint mutex", Mutex::_allow_vm_block_flag, Monitor::_safepoint_check_never);
-  if (_lock == NULL) {
-    return false;
-  }
   return JfrTypeManager::initialize();
 }
 
 void JfrCheckpointManager::register_service_thread(const Thread* thread) {
   _service_thread = thread;
 }
 
-void JfrCheckpointManager::register_full(BufferPtr t, Thread* thread) {
+void JfrCheckpointManager::register_full(BufferPtr buffer, Thread* thread) {
   // nothing here at the moment
-  assert(t != NULL, "invariant");
-  assert(t->acquired_by(thread), "invariant");
-  assert(t->retired(), "invariant");
-}
-
-void JfrCheckpointManager::lock() {
-  assert(!_lock->owned_by_self(), "invariant");
-  _lock->lock_without_safepoint_check();
-}
-
-void JfrCheckpointManager::unlock() {
-  _lock->unlock();
+  assert(buffer != NULL, "invariant");
+  assert(buffer->acquired_by(thread), "invariant");
+  assert(buffer->retired(), "invariant");
 }
 
 #ifdef ASSERT
-bool JfrCheckpointManager::is_locked() const {
-  return _lock->owned_by_self();
-}
-
-static void assert_free_lease(const BufferPtr buffer) {
+static void assert_lease(const BufferPtr buffer) {
   assert(buffer != NULL, "invariant");
   assert(buffer->acquired_by_self(), "invariant");
   assert(buffer->lease(), "invariant");
@@ -172,45 +151,46 @@ static void assert_release(const BufferPtr buffer) {
 }
 #endif // ASSERT
 
-static BufferPtr lease_free(size_t size, JfrCheckpointMspace* mspace, size_t retry_count, Thread* thread) {
+bool JfrCheckpointManager::use_epoch_transition_mspace(const Thread* thread) const {
+  return _service_thread != thread && Atomic::load_acquire(&_checkpoint_epoch_state) != JfrTraceIdEpoch::epoch();
+}
+
+static const size_t lease_retry = 10;
+
+BufferPtr JfrCheckpointManager::lease(JfrCheckpointMspace* mspace, Thread* thread, size_t size /* 0 */) {
+  assert(mspace != NULL, "invariant");
   static const size_t max_elem_size = mspace->min_elem_size(); // min is max
   BufferPtr buffer;
   if (size <= max_elem_size) {
-    BufferPtr buffer = mspace_get_free_lease_with_retry(size, mspace, retry_count, thread);
+    buffer = mspace_get_free_lease_with_retry(size, mspace, lease_retry, thread);
     if (buffer != NULL) {
-      DEBUG_ONLY(assert_free_lease(buffer);)
+      DEBUG_ONLY(assert_lease(buffer);)
       return buffer;
     }
   }
-  buffer = mspace_allocate_transient_lease_to_free(size, mspace, thread);
-  DEBUG_ONLY(assert_free_lease(buffer);)
+  buffer = mspace_allocate_transient_lease_to_full(size, mspace, thread);
+  DEBUG_ONLY(assert_lease(buffer);)
   return buffer;
 }
 
-bool JfrCheckpointManager::use_epoch_transition_mspace(const Thread* thread) const {
-  return _service_thread != thread && Atomic::load_acquire(&_checkpoint_epoch_state) != JfrTraceIdEpoch::epoch();
-}
-
-static const size_t lease_retry = 10;
-
-BufferPtr JfrCheckpointManager::lease_buffer(Thread* thread, size_t size /* 0 */) {
+BufferPtr JfrCheckpointManager::lease(Thread* thread, size_t size /* 0 */) {
   JfrCheckpointManager& manager = instance();
-  if (manager.use_epoch_transition_mspace(thread)) {
-    return lease_free(size, manager._epoch_transition_mspace, lease_retry, thread);
-  }
-  return lease_free(size, manager._free_list_mspace, lease_retry, thread);
+  JfrCheckpointMspace* const mspace = manager.use_epoch_transition_mspace(thread) ?
+                                        manager._epoch_transition_mspace :
+                                          manager._free_list_mspace;
+  return lease(mspace, thread, size);
 }
 
 JfrCheckpointMspace* JfrCheckpointManager::lookup(BufferPtr old) const {
   assert(old != NULL, "invariant");
-  return _free_list_mspace->in_free_list(old) ? _free_list_mspace : _epoch_transition_mspace;
+  return _free_list_mspace->in_mspace(old) ? _free_list_mspace : _epoch_transition_mspace;
 }
 
-BufferPtr JfrCheckpointManager::lease_buffer(BufferPtr old, Thread* thread, size_t size /* 0 */) {
+BufferPtr JfrCheckpointManager::lease(BufferPtr old, Thread* thread, size_t size /* 0 */) {
   assert(old != NULL, "invariant");
   JfrCheckpointMspace* mspace = instance().lookup(old);
   assert(mspace != NULL, "invariant");
-  return lease_free(size, mspace, lease_retry, thread);
+  return lease(mspace, thread, size);
 }
 
 /*
@@ -219,10 +199,14 @@ BufferPtr JfrCheckpointManager::lease_buffer(BufferPtr old, Thread* thread, size
  * The buffer is effectively invalidated for the thread post-return,
  * and the caller should take means to ensure that it is not referenced.
  */
-static void release(BufferPtr const buffer, Thread* thread) {
+static void release(BufferPtr buffer, Thread* thread) {
   DEBUG_ONLY(assert_release(buffer);)
   buffer->clear_lease();
-  buffer->release();
+  if (buffer->transient()) {
+    buffer->set_retired();
+  } else {
+    buffer->release();
+  }
 }
 
 BufferPtr JfrCheckpointManager::flush(BufferPtr old, size_t used, size_t requested, Thread* thread) {
@@ -235,7 +219,7 @@ BufferPtr JfrCheckpointManager::flush(BufferPtr old, size_t used, size_t request
     return NULL;
   }
   // migration of in-flight information
-  BufferPtr const new_buffer = lease_buffer(old, thread, used + requested);
+  BufferPtr const new_buffer = lease(old, thread, used + requested);
   if (new_buffer != NULL) {
     migrate_outstanding_writes(old, new_buffer, used, requested);
   }
@@ -335,18 +319,22 @@ class CheckpointWriteOp {
   size_t processed() const { return _processed; }
 };
 
-typedef CheckpointWriteOp<JfrCheckpointMspace::Type> WriteOperation;
-typedef ReleaseOp<JfrCheckpointMspace> CheckpointReleaseOperation;
+typedef CheckpointWriteOp<JfrCheckpointManager::Buffer> WriteOperation;
+typedef ReleaseOp<JfrCheckpointMspace> CheckpointReleaseFreeOperation;
+typedef ScavengingReleaseOp<JfrCheckpointMspace> CheckpointReleaseFullOperation;
 
-template <template <typename> class WriterHost, template <typename, typename, typename> class CompositeOperation>
+template <template <typename> class WriterHost>
 static size_t write_mspace(JfrCheckpointMspace* mspace, JfrChunkWriter& chunkwriter) {
   assert(mspace != NULL, "invariant");
   WriteOperation wo(chunkwriter);
   WriterHost<WriteOperation> wh(wo);
-  CheckpointReleaseOperation cro(mspace, Thread::current(), false);
-  CompositeOperation<WriterHost<WriteOperation>, CheckpointReleaseOperation, CompositeOperationAnd> co(&wh, &cro);
-  assert(mspace->is_full_empty(), "invariant");
-  process_free_list(co, mspace);
+  CheckpointReleaseFreeOperation free_release_op(mspace);
+  CompositeOperation<WriterHost<WriteOperation>, CheckpointReleaseFreeOperation> free_op(&wh, &free_release_op);
+  process_free_list(free_op, mspace);
+  CheckpointReleaseFullOperation full_release_op(mspace);
+  MutexedWriteOp<WriteOperation> full_write_op(wo);
+  CompositeOperation<MutexedWriteOp<WriteOperation>, CheckpointReleaseFullOperation> full_op(&full_write_op, &full_release_op);
+  process_full_list(full_op, mspace);
   return wo.processed();
 }
 
@@ -369,52 +357,66 @@ void JfrCheckpointManager::synchronize_checkpoint_manager_with_current_epoch() {
 }
 
 size_t JfrCheckpointManager::write() {
-  const size_t processed = write_mspace<MutexedWriteOp, CompositeOperation>(_free_list_mspace, _chunkwriter);
+  const size_t processed = write_mspace<MutexedWriteOp>(_free_list_mspace, _chunkwriter);
   synchronize_checkpoint_manager_with_current_epoch();
   return processed;
 }
 
 size_t JfrCheckpointManager::write_epoch_transition_mspace() {
-  return write_mspace<ExclusiveOp, CompositeOperation>(_epoch_transition_mspace, _chunkwriter);
+  return write_mspace<ExclusiveOp>(_epoch_transition_mspace, _chunkwriter);
 }
 
-typedef DiscardOp<DefaultDiscarder<JfrBuffer> > DiscardOperation;
+typedef DiscardOp<DefaultDiscarder<JfrCheckpointManager::Buffer> > DiscardOperation;
+typedef ExclusiveDiscardOp<DefaultDiscarder<JfrCheckpointManager::Buffer> > DiscardOperationEpochTransitionMspace;
+typedef CompositeOperation<DiscardOperation, CheckpointReleaseFreeOperation> DiscardFreeOperation;
+typedef CompositeOperation<DiscardOperation, CheckpointReleaseFullOperation> DiscardFullOperation;
+typedef CompositeOperation<DiscardOperationEpochTransitionMspace, CheckpointReleaseFreeOperation> DiscardEpochTransMspaceFreeOperation;
+typedef CompositeOperation<DiscardOperationEpochTransitionMspace, CheckpointReleaseFullOperation> DiscardEpochTransMspaceFullOperation;
+
 size_t JfrCheckpointManager::clear() {
   clear_type_set();
-  DiscardOperation discarder(mutexed); // mutexed discard mode
-  process_free_list(discarder, _free_list_mspace);
-  process_free_list(discarder, _epoch_transition_mspace);
+  DiscardOperation mutex_discarder(mutexed);
+  CheckpointReleaseFreeOperation free_release_op(_free_list_mspace);
+  DiscardFreeOperation free_op(&mutex_discarder, &free_release_op);
+  process_free_list(free_op, _free_list_mspace);
+  CheckpointReleaseFullOperation full_release_op(_free_list_mspace);
+  DiscardFullOperation full_op(&mutex_discarder, &full_release_op);
+  process_full_list(full_op, _free_list_mspace);
+  DiscardOperationEpochTransitionMspace epoch_transition_discarder(mutexed);
+  CheckpointReleaseFreeOperation epoch_free_release_op(_epoch_transition_mspace);
+  DiscardEpochTransMspaceFreeOperation epoch_free_op(&epoch_transition_discarder, &epoch_free_release_op);
+  process_free_list(epoch_free_op, _epoch_transition_mspace);
+  CheckpointReleaseFullOperation epoch_full_release_op(_epoch_transition_mspace);
+  DiscardEpochTransMspaceFullOperation epoch_full_op(&epoch_transition_discarder, &epoch_full_release_op);
+  process_full_list(epoch_full_op, _epoch_transition_mspace);
   synchronize_checkpoint_manager_with_current_epoch();
-  return discarder.elements();
+  return mutex_discarder.elements() + epoch_transition_discarder.elements();
 }
 
 // Optimization for write_static_type_set() and write_threads() is to write
 // directly into the epoch transition mspace because we will immediately
 // serialize and reset this mspace post-write.
-static JfrBuffer* get_epoch_transition_buffer(JfrCheckpointMspace* mspace, Thread* t) {
-  assert(mspace != NULL, "invariant");
-  JfrBuffer* const buffer = mspace->free_head();
-  assert(buffer != NULL, "invariant");
-  buffer->acquire(t);
-  buffer->set_lease();
-  DEBUG_ONLY(assert_free_lease(buffer);)
+BufferPtr JfrCheckpointManager::epoch_transition_buffer(Thread* thread) {
+  assert(_epoch_transition_mspace->free_list_is_nonempty(), "invariant");
+  BufferPtr const buffer = lease(_epoch_transition_mspace, thread, _epoch_transition_mspace->min_elem_size());
+  DEBUG_ONLY(assert_lease(buffer);)
   return buffer;
 }
 
 size_t JfrCheckpointManager::write_static_type_set() {
-  Thread* const t = Thread::current();
-  ResourceMark rm(t);
-  HandleMark hm(t);
-  JfrCheckpointWriter writer(t, get_epoch_transition_buffer(_epoch_transition_mspace, t), STATICS);
+  Thread* const thread = Thread::current();
+  ResourceMark rm(thread);
+  HandleMark hm(thread);
+  JfrCheckpointWriter writer(thread, epoch_transition_buffer(thread), STATICS);
   JfrTypeManager::write_static_types(writer);
   return writer.used_size();
 }
 
 size_t JfrCheckpointManager::write_threads() {
-  Thread* const t = Thread::current();
-  ResourceMark rm(t);
-  HandleMark hm(t);
-  JfrCheckpointWriter writer(t, get_epoch_transition_buffer(_epoch_transition_mspace, t), THREADS);
+  Thread* const thread = Thread::current();
+  ResourceMark rm(thread);
+  HandleMark hm(thread);
+  JfrCheckpointWriter writer(thread, epoch_transition_buffer(thread), THREADS);
   JfrTypeManager::write_threads(writer);
   return writer.used_size();
 }
@@ -442,20 +444,20 @@ void JfrCheckpointManager::clear_type_set() {
 
 void JfrCheckpointManager::write_type_set() {
   assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
+  Thread* const thread = Thread::current();
   if (LeakProfiler::is_running()) {
-    Thread* const t = Thread::current();
     // can safepoint here
-    MutexLocker cld_lock(t, ClassLoaderDataGraph_lock);
-    MutexLocker module_lock(t, Module_lock);
-    JfrCheckpointWriter leakp_writer(t);
-    JfrCheckpointWriter writer(t);
+    MutexLocker cld_lock(thread, ClassLoaderDataGraph_lock);
+    MutexLocker module_lock(thread, Module_lock);
+    JfrCheckpointWriter leakp_writer(thread);
+    JfrCheckpointWriter writer(thread);
     JfrTypeSet::serialize(&writer, &leakp_writer, false, false);
     ObjectSampleCheckpoint::on_type_set(leakp_writer);
   } else {
     // can safepoint here
     MutexLocker cld_lock(ClassLoaderDataGraph_lock);
     MutexLocker module_lock(Module_lock);
-    JfrCheckpointWriter writer(Thread::current());
+    JfrCheckpointWriter writer(thread);
     JfrTypeSet::serialize(&writer, NULL, false, false);
   }
   write();
@@ -489,27 +491,27 @@ size_t JfrCheckpointManager::flush_type_set() {
   if (is_constant_pending()) {
     WriteOperation wo(_chunkwriter);
     FlushOperation fo(wo);
-    assert(_free_list_mspace->is_full_empty(), "invariant");
     process_free_list(fo, _free_list_mspace);
+    process_full_list(fo, _free_list_mspace);
   }
   return elements;
 }
 
-void JfrCheckpointManager::create_thread_blob(Thread* t) {
-  JfrTypeManager::create_thread_blob(t);
+void JfrCheckpointManager::create_thread_blob(Thread* thread) {
+  JfrTypeManager::create_thread_blob(thread);
 }
 
-void JfrCheckpointManager::write_thread_checkpoint(Thread* t) {
-  JfrTypeManager::write_thread_checkpoint(t);
+void JfrCheckpointManager::write_thread_checkpoint(Thread* thread) {
+  JfrTypeManager::write_thread_checkpoint(thread);
 }
 
 class JfrNotifyClosure : public ThreadClosure {
  public:
-  void do_thread(Thread* t) {
-    assert(t != NULL, "invariant");
-    assert(t->is_Java_thread(), "invariant");
+  void do_thread(Thread* thread) {
+    assert(thread != NULL, "invariant");
+    assert(thread->is_Java_thread(), "invariant");
     assert_locked_or_safepoint(Threads_lock);
-    JfrJavaEventWriter::notify((JavaThread*)t);
+    JfrJavaEventWriter::notify((JavaThread*)thread);
   }
 };
 

src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
 #include "jfr/recorder/storage/jfrBuffer.hpp"
 #include "jfr/recorder/storage/jfrMemorySpace.hpp"
 #include "jfr/recorder/storage/jfrMemorySpaceRetrieval.hpp"
+#include "jfr/utilities/jfrLinkedList.hpp"
 
 class JfrCheckpointManager;
 class JfrChunkWriter;
@@ -44,7 +45,7 @@ struct JfrCheckpointEntry {
   juint nof_segments;
 };
 
-typedef JfrMemorySpace<JfrBuffer, JfrMspaceSequentialRetrieval, JfrCheckpointManager> JfrCheckpointMspace;
+typedef JfrMemorySpace<JfrCheckpointManager, JfrMspaceRetrieval, JfrLinkedList<JfrBuffer> > JfrCheckpointMspace;
 
 //
 // Responsible for maintaining checkpoints and by implication types.
@@ -53,28 +54,29 @@ typedef JfrMemorySpace<JfrBuffer, JfrMspaceSequentialRetrieval, JfrCheckpointMan
 //
 class JfrCheckpointManager : public JfrCHeapObj {
  public:
-  typedef JfrCheckpointMspace::Type Buffer;
+  size_t flush_type_set();
+  static void create_thread_blob(Thread* thread);
+  static void write_thread_checkpoint(Thread* thread);
+  void register_service_thread(const Thread* thread);
+  typedef JfrCheckpointMspace::Node Buffer;
+  typedef JfrCheckpointMspace::NodePtr BufferPtr;
+
  private:
   JfrCheckpointMspace* _free_list_mspace;
   JfrCheckpointMspace* _epoch_transition_mspace;
-  Mutex* _lock;
   const Thread* _service_thread;
   JfrChunkWriter& _chunkwriter;
   bool _checkpoint_epoch_state;
 
-  // mspace callback
-  void register_full(Buffer* t, Thread* thread);
-  void lock();
-  void unlock();
-  DEBUG_ONLY(bool is_locked() const;)
-
-  JfrCheckpointMspace* lookup(Buffer* old) const;
-  bool use_epoch_transition_mspace(const Thread* t) const;
+  JfrCheckpointMspace* lookup(BufferPtr old) const;
+  bool use_epoch_transition_mspace(const Thread* thread) const;
   size_t write_epoch_transition_mspace();
+  BufferPtr epoch_transition_buffer(Thread* thread);
 
-  static Buffer* lease_buffer(Thread* t, size_t size = 0);
-  static Buffer* lease_buffer(Buffer* old, Thread* t, size_t size = 0);
-  static Buffer* flush(Buffer* old, size_t used, size_t requested, Thread* t);
+  static BufferPtr lease(Thread* thread, size_t size = 0);
+  static BufferPtr lease(BufferPtr old, Thread* thread, size_t size = 0);
+  static BufferPtr lease(JfrCheckpointMspace* mspace, Thread* thread, size_t size = 0);
+  static BufferPtr flush(BufferPtr old, size_t used, size_t requested, Thread* thread);
 
   size_t clear();
   size_t write();
@@ -102,11 +104,8 @@ class JfrCheckpointManager : public JfrCHeapObj {
   void on_rotation();
   static void destroy();
 
- public:
-  size_t flush_type_set();
-  static void create_thread_blob(Thread* t);
-  static void write_thread_checkpoint(Thread* t);
-  void register_service_thread(const Thread* t);
+  // mspace callback
+  void register_full(BufferPtr buffer, Thread* thread);
 
   friend class Jfr;
   friend class JfrRecorder;
@@ -115,7 +114,7 @@ class JfrCheckpointManager : public JfrCHeapObj {
   friend class JfrCheckpointWriter;
   friend class JfrSerializer;
   friend class JfrStackTraceRepository;
-  template <typename, template <typename> class, typename>
+  template <typename, template <typename> class, typename, typename>
   friend class JfrMemorySpace;
 };
 

src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointWriter.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,7 +32,7 @@ JfrCheckpointFlush::JfrCheckpointFlush(Type* old, size_t used, size_t requested,
   _result(JfrCheckpointManager::flush(old, used, requested, t)) {}
 
 JfrCheckpointWriter::JfrCheckpointWriter(JfrCheckpointType type /* GENERIC */) :
-  JfrCheckpointWriterBase(JfrCheckpointManager::lease_buffer(Thread::current()), Thread::current()),
+  JfrCheckpointWriterBase(JfrCheckpointManager::lease(Thread::current()), Thread::current()),
   _time(JfrTicks::now()),
   _offset(0),
   _count(0),
@@ -46,7 +46,7 @@ JfrCheckpointWriter::JfrCheckpointWriter(JfrCheckpointType type /* GENERIC */) :
 }
 
 JfrCheckpointWriter::JfrCheckpointWriter(Thread* t, bool header /* true */, JfrCheckpointType type /* GENERIC */) :
-  JfrCheckpointWriterBase(JfrCheckpointManager::lease_buffer(t), t),
+  JfrCheckpointWriterBase(JfrCheckpointManager::lease(t), t),
   _time(JfrTicks::now()),
   _offset(0),
   _count(0),

src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeManager.cpp

@@ -28,8 +28,8 @@
 #include "jfr/recorder/checkpoint/types/jfrType.hpp"
 #include "jfr/recorder/checkpoint/types/jfrTypeManager.hpp"
 #include "jfr/recorder/jfrRecorder.hpp"
-#include "jfr/utilities/jfrDoublyLinkedList.hpp"
 #include "jfr/utilities/jfrIterator.hpp"
+#include "jfr/utilities/jfrLinkedList.inline.hpp"
 #include "memory/resourceArea.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/safepoint.hpp"
@@ -38,38 +38,20 @@
 #include "utilities/exceptions.hpp"
 
 class JfrSerializerRegistration : public JfrCHeapObj {
+ public:
+  JfrSerializerRegistration* _next; // list support
  private:
-  JfrSerializerRegistration* _next;
-  JfrSerializerRegistration* _prev;
   JfrSerializer* _serializer;
   mutable JfrBlobHandle _cache;
   JfrTypeId _id;
   bool _permit_cache;
-
  public:
   JfrSerializerRegistration(JfrTypeId id, bool permit_cache, JfrSerializer* serializer) :
-    _next(NULL), _prev(NULL), _serializer(serializer), _cache(), _id(id), _permit_cache(permit_cache) {}
-
+    _next(NULL), _serializer(serializer), _cache(), _id(id), _permit_cache(permit_cache) {}
   ~JfrSerializerRegistration() {
     delete _serializer;
   }
 
-  JfrSerializerRegistration* next() const {
-    return _next;
-  }
-
-  void set_next(JfrSerializerRegistration* next) {
-    _next = next;
-  }
-
-  JfrSerializerRegistration* prev() const {
-    return _prev;
-  }
-
-  void set_prev(JfrSerializerRegistration* prev) {
-    _prev = prev;
-  }
-
   JfrTypeId id() const {
     return _id;
   }
@@ -155,34 +137,50 @@ class SerializerRegistrationGuard : public StackObj {
 
 Semaphore SerializerRegistrationGuard::_mutex_semaphore(1);
 
-typedef JfrDoublyLinkedList<JfrSerializerRegistration> List;
-typedef StopOnNullIterator<const List> Iterator;
+typedef JfrLinkedList<JfrSerializerRegistration> List;
 static List types;
 
 void JfrTypeManager::destroy() {
   SerializerRegistrationGuard guard;
-  Iterator iter(types);
   JfrSerializerRegistration* registration;
-  while (iter.has_next()) {
-    registration = types.remove(iter.next());
+  while (types.is_nonempty()) {
+    registration = types.remove();
     assert(registration != NULL, "invariant");
     delete registration;
   }
 }
 
-void JfrTypeManager::on_rotation() {
-  const Iterator iter(types);
-  while (iter.has_next()) {
-    iter.next()->on_rotation();
+class InvokeOnRotation {
+ public:
+  bool process(const JfrSerializerRegistration* r) {
+    assert(r != NULL, "invariant");
+    r->on_rotation();
+    return true;
   }
+};
+
+void JfrTypeManager::on_rotation() {
+  InvokeOnRotation ior;
+  types.iterate(ior);
 }
 
 #ifdef ASSERT
-static void assert_not_registered_twice(JfrTypeId id, List& list) {
-  const Iterator iter(list);
-  while (iter.has_next()) {
-    assert(iter.next()->id() != id, "invariant");
+
+class Diversity {
+ private:
+  const JfrTypeId _id;
+ public:
+  Diversity(JfrTypeId id) : _id(id) {}
+  bool process(const JfrSerializerRegistration* r) {
+    assert(r != NULL, "invariant");
+    assert(r->id() != _id, "invariant");
+    return true;
   }
+};
+
+static void assert_not_registered_twice(JfrTypeId id, List& list) {
+  Diversity d(id);
+  types.iterate(d);
 }
 #endif
 
@@ -199,7 +197,7 @@ static bool register_static_type(JfrTypeId id, bool permit_cache, JfrSerializer*
     JfrCheckpointWriter writer(STATICS);
     registration->invoke(writer);
   }
-  types.prepend(registration);
+  types.add(registration);
   return true;
 }
 
@@ -229,11 +227,20 @@ bool JfrSerializer::register_serializer(JfrTypeId id, bool permit_cache, JfrSeri
   return register_static_type(id, permit_cache, serializer);
 }
 
+class InvokeSerializer {
+ private:
+   JfrCheckpointWriter& _writer;
+ public:
+  InvokeSerializer(JfrCheckpointWriter& writer) : _writer(writer) {}
+  bool process(const JfrSerializerRegistration* r) {
+    assert(r != NULL, "invariant");
+    r->invoke(_writer);
+    return true;
+  }
+};
 
 void JfrTypeManager::write_static_types(JfrCheckpointWriter& writer) {
+  InvokeSerializer is(writer);
   SerializerRegistrationGuard guard;
-  const Iterator iter(types);
-  while (iter.has_next()) {
-    iter.next()->invoke(writer);
-  }
+  types.iterate(is);
 }

src/hotspot/share/jfr/recorder/service/jfrPostBox.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -40,7 +40,6 @@ enum JFR_Msg {
   MSG_WAKEUP,
   MSG_SHUTDOWN,
   MSG_VM_ERROR,
-  MSG_DEADBUFFER,
   MSG_FLUSHPOINT,
   MSG_NO_OF_MSGS
 };
@@ -55,15 +54,14 @@ enum JFR_Msg {
  *  MSG_STOP (2)            ; MSGBIT(MSG_STOP) == (1 << 0x2) == 0x4
  *  MSG_ROTATE (3)          ; MSGBIT(MSG_ROTATE) == (1 << 0x3) == 0x8
  *  MSG_VM_ERROR (8)        ; MSGBIT(MSG_VM_ERROR) == (1 << 0x8) == 0x100
- *  MSG_FLUSHPOINT (10)     ; MSGBIT(MSG_FLUSHPOINT) == (1 << 0xa) == 0x400
+ *  MSG_FLUSHPOINT (9)     ; MSGBIT(MSG_FLUSHPOINT) == (1 << 0x9) == 0x200
  *
  *  Asynchronous messages (posting thread returns immediately upon deposit):
  *
  *  MSG_FULLBUFFER (4)      ; MSGBIT(MSG_FULLBUFFER) == (1 << 0x4) == 0x10
  *  MSG_CHECKPOINT (5)      ; MSGBIT(CHECKPOINT) == (1 << 0x5) == 0x20
  *  MSG_WAKEUP (6)          ; MSGBIT(WAKEUP) == (1 << 0x6) == 0x40
  *  MSG_SHUTDOWN (7)        ; MSGBIT(MSG_SHUTDOWN) == (1 << 0x7) == 0x80
- *  MSG_DEADBUFFER (9)      ; MSGBIT(MSG_DEADBUFFER) == (1 << 0x9) == 0x200
  */
 
 class JfrPostBox : public JfrCHeapObj {

src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp

@@ -657,10 +657,6 @@ void JfrRecorderService::process_full_buffers() {
   }
 }
 
-void JfrRecorderService::scavenge() {
-  _storage.scavenge();
-}
-
 void JfrRecorderService::evaluate_chunk_size_for_rotation() {
   JfrChunkRotation::evaluate(_chunkwriter);
 }

src/hotspot/share/jfr/recorder/service/jfrRecorderThreadLoop.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -42,7 +42,6 @@ void recorderthread_entry(JavaThread* thread, Thread* unused) {
   #define ROTATE (msgs & (MSGBIT(MSG_ROTATE)|MSGBIT(MSG_STOP)))
   #define FLUSHPOINT (msgs & (MSGBIT(MSG_FLUSHPOINT)))
   #define PROCESS_FULL_BUFFERS (msgs & (MSGBIT(MSG_ROTATE)|MSGBIT(MSG_STOP)|MSGBIT(MSG_FULLBUFFER)))
-  #define SCAVENGE (msgs & (MSGBIT(MSG_DEADBUFFER)))
 
   JfrPostBox& post_box = JfrRecorderThread::post_box();
   log_debug(jfr, system)("Recorder thread STARTED");
@@ -63,9 +62,6 @@ void recorderthread_entry(JavaThread* thread, Thread* unused) {
       if (PROCESS_FULL_BUFFERS) {
         service.process_full_buffers();
       }
-      if (SCAVENGE) {
-        service.scavenge();
-      }
       // Check amount of data written to chunk already
       // if it warrants asking for a new chunk
       service.evaluate_chunk_size_for_rotation();

src/hotspot/share/jfr/recorder/storage/jfrBuffer.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,7 +29,6 @@
 static const u1* const TOP_CRITICAL_SECTION = NULL;
 
 JfrBuffer::JfrBuffer() : _next(NULL),
-                         _prev(NULL),
                          _identity(NULL),
                          _pos(NULL),
                          _top(NULL),
@@ -54,7 +53,6 @@ bool JfrBuffer::initialize(size_t header_size, size_t size) {
 void JfrBuffer::reinitialize(bool exclusion /* false */) {
   acquire_critical_section_top();
   assert(!lease(), "invariant");
-  assert(!transient(), "invariant");
   if (exclusion != excluded()) {
     // update
     if (exclusion) {
@@ -124,6 +122,13 @@ bool JfrBuffer::try_acquire(const void* id) {
   return current_id == NULL && Atomic::cmpxchg(&_identity, current_id, id) == current_id;
 }
 
+void JfrBuffer::set_identity(const void* id) {
+  assert(id != NULL, "invariant");
+  assert(_identity == NULL, "invariant");
+  OrderAccess::storestore();
+  _identity = id;
+}
+
 void JfrBuffer::release() {
   assert(identity() != NULL, "invariant");
   Atomic::release_store(&_identity, (const void*)NULL);
@@ -260,13 +265,11 @@ bool JfrBuffer::retired() const {
 }
 
 void JfrBuffer::set_retired() {
-  assert(acquired_by_self(), "invariant");
   set(&_flags, RETIRED);
 }
 
 void JfrBuffer::clear_retired() {
   if (retired()) {
-    assert(identity() != NULL, "invariant");
     clear(&_flags, RETIRED);
   }
 }

src/hotspot/share/jfr/recorder/storage/jfrBuffer.hpp

@@ -60,9 +60,9 @@
 //
 
 class JfrBuffer {
+ public:
+  JfrBuffer* _next; // list support
  private:
-  JfrBuffer* _next;
-  JfrBuffer* _prev;
   const void* _identity;
   u1* _pos;
   mutable const u1* _top;
@@ -77,22 +77,6 @@ class JfrBuffer {
   bool initialize(size_t header_size, size_t size);
   void reinitialize(bool exclusion = false);
 
-  JfrBuffer* next() const {
-    return _next;
-  }
-
-  JfrBuffer* prev() const {
-    return _prev;
-  }
-
-  void set_next(JfrBuffer* next) {
-    _next = next;
-  }
-
-  void set_prev(JfrBuffer* prev) {
-    _prev = prev;
-  }
-
   const u1* start() const {
     return ((const u1*)this) + _header_size;
   }
@@ -157,6 +141,9 @@ class JfrBuffer {
     return Atomic::load_acquire(&_identity);
   }
 
+  // use only if implied owner already
+  void set_identity(const void* id);
+
   void acquire(const void* id);
   bool try_acquire(const void* id);
   bool acquired_by(const void* id) const;
@@ -183,17 +170,4 @@ class JfrBuffer {
   void clear_excluded();
 };
 
-class JfrAgeNode : public JfrBuffer {
- private:
-  JfrBuffer* _retired;
- public:
-  JfrAgeNode() : _retired(NULL) {}
-  void set_retired_buffer(JfrBuffer* retired) {
-    _retired = retired;
-  }
-  JfrBuffer* retired_buffer() const {
-    return _retired;
-  }
-};
-
 #endif // SHARE_JFR_RECORDER_STORAGE_JFRBUFFER_HPP

src/hotspot/share/jfr/recorder/storage/jfrFullStorage.hpp

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_JFR_RECORDER_STORAGE_JFRFULLSTORAGE_HPP
+#define SHARE_JFR_RECORDER_STORAGE_JFRFULLSTORAGE_HPP
+
+#include "jfr/utilities/jfrAllocation.hpp"
+#include "jfr/utilities/jfrConcurrentQueue.hpp"
+
+class JfrStorageControl;
+
+/*
+ * For full storage management.
+ *
+ * In essence, full storage is added to a FIFO queue, where the insertion order
+ * is used to represent the "is older" relation. Removes oldest data first.
+ *
+ * FullType     the type of the data value to be stored in the list.
+ *
+ * NodeType     template class for the node to store a value of FullType.
+ *
+ * AllocPolicy  memory alloction.
+ */
+template <typename FullType, template <typename> class NodeType, typename AllocPolicy = JfrCHeapObj>
+class JfrFullStorage : public AllocPolicy {
+ public:
+  typedef FullType Value;
+  typedef NodeType<Value>* NodePtr;
+  typedef NodeType<Value> Node;
+  JfrFullStorage(JfrStorageControl& control);
+  ~JfrFullStorage();
+  bool initialize(size_t free_list_prealloc_count);
+  bool is_empty() const;
+  bool is_nonempty() const;
+  bool add(Value value);
+  Value remove();
+  template <typename Callback>
+  void iterate(Callback& cb);
+ private:
+  JfrStorageControl& _control;
+  JfrConcurrentQueue<Node, AllocPolicy>* _free_node_list;
+  JfrConcurrentQueue<Node, AllocPolicy>* _queue;
+  NodePtr acquire();
+  void release(NodePtr node);
+};
+
+#endif // SHARE_JFR_RECORDER_STORAGE_JFRFULLSTORAGE_HPP

src/hotspot/share/jfr/recorder/storage/jfrFullStorage.inline.hpp

@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_JFR_RECORDER_STORAGE_JFRSTORAGEFULLLIST_INLINE_HPP
+#define SHARE_JFR_RECORDER_STORAGE_JFRSTORAGEFULLLIST_INLINE_HPP
+
+#include "jfr/recorder/storage/jfrStorageControl.hpp"
+#include "jfr/recorder/storage/jfrFullStorage.hpp"
+#include "jfr/utilities/jfrConcurrentQueue.inline.hpp"
+
+template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
+JfrFullStorage<ValueType, NodeType, AllocPolicy>
+::JfrFullStorage(JfrStorageControl& control) : _control(control), _free_node_list(NULL), _queue(NULL) {}
+
+template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
+JfrFullStorage<ValueType, NodeType, AllocPolicy>::~JfrFullStorage() {
+  NodePtr node;
+  while (_free_node_list->is_nonempty()) {
+    node = _free_node_list->remove();
+    delete node;
+  }
+  while (_queue->is_nonempty()) {
+    node = _queue->remove();
+    delete node;
+  }
+}
+
+template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
+bool JfrFullStorage<ValueType, NodeType, AllocPolicy>::initialize(size_t free_list_prealloc_count) {
+  assert(_free_node_list == NULL, "invariant");
+  _free_node_list = new JfrConcurrentQueue<Node>();
+  if (_free_node_list == NULL || !_free_node_list->initialize()) {
+    return false;
+  }
+  for (size_t i = 0; i < free_list_prealloc_count; ++i) {
+    NodePtr node = new (ResourceObj::C_HEAP, mtTracing) Node();
+    if (node == NULL) {
+      return false;
+    }
+    _free_node_list->add(node);
+  }
+  assert(_queue == NULL, "invariant");
+  _queue = new JfrConcurrentQueue<Node>();
+  return _queue != NULL && _queue->initialize();
+}
+
+template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
+inline bool JfrFullStorage<ValueType, NodeType, AllocPolicy>::is_empty() const {
+  return _queue->is_empty();
+}
+
+template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
+inline bool JfrFullStorage<ValueType, NodeType, AllocPolicy>::is_nonempty() const {
+  return !is_empty();
+}
+
+template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
+inline typename JfrFullStorage<ValueType, NodeType, AllocPolicy>::NodePtr
+JfrFullStorage<ValueType, NodeType, AllocPolicy>::acquire() {
+  NodePtr node = _free_node_list->remove();
+  return node != NULL ? node : new (ResourceObj::C_HEAP, mtTracing) Node();
+}
+
+template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
+inline void JfrFullStorage<ValueType, NodeType, AllocPolicy>
+::release(typename JfrFullStorage<ValueType, NodeType, AllocPolicy>::NodePtr node) {
+  assert(node != NULL, "invariant");
+  _free_node_list->add(node);
+}
+
+template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
+inline bool JfrFullStorage<ValueType, NodeType, AllocPolicy>::add(ValueType value) {
+  assert(value != NULL, "invariant");
+  NodePtr node = acquire();
+  assert(node != NULL, "invariant");
+  node->set_value(value);
+  const bool notify = _control.increment_full();
+  _queue->add(node);
+  return notify;
+}
+
+template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
+inline ValueType JfrFullStorage<ValueType, NodeType, AllocPolicy>::remove() {
+  Value value = NULL;
+  NodePtr node = _queue->remove();
+  if (node != NULL) {
+    _control.decrement_full();
+    value = node->value();
+    release(node);
+  }
+  return value;
+}
+
+template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
+template <typename Callback>
+void JfrFullStorage<ValueType, NodeType, AllocPolicy>::iterate(Callback& cb) {
+  _queue->iterate(cb);
+}
+
+#endif // SHARE_JFR_RECORDER_STORAGE_JFRSTORAGEFULLLIST_INLINE_HPP

src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,82 +25,68 @@
 #define SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_HPP
 
 #include "jfr/utilities/jfrAllocation.hpp"
-#include "jfr/utilities/jfrDoublyLinkedList.hpp"
-#include "jfr/utilities/jfrIterator.hpp"
 
-template <typename T, template <typename> class RetrievalType, typename Callback>
+template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType = FreeListType>
 class JfrMemorySpace : public JfrCHeapObj {
  public:
-  typedef T Type;
-  typedef RetrievalType<JfrMemorySpace<T, RetrievalType, Callback> > Retrieval;
-  typedef JfrDoublyLinkedList<Type> List;
-  typedef StopOnNullIterator<List> Iterator;
+  typedef FreeListType FreeList;
+  typedef FullListType FullList;
+  typedef typename FreeListType::Node Node;
+  typedef typename FreeListType::NodePtr NodePtr;
  private:
-  List _free;
-  List _full;
-  size_t _min_elem_size;
-  size_t _limit_size;
-  size_t _cache_count;
+  FreeList _free_list;
+  FullList _full_list;
+  const size_t _min_elem_size;
+  const size_t _limit_size;
+  const size_t _free_list_cache_count;
+  size_t _free_list_count;
   Callback* _callback;
 
-  bool should_populate_cache() const { return _free.count() < _cache_count; }
+  bool should_populate_free_list() const;
 
  public:
-  JfrMemorySpace(size_t min_elem_size, size_t limit_size, size_t cache_count, Callback* callback);
+  JfrMemorySpace(size_t min_elem_size, size_t limit_size, size_t free_list_cache_count, Callback* callback);
   ~JfrMemorySpace();
   bool initialize();
 
-  size_t min_elem_size() const { return _min_elem_size; }
-  size_t limit_size() const { return _limit_size; }
-
-  bool has_full() const { return _full.head() != NULL; }
-  bool has_free() const { return _free.head() != NULL; }
-  bool is_full_empty() const { return !has_full(); }
-  bool is_free_empty() const { return !has_free(); }
-
-  size_t full_count() const { return _full.count(); }
-  size_t free_count() const { return _free.count(); }
-
-  List& full() { return _full; }
-  const List& full() const { return _full; }
-  List& free() { return _free; }
-  const List& free() const { return _free; }
-
-  Type* full_head() { return _full.head(); }
-  Type* full_tail() { return _full.tail(); }
-  Type* free_head() { return _free.head(); }
-  Type* free_tail() { return _free.tail(); }
-
-  void insert_free_head(Type* t) { _free.prepend(t); }
-  void insert_free_tail(Type* t) { _free.append(t); }
-  void insert_free_tail(Type* t, Type* tail, size_t count) { _free.append_list(t, tail, count); }
-  void insert_full_head(Type* t) { _full.prepend(t); }
-  void insert_full_tail(Type* t) { _full.append(t); }
-  void insert_full_tail(Type* t, Type* tail, size_t count) { _full.append_list(t, tail, count); }
-
-  Type* remove_free(Type* t) { return _free.remove(t); }
-  Type* remove_full(Type* t) { return _full.remove(t); }
-  Type* remove_free_tail() { _free.remove(_free.tail()); }
-  Type* remove_full_tail() { return _full.remove(_full.tail()); }
-  Type* clear_full(bool return_tail = false) { return _full.clear(return_tail); }
-  Type* clear_free(bool return_tail = false) { return _free.clear(return_tail); }
-  void release_full(Type* t);
-  void release_free(Type* t);
-
-  void register_full(Type* t, Thread* thread) { _callback->register_full(t, thread); }
-  void lock() { _callback->lock(); }
-  void unlock() { _callback->unlock(); }
-  DEBUG_ONLY(bool is_locked() const { return _callback->is_locked(); })
-
-  Type* allocate(size_t size);
-  void deallocate(Type* t);
-  Type* get(size_t size, Thread* thread) { return Retrieval::get(size, this, thread); }
-
-  template <typename IteratorCallback, typename IteratorType>
-  void iterate(IteratorCallback& callback, bool full = true, jfr_iter_direction direction = forward);
-
-  bool in_full_list(const Type* t) const { return _full.in_list(t); }
-  bool in_free_list(const Type* t) const { return _free.in_list(t); }
+  size_t min_elem_size() const;
+  size_t limit_size() const;
+
+  NodePtr allocate(size_t size);
+  void deallocate(NodePtr node);
+
+  NodePtr acquire(Thread* thread, size_t size = 0);
+  void release(NodePtr node);
+
+  FreeList& free_list();
+  const FreeList& free_list() const;
+
+  FullList& full_list();
+  const FullList& full_list() const;
+
+  bool free_list_is_empty() const;
+  bool full_list_is_empty() const;
+  bool free_list_is_nonempty() const;
+  bool full_list_is_nonempty() const;
+  bool in_free_list(const Node* node) const;
+  bool in_full_list(const Node* node) const;
+  bool in_mspace(const Node* node) const;
+
+  void add_to_free_list(NodePtr node);
+  void add_to_full_list(NodePtr node);
+
+  NodePtr remove_from_free_list();
+  NodePtr remove_from_full_list();
+
+  NodePtr clear_free_list();
+  NodePtr clear_full_list();
+
+  template <typename Processor>
+  void iterate(Processor& processor, bool full_list = true);
+
+  void decrement_free_list_count();
+
+  void register_full(NodePtr node, Thread* thread);
 };
 
 #endif // SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_HPP

src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,89 +25,51 @@
 #ifndef SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACERETRIEVAL_HPP
 #define SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACERETRIEVAL_HPP
 
-#include "memory/allocation.hpp"
-#include "jfr/recorder/repository/jfrChunkWriter.hpp"
-#include "jfr/recorder/storage/jfrBuffer.hpp"
-#include "jfr/utilities/jfrAllocation.hpp"
-#include "jfr/utilities/jfrTypes.hpp"
+#include "jfr/utilities/jfrIterator.hpp"
 
-/*
-* Some policy classes for getting mspace memory
-*/
+/* Some policy classes for getting mspace memory. */
 
 template <typename Mspace>
-class JfrMspaceRetrieval : AllStatic {
+class JfrMspaceRetrieval {
  public:
-  typedef typename Mspace::Type Type;
-  static Type* get(size_t size, Mspace* mspace, typename Mspace::Iterator& iterator, Thread* thread) {
+  typedef typename Mspace::Node Node;
+  static Node* acquire(Mspace* mspace, Thread* thread, size_t size) {
+    StopOnNullCondition<typename Mspace::FreeList> iterator(mspace->free_list());
     while (iterator.has_next()) {
-      Type* const t = iterator.next();
-      if (t->retired()) continue;
-      if (t->try_acquire(thread)) {
-        assert(!t->retired(), "invariant");
-        if (t->free_size() >= size) {
-          return t;
+      Node* const node = iterator.next();
+      if (node->retired()) continue;
+      if (node->try_acquire(thread)) {
+        assert(!node->retired(), "invariant");
+        if (node->free_size() >= size) {
+          return node;
         }
-        t->set_retired();
-        mspace->register_full(t, thread);
+        node->set_retired();
+        mspace->register_full(node, thread);
       }
     }
     return NULL;
   }
 };
 
 template <typename Mspace>
-class JfrMspaceAlternatingRetrieval {
- private:
-   // provides stochastic distribution over "deque" endpoints; racy is ok here
-  static bool _last_access;
+class JfrMspaceRemoveRetrieval : AllStatic {
  public:
-  typedef typename Mspace::Type Type;
-  static Type* get(size_t size, Mspace* mspace, Thread* thread) {
-    typename Mspace::Iterator iterator(mspace->free(), (_last_access = !_last_access) ? forward : backward);
-    return JfrMspaceRetrieval<Mspace>::get(size, mspace, iterator, thread);
-  }
-};
-
-template <typename Mspace>
-bool JfrMspaceAlternatingRetrieval<Mspace>::_last_access = false;
-
-template <typename Mspace>
-class JfrMspaceSequentialRetrieval {
- public:
-  typedef typename Mspace::Type Type;
-  static Type* get(size_t size, Mspace* mspace, Thread* thread) {
-    typename Mspace::Iterator iterator(mspace->free());
-    return JfrMspaceRetrieval<Mspace>::get(size, mspace, iterator, thread);
-  }
-};
-
-template <typename Mspace>
-class JfrExclusiveRetrieval : AllStatic {
-public:
-  typedef typename Mspace::Type Type;
-  static Type* get(size_t size, Mspace* mspace, typename Mspace::Iterator& iterator, Thread* thread) {
-    assert(mspace->is_locked(), "invariant");
-    if (iterator.has_next()) {
-      Type* const t = iterator.next();
-      assert(!t->retired(), "invariant");
-      assert(t->identity() == NULL, "invariant");
-      assert(t->free_size() >= size, "invariant");
-      t->acquire(thread);
-      return t;
+  typedef typename Mspace::Node Node;
+  static Node* acquire(Mspace* mspace, Thread* thread, size_t size) {
+    StopOnNullConditionRemoval<typename Mspace::FreeList> iterator(mspace->free_list());
+    // it is the iterator that removes the nodes
+    while (iterator.has_next()) {
+      Node* const node = iterator.next();
+      if (node == NULL) return NULL;
+      mspace->decrement_free_list_count();
+      assert(node->free_size() >= size, "invariant");
+      assert(!node->retired(), "invariant");
+      assert(node->identity() == NULL, "invariant");
+      node->set_identity(thread);
+      return node;
     }
     return NULL;
   }
 };
 
-template <typename Mspace>
-class JfrThreadLocalRetrieval {
-public:
-  typedef typename Mspace::Type Type;
-  static Type* get(size_t size, Mspace* mspace, Thread* thread) {
-    typename Mspace::Iterator iterator(mspace->free(), forward);
-    return JfrExclusiveRetrieval<Mspace>::get(size, mspace, iterator, thread);
-  }
-};
-
 #endif // SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACERETRIEVAL_HPP

src/hotspot/share/jfr/recorder/storage/jfrMemorySpaceRetrieval.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,52 +25,51 @@
 #define SHARE_JFR_RECORDER_STORAGE_JFRSTORAGE_HPP
 
 #include "jfr/recorder/storage/jfrBuffer.hpp"
+#include "jfr/recorder/storage/jfrFullStorage.hpp"
 #include "jfr/recorder/storage/jfrMemorySpace.hpp"
 #include "jfr/recorder/storage/jfrMemorySpaceRetrieval.hpp"
+#include "jfr/utilities/jfrConcurrentQueue.hpp"
+#include "jfr/utilities/jfrLinkedList.hpp"
+#include "jfr/utilities/jfrNode.hpp"
+#include "jfr/utilities/jfrRelation.hpp"
 
 class JfrChunkWriter;
 class JfrPostBox;
 class JfrStorage;
 class JfrStorageControl;
 
-typedef JfrMemorySpace<JfrBuffer, JfrMspaceAlternatingRetrieval, JfrStorage> JfrStorageMspace;
-typedef JfrMemorySpace<JfrBuffer, JfrThreadLocalRetrieval, JfrStorage> JfrThreadLocalMspace;
-typedef JfrMemorySpace<JfrAgeNode, JfrMspaceSequentialRetrieval, JfrStorage> JfrStorageAgeMspace;
+typedef JfrMemorySpace<JfrStorage, JfrMspaceRetrieval, JfrLinkedList<JfrBuffer> > JfrStorageMspace;
+typedef JfrMemorySpace<JfrStorage, JfrMspaceRemoveRetrieval, JfrConcurrentQueue<JfrBuffer>, JfrLinkedList<JfrBuffer> > JfrThreadLocalMspace;
+typedef JfrFullStorage<JfrBuffer*, JfrValueNode> JfrFullList;
 
 //
 // Responsible for providing backing storage for writing events.
 //
 class JfrStorage : public JfrCHeapObj {
  public:
-  typedef JfrStorageMspace::Type Buffer;
+  typedef JfrStorageMspace::Node    Buffer;
+  typedef JfrStorageMspace::NodePtr BufferPtr;
+
  private:
   JfrStorageControl* _control;
   JfrStorageMspace* _global_mspace;
   JfrThreadLocalMspace* _thread_local_mspace;
-  JfrStorageMspace* _transient_mspace;
-  JfrStorageAgeMspace* _age_mspace;
+  JfrFullList* _full_list;
   JfrChunkWriter& _chunkwriter;
   JfrPostBox& _post_box;
 
-  // mspace callbacks
-  void register_full(Buffer* t, Thread* thread);
-  void lock();
-  void unlock();
-  DEBUG_ONLY(bool is_locked() const;)
-
-  Buffer* acquire_large(size_t size, Thread* t);
-  Buffer* acquire_transient(size_t size, Thread* thread);
-  bool flush_regular_buffer(Buffer* const buffer, Thread* t);
-  Buffer* flush_regular(Buffer* cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* t);
-  Buffer* flush_large(Buffer* cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* t);
-  Buffer* provision_large(Buffer* cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* t);
-  void release(Buffer* buffer, Thread* t);
+  BufferPtr acquire_large(size_t size, Thread* thread);
+  BufferPtr acquire_transient(size_t size, Thread* thread);
+  bool flush_regular_buffer(BufferPtr buffer, Thread* thread);
+  BufferPtr flush_regular(BufferPtr cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* thread);
+  BufferPtr flush_large(BufferPtr cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* thread);
+  BufferPtr provision_large(BufferPtr cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* thread);
+  void release(BufferPtr buffer, Thread* thread);
 
   size_t clear();
   size_t clear_full();
   size_t write_full();
   size_t write_at_safepoint();
-  size_t scavenge();
 
   JfrStorage(JfrChunkWriter& cw, JfrPostBox& post_box);
   ~JfrStorage();
@@ -80,19 +79,21 @@ class JfrStorage : public JfrCHeapObj {
   bool initialize();
   static void destroy();
 
+  // mspace callback
+  void register_full(BufferPtr buffer, Thread* thread);
+
  public:
-  static Buffer* acquire_thread_local(Thread* t, size_t size = 0);
-  static void release_thread_local(Buffer* buffer, Thread* t);
-  void release_large(Buffer* const buffer, Thread* t);
-  static Buffer* flush(Buffer* cur, size_t used, size_t req, bool native, Thread* t);
-  void discard_oldest(Thread* t);
+  static BufferPtr acquire_thread_local(Thread* thread, size_t size = 0);
+  static void release_thread_local(BufferPtr buffer, Thread* thread);
+  void release_large(BufferPtr buffer, Thread* thread);
+  static BufferPtr flush(BufferPtr cur, size_t used, size_t req, bool native, Thread* thread);
+  void discard_oldest(Thread* thread);
   static JfrStorageControl& control();
-
   size_t write();
 
   friend class JfrRecorder;
   friend class JfrRecorderService;
-  template <typename, template <typename> class, typename>
+  template <typename, template <typename> class, typename, typename>
   friend class JfrMemorySpace;
 };
 

src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,40 +25,15 @@
 #include "precompiled.hpp"
 #include "jfr/recorder/storage/jfrStorageControl.hpp"
 #include "runtime/atomic.hpp"
-#include "runtime/mutexLocker.hpp"
-
-// returns the updated value
-static jlong atomic_add(size_t value, size_t volatile* const dest) {
-  size_t compare_value;
-  size_t exchange_value;
-  do {
-    compare_value = *dest;
-    exchange_value = compare_value + value;
-  } while (Atomic::cmpxchg(dest, compare_value, exchange_value) != compare_value);
-  return exchange_value;
-}
-
-static jlong atomic_dec(size_t volatile* const dest) {
-  size_t compare_value;
-  size_t exchange_value;
-  do {
-    compare_value = *dest;
-    assert(compare_value >= 1, "invariant");
-    exchange_value = compare_value - 1;
-  } while (Atomic::cmpxchg(dest, compare_value, exchange_value) != compare_value);
-  return exchange_value;
-}
 
 const size_t max_lease_factor = 2;
 JfrStorageControl::JfrStorageControl(size_t global_count_total, size_t in_memory_discard_threshold) :
   _global_count_total(global_count_total),
   _full_count(0),
   _global_lease_count(0),
-  _dead_count(0),
   _to_disk_threshold(0),
   _in_memory_discard_threshold(in_memory_discard_threshold),
   _global_lease_threshold(global_count_total / max_lease_factor),
-  _scavenge_threshold(0),
   _to_disk(false) {}
 
 bool JfrStorageControl::to_disk() const {
@@ -73,21 +48,24 @@ size_t JfrStorageControl::full_count() const {
   return _full_count;
 }
 
-// mutexed access
-size_t JfrStorageControl::increment_full() {
-  assert(JfrBuffer_lock->owned_by_self(), "invariant");
-  return ++_full_count;
+bool JfrStorageControl::increment_full() {
+  const size_t result = Atomic::add(&_full_count, (size_t)1);
+  return to_disk() && result > _to_disk_threshold;
 }
 
 size_t JfrStorageControl::decrement_full() {
-  assert(JfrBuffer_lock->owned_by_self(), "invariant");
   assert(_full_count > 0, "invariant");
-  return --_full_count;
+  size_t current;
+  size_t exchange;
+  do {
+    current = _full_count;
+    exchange = current - 1;
+  } while (Atomic::cmpxchg(&_full_count, current, exchange) != current);
+  return exchange;
 }
 
 void JfrStorageControl::reset_full() {
-  assert(JfrBuffer_lock->owned_by_self(), "invariant");
-  _full_count = 0;
+  Atomic::store(&_full_count, (size_t)0);
 }
 
 bool JfrStorageControl::should_post_buffer_full_message() const {
@@ -98,42 +76,24 @@ bool JfrStorageControl::should_discard() const {
   return !to_disk() && full_count() >= _in_memory_discard_threshold;
 }
 
-// concurrent with accuracy requirement
-
 size_t JfrStorageControl::global_lease_count() const {
   return Atomic::load(&_global_lease_count);
 }
 
 size_t JfrStorageControl::increment_leased() {
-  return atomic_add(1, &_global_lease_count);
+  return Atomic::add(&_global_lease_count, (size_t)1);
 }
 
 size_t JfrStorageControl::decrement_leased() {
-  return atomic_dec(&_global_lease_count);
+  size_t current;
+  size_t exchange;
+  do {
+    current = _global_lease_count;
+    exchange = current - 1;
+  } while (Atomic::cmpxchg(&_global_lease_count, current, exchange) != current);
+  return exchange;
 }
 
 bool JfrStorageControl::is_global_lease_allowed() const {
   return global_lease_count() <= _global_lease_threshold;
 }
-
-// concurrent with lax requirement
-
-size_t JfrStorageControl::dead_count() const {
-  return _dead_count;
-}
-
-size_t JfrStorageControl::increment_dead() {
-  return atomic_add(1, &_dead_count);
-}
-
-size_t JfrStorageControl::decrement_dead() {
-  return atomic_dec(&_dead_count);
-}
-
-bool JfrStorageControl::should_scavenge() const {
-  return dead_count() >= _scavenge_threshold;
-}
-
-void JfrStorageControl::set_scavenge_threshold(size_t number_of_dead_buffers) {
-  _scavenge_threshold = number_of_dead_buffers;
-}

src/hotspot/share/jfr/recorder/storage/jfrStorage.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,11 +32,9 @@ class JfrStorageControl : public JfrCHeapObj {
   size_t _global_count_total;
   size_t _full_count;
   volatile size_t _global_lease_count;
-  volatile size_t _dead_count;
   size_t _to_disk_threshold;
   size_t _in_memory_discard_threshold;
   size_t _global_lease_threshold;
-  size_t _scavenge_threshold;
   bool _to_disk;
 
  public:
@@ -46,7 +44,7 @@ class JfrStorageControl : public JfrCHeapObj {
   bool to_disk() const;
 
   size_t full_count() const;
-  size_t increment_full();
+  bool increment_full();
   size_t decrement_full();
   void   reset_full();
   bool should_post_buffer_full_message() const;
@@ -56,13 +54,6 @@ class JfrStorageControl : public JfrCHeapObj {
   size_t increment_leased();
   size_t decrement_leased();
   bool is_global_lease_allowed() const;
-
-  size_t dead_count() const;
-  size_t increment_dead();
-  size_t decrement_dead();
-
-  void set_scavenge_threshold(size_t number_of_dead_buffers);
-  bool should_scavenge() const;
 };
 
 #endif // SHARE_JFR_RECORDER_STORAGE_JFRSTORAGECONTROL_HPP

src/hotspot/share/jfr/recorder/storage/jfrStorageControl.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -189,4 +189,15 @@ class DiscardOp {
   size_t size() const { return _operation.size(); }
 };
 
+template <typename Operation>
+class ExclusiveDiscardOp : private DiscardOp<Operation> {
+public:
+  typedef typename Operation::Type Type;
+  ExclusiveDiscardOp(jfr_operation_mode mode = concurrent) : DiscardOp<Operation>(mode) {}
+  bool process(Type* t);
+  size_t processed() const { return DiscardOp<Operation>::processed(); }
+  size_t elements() const { return DiscardOp<Operation>::elements(); }
+  size_t size() const { return DiscardOp<Operation>::size(); }
+};
+
 #endif // SHARE_JFR_RECORDER_STORAGE_JFRSTORAGEUTILS_HPP

src/hotspot/share/jfr/recorder/storage/jfrStorageControl.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -52,15 +52,24 @@ inline size_t get_unflushed_size(const u1* top, Type* t) {
 
 template <typename Operation>
 inline bool ConcurrentWriteOp<Operation>::process(typename Operation::Type* t) {
+  const bool is_retired = t->retired();
   // acquire_critical_section_top() must be read before pos() for stable access
-  const u1* const top = t->acquire_critical_section_top();
+  const u1* const top = is_retired ? t->top() : t->acquire_critical_section_top();
   const size_t unflushed_size = get_unflushed_size(top, t);
   if (unflushed_size == 0) {
-    t->release_critical_section_top(top);
+    if (is_retired) {
+      t->set_top(top);
+    } else {
+      t->release_critical_section_top(top);
+    }
     return true;
   }
   const bool result = _operation.write(t, top, unflushed_size);
-  t->release_critical_section_top(top + unflushed_size);
+  if (is_retired) {
+    t->set_top(top + unflushed_size);
+  } else {
+    t->release_critical_section_top(top + unflushed_size);
+  }
   return result;
 }
 
@@ -119,4 +128,12 @@ inline bool DiscardOp<Operation>::process(typename Operation::Type* t) {
   return result;
 }
 
+template <typename Operation>
+inline bool ExclusiveDiscardOp<Operation>::process(typename Operation::Type* t) {
+  retired_sensitive_acquire(t);
+  assert(t->acquired_by_self() || t->retired(), "invariant");
+  // User is required to ensure proper release of the acquisition
+  return DiscardOp<Operation>::process(t);
+}
+
 #endif // SHARE_JFR_RECORDER_STORAGE_JFRSTORAGEUTILS_INLINE_HPP

src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,14 +31,14 @@
 #include "jfr/recorder/storage/jfrStorageUtils.inline.hpp"
 #include "jfr/recorder/stringpool/jfrStringPool.hpp"
 #include "jfr/recorder/stringpool/jfrStringPoolWriter.hpp"
+#include "jfr/utilities/jfrLinkedList.inline.hpp"
 #include "jfr/utilities/jfrTypes.hpp"
 #include "logging/log.hpp"
 #include "runtime/atomic.hpp"
-#include "runtime/mutexLocker.hpp"
 #include "runtime/safepoint.hpp"
 #include "runtime/thread.inline.hpp"
 
-typedef JfrStringPool::Buffer* BufferPtr;
+typedef JfrStringPool::BufferPtr BufferPtr;
 
 static JfrStringPool* _instance = NULL;
 static uint64_t store_generation = 0;
@@ -48,6 +48,7 @@ inline void set_generation(uint64_t value, uint64_t* const dest) {
   assert(dest != NULL, "invariant");
   Atomic::release_store(dest, value);
 }
+
 static void increment_store_generation() {
   const uint64_t current_serialized = Atomic::load_acquire(&serialized_generation);
   const uint64_t current_stored = Atomic::load_acquire(&store_generation);
@@ -88,14 +89,11 @@ void JfrStringPool::destroy() {
   _instance = NULL;
 }
 
-JfrStringPool::JfrStringPool(JfrChunkWriter& cw) : _free_list_mspace(NULL), _lock(NULL), _chunkwriter(cw) {}
+JfrStringPool::JfrStringPool(JfrChunkWriter& cw) : _mspace(NULL), _chunkwriter(cw) {}
 
 JfrStringPool::~JfrStringPool() {
-  if (_free_list_mspace != NULL) {
-    delete _free_list_mspace;
-  }
-  if (_lock != NULL) {
-    delete _lock;
+  if (_mspace != NULL) {
+    delete _mspace;
   }
 }
 
@@ -104,14 +102,9 @@ static const size_t string_pool_cache_count = 2;
 static const size_t string_pool_buffer_size = 512 * K;
 
 bool JfrStringPool::initialize() {
-  assert(_free_list_mspace == NULL, "invariant");
-  _free_list_mspace = new JfrStringPoolMspace(string_pool_buffer_size, unlimited_mspace_size, string_pool_cache_count, this);
-  if (_free_list_mspace == NULL || !_free_list_mspace->initialize()) {
-    return false;
-  }
-  assert(_lock == NULL, "invariant");
-  _lock = new Mutex(Monitor::leaf - 1, "Checkpoint mutex", Mutex::_allow_vm_block_flag, Monitor::_safepoint_check_never);
-  return _lock != NULL;
+  assert(_mspace == NULL, "invariant");
+  _mspace = create_mspace<JfrStringPoolMspace>(string_pool_buffer_size, unlimited_mspace_size, string_pool_cache_count, this);
+  return _mspace != NULL;
 }
 
 /*
@@ -125,7 +118,11 @@ static void release(BufferPtr buffer, Thread* thread) {
   assert(buffer->lease(), "invariant");
   assert(buffer->acquired_by_self(), "invariant");
   buffer->clear_lease();
-  buffer->release();
+  if (buffer->transient()) {
+    buffer->set_retired();
+  } else {
+    buffer->release();
+  }
 }
 
 BufferPtr JfrStringPool::flush(BufferPtr old, size_t used, size_t requested, Thread* thread) {
@@ -137,7 +134,7 @@ BufferPtr JfrStringPool::flush(BufferPtr old, size_t used, size_t requested, Thr
     return NULL;
   }
   // migration of in-flight information
-  BufferPtr const new_buffer = lease_buffer(thread, used + requested);
+  BufferPtr const new_buffer = lease(thread, used + requested);
   if (new_buffer != NULL) {
     migrate_outstanding_writes(old, new_buffer, used, requested);
   }
@@ -147,10 +144,10 @@ BufferPtr JfrStringPool::flush(BufferPtr old, size_t used, size_t requested, Thr
 
 static const size_t lease_retry = 10;
 
-BufferPtr JfrStringPool::lease_buffer(Thread* thread, size_t size /* 0 */) {
-  BufferPtr buffer = mspace_get_free_lease_with_retry(size, instance()._free_list_mspace, lease_retry, thread);
+BufferPtr JfrStringPool::lease(Thread* thread, size_t size /* 0 */) {
+  BufferPtr buffer = mspace_get_free_lease_with_retry(size, instance()._mspace, lease_retry, thread);
   if (buffer == NULL) {
-    buffer = mspace_allocate_transient_lease_to_free(size,  instance()._free_list_mspace, thread);
+    buffer = mspace_allocate_transient_lease_to_full(size,  instance()._mspace, thread);
   }
   assert(buffer->acquired_by_self(), "invariant");
   assert(buffer->lease(), "invariant");
@@ -210,18 +207,23 @@ typedef StringPoolOp<UnBufferedWriteToChunk> WriteOperation;
 typedef StringPoolOp<StringPoolDiscarderStub> DiscardOperation;
 typedef ExclusiveOp<WriteOperation> ExclusiveWriteOperation;
 typedef ExclusiveOp<DiscardOperation> ExclusiveDiscardOperation;
-typedef ReleaseOp<JfrStringPoolMspace> StringPoolReleaseOperation;
-typedef CompositeOperation<ExclusiveWriteOperation, StringPoolReleaseOperation> StringPoolWriteOperation;
-typedef CompositeOperation<ExclusiveDiscardOperation, StringPoolReleaseOperation> StringPoolDiscardOperation;
+typedef ReleaseOp<JfrStringPoolMspace> StringPoolReleaseFreeOperation;
+typedef ScavengingReleaseOp<JfrStringPoolMspace> StringPoolReleaseFullOperation;
+typedef CompositeOperation<ExclusiveWriteOperation, StringPoolReleaseFreeOperation> StringPoolWriteFreeOperation;
+typedef CompositeOperation<ExclusiveWriteOperation, StringPoolReleaseFullOperation> StringPoolWriteFullOperation;
+typedef CompositeOperation<ExclusiveDiscardOperation, StringPoolReleaseFreeOperation> StringPoolDiscardFreeOperation;
+typedef CompositeOperation<ExclusiveDiscardOperation, StringPoolReleaseFullOperation> StringPoolDiscardFullOperation;
 
 size_t JfrStringPool::write() {
   Thread* const thread = Thread::current();
   WriteOperation wo(_chunkwriter, thread);
   ExclusiveWriteOperation ewo(wo);
-  StringPoolReleaseOperation spro(_free_list_mspace, thread, false);
-  StringPoolWriteOperation spwo(&ewo, &spro);
-  assert(_free_list_mspace->is_full_empty(), "invariant");
-  process_free_list(spwo, _free_list_mspace);
+  StringPoolReleaseFreeOperation free_release_op(_mspace);
+  StringPoolWriteFreeOperation free_op(&ewo, &free_release_op);
+  process_free_list(free_op, _mspace);
+  StringPoolReleaseFullOperation full_release_op(_mspace);
+  StringPoolWriteFullOperation full_op(&ewo, &full_release_op);
+  process_full_list(full_op, _mspace);
   return wo.processed();
 }
 
@@ -234,31 +236,18 @@ size_t JfrStringPool::clear() {
   increment_serialized_generation();
   DiscardOperation discard_operation;
   ExclusiveDiscardOperation edo(discard_operation);
-  StringPoolReleaseOperation spro(_free_list_mspace, Thread::current(), false);
-  StringPoolDiscardOperation spdo(&edo, &spro);
-  assert(_free_list_mspace->is_full_empty(), "invariant");
-  process_free_list(spdo, _free_list_mspace);
+  StringPoolReleaseFreeOperation free_release_op(_mspace);
+  StringPoolDiscardFreeOperation free_op(&edo, &free_release_op);
+  process_free_list(free_op, _mspace);
+  StringPoolReleaseFullOperation full_release_op(_mspace);
+  StringPoolDiscardFullOperation full_op(&edo, &full_release_op);
+  process_full_list(full_op, _mspace);
   return discard_operation.processed();
 }
 
-void JfrStringPool::register_full(BufferPtr t, Thread* thread) {
+void JfrStringPool::register_full(BufferPtr buffer, Thread* thread) {
   // nothing here at the moment
-  assert(t != NULL, "invariant");
-  assert(t->acquired_by(thread), "invariant");
-  assert(t->retired(), "invariant");
-}
-
-void JfrStringPool::lock() {
-  assert(!_lock->owned_by_self(), "invariant");
-  _lock->lock_without_safepoint_check();
-}
-
-void JfrStringPool::unlock() {
-  _lock->unlock();
-}
-
-#ifdef ASSERT
-bool JfrStringPool::is_locked() const {
-  return _lock->owned_by_self();
+  assert(buffer != NULL, "invariant");
+  assert(buffer->acquired_by(thread), "invariant");
+  assert(buffer->retired(), "invariant");
 }
-#endif

src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,12 +29,12 @@
 #include "jfr/recorder/storage/jfrMemorySpace.hpp"
 #include "jfr/recorder/storage/jfrMemorySpaceRetrieval.hpp"
 #include "jfr/recorder/stringpool/jfrStringPoolBuffer.hpp"
+#include "jfr/utilities/jfrLinkedList.hpp"
 
 class JfrChunkWriter;
 class JfrStringPool;
-class Mutex;
 
-typedef JfrMemorySpace<JfrStringPoolBuffer, JfrMspaceSequentialRetrieval, JfrStringPool> JfrStringPoolMspace;
+typedef JfrMemorySpace<JfrStringPool, JfrMspaceRetrieval, JfrLinkedList<JfrStringPoolBuffer> > JfrStringPoolMspace;
 
 //
 // Although called JfrStringPool, a more succinct description would be
@@ -48,21 +48,15 @@ class JfrStringPool : public JfrCHeapObj {
   size_t write();
   size_t write_at_safepoint();
   size_t clear();
+  typedef JfrStringPoolMspace::Node    Buffer;
+  typedef JfrStringPoolMspace::NodePtr BufferPtr;
 
-  typedef JfrStringPoolMspace::Type Buffer;
  private:
-  JfrStringPoolMspace* _free_list_mspace;
-  Mutex* _lock;
+  JfrStringPoolMspace* _mspace;
   JfrChunkWriter& _chunkwriter;
 
-  // mspace callback
-  void register_full(Buffer* t, Thread* thread);
-  void lock();
-  void unlock();
-  DEBUG_ONLY(bool is_locked() const;)
-
-  static Buffer* lease_buffer(Thread* thread, size_t size = 0);
-  static Buffer* flush(Buffer* old, size_t used, size_t requested, Thread* t);
+  static BufferPtr lease(Thread* thread, size_t size = 0);
+  static BufferPtr flush(BufferPtr old, size_t used, size_t requested, Thread* thread);
 
   JfrStringPool(JfrChunkWriter& cw);
   ~JfrStringPool();
@@ -73,11 +67,14 @@ class JfrStringPool : public JfrCHeapObj {
   static void destroy();
   static bool is_modified();
 
+  // mspace callback
+  void register_full(BufferPtr buffer, Thread* thread);
+
   friend class JfrRecorder;
   friend class JfrRecorderService;
   friend class JfrStringPoolFlush;
   friend class JfrStringPoolWriter;
-  template <typename, template <typename> class, typename>
+  template <typename, template <typename> class, typename, typename>
   friend class JfrMemorySpace;
 };
 

src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -41,6 +41,11 @@ class JfrStringPoolBuffer : public JfrBuffer {
   void increment(uint64_t value);
   void set_string_pos(uint64_t value);
   void set_string_top(uint64_t value);
+
+  template <typename, typename>
+  friend class JfrLinkedList;
+  template <typename, typename>
+  friend class JfrConcurrentLinkedList;
 };
 
 #endif // SHARE_JFR_RECORDER_STRINGPOOL_JFRSTRINGPOOLBUFFER_HPP

src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,11 +28,11 @@
 #include "jfr/writers/jfrEventWriterHost.inline.hpp"
 #include "jfr/writers/jfrMemoryWriterHost.inline.hpp"
 
-JfrStringPoolFlush::JfrStringPoolFlush(Type* old, size_t used, size_t requested, Thread* t) :
-  _result(JfrStringPool::flush(old, used, requested, t)) {}
+JfrStringPoolFlush::JfrStringPoolFlush(Type* old, size_t used, size_t requested, Thread* thread) :
+  _result(JfrStringPool::flush(old, used, requested, thread)) {}
 
 JfrStringPoolWriter::JfrStringPoolWriter(Thread* thread) :
-  JfrStringPoolWriterBase(JfrStringPool::lease_buffer(thread), thread), _nof_strings(0) {}
+  JfrStringPoolWriterBase(JfrStringPool::lease(thread), thread), _nof_strings(0) {}
 
 JfrStringPoolWriter::~JfrStringPoolWriter() {
   assert(this->is_acquired(), "invariant");

src/hotspot/share/jfr/recorder/stringpool/jfrStringPoolBuffer.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,8 +31,8 @@
 #include "runtime/thread.inline.hpp"
 #include "utilities/debug.hpp"
 
-JfrFlush::JfrFlush(JfrStorage::Buffer* old, size_t used, size_t requested, Thread* t) :
-  _result(JfrStorage::flush(old, used, requested, true, t)) {
+JfrFlush::JfrFlush(JfrStorage::BufferPtr old, size_t used, size_t requested, Thread* thread) :
+  _result(JfrStorage::flush(old, used, requested, true, thread)) {
 }
 
 template <typename T>
@@ -61,24 +61,24 @@ bool jfr_has_stacktrace_enabled(JfrEventId id) {
   return JfrEventSetting::has_stacktrace(id);
 }
 
-void jfr_conditional_flush(JfrEventId id, size_t size, Thread* t) {
-  if (t->jfr_thread_local()->has_native_buffer()) {
-    JfrStorage::Buffer* const buffer = t->jfr_thread_local()->native_buffer();
+void jfr_conditional_flush(JfrEventId id, size_t size, Thread* thread) {
+  if (thread->jfr_thread_local()->has_native_buffer()) {
+    JfrStorage::BufferPtr buffer = thread->jfr_thread_local()->native_buffer();
     if (LessThanSize<JfrStorage::Buffer>::evaluate(buffer, size)) {
-      JfrFlush f(buffer, 0, 0, t);
+      JfrFlush f(buffer, 0, 0, thread);
     }
   }
 }
 
-bool jfr_save_stacktrace(Thread* t) {
-  JfrThreadLocal* const tl = t->jfr_thread_local();
+bool jfr_save_stacktrace(Thread* thread) {
+  JfrThreadLocal* const tl = thread->jfr_thread_local();
   if (tl->has_cached_stack_trace()) {
     return false; // no ownership
   }
-  tl->set_cached_stack_trace_id(JfrStackTraceRepository::record(t));
+  tl->set_cached_stack_trace_id(JfrStackTraceRepository::record(thread));
   return true;
 }
 
-void jfr_clear_stacktrace(Thread* t) {
-  t->jfr_thread_local()->clear_cached_stack_trace();
+void jfr_clear_stacktrace(Thread* thread) {
+  thread->jfr_thread_local()->clear_cached_stack_trace();
 }

src/hotspot/share/jfr/recorder/stringpool/jfrStringPoolWriter.cpp

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_HPP
+#define SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_HPP
+
+#include "jfr/utilities/jfrAllocation.hpp"
+
+/*
+* This implementation is a derivation from Harris
+* https://www.cl.cam.ac.uk/research/srg/netos/papers/2001-caslists.pdf
+*
+* A concurrent LIFO structure can be built using the pair:
+*
+*   insert_head() and remove()
+*
+* The LIFO algorithm is non-blocking, more specifically wait-free.
+* When combined with a system for safe memory reclamation, where a thread will require
+* to know if other threads are possibly reading the memory that is to be reclaimed (more below),
+* a potential wait point is introduced, so technically, we are no longer wait-free.
+* The combination is still lock-free, but since it is no longer pure non-blocking,
+* we instead say the solution is concurrent.
+*
+* It is also possible to build a FIFO structure using the pair:
+*
+*   insert_tail() and remove()
+*
+* To allow FIFO, the solution extends support to mark, or reserve a node, not only as part of deletions
+* as with the LIFO case, but also, to enable tail insertions.
+*
+* Compared to the LIFO algorithm, the FIFO algorithm is not non-blocking, because inserts to the tail block,
+* making it not lock-free. remove() is lock-free up until the last node in the list. In practice, the FIFO
+* solution can be used in certain ways that very closely approximate non-blocking, for example, situations
+* involving a single producer and multiple consumers.
+*
+* Although the FIFO algorithm is not non-blocking, it includes an optimization for remove() that is attractive:
+* In the LIFO case, a slow path taken as the result of a failed excision would have to re-traverse the list
+* to find the updated adjacent node pair for the already marked node. However, that node might already have
+* been excised by some other thread, letting the thread potentially traverse the entire list just to discover
+* it is no longer present (not an issue if the list is ordered by a key, then traversal is only to node >= key).
+* In the FIFO case, premised on the invariant that inserts only come in from the tail, it is possible to prove
+* a failed cas not to be the result of a new node inserted as with the LIFO case. With FIFO, there is only a single
+* failure mode, i.e. some other thread excised the node already. Therefore, in the FIFO case, we skip the slow-path search pass.
+*
+* We say that the FIFO solution is "mostly" concurrent, in certain situations.
+*
+* Safe memory reclamation is based on a reference tracking scheme based on versions, implemented using JfrVersion.
+* An access to the list is "version controlled", with clients checking out the latest version of the list.
+* Destructive modifications made by clients, i.e. deletions, are committed to describe new versions of the list.
+* Before reclamation, a client inspects the versioning system to ensure checkouts for versions strictly
+* less than the version of the modification have all been relinquished. See utilities/JfrVersion.hpp.
+*
+* Insertions can only take place from one end of the list, head or tail, exclusively.
+* Specializations, a.k.a clients, must ensure this requirement.
+*/
+
+template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy = JfrCHeapObj>
+class JfrConcurrentLinkedListHost : public AllocPolicy {
+ private:
+  Client* _client;
+  typedef typename Client::Node Node;
+  typedef Node* NodePtr;
+  typedef const Node* ConstNodePtr;
+  typedef typename Client::VersionSystem::Type VersionType;
+  typedef typename Client::VersionSystem::Handle VersionHandle;
+ public:
+  JfrConcurrentLinkedListHost(Client* client);
+  bool initialize();
+  void insert_head(NodePtr node, NodePtr head, ConstNodePtr tail) const;
+  void insert_tail(NodePtr node, NodePtr head, NodePtr last, ConstNodePtr tail) const;
+  NodePtr remove(NodePtr head, ConstNodePtr tail, NodePtr last = NULL, bool insert_is_head = true);
+  template <typename Callback>
+  void iterate(NodePtr head, ConstNodePtr tail, Callback& cb);
+  bool in_list(ConstNodePtr node, NodePtr head, ConstNodePtr tail) const;
+};
+
+#endif // SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_HPP

src/hotspot/share/jfr/support/jfrFlush.cpp

@@ -0,0 +1,299 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP
+#define SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP
+
+#include "jfr/utilities/jfrConcurrentLinkedListHost.hpp"
+#include "jfr/utilities/jfrRelation.hpp"
+#include "jfr/utilities/jfrTypes.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/os.inline.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+/*
+ * The removal marker (i.e. the excision bit) is represented by '( )' as part of state description comments:
+ * node --> next becomes (node) --> next, when node is logically deleted.
+ */
+template <typename Node>
+inline Node* mark_for_removal(Node* node) {
+  assert(node != NULL, "invariant");
+  const Node* next = node->_next;
+  assert(next != NULL, "invariant");
+  Node* const unmasked_next = unmask(next);
+  return next == unmasked_next && cas(&node->_next, unmasked_next, set_excision_bit(unmasked_next)) ? unmasked_next : NULL;
+}
+
+/*
+ * The insertion marker (i.e. the insertion bit) is represented by '[ ]' as part of state description comments:
+ * "node --> next" becomes "[node} --> next", in an attempt to convey node as being exlusively reserved.
+ */
+template <typename Node>
+inline bool mark_for_insertion(Node* node, const Node* tail) {
+  assert(node != NULL, "invariant");
+  return node->_next == tail && cas(&node->_next, const_cast<Node*>(tail), set_insertion_bit(tail));
+}
+
+/*
+ * Find a predecessor and successor node pair where successor covers predecessor (adjacency).
+ */
+template <typename Node, typename VersionHandle, template <typename> class SearchPolicy>
+Node* find_adjacent(Node* head, const Node* tail, Node** predecessor, VersionHandle& version_handle, SearchPolicy<Node>& predicate) {
+  assert(head != NULL, "invariant");
+  assert(tail != NULL, "invariant");
+  assert(head != tail, "invariant");
+  while (true) {
+    Node* predecessor_next;
+    Node* current = head;
+    version_handle.checkout();
+    assert(version_handle.is_tracked(), "invariant");
+    Node* next = Atomic::load_acquire(&current->_next);
+    do {
+      assert(next != NULL, "invariant");
+      Node* const unmasked_next = unmask(next);
+      // 1A: Locate the first node to keep as predecessor.
+      if (!is_marked_for_removal(next)) {
+        *predecessor = current;
+        predecessor_next = unmasked_next;
+      }
+      // 1B: Locate the next node to keep as successor.
+      current = unmasked_next;
+      if (current == tail) break;
+      next = current->_next;
+    } while (predicate(current, next));
+    // current represents the successor node from here on out.
+    // 2: Check predecessor and successor node pair for adjacency.
+    if (predecessor_next == current) {
+      // Invariant: predecessor --> successor
+      return current;
+    }
+    // 3: Successor does not (yet) cover predecessor.
+    // Invariant: predecessor --> (logically excised nodes) --> successor
+    // Physically excise one or more logically excised nodes in-between.
+    if (cas(&(*predecessor)->_next, predecessor_next, current)) {
+      // Invariant: predecessor --> successor
+      return current;
+    }
+  }
+}
+
+template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
+JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::JfrConcurrentLinkedListHost(Client* client) : _client(client) {}
+
+template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
+bool JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::initialize() {
+  return true;
+}
+
+template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
+void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::insert_head(typename Client::Node* node,
+                                                                                 typename Client::Node* head,
+                                                                                 const typename Client::Node* tail) const {
+  Node* predecessor;
+  Node* successor;
+  HeadNode<Node> predicate(node);
+  VersionHandle version_handle = _client->get_version_handle();
+  while (true) {
+    // Find an adjacent predecessor and successor node pair.
+    successor = find_adjacent<Node, VersionHandle, HeadNode>(head, tail, &predecessor, version_handle, predicate);
+    assert(version_handle.is_tracked(), "invariant");
+    // Invariant (adjacency): predecessor --> successor
+    // Invariant (optional: key-based total order): predecessor->key() < key  && key <= successor->key().
+    // We can now attempt to insert the new node in-between.
+    node->_next = successor;
+    if (cas(&predecessor->_next, successor, node)) {
+      // Invariant: predecessor --> node --> successor
+      // An insert to head is a benign modification and will not need to be committed to the version control system.
+      return;
+    }
+  }
+}
+
+template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
+void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::insert_tail(typename Client::Node* node,
+                                                                                 typename Client::Node* head,
+                                                                                 typename Client::Node* last,
+                                                                                 const typename Client::Node* tail) const {
+  assert(node != NULL, "invariant");
+  assert(head != NULL, "invariant");
+  assert(last != NULL, "invarinat");
+  assert(tail != NULL, "invariant");
+  // Mark the new node to be inserted with the insertion marker already.
+  node->_next = set_insertion_bit(const_cast<NodePtr>(tail));
+  // Invariant: [node]--> tail
+  assert(is_marked_for_insertion(node->_next), "invariant");
+  NodePtr predecessor;
+  LastNode<Node> predicate;
+  VersionHandle version_handle = _client->get_version_handle();
+  while (true) {
+    // Find an adjacent predecessor and successor node pair, where the successor == tail
+    const NodePtr successor = find_adjacent<Node, VersionHandle, LastNode>(last, tail, &predecessor, version_handle, predicate);
+    assert(version_handle.is_tracked(), "invariant");
+    assert(successor == tail, "invariant");
+    // Invariant: predecessor --> successor
+    // We first attempt to mark the predecessor node to signal our intent of performing an insertion.
+    if (mark_for_insertion(predecessor, tail)) {
+      break;
+    }
+  }
+  // Predecessor node is claimed for insertion.
+  // Invariant: [predecessor] --> tail
+  assert(is_marked_for_insertion(predecessor->_next), "invariant");
+  assert(predecessor != head, "invariant");
+  if (Atomic::load_acquire(&last->_next) == predecessor) {
+    /* Even after we store the new node into the last->_next field, there is no race
+       because it is also marked with the insertion bit. */
+    last->_next = node;
+    // Invariant: last --> [node] --> tail
+    OrderAccess::storestore();
+    // Perform the link with the predecessor node, which by this store becomes visible for removal.
+    predecessor->_next = node;
+    // Invariant: predecessor --> [node] --> tail
+  } else {
+    assert(last == predecessor, "invariant");
+    last->_next = node;
+    // Invariant: last --> [node] --> tail
+    OrderAccess::storestore();
+    /* This implies the list is logically empty from the removal perspective.
+       cas is not needed here because inserts must not come in from the head side
+       concurrently with inserts from tail which are currently blocked by us.
+       Invariant (logical): head --> tail. */
+    head->_next = node;
+    // Invariant: head --> [node] --> tail
+  }
+  version_handle.release(); // release_store_fence
+  // Publish the inserted node by removing the insertion marker.
+  node->_next = const_cast<NodePtr>(tail);
+  // Invariant: last --> node --> tail (possibly also head --> node --> tail)
+}
+
+template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
+typename Client::Node* JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::remove(typename Client::Node* head,
+                                                                                              const typename Client::Node* tail,
+                                                                                              typename Client::Node* last /* NULL */,
+                                                                                              bool insert_is_head /* true */) {
+  assert(head != NULL, "invariant");
+  assert(tail != NULL, "invariant");
+  assert(head != tail, "invariant");
+  NodePtr predecessor;
+  NodePtr successor;
+  NodePtr successor_next;
+  SearchPolicy<Node> predicate;
+  VersionHandle version_handle = _client->get_version_handle();
+  while (true) {
+    // Find an adjacent predecessor and successor node pair.
+    successor = find_adjacent<Node, VersionHandle, SearchPolicy>(head, tail, &predecessor, version_handle, predicate);
+    assert(version_handle.is_tracked(), "invariant");
+    if (successor == tail) {
+      return NULL;
+    }
+    // Invariant: predecessor --> successor
+    // Invariant (optional: key-based total order): predecessor->key() < key  && key <= successor->key()
+    // It is the successor node that is to be removed.
+    // We first attempt to reserve (logically excise) the successor node.
+    successor_next = mark_for_removal(successor);
+    if (successor_next != NULL) {
+      break;
+    }
+  }
+  // Invariant: predecessor --> (successor) --> successor_next
+  // Successor node now logically excised.
+  assert(is_marked_for_removal(successor->_next), "invariant");
+  // Now attempt to physically excise the successor node.
+  // If the cas fails, we can optimize for the slow path if we know we are not performing
+  // insertions from the head. Then a failed cas results not from new a node being inserted,
+  // but only because another thread excised us already.
+  if (!cas(&predecessor->_next, successor, successor_next) && insert_is_head) {
+    // Physically excise using slow path, can be completed asynchronously by other threads.
+    Identity<Node> excise(successor);
+    find_adjacent<Node, VersionHandle, Identity>(head, tail, &predecessor, version_handle, excise);
+    assert(version_handle.is_tracked(), "invariant");
+  }
+  if (last != NULL && Atomic::load_acquire(&last->_next) == successor) {
+    guarantee(!insert_is_head, "invariant");
+    guarantee(successor_next == tail, "invariant");
+    LastNode<Node> excise;
+    find_adjacent<Node, VersionHandle, LastNode>(last, tail, &predecessor, version_handle, excise);
+    // Invariant: successor excised from last list
+  }
+  // Increment the current version so we can track when other threads have seen this update.
+  VersionType version = version_handle.increment();
+  version_handle.release(); // release_store_fence
+  // Rendezvous with checkouts for versions less than this version.
+  version_handle.await(version);
+  // At this point we know there can be no references onto the excised node. It is safe, enjoy it.
+  return successor;
+}
+
+template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
+bool JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::in_list(const typename Client::Node* node,
+                                                                             typename Client::Node* head,
+                                                                             const typename Client::Node* tail) const {
+  assert(head != NULL, "invariant");
+  assert(tail != NULL, "invariant");
+  assert(head != tail, "invariant");
+  VersionHandle version_handle = _client->get_version_handle();
+  const Node* current = head;
+  version_handle.checkout();
+  assert(version_handle.is_tracked(), "invariant");
+  const Node* next = Atomic::load_acquire(&current->_next);
+  while (true) {
+    if (!is_marked_for_removal(next)) {
+      if (current == node) {
+        return true;
+      }
+    }
+    current = unmask(next);
+    if (current == tail) break;
+    next = current->_next;
+  }
+  return false;
+}
+
+template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
+template <typename Callback>
+inline void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::iterate(typename Client::Node* head,
+                                                                                    const typename Client::Node* tail,
+                                                                                    Callback& cb) {
+  assert(head != NULL, "invariant");
+  assert(tail != NULL, "invariant");
+  assert(head != tail, "invariant");
+  VersionHandle version_handle = _client->get_version_handle();
+  NodePtr current = head;
+  version_handle.checkout();
+  assert(version_handle.is_tracked(), "invariant");
+  NodePtr next = Atomic::load_acquire(&current->_next);
+  while (true) {
+    if (!is_marked_for_removal(next)) {
+      if (!cb.process(current)) {
+        return;
+      }
+    }
+    current = unmask(next);
+    if (current == tail) break;
+    next = current->_next;
+  }
+}
+
+#endif // SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP

src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.hpp

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_HPP
+#define SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_HPP
+
+#include "jfr/utilities/jfrAllocation.hpp"
+#include "jfr/utilities/jfrConcurrentLinkedListHost.hpp"
+#include "jfr/utilities/jfrRelation.hpp"
+#include "jfr/utilities/jfrVersionSystem.hpp"
+
+ /*
+  * This is a thread-safe FIFO structure.
+  * Although not non-blocking, for certain scenarios,
+  * it can act as a close approximation, "mostly" concurrent.
+  * For a more detailed description of its properties,
+  * please see JfrConcurrentLinkedListHost.hpp.
+  */
+template <typename NodeType, typename AllocPolicy = JfrCHeapObj>
+class JfrConcurrentQueue : public AllocPolicy {
+ public:
+  typedef NodeType Node;
+  typedef NodeType* NodePtr;
+  typedef const NodeType* ConstNodePtr;
+  typedef JfrVersionSystem VersionSystem;
+  JfrConcurrentQueue();
+  bool initialize();
+  bool is_empty() const;
+  bool is_nonempty() const;
+  void add(NodePtr node);
+  NodePtr remove();
+  template <typename Callback>
+  void iterate(Callback& cb);
+  bool in_list(const Node* node) const;
+ private:
+  JfrConcurrentLinkedListHost<JfrConcurrentQueue<Node, AllocPolicy>, HeadNode>* _list;
+  Node _head;
+  Node _last;
+  const Node _tail;
+  JfrVersionSystem _version_system;
+  // callback for JfrConcurrentLinkedListHost
+  typename VersionSystem::Handle get_version_handle();
+  template <typename, template <typename> class, typename>
+  friend class JfrConcurrentLinkedListHost;
+};
+
+#endif // SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_HPP
+

src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP
+#define SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP
+
+#include "jfr/utilities/jfrConcurrentQueue.hpp"
+#include "jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
+#include "jfr/utilities/jfrVersionSystem.inline.hpp"
+
+template <typename NodeType, typename AllocPolicy>
+JfrConcurrentQueue<NodeType, AllocPolicy>::JfrConcurrentQueue() : _list(NULL), _head(), _last(), _tail(), _version_system() {
+  _head._next = const_cast<NodePtr>(&_tail);
+  _last._next = const_cast<NodePtr>(&_tail);
+}
+
+template <typename NodeType, typename AllocPolicy>
+bool JfrConcurrentQueue<NodeType, AllocPolicy>::initialize() {
+  assert(_list == NULL, "invariant");
+  _list = new JfrConcurrentLinkedListHost<JfrConcurrentQueue<NodeType, AllocPolicy>, HeadNode, AllocPolicy>(this);
+  return _list != NULL && _list->initialize();
+}
+
+template <typename NodeType, typename AllocPolicy>
+inline bool JfrConcurrentQueue<NodeType, AllocPolicy>::is_empty() const {
+  return Atomic::load_acquire(&_head._next) == &_tail;
+}
+
+template <typename NodeType, typename AllocPolicy>
+inline bool JfrConcurrentQueue<NodeType, AllocPolicy>::is_nonempty() const {
+  return !is_empty();
+}
+
+template <typename NodeType, typename AllocPolicy>
+void JfrConcurrentQueue<NodeType, AllocPolicy>::add(typename JfrConcurrentQueue<NodeType, AllocPolicy>::NodePtr node) {
+  _list->insert_tail(node, &_head, &_last, &_tail);
+}
+
+template <typename NodeType, typename AllocPolicy>
+typename JfrConcurrentQueue<NodeType, AllocPolicy>::NodePtr JfrConcurrentQueue<NodeType, AllocPolicy>::remove() {
+  return _list->remove(&_head, &_tail, &_last, false);
+}
+
+template <typename NodeType, typename AllocPolicy>
+template <typename Callback>
+void JfrConcurrentQueue<NodeType, AllocPolicy>::iterate(Callback& cb) {
+  _list->iterate(&_head, &_tail, cb);
+}
+
+template <typename NodeType, typename AllocPolicy>
+inline JfrVersionSystem::Handle JfrConcurrentQueue<NodeType, AllocPolicy>::get_version_handle() {
+  return _version_system.get_handle();
+}
+
+template <typename NodeType, typename AllocPolicy>
+bool JfrConcurrentQueue<NodeType, AllocPolicy>::in_list(const NodeType* node) const {
+  assert(node != NULL, "invariant");
+  return _list->in_list(node, const_cast<NodePtr>(&_head), &_tail);
+}
+
+#endif // SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP

src/hotspot/share/jfr/utilities/jfrConcurrentQueue.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,81 +27,93 @@
 
 #include "memory/allocation.hpp"
 
-enum jfr_iter_direction {
-  forward = 1,
-  backward
+template <typename List>
+class StopOnNullCondition {
+  typedef typename List::Node Node;
+ private:
+  List& _list;
+  mutable Node* _node;
+ public:
+  StopOnNullCondition(List& list) : _list(list), _node(list.head()) {}
+  bool has_next() const {
+    return _node != NULL;
+  }
+  Node* next() const {
+    assert(_node != NULL, "invariant");
+    Node* temp = _node;
+    _node = (Node*)_node->_next;
+    return temp;
+  }
 };
 
-template <typename Node>
-class StopOnNullCondition : public AllStatic {
+template <typename List>
+class StopOnNullConditionRemoval {
+  typedef typename List::Node Node;
+ private:
+  List& _list;
+  mutable Node* _node;
  public:
-  static bool has_next(const Node* node) {
-    return node != NULL;
+  StopOnNullConditionRemoval(List& list) : _list(list), _node(NULL) {}
+  bool has_next() const {
+    _node = _list.remove();
+    return _node != NULL;
+  }
+  Node* next() const {
+    assert(_node != NULL, "invariant");
+    return _node;
   }
 };
 
 template <typename List, template <typename> class ContinuationPredicate>
 class Navigator {
  public:
   typedef typename List::Node Node;
-  typedef jfr_iter_direction Direction;
-  Navigator(List& list, Direction direction) :
-    _list(list), _node(direction == forward ? list.head() : list.tail()), _direction(direction) {}
+  Navigator(List& list) : _continuation(list) {}
   bool has_next() const {
-    return ContinuationPredicate<Node>::has_next(_node);
-  }
-
-  bool direction_forward() const {
-    return _direction == forward;
+    return _continuation.has_next();
   }
-
   Node* next() const {
-    assert(_node != NULL, "invariant");
-    Node* temp = _node;
-    _node = direction_forward() ? (Node*)_node->next() : (Node*)_node->prev();
-    return temp;
-  }
-
-  void set_direction(Direction direction) {
-    _direction = direction;
-  }
-
-  void reset(Direction direction) {
-    set_direction(direction);
-    _node = direction_forward() ? _list.head() : _list.tail();
+    return _continuation.next();
   }
-
  private:
-  List& _list;
+  ContinuationPredicate<List> _continuation;
   mutable Node* _node;
-  Direction _direction;
 };
 
 template <typename List>
 class NavigatorStopOnNull : public Navigator<List, StopOnNullCondition> {
  public:
-  NavigatorStopOnNull(List& list, jfr_iter_direction direction = forward) : Navigator<List, StopOnNullCondition>(list, direction) {}
+  NavigatorStopOnNull(List& list) : Navigator<List, StopOnNullCondition>(list) {}
+};
+
+template <typename List>
+class NavigatorStopOnNullRemoval : public Navigator<List, StopOnNullConditionRemoval> {
+public:
+  NavigatorStopOnNullRemoval(List& list) : Navigator<List, StopOnNullConditionRemoval>(list) {}
 };
 
 template<typename List, template <typename> class Navigator, typename AP = StackObj>
 class IteratorHost : public AP {
  private:
   Navigator<List> _navigator;
-
  public:
-  typedef typename List::Node Node;
-  typedef jfr_iter_direction Direction;
-  IteratorHost(List& list, Direction direction = forward) : AP(), _navigator(list, direction) {}
-  void reset(Direction direction = forward) { _navigator.reset(direction); }
+  typedef typename List::NodePtr NodePtr;
+  IteratorHost(List& list) : AP(), _navigator(list) {}
+  void reset() { _navigator.reset(); }
   bool has_next() const { return _navigator.has_next(); }
-  Node* next() const { return _navigator.next(); }
-  void set_direction(Direction direction) { _navigator.set_direction(direction); }
+  NodePtr next() const { return _navigator.next(); }
 };
 
 template<typename List, typename AP = StackObj>
 class StopOnNullIterator : public IteratorHost<List, NavigatorStopOnNull, AP> {
  public:
-  StopOnNullIterator(List& list, jfr_iter_direction direction = forward) : IteratorHost<List, NavigatorStopOnNull, AP>(list, direction) {}
+  StopOnNullIterator(List& list) : IteratorHost<List, NavigatorStopOnNull, AP>(list) {}
+};
+
+template<typename List, typename AP = StackObj>
+class StopOnNullIteratorRemoval : public IteratorHost<List, NavigatorStopOnNullRemoval, AP> {
+public:
+  StopOnNullIteratorRemoval(List& list) : IteratorHost<List, NavigatorStopOnNullRemoval, AP>(list) {}
 };
 
 #endif // SHARE_JFR_UTILITIES_JFRITERATOR_HPP

src/hotspot/share/jfr/utilities/jfrConcurrentQueue.inline.hpp

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_JFR_UTILITIES_JFRLINKEDLIST_HPP
+#define SHARE_JFR_UTILITIES_JFRLINKEDLIST_HPP
+
+#include "jfr/utilities/jfrAllocation.hpp"
+
+/*
+ * This linked-list is thread-safe only for add,
+ * not for remove, iterate, excise and in_list.
+ * For multiple producers, single consumer.
+ */
+
+template <typename NodeType, typename AllocPolicy = JfrCHeapObj>
+class JfrLinkedList : public AllocPolicy {
+ public:
+  typedef NodeType Node;
+  typedef NodeType* NodePtr;
+  JfrLinkedList();
+  bool initialize();
+  bool is_empty() const;
+  bool is_nonempty() const;
+  void add(NodePtr node);
+  NodePtr remove();
+  template <typename Callback>
+  void iterate(Callback& cb);
+  NodePtr head() const;
+  NodePtr excise(NodePtr prev, NodePtr node);
+  bool in_list(const NodeType* node) const;
+ private:
+  NodePtr _head;
+};
+
+#endif // SHARE_JFR_UTILITIES_JFRLINKEDLIST_HPP

src/hotspot/share/jfr/utilities/jfrIterator.hpp

@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_JFR_UTILITIES_JFRLINKEDLIST_INLINE_HPP
+#define SHARE_JFR_UTILITIES_JFRLINKEDLIST_INLINE_HPP
+
+#include "jfr/utilities/jfrLinkedList.hpp"
+#include "runtime/atomic.hpp"
+
+template <typename NodeType, typename AllocPolicy>
+JfrLinkedList<NodeType, AllocPolicy>::JfrLinkedList() : _head(NULL) {}
+
+template <typename NodeType, typename AllocPolicy>
+bool JfrLinkedList<NodeType, AllocPolicy>::initialize() {
+  return true;
+}
+
+template <typename NodeType, typename AllocPolicy>
+inline NodeType* JfrLinkedList<NodeType, AllocPolicy>::head() const {
+  return (NodeType*)Atomic::load_acquire(&_head);
+}
+
+template <typename NodeType, typename AllocPolicy>
+inline bool JfrLinkedList<NodeType, AllocPolicy>::is_empty() const {
+  return NULL == head();
+}
+
+template <typename NodeType, typename AllocPolicy>
+inline bool JfrLinkedList<NodeType, AllocPolicy>::is_nonempty() const {
+  return !is_empty();
+}
+
+template <typename NodeType, typename AllocPolicy>
+inline void JfrLinkedList<NodeType, AllocPolicy>::add(NodeType* node) {
+  assert(node != NULL, "invariant");
+  NodePtr next;
+  do {
+    next = head();
+    node->_next = next;
+  } while (Atomic::cmpxchg(&_head, next, node) != next);
+}
+
+template <typename NodeType, typename AllocPolicy>
+inline NodeType* JfrLinkedList<NodeType, AllocPolicy>::remove() {
+  NodePtr node;
+  NodePtr next;
+  do {
+    node = head();
+    if (node == NULL) break;
+    next = (NodePtr)node->_next;
+  } while (Atomic::cmpxchg(&_head, node, next) != node);
+  return node;
+}
+
+template <typename NodeType, typename AllocPolicy>
+template <typename Callback>
+void JfrLinkedList<NodeType, AllocPolicy>::iterate(Callback& cb) {
+  NodePtr current = head();
+  while (current != NULL) {
+    NodePtr next = (NodePtr)current->_next;
+    if (!cb.process(current)) {
+      return;
+    }
+    current = next;
+  }
+}
+
+template <typename NodeType, typename AllocPolicy>
+NodeType* JfrLinkedList<NodeType, AllocPolicy>::excise(NodeType* prev, NodeType* node) {
+  NodePtr next = (NodePtr)node->_next;
+  if (prev == NULL) {
+    prev = Atomic::cmpxchg(&_head, node, next);
+    if (prev == node) {
+      return NULL;
+    }
+  }
+  assert(prev != NULL, "invariant");
+  while (prev->_next != node) {
+    prev = (NodePtr)prev->_next;
+  }
+  assert(prev->_next == node, "invariant");
+  prev->_next = next;
+  return prev;
+}
+
+template <typename NodeType, typename AllocPolicy>
+bool JfrLinkedList<NodeType, AllocPolicy>::in_list(const NodeType* node) const {
+  assert(node != NULL, "invariant");
+  const NodeType* current = head();
+  while (current != NULL) {
+    if (current == node) {
+      return true;
+    }
+    current = (NodeType*)current->_next;
+  }
+  return false;
+}
+
+#endif // SHARE_JFR_UTILITIES_JFRLINKEDLIST_INLINE_HPP

src/hotspot/share/jfr/utilities/jfrLinkedList.hpp

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_JFR_UTILITIES_JFRNODE_HPP
+#define SHARE_JFR_UTILITIES_JFRNODE_HPP
+
+#include "jfr/utilities/jfrTypes.hpp"
+#include "memory/allocation.hpp"
+#include "runtime/atomic.hpp"
+
+const uint64_t JFR_NODE_LOGICAL_EXCISION_BIT = 1;
+const uint64_t JFR_NODE_LOGICAL_INSERTION_BIT = 2;
+const uint64_t JFR_NODE_MASK = ~(JFR_NODE_LOGICAL_INSERTION_BIT | JFR_NODE_LOGICAL_EXCISION_BIT);
+
+template <typename Node>
+inline bool cas(Node** address, Node* current, Node* exchange) {
+  return Atomic::cmpxchg(address, current, exchange) == current;
+}
+
+template <typename Node>
+inline bool is_marked_for_removal(const Node* ptr) {
+  return ((uint64_t)ptr & JFR_NODE_LOGICAL_EXCISION_BIT) == JFR_NODE_LOGICAL_EXCISION_BIT;
+}
+
+template <typename Node>
+inline bool is_marked_for_insertion(const Node* ptr) {
+  return ((uint64_t)ptr & JFR_NODE_LOGICAL_INSERTION_BIT) == JFR_NODE_LOGICAL_INSERTION_BIT;
+}
+
+template <typename Node>
+inline Node* set_excision_bit(const Node* ptr) {
+  return (Node*)(((uint64_t)ptr) | JFR_NODE_LOGICAL_EXCISION_BIT);
+}
+
+template <typename Node>
+inline Node* set_insertion_bit(const Node* ptr) {
+  return (Node*)(((uint64_t)ptr) | JFR_NODE_LOGICAL_INSERTION_BIT);
+}
+
+template <typename Node>
+inline Node* unmask(const Node* ptr) {
+  return (Node*)(((uint64_t)ptr) & JFR_NODE_MASK);
+}
+
+template <typename Derived, typename Version = traceid>
+class JfrLinkedNode : public ResourceObj {
+ public:
+  typedef Version VersionType;
+  Derived* _next;
+  JfrLinkedNode() : _next(NULL) {}
+  JfrLinkedNode(JfrLinkedNode<Derived, VersionType>* next) : _next(next) {}
+};
+
+template <typename V>
+class JfrKeyIsThisNode : public JfrLinkedNode<JfrKeyIsThisNode<V> > {
+ private:
+  V _value;
+ public:
+  typedef V Value;
+  typedef const JfrKeyIsThisNode<V>* Key;
+  JfrKeyIsThisNode(const Value value = NULL) : JfrLinkedNode<JfrKeyIsThisNode<V> >(), _value(value) {}
+  Key key() const { return this; }
+  Value value() const { return _value; }
+  void set_value(Value value) { _value = value; }
+};
+
+template <typename V>
+class JfrValueNode : public JfrLinkedNode<JfrValueNode<V> > {
+ private:
+  V _value;
+ public:
+  typedef V Value;
+  typedef Value Key;
+  JfrValueNode(const Value value = NULL) : JfrLinkedNode<JfrValueNode<V> >(), _value(value) {}
+  Key key() const { return value(); }
+  Value value() const { return _value; }
+  void set_value(Value value) { _value = value; }
+};
+
+template <typename V>
+class JfrKeyIsFreeSizeNode : public JfrLinkedNode<JfrKeyIsFreeSizeNode<V> > {
+ private:
+  V _value;
+ public:
+  typedef V Value;
+  typedef size_t Key;
+  JfrKeyIsFreeSizeNode(const Value value = NULL) : JfrLinkedNode<JfrKeyIsFreeSizeNode<V> >(), _value(value) {}
+  Key key() const { return value()->free_size(); }
+  Value value() const { return _value; }
+  void set_value(Value value) { _value = value; }
+};
+
+#endif // SHARE_JFR_UTILITIES_JFRNODE_HPP