fixed deadlocks in JVMTI S/R support for cthreads and VTMT disabling mechanism

Author: sspitsyn

src/hotspot/share/prims/jvmtiEnvBase.cpp

@@ -655,7 +655,7 @@ JvmtiEnvBase::get_thread_state(oop thread_oop, JavaThread* jt) {
     // We have a JavaThread* so add more state bits.
     JavaThreadState jts = jt->thread_state();
 
-    if (cthread_with_mounted_vthread(jt) && jt->is_cthread_pending_suspend()) {
+    if (jt->is_cthread_pending_suspend()) {
       // Suspended carrier thread with a mounted virtual thread.
       state |= JVMTI_THREAD_STATE_SUSPENDED;
     }
@@ -1455,11 +1455,10 @@ JvmtiEnvBase::suspend_thread(oop thread_oop, JavaThread* java_thread, bool singl
     oop mounted_vt = java_thread->mounted_vthread();
 
     if (single_suspend && JvmtiExport::can_support_virtual_threads() &&
+        !java_lang_VirtualThread::is_instance(thread_oop) &&
         mounted_vt != NULL && thread_oop != mounted_vt) {
       // A case of a carrier thread executing a mounted virtual thread.
-      assert(!java_lang_VirtualThread::is_instance(thread_oop) &&
-             java_lang_VirtualThread::is_instance(mounted_vt),
-             "sanity check");
+      assert(java_lang_VirtualThread::is_instance(mounted_vt), "sanity check");
       if (java_thread->is_cthread_pending_suspend()) {
         return JVMTI_ERROR_THREAD_SUSPENDED;
       }
@@ -1528,20 +1527,10 @@ JvmtiEnvBase::resume_thread(oop thread_oop, JavaThread* java_thread, bool single
   if (java_thread->is_hidden_from_external_view()) {
     return JVMTI_ERROR_NONE;
   }
-
-  oop mounted_vt = java_thread->mounted_vthread();
-  if (single_suspend && JvmtiExport::can_support_virtual_threads() &&
-      mounted_vt != NULL && thread_oop != java_thread->mounted_vthread()) {
-    // A case of a carrier thread executing a mounted virtual thread.
-    assert(!java_lang_VirtualThread::is_instance(thread_oop) &&
-           java_lang_VirtualThread::is_instance(mounted_vt),
-           "sanity check");
-    if (java_thread->is_cthread_pending_suspend()) {
-      java_thread->clear_cthread_pending_suspend();
-      return JVMTI_ERROR_NONE;
-    } else {
-      return JVMTI_ERROR_THREAD_NOT_SUSPENDED;
-    }
+  // A case of a carrier thread executing a mounted virtual thread.
+  if (java_thread->is_cthread_pending_suspend()) {
+    java_thread->clear_cthread_pending_suspend();
+    return JVMTI_ERROR_NONE;
   }
   if (!java_thread->is_being_ext_suspended()) {
     return JVMTI_ERROR_THREAD_NOT_SUSPENDED;

src/hotspot/share/prims/jvmtiThreadState.cpp

@@ -205,11 +205,7 @@ JvmtiVTMTDisabler::disable_VTMT() {
   _VTMT_disable_count++;
 
   // Block while some mount/unmount transitions are in progress.
-  // Thread in VTMT can call JVMTI functions in mount/unmount event handlers.
-  // The JVMTI functions supporting virtual threads normally disable VTMT.
-  // To avoid deadlocks, such VTMT disablers are not blocked on wait below.
-  // Other VTMT disablers are waiting until all VTMT's go through.
-  while (_VTMT_count > 0 && !thread->in_vtmt()) {
+  while (_VTMT_count > 0) {
     ml.wait();
   }
 }
@@ -220,8 +216,7 @@ JvmtiVTMTDisabler::enable_VTMT() {
     return;
   }
   MonitorLocker ml(JvmtiVTMT_lock, Mutex::_no_safepoint_check_flag);
-  assert((JavaThread::current()->in_vtmt() || _VTMT_count == 0) && _VTMT_disable_count > 0,
-         "VTMT sanity check");
+  assert(_VTMT_count == 0 && _VTMT_disable_count > 0, "VTMT sanity check");
 
   if (--_VTMT_disable_count == 0) {
     ml.notify_all();
@@ -247,7 +242,6 @@ JvmtiVTMTDisabler::start_VTMT(jthread vthread, int callsite_tag) {
     ml.wait();
   }
   _VTMT_count++;
-  thread->set_in_vtmt(true);
 }
 
 void
@@ -259,18 +253,16 @@ JvmtiVTMTDisabler::finish_VTMT(jthread vthread, int callsite_tag) {
   MonitorLocker ml(JvmtiVTMT_lock, Mutex::_no_safepoint_check_flag);
 
   _VTMT_count--;
-  thread->set_in_vtmt(false);
 
   // unblock waiting VTMT disablers
   if (_VTMT_disable_count > 0) {
     ml.notify_all();
   }
   if (callsite_tag == 1) { // finish_VTMT for a vthread unmount
+    MonitorLocker ml(thread->SR_lock(), Mutex::_no_safepoint_check_flag);
     if (thread->is_cthread_pending_suspend()) {
       thread->clear_cthread_pending_suspend();
-      // Safe to do without grabbing SR_lock as the thread is current.
       // The JavaThread* will be suspended upon return to Java.
-      MonitorLocker ml(thread->SR_lock(), Mutex::_no_safepoint_check_flag);
       thread->set_external_suspend();
     }
   }

src/hotspot/share/runtime/thread.cpp

@@ -1686,7 +1686,6 @@ void JavaThread::initialize() {
   set_deopt_compiled_method(NULL);
   set_monitor_chunks(NULL);
   _on_thread_list = false;
-  _in_vtmt = false;
   _thread_state = _thread_new;
   _terminated = _not_terminated;
   _suspend_equivalent = false;

src/hotspot/share/runtime/thread.hpp

@@ -1036,7 +1036,6 @@ class JavaThread: public Thread {
   friend class Continuation;
  private:
   bool           _on_thread_list;                // Is set when this JavaThread is added to the Threads list
-  bool           _in_vtmt;
   OopHandle      _threadObj;                     // The Java level thread object
   OopHandle      _vthread;
   OopHandle      _scopedCache;
@@ -1363,8 +1362,6 @@ class JavaThread: public Thread {
   void smr_delete();
   bool on_thread_list() const { return _on_thread_list; }
   void set_on_thread_list() { _on_thread_list = true; }
-  bool in_vtmt() const { return _in_vtmt; }
-  void set_in_vtmt(bool val) { _in_vtmt = val; }
 
   // thread has called JavaThread::exit() or is terminated
   bool is_exiting() const;

src/java.base/share/classes/java/lang/VirtualThread.java

@@ -907,25 +907,25 @@ private void notifyJvmtiMountBegin(boolean firstRun) {
 
     private void notifyJvmtiMountEnd(boolean firstRun) {
         Thread carrier = Thread.currentCarrierThread();
+        notifyVTMTFinish(this, 0);
         if (firstRun) {
             notifyStarted(carrier, this);
         }
         notifyMount(carrier, this);
-        notifyVTMTFinish(this, 0);
     }
 
     private void notifyJvmtiUnmountBegin() {
-        notifyVTMTStart(this, 1);
         notifyUnmount(Thread.currentCarrierThread(), this);
+        notifyVTMTStart(this, 1);
     }
 
     private void notifyJvmtiUnmountEnd() {
         notifyVTMTFinish(this, 1);
     }
 
     private void notifyJvmtiTerminate() {
-        notifyVTMTStart(this, 0);
         notifyTerminated(Thread.currentCarrierThread(), this);
+        notifyVTMTStart(this, 0);
         notifyVTMTFinish(this, 0);
     }
 

test/hotspot/jtreg/vmTestbase/nsk/jvmti/SuspendThread/suspendvthr001.java

@@ -94,7 +94,7 @@ private int test_vthreads() {
         if (status != Consts.TEST_PASSED) {
             return status;
         }
-        sleep(100); // let tested vthreads work while they are tested in native agent
+        sleep(3000); // let tested vthreads work while they are tested in native agent
 
         System.out.println("\n## Java: runIt: Finishing vthreads");
         try {