8264954: unified handling for VectorMask object re-materialization during de-optimization

Reviewed-by: vlivanov

Author: Jatin Bhateja

src/hotspot/share/opto/vector.cpp

@@ -32,6 +32,15 @@
 #include "opto/vector.hpp"
 #include "utilities/macros.hpp"
 
+static bool is_vector_mask(ciKlass* klass) {
+  return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
+}
+
+static bool is_vector_shuffle(ciKlass* klass) {
+  return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
+}
+
+
 void PhaseVector::optimize_vector_boxes() {
   Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]);
 
@@ -252,6 +261,17 @@ void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) {
 #endif // ASSERT
                                                first_ind, n_fields);
     sobj->init_req(0, C->root());
+
+    // If a mask is feeding into a safepoint, then its value should be
+    // packed into a boolean/byte vector first, this will simplify the
+    // re-materialization logic for both predicated and non-predicated
+    // targets.
+    bool is_mask = is_vector_mask(iklass);
+    if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) {
+      const TypeVect* vt = vec_value->bottom_type()->is_vect();
+      BasicType bt = vt->element_basic_type();
+      vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length()));
+    }
     sfpt->add_req(vec_value);
 
     sobj = gvn.transform(sobj);
@@ -305,14 +325,6 @@ Node* PhaseVector::expand_vbox_node_helper(Node* vbox,
   }
 }
 
-static bool is_vector_mask(ciKlass* klass) {
-  return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
-}
-
-static bool is_vector_shuffle(ciKlass* klass) {
-  return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
-}
-
 Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc,
                                           Node* value,
                                           const TypeInstPtr* box_type,

src/hotspot/share/prims/vectorSupport.cpp

@@ -65,6 +65,8 @@ BasicType VectorSupport::klass2bt(InstanceKlass* ik) {
 
   if (is_vector_shuffle(ik)) {
     return T_BYTE;
+  } else if (is_vector_mask(ik)) {
+    return T_BOOLEAN;
   } else { // vector and mask
     oop value = ik->java_mirror()->obj_field(fd.offset());
     BasicType elem_bt = java_lang_Class::as_BasicType(value);
@@ -86,48 +88,34 @@ jint VectorSupport::klass2length(InstanceKlass* ik) {
   return vlen;
 }
 
-void VectorSupport::init_payload_element(typeArrayOop arr, bool is_mask, BasicType elem_bt, int index, address addr) {
-  if (is_mask) {
-    // Masks require special handling: when boxed they are packed and stored in boolean
-    // arrays, but in scalarized form they have the same size as corresponding vectors.
-    // For example, Int512Mask is represented in memory as boolean[16], but
-    // occupies the whole 512-bit vector register when scalarized.
-    // (In generated code, the conversion is performed by VectorStoreMask.)
-    //
-    // TODO: revisit when predicate registers are fully supported.
-    switch (elem_bt) {
-      case T_BYTE:   arr->bool_at_put(index,  (*(jbyte*)addr) != 0); break;
-      case T_SHORT:  arr->bool_at_put(index, (*(jshort*)addr) != 0); break;
-      case T_INT:    // fall-through
-      case T_FLOAT:  arr->bool_at_put(index,   (*(jint*)addr) != 0); break;
-      case T_LONG:   // fall-through
-      case T_DOUBLE: arr->bool_at_put(index,  (*(jlong*)addr) != 0); break;
-
-      default: fatal("unsupported: %s", type2name(elem_bt));
-    }
-  } else {
-    switch (elem_bt) {
-      case T_BYTE:   arr->  byte_at_put(index,   *(jbyte*)addr); break;
-      case T_SHORT:  arr-> short_at_put(index,  *(jshort*)addr); break;
-      case T_INT:    arr->   int_at_put(index,    *(jint*)addr); break;
-      case T_FLOAT:  arr-> float_at_put(index,  *(jfloat*)addr); break;
-      case T_LONG:   arr->  long_at_put(index,   *(jlong*)addr); break;
-      case T_DOUBLE: arr->double_at_put(index, *(jdouble*)addr); break;
-
-      default: fatal("unsupported: %s", type2name(elem_bt));
-    }
+// Masks require special handling: when boxed they are packed and stored in boolean
+// arrays, but in scalarized form they have the same size as corresponding vectors.
+// For example, Int512Mask is represented in memory as boolean[16], but
+// occupies the whole 512-bit vector register when scalarized.
+// During scalarization inserting a VectorStoreMask node between mask
+// and safepoint node always ensures the existence of masks in a boolean array.
+
+void VectorSupport::init_payload_element(typeArrayOop arr, BasicType elem_bt, int index, address addr) {
+  switch (elem_bt) {
+    case T_BOOLEAN: arr->  byte_at_put(index,   *(jboolean*)addr); break;
+    case T_BYTE:   arr->  byte_at_put(index,   *(jbyte*)addr); break;
+    case T_SHORT:  arr-> short_at_put(index,  *(jshort*)addr); break;
+    case T_INT:    arr->   int_at_put(index,    *(jint*)addr); break;
+    case T_FLOAT:  arr-> float_at_put(index,  *(jfloat*)addr); break;
+    case T_LONG:   arr->  long_at_put(index,   *(jlong*)addr); break;
+    case T_DOUBLE: arr->double_at_put(index, *(jdouble*)addr); break;
+
+    default: fatal("unsupported: %s", type2name(elem_bt));
   }
 }
 
 Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS) {
-  bool is_mask = is_vector_mask(ik);
-
   int num_elem = klass2length(ik);
   BasicType elem_bt = klass2bt(ik);
   int elem_size = type2aelembytes(elem_bt);
 
   // On-heap vector values are represented as primitive arrays.
-  TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(is_mask ? T_BOOLEAN : elem_bt));
+  TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(elem_bt));
 
   typeArrayOop arr = tak->allocate(num_elem, CHECK_NH); // safepoint
 
@@ -140,13 +128,13 @@ Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* f
       int off   = (i * elem_size) % VMRegImpl::stack_slot_size;
 
       address elem_addr = reg_map->location(vreg, vslot) + off; // assumes little endian element order
-      init_payload_element(arr, is_mask, elem_bt, i, elem_addr);
+      init_payload_element(arr, elem_bt, i, elem_addr);
     }
   } else {
     // Value was directly saved on the stack.
     address base_addr = ((address)fr->unextended_sp()) + location.stack_offset();
     for (int i = 0; i < num_elem; i++) {
-      init_payload_element(arr, is_mask, elem_bt, i, base_addr + i * elem_size);
+      init_payload_element(arr, elem_bt, i, base_addr + i * elem_size);
     }
   }
   return Handle(THREAD, arr);

src/hotspot/share/prims/vectorSupport.hpp

@@ -42,7 +42,7 @@ class VectorSupport : AllStatic {
   static Handle allocate_vector_payload(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ScopeValue* payload, TRAPS);
   static Handle allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS);
 
-  static void init_payload_element(typeArrayOop arr, bool is_mask, BasicType elem_bt, int index, address addr);
+  static void init_payload_element(typeArrayOop arr, BasicType elem_bt, int index, address addr);
 
   static BasicType klass2bt(InstanceKlass* ik);
   static jint klass2length(InstanceKlass* ik);