diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index bbcb4c59d3a..1ccc5cf0abf 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -3744,12 +3744,12 @@ Node* GraphKit::array_lh_test(Node* klass, jint mask, jint val, bool eq) { return _gvn.transform(new BoolNode(cmp, eq ? BoolTest::eq : BoolTest::ne)); } -Node* GraphKit::flat_array_test(Node* ary, bool flat) { +Node* GraphKit::flat_array_test(Node* array_or_klass, bool flat) { // We can't use immutable memory here because the mark word is mutable. // PhaseIdealLoop::move_flat_array_check_out_of_loop will make sure the // check is moved out of loops (mainly to enable loop unswitching). Node* mem = UseArrayMarkWordCheck ? memory(Compile::AliasIdxRaw) : immutable_memory(); - Node* cmp = _gvn.transform(new FlatArrayCheckNode(C, mem, ary)); + Node* cmp = _gvn.transform(new FlatArrayCheckNode(C, mem, array_or_klass)); record_for_igvn(cmp); // Give it a chance to be optimized out by IGVN return _gvn.transform(new BoolNode(cmp, flat ? BoolTest::eq : BoolTest::ne)); } diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp index 32c0c50a986..dd98bd355d0 100644 --- a/src/hotspot/share/opto/graphKit.hpp +++ b/src/hotspot/share/opto/graphKit.hpp @@ -875,7 +875,7 @@ class GraphKit : public Phase { Node* inline_type_test(Node* obj, bool is_inline = true); Node* is_val_mirror(Node* mirror); Node* array_lh_test(Node* kls, jint mask, jint val, bool eq = true); - Node* flat_array_test(Node* ary, bool flat = true); + Node* flat_array_test(Node* array_or_klass, bool flat = true); Node* null_free_array_test(Node* klass, bool null_free = true); Node* inline_array_null_guard(Node* ary, Node* val, int nargs, bool safe_for_replace = false); diff --git a/src/hotspot/share/opto/ifnode.cpp b/src/hotspot/share/opto/ifnode.cpp index f1466430587..ff42974aa20 100644 --- a/src/hotspot/share/opto/ifnode.cpp +++ b/src/hotspot/share/opto/ifnode.cpp @@ -1239,7 +1239,7 @@ bool IfNode::is_flat_array_check(PhaseTransform* phase, Node** array) { Node* cmp = bol->in(1); if (cmp->isa_FlatArrayCheck()) { if (array != NULL) { - *array = cmp->in(FlatArrayCheckNode::Array); + *array = cmp->in(FlatArrayCheckNode::ArrayOrKlass); } return true; } diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index efb56e571eb..b7e8ee29f10 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -3680,8 +3680,6 @@ Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region, case ObjectArray: query = Klass::layout_helper_is_objArray(layout_con); break; case NonObjectArray: query = !Klass::layout_helper_is_objArray(layout_con); break; case TypeArray: query = Klass::layout_helper_is_typeArray(layout_con); break; - case FlatArray: query = Klass::layout_helper_is_flatArray(layout_con); break; - case NonFlatArray: query = !Klass::layout_helper_is_flatArray(layout_con); break; case AnyArray: query = Klass::layout_helper_is_array(layout_con); break; case NonArray: query = !Klass::layout_helper_is_array(layout_con); break; default: @@ -3713,13 +3711,6 @@ Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region, btest = BoolTest::eq; break; } - case FlatArray: - case NonFlatArray: { - value = 0; - layout_val = _gvn.transform(new AndINode(layout_val, intcon(Klass::_lh_array_tag_flat_value_bit_inplace))); - btest = (kind == FlatArray) ? BoolTest::ne : BoolTest::eq; - break; - } case AnyArray: value = Klass::_lh_neutral_value; btest = BoolTest::lt; break; case NonArray: value = Klass::_lh_neutral_value; btest = BoolTest::gt; break; default: @@ -3949,14 +3940,14 @@ bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) { bailout->add_req(control()); set_control(top()); } else { - generate_non_flatArray_guard(klass_node, bailout); + generate_fair_guard(flat_array_test(klass_node, /* flat = */ false), bailout); } } else if (UseFlatArray && (orig_t == NULL || !orig_t->is_not_flat()) && // If dest is flat, src must be flat as well (guaranteed by src <: dest check if validated). ((!klass->is_flat_array_klass() && klass->can_be_inline_array_klass()) || !can_validate)) { // Src might be flat and dest might not be flat. Go to the slow path if src is flat. // TODO 8251971: Optimize for the case when src/dest are later found to be both flat. - generate_flatArray_guard(original_kls, bailout); + generate_fair_guard(flat_array_test(original_kls), bailout); if (orig_t != NULL) { orig_t = orig_t->cast_to_not_flat(); original = _gvn.transform(new CheckCastPPNode(control(), original, orig_t)); @@ -4665,7 +4656,7 @@ bool LibraryCallKit::inline_native_clone(bool is_virtual) { (ary_ptr == NULL || (!ary_ptr->is_not_flat() && (!ary_ptr->is_flat() || ary_ptr->elem()->inline_klass()->contains_oops())))) { // Flattened inline type array may have object field that would require a // write barrier. Conservatively, go to slow path. - generate_flatArray_guard(obj_klass, slow_region); + generate_fair_guard(flat_array_test(obj_klass), slow_region); } if (!stopped()) { @@ -5177,7 +5168,7 @@ bool LibraryCallKit::inline_arraycopy() { if (top_src != NULL && top_src->is_flat()) { // Src is flat, check that dest is flat as well if (top_dest != NULL && !top_dest->is_flat()) { - generate_non_flatArray_guard(dest_klass, slow_region); + generate_fair_guard(flat_array_test(dest_klass, /* flat = */ false), slow_region); // Since dest is flat and src <: dest, dest must have the same type as src. top_dest = TypeOopPtr::make_from_klass(top_src->klass())->isa_aryptr(); assert(top_dest->is_flat(), "dest must be flat"); @@ -5187,7 +5178,7 @@ bool LibraryCallKit::inline_arraycopy() { // Src might be flat and dest might not be flat. Go to the slow path if src is flat. // TODO 8251971: Optimize for the case when src/dest are later found to be both flat. assert(top_dest == NULL || !top_dest->is_flat(), "dest array must not be flat"); - generate_flatArray_guard(load_object_klass(src), slow_region); + generate_fair_guard(flat_array_test(src), slow_region); if (top_src != NULL) { top_src = top_src->cast_to_not_flat(); src = _gvn.transform(new CheckCastPPNode(control(), src, top_src)); diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index 546718cd050..8dc393a9599 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -171,9 +171,7 @@ class LibraryCallKit : public GraphKit { NonArray, ObjectArray, NonObjectArray, - TypeArray, - FlatArray, - NonFlatArray + TypeArray }; Node* generate_hidden_class_guard(Node* kls, RegionNode* region); @@ -193,14 +191,6 @@ class LibraryCallKit : public GraphKit { Node* generate_typeArray_guard(Node* kls, RegionNode* region) { return generate_array_guard_common(kls, region, TypeArray); } - Node* generate_flatArray_guard(Node* kls, RegionNode* region) { - assert(UseFlatArray, "can never be flattened"); - return generate_array_guard_common(kls, region, FlatArray); - } - Node* generate_non_flatArray_guard(Node* kls, RegionNode* region) { - assert(UseFlatArray, "can never be flattened"); - return generate_array_guard_common(kls, region, NonFlatArray); - } Node* generate_array_guard_common(Node* kls, RegionNode* region, ArrayKind kind); Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region); CallJavaNode* generate_method_call(vmIntrinsics::ID method_id, diff --git a/src/hotspot/share/opto/loopUnswitch.cpp b/src/hotspot/share/opto/loopUnswitch.cpp index 2546d16f02b..7ba4ec5767b 100644 --- a/src/hotspot/share/opto/loopUnswitch.cpp +++ b/src/hotspot/share/opto/loopUnswitch.cpp @@ -323,8 +323,8 @@ IfNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop, assert(cmp->req() == 3, "unexpected number of inputs for FlatArrayCheck"); cmp->add_req_batch(C->top(), unswitch_iffs.size() - 1); for (uint i = 0; i < unswitch_iffs.size(); i++) { - Node* array = unswitch_iffs.at(i)->in(1)->in(1)->in(FlatArrayCheckNode::Array); - cmp->set_req(FlatArrayCheckNode::Array + i, array); + Node* array = unswitch_iffs.at(i)->in(1)->in(1)->in(FlatArrayCheckNode::ArrayOrKlass); + cmp->set_req(FlatArrayCheckNode::ArrayOrKlass + i, array); } } diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index c2e39415e5b..cdb2384ae9b 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -1012,7 +1012,7 @@ void PhaseIdealLoop::move_flat_array_check_out_of_loop(Node* n) { return; } Node* mem = n->in(FlatArrayCheckNode::Memory); - Node* array = n->in(FlatArrayCheckNode::Array)->uncast(); + Node* array = n->in(FlatArrayCheckNode::ArrayOrKlass)->uncast(); IdealLoopTree* check_loop = get_loop(get_ctrl(n)); IdealLoopTree* ary_loop = get_loop(get_ctrl(array)); diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp index c2d99b39ff5..7c94f82285a 100644 --- a/src/hotspot/share/opto/macro.cpp +++ b/src/hotspot/share/opto/macro.cpp @@ -2697,14 +2697,15 @@ void PhaseMacroExpand::expand_subtypecheck_node(SubTypeCheckNode *check) { // ... // } void PhaseMacroExpand::expand_flatarraycheck_node(FlatArrayCheckNode* check) { - if (UseArrayMarkWordCheck) { + bool array_inputs = _igvn.type(check->in(FlatArrayCheckNode::ArrayOrKlass))->isa_oopptr() != NULL; + if (UseArrayMarkWordCheck && array_inputs) { Node* mark = MakeConX(0); Node* locked_bit = MakeConX(markWord::unlocked_value); Node* mem = check->in(FlatArrayCheckNode::Memory); - for (uint i = FlatArrayCheckNode::Array; i < check->req(); ++i) { + for (uint i = FlatArrayCheckNode::ArrayOrKlass; i < check->req(); ++i) { Node* ary = check->in(i); - if (ary->is_top()) continue; - const TypeAryPtr* t = _igvn.type(ary)->isa_aryptr(); + const TypeOopPtr* t = _igvn.type(ary)->isa_oopptr(); + assert(t != NULL, "Mixing array and klass inputs"); assert(!t->is_flat() && !t->is_not_flat(), "Should have been optimized out"); Node* mark_adr = basic_plus_adr(ary, oopDesc::mark_offset_in_bytes()); Node* mark_load = _igvn.transform(LoadNode::make(_igvn, NULL, mem, mark_adr, mark_adr->bottom_type()->is_ptr(), TypeX_X, TypeX_X->basic_type(), MemNode::unordered)); @@ -2734,9 +2735,8 @@ void PhaseMacroExpand::expand_flatarraycheck_node(FlatArrayCheckNode* check) { // Locked: Load prototype header from klass ctrl = _igvn.transform(new IfFalseNode(iff)); Node* proto = MakeConX(0); - for (uint i = FlatArrayCheckNode::Array; i < check->req(); ++i) { + for (uint i = FlatArrayCheckNode::ArrayOrKlass; i < check->req(); ++i) { Node* ary = check->in(i); - if (ary->is_top()) continue; // Make loads control dependent to make sure they are only executed if array is locked Node* klass_adr = basic_plus_adr(ary, oopDesc::klass_offset_in_bytes()); Node* klass = _igvn.transform(LoadKlassNode::make(_igvn, ctrl, C->immutable_memory(), klass_adr, TypeInstPtr::KLASS, TypeInstKlassPtr::OBJECT)); @@ -2761,13 +2761,18 @@ void PhaseMacroExpand::expand_flatarraycheck_node(FlatArrayCheckNode* check) { } else { // Fall back to layout helper check Node* lhs = intcon(0); - for (uint i = FlatArrayCheckNode::Array; i < check->req(); ++i) { - Node* ary = check->in(i); - if (ary->is_top()) continue; - const TypeAryPtr* t = _igvn.type(ary)->isa_aryptr(); + for (uint i = FlatArrayCheckNode::ArrayOrKlass; i < check->req(); ++i) { + Node* array_or_klass = check->in(i); + Node* klass = NULL; + const TypePtr* t = _igvn.type(array_or_klass)->is_ptr(); assert(!t->is_flat() && !t->is_not_flat(), "Should have been optimized out"); - Node* klass_adr = basic_plus_adr(ary, oopDesc::klass_offset_in_bytes()); - Node* klass = transform_later(LoadKlassNode::make(_igvn, NULL, C->immutable_memory(), klass_adr, TypeInstPtr::KLASS, TypeInstKlassPtr::OBJECT)); + if (t->isa_oopptr() != NULL) { + Node* klass_adr = basic_plus_adr(array_or_klass, oopDesc::klass_offset_in_bytes()); + klass = transform_later(LoadKlassNode::make(_igvn, NULL, C->immutable_memory(), klass_adr, TypeInstPtr::KLASS, TypeInstKlassPtr::OBJECT)); + } else { + assert(t->isa_aryklassptr(), "Unexpected input type"); + klass = array_or_klass; + } Node* lh_addr = basic_plus_adr(klass, in_bytes(Klass::layout_helper_offset())); Node* lh_val = _igvn.transform(LoadNode::make(_igvn, NULL, C->immutable_memory(), lh_addr, lh_addr->bottom_type()->is_ptr(), TypeInt::INT, T_INT, MemNode::unordered)); lhs = _igvn.transform(new OrINode(lhs, lh_val)); diff --git a/src/hotspot/share/opto/subnode.cpp b/src/hotspot/share/opto/subnode.cpp index adb49ceb2bf..95448dc25b7 100644 --- a/src/hotspot/share/opto/subnode.cpp +++ b/src/hotspot/share/opto/subnode.cpp @@ -1396,19 +1396,17 @@ Node *CmpDNode::Ideal(PhaseGVN *phase, bool can_reshape){ //------------------------------Value------------------------------------------ const Type* FlatArrayCheckNode::Value(PhaseGVN* phase) const { bool all_not_flat = true; - for (uint i = Array; i < req(); ++i) { - Node* array = in(i); - if (!array->is_top()) { - const Type* t = phase->type(array); - if (t == Type::TOP) { - return Type::TOP; - } else if (t->is_aryptr()->is_flat()) { - // One of the input arrays is flat, check always passes - return TypeInt::CC_EQ; - } else if (!t->is_aryptr()->is_not_flat()) { - // One of the input arrays might be flat - all_not_flat = false; - } + for (uint i = ArrayOrKlass; i < req(); ++i) { + const Type* t = phase->type(in(i)); + if (t == Type::TOP) { + return Type::TOP; + } + if (t->is_ptr()->is_flat()) { + // One of the input arrays is flat, check always passes + return TypeInt::CC_EQ; + } else if (!t->is_ptr()->is_not_flat()) { + // One of the input arrays might be flat + all_not_flat = false; } } if (all_not_flat) { @@ -1421,11 +1419,11 @@ const Type* FlatArrayCheckNode::Value(PhaseGVN* phase) const { //------------------------------Ideal------------------------------------------ Node* FlatArrayCheckNode::Ideal(PhaseGVN* phase, bool can_reshape) { bool changed = false; - // Remove array inputs that are known to be non-flat - for (uint i = Array; i < req(); ++i) { - const TypeAryPtr* t = phase->type(in(i))->isa_aryptr(); - if (t != NULL && t->is_not_flat()) { - set_req(i, phase->C->top()); + // Remove inputs that are known to be non-flat + for (uint i = ArrayOrKlass; i < req(); ++i) { + const Type* t = phase->type(in(i)); + if (t->isa_ptr() && t->is_ptr()->is_not_flat()) { + del_req(i--); changed = true; } } diff --git a/src/hotspot/share/opto/subnode.hpp b/src/hotspot/share/opto/subnode.hpp index 38beaf54af3..29a06248ae3 100644 --- a/src/hotspot/share/opto/subnode.hpp +++ b/src/hotspot/share/opto/subnode.hpp @@ -282,15 +282,16 @@ class CmpD3Node : public CmpDNode { }; //--------------------------FlatArrayCheckNode--------------------------------- -// Returns true if one of the input arrays (there can be multiple) is flat. +// Returns true if one of the input array objects or array klass ptrs (there +// can be multiple) is flat. class FlatArrayCheckNode : public CmpNode { public: enum { Control, Memory, - Array + ArrayOrKlass }; - FlatArrayCheckNode(Compile* C, Node* mem, Node* array) : CmpNode(mem, array) { + FlatArrayCheckNode(Compile* C, Node* mem, Node* array_or_klass) : CmpNode(mem, array_or_klass) { init_class_id(Class_FlatArrayCheck); init_flags(Flag_is_macro); C->add_macro_node(this); diff --git a/src/hotspot/share/opto/type.cpp b/src/hotspot/share/opto/type.cpp index 73975e37fdf..2223897abc0 100644 --- a/src/hotspot/share/opto/type.cpp +++ b/src/hotspot/share/opto/type.cpp @@ -5907,7 +5907,7 @@ const Type *TypeInstKlassPtr::xmeet( const Type *t ) const { // below the centerline when the superclass is exact. We need to // do the same here. if (klass()->equals(ciEnv::current()->Object_klass()) && !klass_is_exact()) { - return TypeAryKlassPtr::make(ptr, tp->elem(), tp->klass(), offset, tp->is_not_flat(), tp->is_not_null_free(), tp->null_free()); + return TypeAryKlassPtr::make(ptr, tp->elem(), tp->klass(), offset, tp->is_not_flat(), tp->is_not_null_free(), tp->is_null_free()); } else { // cannot subclass, so the meet has to fall badly below the centerline ptr = NotNull; @@ -5926,7 +5926,7 @@ const Type *TypeInstKlassPtr::xmeet( const Type *t ) const { if (klass()->equals(ciEnv::current()->Object_klass())) { // that is, tp's array type is a subtype of my klass return TypeAryKlassPtr::make(ptr, - tp->elem(), tp->klass(), offset, tp->is_not_flat(), tp->is_not_null_free(), tp->null_free()); + tp->elem(), tp->klass(), offset, tp->is_not_flat(), tp->is_not_null_free(), tp->is_null_free()); } } // The other case cannot happen, since I cannot be a subtype of an array. diff --git a/src/hotspot/share/opto/type.hpp b/src/hotspot/share/opto/type.hpp index 709f4cfb4f5..75e4d801977 100644 --- a/src/hotspot/share/opto/type.hpp +++ b/src/hotspot/share/opto/type.hpp @@ -1085,9 +1085,11 @@ class TypePtr : public Type { virtual bool maybe_null() const { return meet_ptr(Null) == ptr(); } virtual bool can_be_inline_type() const { return false; } - virtual bool flatten_array() const { return false; } - virtual bool is_not_flat() const { return false; } - virtual bool is_not_null_free() const { return false; } + virtual bool flatten_array() const { return false; } + virtual bool is_flat() const { return false; } + virtual bool is_not_flat() const { return false; } + virtual bool is_null_free() const { return false; } + virtual bool is_not_null_free() const { return false; } // Tests for relation to centerline of type lattice: static bool above_centerline(PTR ptr) { return (ptr <= AnyNull); } @@ -1664,9 +1666,10 @@ class TypeAryKlassPtr : public TypeKlassPtr { return TypeKlassPtr::empty() || _elem->empty(); } - virtual bool is_not_flat() const { return _not_flat; } - virtual bool is_not_null_free() const { return _not_null_free; } - bool null_free() const { return _null_free; } + bool is_flat() const { return klass()->is_flat_array_klass(); } + bool is_not_flat() const { return _not_flat; } + bool is_null_free() const { return _null_free; } + bool is_not_null_free() const { return _not_null_free; } #ifndef PRODUCT virtual void dump2( Dict &d, uint depth, outputStream *st ) const; // Specialized per-Type dumping