@@ -2259,55 +2259,88 @@ void SuperWord::co_locate_pack(Node_List* pk) {
22592259 _igvn.replace_input_of (ld, MemNode::Memory, upper_insert_pt);
22602260 }
22612261 }
2262- } else if (pk->at (0 )->is_Load ()) { // load
2263- // all loads in the pack should have the same memory state. By default,
2262+ } else if (pk->at (0 )->is_Load ()) { // Load pack
2263+ // All loads in the pack should have the same memory state. By default,
22642264 // we use the memory state of the last load. However, if any load could
22652265 // not be moved down due to the dependence constraint, we use the memory
22662266 // state of the first load.
2267- Node* last_mem = pk->at (0 )->in (MemNode::Memory);
2268- Node* first_mem = last_mem;
2269- // Walk the memory graph from the current first load until the
2270- // start of the loop and check if nodes on the way are memory
2271- // edges of loads in the pack. The last one we encounter is the
2272- // first load.
2273- for (Node* current = first_mem; in_bb (current); current = current->is_Phi () ? current->in (LoopNode::EntryControl) : current->in (MemNode::Memory)) {
2274- assert (current->is_Mem () || (current->is_Phi () && current->in (0 ) == bb ()), " unexpected memory"
2275- for (uint i = 1 ; i < pk->size (); i++) {
2276- Node* ld = pk->at (i);
2277- if (ld->in (MemNode::Memory) == current) {
2278- first_mem = current;
2279- break ;
2280- }
2267+ Node* mem_input = pick_mem_state (pk);
2268+ _igvn.hash_delete (mem_input);
2269+ // Give each load the same memory state
2270+ for (uint i = 0 ; i < pk->size (); i++) {
2271+ LoadNode* ld = pk->at (i)->as_Load ();
2272+ _igvn.replace_input_of (ld, MemNode::Memory, mem_input);
2273+ }
2274+ }
2275+ }
2276+
2277+ // Finds the first and last memory state and then picks either of them by checking dependence constraints.
2278+ // If a store is dependent on an earlier load then we need to pick the memory state of the first load and cannot
2279+ // pick the memory state of the last load.
2280+ Node* SuperWord::pick_mem_state (Node_List* pk) {
2281+ Node* first_mem = find_first_mem_state (pk);
2282+ Node* last_mem = find_last_mem_state (pk, first_mem);
2283+
2284+ for (uint i = 0 ; i < pk->size (); i++) {
2285+ Node* ld = pk->at (i);
2286+ for (Node* current = last_mem; current != ld->in (MemNode::Memory); current = current->in (MemNode::Memory)) {
2287+ assert (current->is_Mem () && in_bb (current), " unexpected memory"
2288+ assert (current != first_mem, " corrupted memory graph"
2289+ if (!independent (current, ld)) {
2290+ #ifdef ASSERT
2291+ // Added assertion code since no case has been observed that should pick the first memory state.
2292+ // Remove the assertion code whenever we find a (valid) case that really needs the first memory state.
2293+ pk->dump ();
2294+ first_mem->dump ();
2295+ last_mem->dump ();
2296+ current->dump ();
2297+ ld->dump ();
2298+ ld->in (MemNode::Memory)->dump ();
2299+ assert (false , " never observed that first memory should be picked"
2300+ #endif
2301+ return first_mem; // A later store depends on this load, pick memory state of first load
22812302 }
22822303 }
2283- // Find the last load by going over the pack again and walking
2284- // the memory graph from the loads of the pack to the memory of
2285- // the first load. If we encounter the memory of the current last
2286- // load, then we started from further down in the memory graph and
2287- // the load we started from is the last load. Check for dependence
2288- // constraints in that loop as well.
2289- bool schedule_last = true ;
2290- for (uint i = 0 ; i < pk->size (); i++) {
2304+ }
2305+ return last_mem;
2306+ }
2307+
2308+ // Walk the memory graph from the current first load until the
2309+ // start of the loop and check if nodes on the way are memory
2310+ // edges of loads in the pack. The last one we encounter is the
2311+ // first load.
2312+ Node* SuperWord::find_first_mem_state (Node_List* pk) {
2313+ Node* first_mem = pk->at (0 )->in (MemNode::Memory);
2314+ for (Node* current = first_mem; in_bb (current); current = current->is_Phi () ? current->in (LoopNode::EntryControl) : current->in (MemNode::Memory)) {
2315+ assert (current->is_Mem () || (current->is_Phi () && current->in (0 ) == bb ()), " unexpected memory"
2316+ for (uint i = 1 ; i < pk->size (); i++) {
22912317 Node* ld = pk->at (i);
2292- for (Node* current = ld->in (MemNode::Memory); current != first_mem; current = current->in (MemNode::Memory)) {
2293- assert (current->is_Mem () && in_bb (current), " unexpected memory"
2294- if (current->in (MemNode::Memory) == last_mem) {
2295- last_mem = ld->in (MemNode::Memory);
2296- }
2297- if (!independent (current, ld)) {
2298- schedule_last = false ; // a later store depends on this load
2299- }
2318+ if (ld->in (MemNode::Memory) == current) {
2319+ first_mem = current;
2320+ break ;
23002321 }
23012322 }
2323+ }
2324+ return first_mem;
2325+ }
23022326
2303- Node* mem_input = schedule_last ? last_mem : first_mem;
2304- _igvn.hash_delete (mem_input);
2305- // Give each load the same memory state
2306- for (uint i = 0 ; i < pk->size (); i++) {
2307- LoadNode* ld = pk->at (i)->as_Load ();
2308- _igvn.replace_input_of (ld, MemNode::Memory, mem_input);
2327+ // Find the last load by going over the pack again and walking
2328+ // the memory graph from the loads of the pack to the memory of
2329+ // the first load. If we encounter the memory of the current last
2330+ // load, then we started from further down in the memory graph and
2331+ // the load we started from is the last load.
2332+ Node* SuperWord::find_last_mem_state (Node_List* pk, Node* first_mem) {
2333+ Node* last_mem = pk->at (0 )->in (MemNode::Memory);
2334+ for (uint i = 0 ; i < pk->size (); i++) {
2335+ Node* ld = pk->at (i);
2336+ for (Node* current = ld->in (MemNode::Memory); current != first_mem; current = current->in (MemNode::Memory)) {
2337+ assert (current->is_Mem () && in_bb (current), " unexpected memory"
2338+ if (current->in (MemNode::Memory) == last_mem) {
2339+ last_mem = ld->in (MemNode::Memory);
2340+ }
23092341 }
23102342 }
2343+ return last_mem;
23112344}
23122345
23132346#ifndef PRODUCT
0 commit comments