8302217: [lworld] Infinite invocation of PhiNode::push_inline_types_through

src/hotspot/share/opto/callGenerator.cpp

@@ -701,6 +701,7 @@ void CallGenerator::do_late_inline_helper() {
         // field of the inline type. Build InlineTypeNodes from the inline type arguments.
         GraphKit arg_kit(jvms, &gvn);
         Node* vt = InlineTypeNode::make_from_multi(&arg_kit, call, t->inline_klass(), j, /* in= */ true, /* null_free= */ !t->maybe_null());
+        map = arg_kit.map();
         map->set_control(arg_kit.control());
         map->set_argument(jvms, i1, vt);
       } else {

src/hotspot/share/opto/cfgnode.cpp

@@ -695,7 +695,9 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) {
       if (add_to_worklist) {
         igvn->add_users_to_worklist(this); // Check for further allowed opts
       }
-      for (DUIterator_Last imin, i = last_outs(imin); i >= imin; --i) {
+      uint nb_edges = 1;
+      for (DUIterator_Last imin, i = last_outs(imin); i >= imin; i -= nb_edges) {
+        nb_edges = 1;
         Node* n = last_out(i);
         igvn->hash_delete(n); // Remove from worklist before modifying edges
         if (n->outcnt() == 0) {
@@ -2140,6 +2142,203 @@ bool PhiNode::wait_for_region_igvn(PhaseGVN* phase) {
 
 // Push inline type input nodes (and null) down through the phi recursively (can handle data loops).
 InlineTypeNode* PhiNode::push_inline_types_down(PhaseGVN* phase, bool can_reshape, ciInlineKlass* inline_klass) {
+  if (can_reshape) {
+    ResourceMark rm;
+    Unique_Node_List wq, wq2;
+    Node_List clones;
+    wq.push(this);
+    for (uint i = 0; i < wq.size(); ++i) {
+      Node* n = wq.at(i);
+      if (n->is_Phi()) {
+        for (uint j = 1; j < n->req(); ++j) {
+          Node* in = n->in(j);
+          if (in == nullptr) {
+            continue;
+          }
+          wq.push(in);
+        }
+      } else if (n->is_ConstraintCast()) {
+        Node* in = n->in(1);
+        if (in == nullptr) {
+          continue;
+        }
+        wq.push(in);
+      }
+    }
+    for (int i = wq.size() - 1; i >= 0; i--) {
+      Node* n = wq.at(i);
+      if (!n->is_InlineType()) {
+        wq.remove(i);
+      }
+    }
+    uint init_nodes = wq.size();
+    for (uint i = 0; i < wq.size(); ++i) {
+      Node* n = wq.at(i);
+      if (n->is_Phi()) {
+        for (uint j = 1; j < n->req(); ++j) {
+          Node* in = n->in(j);
+          if (in == nullptr) {
+            continue;
+          }
+          wq.push(in);
+        }
+      } else if (n->is_ConstraintCast()) {
+        Node* in = n->in(1);
+        if (in == nullptr) {
+          continue;
+        }
+        wq.push(in);
+      } else if (n->is_InlineType()) {
+        Node* buf = n->as_InlineType()->get_oop();
+        if (buf == nullptr) {
+          continue;
+        }
+        wq.push(buf);
+        wq2.push(n);
+      }
+    }
+    for (uint i = 0; i < wq2.size(); ++i) {
+      Node* n = wq2.at(i);
+      for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
+        Node* u = n->fast_out(i);
+        if (wq.member(u)) {
+          wq2.push(u);
+        }
+      }
+    }
+    for (uint i = 0; i < wq2.size(); ++i) {
+      Node* n = wq2.at(i);
+      assert(clones[n->_idx] == nullptr, "");
+      if (n->is_InlineType()) {
+        clones.map(n->_idx, n->as_InlineType()->get_oop());
+      } else {
+        Node* clone = n->clone();
+        phase->is_IterGVN()->register_new_node_with_optimizer(clone);
+        clones.map(n->_idx, clone);
+      }
+    }
+    auto get_clone = [&](Node* n) {
+      Node* clone = nullptr;
+      while (true) {
+        Node* m = clones[n->_idx];
+        if (m == nullptr) {
+          return clone;
+        }
+        clone = m;
+        n = clone;
+      }
+    };
+    for (uint i = 0; i < wq2.size(); ++i) {
+      Node* n = wq2.at(i);
+      Node* n_clone = get_clone(n);
+      assert(n_clone != nullptr, "");
+      if (n->is_Phi()) {
+        for (uint j = 1; j < n->req(); ++j) {
+          Node* in = n->in(j);
+          Node* in_clone = get_clone(in);
+          if (in_clone != nullptr) {
+            n_clone->set_req(j, in_clone);
+          }
+        }
+      } else if (n->is_ConstraintCast()) {
+        Node* in = n->in(1);
+        Node* in_clone = get_clone(in);
+        assert(in_clone != nullptr, "");
+        n_clone->set_req(1, in_clone);
+      } else if (n->is_InlineType()) {
+        Node* in = n->as_InlineType()->get_oop();
+        Node* in_clone = get_clone(in);
+        if (in_clone != nullptr) {
+          phase->is_IterGVN()->rehash_node_delayed(n);
+          n->as_InlineType()->set_oop(*phase, in_clone);
+        }
+      }
+    }
+#ifdef ASSERT
+    {
+      uint vts_to_skip = 0;
+      uint before_phis = 0;
+      uint before_casts = 0;
+      for (uint i = 0; i < wq.size(); ++i) {
+        Node* n = wq.at(i);
+        if (n->is_Phi()) {
+          before_phis++;
+        } else if (n->is_ConstraintCast()) {
+          before_casts++;
+        } else if (n->is_InlineType()) {
+          Node* buf = n->as_InlineType()->get_oop();
+          if (buf == nullptr) {
+            continue;
+          }
+          if (i >= init_nodes) {
+            vts_to_skip++;
+          }
+        }
+      }
+
+      Unique_Node_List after;
+      after.push(this);
+      for (uint i = 0; i < after.size(); ++i) {
+        Node* n = after.at(i);
+        if (n->is_Phi()) {
+          for (uint j = 1; j < n->req(); ++j) {
+            Node* in = n->in(j);
+            if (in == nullptr) {
+              continue;
+            }
+            after.push(in);
+          }
+        } else if (n->is_ConstraintCast()) {
+          Node* in = n->in(1);
+          if (in == nullptr) {
+            continue;
+          }
+          after.push(in);
+        }
+      }
+      for (int i = after.size() - 1; i >= 0; i--) {
+        Node* n = after.at(i);
+        if (!n->is_InlineType()) {
+          after.remove(i);
+        }
+      }
+      uint after_phis = 0;
+      uint after_casts = 0;
+      uint init_nodes = after.size();
+      for (uint i = 0; i < after.size(); ++i) {
+        Node* n = after.at(i);
+        if (n->is_Phi()) {
+          after_phis++;
+          for (uint j = 1; j < n->req(); ++j) {
+            Node* in = n->in(j);
+            if (in == nullptr) {
+              continue;
+            }
+            after.push(in);
+          }
+        } else if (n->is_ConstraintCast()) {
+          after_casts++;
+          Node* in = n->in(1);
+          if (in == nullptr) {
+            continue;
+          }
+          after.push(in);
+        } else if (n->is_InlineType()) {
+          assert(i < init_nodes, "");
+          Node* buf = n->as_InlineType()->get_oop();
+          if (buf == nullptr) {
+            continue;
+          }
+          after.push(buf);
+        }
+      }
+      assert(after.size() + vts_to_skip == wq.size(), "");
+      assert(before_casts == after_casts, "");
+      assert(before_phis == after_phis, "");
+    }
+#endif
+  }
+
   assert(inline_klass != nullptr, "must be");
   InlineTypeNode* vt = InlineTypeNode::make_null(*phase, inline_klass, /* transform = */ false)->clone_with_phis(phase, in(0), nullptr, !_type->maybe_null(), true);
   if (can_reshape) {
@@ -2171,7 +2370,6 @@ InlineTypeNode* PhiNode::push_inline_types_down(PhaseGVN* phase, bool can_reshap
     }
     while (casts.size() != 0) {
       // Push the cast(s) through the InlineTypeNode
-      // TODO 8302217 Can we avoid cloning? See InlineTypeNode::clone_if_required
       Node* cast = casts.pop()->clone();
       cast->set_req_X(1, n->as_InlineType()->get_oop(), phase);
       n = n->clone();
@@ -2951,21 +3149,6 @@ bool PhiNode::can_push_inline_types_down(PhaseGVN* phase, const bool can_reshape
   }
   inline_klass = nullptr;
 
-  // TODO 8302217 We need to prevent endless pushing through
-  bool only_phi = (outcnt() != 0);
-  for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
-    Node* n = fast_out(i);
-    if (n->is_InlineType() && n->in(1) == this) {
-      return false;
-    }
-    if (!n->is_Phi()) {
-      only_phi = false;
-    }
-  }
-  if (only_phi) {
-    return false;
-  }
-
   ResourceMark rm;
   Unique_Node_List worklist;
   worklist.push(this);

src/hotspot/share/opto/compile.cpp

@@ -2135,8 +2135,6 @@ void Compile::process_inline_types(PhaseIterGVN &igvn, bool remove) {
         // Verify that inline type is buffered when replacing by oop
         else if (u->is_InlineType()) {
           // InlineType uses don't need buffering because they are about to be replaced as well
-        } else if (u->is_Phi()) {
-          // TODO 8302217 Remove this once InlineTypeNodes are reliably pushed through
         } else {
           must_be_buffered = true;
         }

src/hotspot/share/opto/loopopts.cpp

@@ -798,10 +798,6 @@ Node *PhaseIdealLoop::conditional_move( Node *region ) {
     for (uint j = 1; j < region->req(); j++) {
       Node *proj = region->in(j);
       Node *inp = phi->in(j);
-      if (inp->isa_InlineType()) {
-        // TODO 8302217 This prevents PhiNode::push_inline_types_through
-        return nullptr;
-      }
       if (get_ctrl(inp) == proj) { // Found local op
         cost++;
         // Check for a chain of dependent ops; these will all become

test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorld.java

@@ -456,7 +456,6 @@ public void test8_verifier() {
     }
 
     // merge of inline types in a loop, stored in an object local
-    /* TODO 8302217: Enable again when this is fixed.
     @Test
     public Object test9() {
         Object o = valueField1;
@@ -471,7 +470,6 @@ public Object test9() {
     public void test9_verifier() {
         Asserts.assertEQ(test9(), MyValue1.setX(valueField1, valueField1.x + 7));
     }
-    */
 
     // merge of inline types in an object local
     @ForceInline