8357781: Deep recursion in PhaseCFG::set_next_call leads to stack overflow

Reviewed-by: thartmann, kvn, mhaessig

src/hotspot/share/opto/block.hpp

@@ -447,7 +447,7 @@ class PhaseCFG : public Phase {
   Block* hoist_to_cheaper_block(Block* LCA, Block* early, Node* self);
 
   bool schedule_local(Block* block, GrowableArray<int>& ready_cnt, VectorSet& next_call, intptr_t* recacl_pressure_nodes);
-  void set_next_call(Block* block, Node* n, VectorSet& next_call);
+  void set_next_call(const Block* block, Node* n, VectorSet& next_call) const;
   void needed_for_next_call(Block* block, Node* this_call, VectorSet& next_call);
 
   // Perform basic-block local scheduling

src/hotspot/share/opto/lcm.cpp

@@ -795,13 +795,19 @@ void PhaseCFG::adjust_register_pressure(Node* n, Block* block, intptr_t* recalc_
 }
 
 //------------------------------set_next_call----------------------------------
-void PhaseCFG::set_next_call(Block* block, Node* n, VectorSet& next_call) {
-  if( next_call.test_set(n->_idx) ) return;
-  for( uint i=0; i<n->len(); i++ ) {
-    Node *m = n->in(i);
-    if( !m ) continue;  // must see all nodes in block that precede call
-    if (get_block_for_node(m) == block) {
-      set_next_call(block, m, next_call);
+void PhaseCFG::set_next_call(const Block* block, Node* init, VectorSet& next_call) const {
+  Node_List worklist;
+  worklist.push(init);
+
+  while (worklist.size() > 0) {
+    Node* n = worklist.pop();
+    if (next_call.test_set(n->_idx)) continue;
+    for (uint i = 0; i < n->len(); i++) {
+      Node* m = n->in(i);
+      if (m == nullptr) continue;  // must see all nodes in block that precede call
+      if (get_block_for_node(m) == block) {
+        worklist.push(m);
+      }
     }
   }
 }

test/hotspot/jtreg/compiler/c2/StackOverflowInSetNextCall.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+/**
+ * @test
+ * @bug 8357781
+ * @summary Triggered a stack overflow in PhaseCFG::set_next_call due to a legitimately big (mostly deep and not wide) graph.
+ *
+ * @run main/othervm -Xcomp -XX:LoopUnrollLimit=8192 -XX:CompileCommand=compileonly,StackOverflowInSetNextCall::test StackOverflowInSetNextCall
+ * @run main StackOverflowInSetNextCall
+ *
+ */
+
+import java.util.Arrays;
+
+public class StackOverflowInSetNextCall {
+    public static Double d;
+
+    static Long[] arr = new Long[500];
+
+    public static double test() {
+        long x = 0;
+        for (int i = 0; i < 100; i++) {
+            for (int j = 0; j < 100; j++) {
+                if (!Arrays.equals(arr, arr)) {
+                    for (int k = 0; k < 100; k++) {
+                        for (int l = 0; l < 100; ++l) {
+                            x -= (j - k) * x;
+                            for (int m = 0; m < 100; m++) {
+                                x += x;
+                                x += x - (x - arr[i]);
+                            }
+                        }
+                        d = 0.0d;
+                    }
+                }
+            }
+        }
+        return x;
+    }
+
+    public static void main(String[] args) {
+        for (int i = 0; i < 400; ++i) {
+            test();
+        }
+    }
+}