8160717: MethodHandles.loop() does not check for excessive signature

Reviewed-by: psandoz, redestad

jdk/src/java.base/share/classes/java/lang/invoke/InvokerBytecodeGenerator.java

@@ -751,11 +751,25 @@ class InvokerBytecodeGenerator {
         classFileEpilogue();
         bogusMethod(lambdaForm);
 
-        final byte[] classFile = cw.toByteArray();
+        final byte[] classFile;
+        try {
+            classFile = cw.toByteArray();
+        } catch (RuntimeException e) {
+            // ASM throws RuntimeException if something goes wrong - capture these and wrap them in a meaningful
+            // exception to support falling back to LambdaForm interpretation
+            throw new BytecodeGenerationException(e);
+        }
         maybeDump(className, classFile);
         return classFile;
     }
 
+    @SuppressWarnings("serial")
+    static final class BytecodeGenerationException extends RuntimeException {
+        BytecodeGenerationException(Exception cause) {
+            super(cause);
+        }
+    }
+
     void emitArrayLoad(Name name)   { emitArrayOp(name, Opcodes.AALOAD);      }
     void emitArrayStore(Name name)  { emitArrayOp(name, Opcodes.AASTORE);     }
     void emitArrayLength(Name name) { emitArrayOp(name, Opcodes.ARRAYLENGTH); }

jdk/src/java.base/share/classes/java/lang/invoke/LambdaForm.java

@@ -25,6 +25,7 @@
 
 package java.lang.invoke;
 
+import jdk.internal.perf.PerfCounter;
 import jdk.internal.vm.annotation.DontInline;
 import jdk.internal.vm.annotation.Stable;
 import sun.invoke.util.Wrapper;
@@ -39,8 +40,7 @@ import java.util.HashMap;
 
 import static java.lang.invoke.LambdaForm.BasicType.*;
 import static java.lang.invoke.MethodHandleNatives.Constants.REF_invokeStatic;
-import static java.lang.invoke.MethodHandleStatics.debugEnabled;
+import static java.lang.invoke.MethodHandleStatics.*;
-import static java.lang.invoke.MethodHandleStatics.newInternalError;
 
 /**
  * The symbolic, non-executable form of a method handle's invocation semantics.
@@ -396,7 +396,7 @@ class LambdaForm {
     /** Customize LambdaForm for a particular MethodHandle */
     LambdaForm customize(MethodHandle mh) {
         LambdaForm customForm = new LambdaForm(debugName, arity, names, result, forceInline, mh);
-        if (COMPILE_THRESHOLD > 0 && isCompiled) {
+        if (COMPILE_THRESHOLD >= 0 && isCompiled) {
             // If shared LambdaForm has been compiled, compile customized version as well.
             customForm.compileToBytecode();
         }
@@ -411,7 +411,7 @@ class LambdaForm {
         }
         assert(transformCache != null); // Customized LambdaForm should always has a link to uncustomized version.
         LambdaForm uncustomizedForm = (LambdaForm)transformCache;
-        if (COMPILE_THRESHOLD > 0 && isCompiled) {
+        if (COMPILE_THRESHOLD >= 0 && isCompiled) {
             // If customized LambdaForm has been compiled, compile uncustomized version as well.
             uncustomizedForm.compileToBytecode();
         }
@@ -717,7 +717,7 @@ class LambdaForm {
      * as a sort of pre-invocation linkage step.)
      */
     public void prepare() {
-        if (COMPILE_THRESHOLD == 0 && !isCompiled) {
+        if (COMPILE_THRESHOLD == 0 && !forceInterpretation() && !isCompiled) {
             compileToBytecode();
         }
         if (this.vmentry != null) {
@@ -736,10 +736,22 @@ class LambdaForm {
         // TO DO: Maybe add invokeGeneric, invokeWithArguments
     }
 
+    private static @Stable PerfCounter LF_FAILED;
+
+    private static PerfCounter failedCompilationCounter() {
+        if (LF_FAILED == null) {
+            LF_FAILED = PerfCounter.newPerfCounter("java.lang.invoke.failedLambdaFormCompilations");
+        }
+        return LF_FAILED;
+    }
+
     /** Generate optimizable bytecode for this form. */
-    MemberName compileToBytecode() {
+    void compileToBytecode() {
+        if (forceInterpretation()) {
+            return; // this should not be compiled
+        }
         if (vmentry != null && isCompiled) {
-            return vmentry;  // already compiled somehow
+            return;  // already compiled somehow
         }
         MethodType invokerType = methodType();
         assert(vmentry == null || vmentry.getMethodType().basicType().equals(invokerType));
@@ -748,9 +760,16 @@ class LambdaForm {
             if (TRACE_INTERPRETER)
                 traceInterpreter("compileToBytecode", this);
             isCompiled = true;
-            return vmentry;
+        } catch (InvokerBytecodeGenerator.BytecodeGenerationException bge) {
-        } catch (Error | Exception ex) {
+            // bytecode generation failed - mark this LambdaForm as to be run in interpretation mode only
-            throw newInternalError(this.toString(), ex);
+            invocationCounter = -1;
+            failedCompilationCounter().increment();
+            if (LOG_LF_COMPILATION_FAILURE) {
+                System.out.println("LambdaForm compilation failed: " + this);
+                bge.printStackTrace(System.out);
+            }
+        } catch (Error | Exception e) {
+            throw newInternalError(this.toString(), e);
         }
     }
 
@@ -856,7 +875,11 @@ class LambdaForm {
     static {
         COMPILE_THRESHOLD = Math.max(-1, MethodHandleStatics.COMPILE_THRESHOLD);
     }
-    private int invocationCounter = 0;
+    private int invocationCounter = 0; // a value of -1 indicates LambdaForm interpretation mode forever
+
+    private boolean forceInterpretation() {
+        return invocationCounter == -1;
+    }
 
     @Hidden
     @DontInline
@@ -896,7 +919,7 @@ class LambdaForm {
 
     private void checkInvocationCounter() {
         if (COMPILE_THRESHOLD != 0 &&
-            invocationCounter < COMPILE_THRESHOLD) {
+            !forceInterpretation() && invocationCounter < COMPILE_THRESHOLD) {
             invocationCounter++;  // benign race
             if (invocationCounter >= COMPILE_THRESHOLD) {
                 // Replace vmentry with a bytecode version of this LF.
@@ -906,7 +929,7 @@ class LambdaForm {
     }
     Object interpretWithArgumentsTracing(Object... argumentValues) throws Throwable {
         traceInterpreter("[ interpretWithArguments", this, argumentValues);
-        if (invocationCounter < COMPILE_THRESHOLD) {
+        if (!forceInterpretation() && invocationCounter < COMPILE_THRESHOLD) {
             int ctr = invocationCounter++;  // benign race
             traceInterpreter("| invocationCounter", ctr);
             if (invocationCounter >= COMPILE_THRESHOLD) {

jdk/src/java.base/share/classes/java/lang/invoke/MethodHandleStatics.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,10 +25,11 @@
 
 package java.lang.invoke;
 
-import java.util.Properties;
 import jdk.internal.misc.Unsafe;
 import sun.security.action.GetPropertyAction;
 
+import java.util.Properties;
+
 /**
  * This class consists exclusively of static names internal to the
  * method handle implementation.
@@ -46,6 +47,7 @@ import sun.security.action.GetPropertyAction;
     static final boolean TRACE_INTERPRETER;
     static final boolean TRACE_METHOD_LINKAGE;
     static final int COMPILE_THRESHOLD;
+    static final boolean LOG_LF_COMPILATION_FAILURE;
     static final int DONT_INLINE_THRESHOLD;
     static final int PROFILE_LEVEL;
     static final boolean PROFILE_GWT;
@@ -64,6 +66,8 @@ import sun.security.action.GetPropertyAction;
                 props.getProperty("java.lang.invoke.MethodHandle.TRACE_METHOD_LINKAGE"));
         COMPILE_THRESHOLD = Integer.parseInt(
                 props.getProperty("java.lang.invoke.MethodHandle.COMPILE_THRESHOLD", "0"));
+        LOG_LF_COMPILATION_FAILURE = Boolean.parseBoolean(
+                props.getProperty("java.lang.invoke.MethodHandle.LOG_LF_COMPILATION_FAILURE", "false"));
         DONT_INLINE_THRESHOLD = Integer.parseInt(
                 props.getProperty("java.lang.invoke.MethodHandle.DONT_INLINE_THRESHOLD", "30"));
         PROFILE_LEVEL = Integer.parseInt(
@@ -87,7 +91,8 @@ import sun.security.action.GetPropertyAction;
         return (DEBUG_METHOD_HANDLE_NAMES |
                 DUMP_CLASS_FILES |
                 TRACE_INTERPRETER |
-                TRACE_METHOD_LINKAGE);
+                TRACE_METHOD_LINKAGE |
+                LOG_LF_COMPILATION_FAILURE);
     }
 
     // handy shared exception makers (they simplify the common case code)

jdk/test/java/lang/invoke/LoopCombinatorLongSignatureTest.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2015, 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/* @test
+ * @bug 8160717
+ * @run main/othervm -ea -esa -Djava.lang.invoke.MethodHandle.COMPILE_THRESHOLD=-1 test.java.lang.invoke.LoopCombinatorLongSignatureTest
+ * @run main/othervm -ea -esa test.java.lang.invoke.LoopCombinatorLongSignatureTest
+ */
+
+package test.java.lang.invoke;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+
+/**
+ * If a loop with an excessive amount of clauses is created, so that the number of parameters to the resulting loop
+ * handle exceeds the allowed maximum, an IAE must be signalled. The test is run first in LambdaForm interpretation mode
+ * and then in default mode, wherein bytecode generation falls back to LFI mode due to excessively long methods.
+ */
+public class LoopCombinatorLongSignatureTest {
+
+    static final MethodHandle INIT = MethodHandles.constant(int.class, 0);
+    static final MethodHandle STEP = MethodHandles.identity(int.class);
+    static final MethodHandle PRED_F = MethodHandles.constant(boolean.class, false);
+    static final MethodHandle PRED_T = MethodHandles.constant(boolean.class, true);
+    static final MethodHandle FINI = MethodHandles.identity(int.class);
+
+    static final int ARG_LIMIT = 254; // for internal reasons, this is the maximum allowed number of arguments
+
+    public static void main(String[] args) {
+        for (int loopArgs = 0; loopArgs < 2; ++loopArgs) {
+            testLongSignature(loopArgs, false);
+            testLongSignature(loopArgs, true);
+        }
+    }
+
+    static void testLongSignature(int loopArgs, boolean excessive) {
+        int nClauses = ARG_LIMIT - loopArgs + (excessive ? 1 : 0);
+
+        System.out.print((excessive ? "(EXCESSIVE)" : "(LONG     )") + " arguments: " + loopArgs + ", clauses: " + nClauses + " -> ");
+
+        // extend init to denote what arguments the loop should accept
+        Class<?>[] argTypes = new Class<?>[loopArgs];
+        Arrays.fill(argTypes, int.class);
+        MethodHandle init = MethodHandles.dropArguments(INIT, 0, argTypes);
+
+        // build clauses
+        MethodHandle[][] clauses = new MethodHandle[nClauses][];
+        MethodHandle[] clause = {init, STEP, PRED_T, FINI};
+        MethodHandle[] fclause = {init, STEP, PRED_F, FINI};
+        Arrays.fill(clauses, clause);
+        clauses[nClauses - 1] = fclause; // make the last clause terminate the loop
+
+        try {
+            MethodHandle loop = MethodHandles.loop(clauses);
+            if (excessive) {
+                throw new AssertionError("loop construction should have failed");
+            } else {
+                int r;
+                if (loopArgs == 0) {
+                    r = (int) loop.invoke();
+                } else {
+                    Object[] args = new Object[loopArgs];
+                    Arrays.fill(args, 0);
+                    r = (int) loop.invokeWithArguments(args);
+                }
+                System.out.println("SUCCEEDED (OK) -> " + r);
+            }
+        } catch (IllegalArgumentException iae) {
+            if (excessive) {
+                System.out.println("FAILED    (OK)");
+            } else {
+                iae.printStackTrace(System.out);
+                throw new AssertionError("loop construction should not have failed (see above)");
+            }
+        } catch (Throwable t) {
+            t.printStackTrace(System.out);
+            throw new AssertionError("unexpected failure (see above)");
+        }
+    }
+
+}