8203436: javac should fail early when emitting illegal signature attributes

Check that signature attributes do not contain non-denotable types

Reviewed-by: vromero

src/jdk.compiler/share/classes/com/sun/tools/javac/code/Types.java

@@ -4989,6 +4989,20 @@ public class Types {
 
     public static abstract class SignatureGenerator {
 
+        public static class InvalidSignatureException extends RuntimeException {
+            private static final long serialVersionUID = 0;
+
+            private final Type type;
+
+            InvalidSignatureException(Type type) {
+                this.type = type;
+            }
+
+            public Type type() {
+                return type;
+            }
+        }
+
         private final Types types;
 
         protected abstract void append(char ch);
@@ -5033,6 +5047,9 @@ public class Types {
                     append('V');
                     break;
                 case CLASS:
+                    if (type.isCompound()) {
+                        throw new InvalidSignatureException(type);
+                    }
                     append('L');
                     assembleClassSig(type);
                     append(';');
@@ -5075,6 +5092,9 @@ public class Types {
                     break;
                 }
                 case TYPEVAR:
+                    if (((TypeVar)type).isCaptured()) {
+                        throw new InvalidSignatureException(type);
+                    }
                     append('T');
                     append(type.tsym.name);
                     append(';');

src/jdk.compiler/share/classes/com/sun/tools/javac/comp/Check.java

@@ -2408,22 +2408,6 @@ public class Check {
         checkCompatibleConcretes(pos, c);
     }
 
-    void checkConflicts(DiagnosticPosition pos, Symbol sym, TypeSymbol c) {
-        for (Type ct = c.type; ct != Type.noType ; ct = types.supertype(ct)) {
-            for (Symbol sym2 : ct.tsym.members().getSymbolsByName(sym.name, NON_RECURSIVE)) {
-                // VM allows methods and variables with differing types
-                if (sym.kind == sym2.kind &&
-                    types.isSameType(types.erasure(sym.type), types.erasure(sym2.type)) &&
-                    sym != sym2 &&
-                    (sym.flags() & Flags.SYNTHETIC) != (sym2.flags() & Flags.SYNTHETIC) &&
-                    (sym.flags() & BRIDGE) == 0 && (sym2.flags() & BRIDGE) == 0) {
-                    syntheticError(pos, (sym2.flags() & SYNTHETIC) == 0 ? sym2 : sym);
-                    return;
-                }
-            }
-        }
-    }
-
     /** Check that all non-override equivalent methods accessible from 'site'
      *  are mutually compatible (JLS 8.4.8/9.4.1).
      *
@@ -2710,14 +2694,6 @@ public class Check {
                 fullName.contains(".internal.");
     }
 
-    /** Report a conflict between a user symbol and a synthetic symbol.
-     */
-    private void syntheticError(DiagnosticPosition pos, Symbol sym) {
-        if (!sym.type.isErroneous()) {
-            log.error(pos, Errors.SyntheticNameConflict(sym, sym.location()));
-        }
-    }
-
     /** Check that class c does not implement directly or indirectly
      *  the same parameterized interface with two different argument lists.
      *  @param pos          Position to be used for error reporting.

src/jdk.compiler/share/classes/com/sun/tools/javac/comp/Lower.java

@@ -32,6 +32,7 @@ import com.sun.tools.javac.code.Kinds.KindSelector;
 import com.sun.tools.javac.code.Scope.WriteableScope;
 import com.sun.tools.javac.jvm.*;
 import com.sun.tools.javac.main.Option.PkgInfo;
+import com.sun.tools.javac.resources.CompilerProperties.Fragments;
 import com.sun.tools.javac.tree.*;
 import com.sun.tools.javac.util.*;
 import com.sun.tools.javac.util.JCDiagnostic.DiagnosticPosition;
@@ -721,14 +722,14 @@ public class Lower extends TreeTranslator {
 
         @Override
         public void visitMethodDef(JCMethodDecl that) {
-            chk.checkConflicts(that.pos(), that.sym, currentClass);
+            checkConflicts(that.pos(), that.sym, currentClass);
             super.visitMethodDef(that);
         }
 
         @Override
         public void visitVarDef(JCVariableDecl that) {
             if (that.sym.owner.kind == TYP) {
-                chk.checkConflicts(that.pos(), that.sym, currentClass);
+                checkConflicts(that.pos(), that.sym, currentClass);
             }
             super.visitVarDef(that);
         }
@@ -744,6 +745,30 @@ public class Lower extends TreeTranslator {
                 currentClass = prevCurrentClass;
             }
         }
+
+        void checkConflicts(DiagnosticPosition pos, Symbol sym, TypeSymbol c) {
+            for (Type ct = c.type; ct != Type.noType ; ct = types.supertype(ct)) {
+                for (Symbol sym2 : ct.tsym.members().getSymbolsByName(sym.name, NON_RECURSIVE)) {
+                    // VM allows methods and variables with differing types
+                    if (sym.kind == sym2.kind &&
+                        types.isSameType(types.erasure(sym.type), types.erasure(sym2.type)) &&
+                        sym != sym2 &&
+                        (sym.flags() & Flags.SYNTHETIC) != (sym2.flags() & Flags.SYNTHETIC) &&
+                        (sym.flags() & BRIDGE) == 0 && (sym2.flags() & BRIDGE) == 0) {
+                        syntheticError(pos, (sym2.flags() & SYNTHETIC) == 0 ? sym2 : sym);
+                        return;
+                    }
+                }
+            }
+        }
+
+        /** Report a conflict between a user symbol and a synthetic symbol.
+         */
+        private void syntheticError(DiagnosticPosition pos, Symbol sym) {
+            if (!sym.type.isErroneous()) {
+                log.error(pos, Errors.CannotGenerateClass(sym.location(), Fragments.SyntheticNameConflict(sym, sym.location())));
+            }
+        }
     };
 
     /** Look up a synthetic name in a given scope.

src/jdk.compiler/share/classes/com/sun/tools/javac/jvm/ClassWriter.java

@@ -42,6 +42,7 @@ import com.sun.tools.javac.code.Attribute.RetentionPolicy;
 import com.sun.tools.javac.code.Directive.*;
 import com.sun.tools.javac.code.Symbol.*;
 import com.sun.tools.javac.code.Type.*;
+import com.sun.tools.javac.code.Types.SignatureGenerator.InvalidSignatureException;
 import com.sun.tools.javac.code.Types.UniqueType;
 import com.sun.tools.javac.comp.Check;
 import com.sun.tools.javac.file.PathFileObject;
@@ -49,6 +50,8 @@ import com.sun.tools.javac.jvm.Pool.DynamicMethod;
 import com.sun.tools.javac.jvm.Pool.Method;
 import com.sun.tools.javac.jvm.Pool.MethodHandle;
 import com.sun.tools.javac.jvm.Pool.Variable;
+import com.sun.tools.javac.resources.CompilerProperties.Errors;
+import com.sun.tools.javac.resources.CompilerProperties.Fragments;
 import com.sun.tools.javac.util.*;
 
 import static com.sun.tools.javac.code.Flags.*;
@@ -1689,6 +1692,8 @@ public class ClassWriter extends ClassFile {
                 log.printVerbose("wrote.file", outFile.getName());
             out.close();
             out = null;
+        } catch (InvalidSignatureException ex) {
+            log.error(Errors.CannotGenerateClass(c, Fragments.IllegalSignature(c, ex.type())));
         } finally {
             if (out != null) {
                 // if we are propagating an exception, delete the file

src/jdk.compiler/share/classes/com/sun/tools/javac/resources/compiler.properties

@@ -1120,10 +1120,19 @@ compiler.err.static.imp.only.classes.and.interfaces=\
 compiler.err.string.const.req=\
     constant string expression required
 
+# 0: symbol, 1: fragment
+compiler.err.cannot.generate.class=\
+    error while generating class {0}\n\
+    ({1})
+
 # 0: symbol, 1: symbol
-compiler.err.synthetic.name.conflict=\
+compiler.misc.synthetic.name.conflict=\
     the symbol {0} conflicts with a compiler-synthesized symbol in {1}
 
+# 0: symbol, 1: type
+compiler.misc.illegal.signature=\
+    illegal signature attribute for type {1}
+
 compiler.err.throws.not.allowed.in.intf.annotation=\
     throws clause not allowed in @interface members
 

test/langtools/tools/javac/6521805/T6521805d.out

@@ -1,2 +1,2 @@
-T6521805d.java:18:18: compiler.err.synthetic.name.conflict: this$0, T6521805.Inner
+T6521805d.java:18:18: compiler.err.cannot.generate.class: T6521805.Inner, (compiler.misc.synthetic.name.conflict: this$0, T6521805.Inner)
 1 error

test/langtools/tools/javac/6521805/T6521805e.out

@@ -1,2 +1,2 @@
-Sub.java:10:11: compiler.err.synthetic.name.conflict: this$0, p.Inner
+Sub.java:10:11: compiler.err.cannot.generate.class: p.Inner, (compiler.misc.synthetic.name.conflict: this$0, p.Inner)
 1 error

test/langtools/tools/javac/8203436/T8203436a.java

@@ -0,0 +1,14 @@
+/*
+ * @test /nodynamiccopyright/
+ * @bug 8203436
+ * @summary javac should fail early when emitting illegal signature attributes
+ * @compile/fail/ref=T8203436a.out -XDrawDiagnostics T8203436a.java
+ */
+
+class T8203436a<X> {
+   class Inner { }
+
+   void test(T8203436a<?> outer) {
+      outer.new Inner() { };
+   }
+}

test/langtools/tools/javac/8203436/T8203436a.out

@@ -0,0 +1,2 @@
+- compiler.err.cannot.generate.class: compiler.misc.anonymous.class: T8203436a$1, (compiler.misc.illegal.signature: compiler.misc.anonymous.class: T8203436a$1, compiler.misc.type.captureof: 1, ?)
+1 error

test/langtools/tools/javac/8203436/T8203436b.java

@@ -0,0 +1,19 @@
+/*
+ * @test /nodynamiccopyright/
+ * @bug 8203436
+ * @summary javac should fail early when emitting illegal signature attributes
+ * @compile/fail/ref=T8203436b.out -XDrawDiagnostics T8203436b.java
+ */
+
+class T8203436b<X> {
+    interface A { }
+    interface B { }
+
+    class Inner { }
+
+    <Z extends A & B> T8203436b<Z> m() { return null; }
+
+    void test() {
+        m().new Inner() { };
+    }
+}

test/langtools/tools/javac/8203436/T8203436b.out

@@ -0,0 +1,2 @@
+- compiler.err.cannot.generate.class: compiler.misc.anonymous.class: T8203436b$1, (compiler.misc.illegal.signature: compiler.misc.anonymous.class: T8203436b$1, java.lang.Object&T8203436b.A&T8203436b.B)
+1 error

test/langtools/tools/javac/SynthName2.out

@@ -1,4 +1,4 @@
-SynthName2.java:33:9: compiler.err.synthetic.name.conflict: val$zzz, InnClass
+SynthName2.java:33:9: compiler.err.cannot.generate.class: InnClass, (compiler.misc.synthetic.name.conflict: val$zzz, InnClass)
-SynthName2.java:34:17: compiler.err.synthetic.name.conflict: val$prm1, InnClass
+SynthName2.java:34:17: compiler.err.cannot.generate.class: InnClass, (compiler.misc.synthetic.name.conflict: val$prm1, InnClass)
-SynthName2.java:35:17: compiler.err.synthetic.name.conflict: val$zzz, InnClass
+SynthName2.java:35:17: compiler.err.cannot.generate.class: InnClass, (compiler.misc.synthetic.name.conflict: val$zzz, InnClass)
 3 errors

test/langtools/tools/javac/diags/examples/ErrSyntheticNameConflict.java

@@ -21,7 +21,8 @@
  * questions.
  */
 
-// key: compiler.err.synthetic.name.conflict
+// key: compiler.err.cannot.generate.class
+// key: compiler.misc.synthetic.name.conflict
 
 class ErrSyntheticNameConflict {
 

test/langtools/tools/javac/diags/examples/IllegalSignature.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+// key: compiler.err.cannot.generate.class
+// key: compiler.misc.illegal.signature
+
+class IllegalSignature<X> {
+    class Inner { }
+
+    void test(IllegalSignature<?> outer) {
+        outer.new Inner() { };
+    }
+}