8155977: ObjectInputStream::resolveClass & resolveProxyClass for platform loader

Reviewed-by: acorn, alanb, chegar, dfuchs

jdk/make/mapfiles/libjava/mapfile-vers

@@ -262,7 +262,7 @@ SUNWprivate_1.1 {
 		Java_jdk_internal_reflect_Reflection_getCallerClass__;
 		Java_jdk_internal_reflect_Reflection_getCallerClass__I;
 		Java_jdk_internal_reflect_Reflection_getClassAccessFlags;
-		Java_jdk_internal_misc_VM_latestUserDefinedLoader;
+		Java_jdk_internal_misc_VM_latestUserDefinedLoader0;
                 Java_jdk_internal_misc_VM_getuid;
                 Java_jdk_internal_misc_VM_geteuid;
                 Java_jdk_internal_misc_VM_getgid;

jdk/src/java.base/share/classes/java/io/ObjectInputStream.java

@@ -603,12 +603,12 @@ public class ObjectInputStream
      *     Class.forName(desc.getName(), false, loader)
      * </pre>
      * where <code>loader</code> is determined as follows: if there is a
-     * method on the current thread's stack whose declaring class was
+     * method on the current thread's stack whose declaring class is not a
-     * defined by a user-defined class loader (and was not a generated to
+     * <a href="../lang/ClassLoader.html#builtinLoaders">
-     * implement reflective invocations), then <code>loader</code> is class
+     * <em>platform class</em></a>, then <code>loader</code> is
-     * loader corresponding to the closest such method to the currently
+     * the class loader of such class; otherwise, <code>loader</code>
-     * executing frame; otherwise, <code>loader</code> is
+     * is the {@linkplain ClassLoader#getPlatformClassLoader()
-     * <code>null</code>. If this call results in a
+     * platform class loader}.  If this call results in a
      * <code>ClassNotFoundException</code> and the name of the passed
      * <code>ObjectStreamClass</code> instance is the Java language keyword
      * for a primitive type or void, then the <code>Class</code> object
@@ -666,12 +666,15 @@ public class ObjectInputStream
      * <pre>
      *     Class.forName(i, false, loader)
      * </pre>
-     * where <code>loader</code> is that of the first non-<code>null</code>
+     * where <code>loader</code> is determined as follows: if there is a
-     * class loader up the execution stack, or <code>null</code> if no
+     * method on the current thread's stack whose declaring class is not a
-     * non-<code>null</code> class loaders are on the stack (the same class
+     * <a href="../lang/ClassLoader.html#builtinLoaders">
-     * loader choice used by the <code>resolveClass</code> method).  Unless any
+     * <em>platform class</em></a>, then <code>loader</code> is
-     * of the resolved interfaces are non-public, this same value of
+     * the class loader of such class; otherwise, <code>loader</code>
-     * <code>loader</code> is also the class loader passed to
+     * is the {@linkplain ClassLoader#getPlatformClassLoader()
+     * platform class loader}.
+     * Unless any of the resolved interfaces are non-public, this same value
+     * of <code>loader</code> is also the class loader passed to
      * <code>Proxy.getProxyClass</code>; if non-public interfaces are present,
      * their class loader is passed instead (if more than one non-public
      * interface class loader is encountered, an
@@ -2154,10 +2157,11 @@ public class ObjectInputStream
                                               int ndoubles);
 
     /**
-     * Returns the first non-null class loader (not counting class loaders of
+     * Returns the first non-null and non-platform class loader
-     * generated reflection implementation classes) up the execution stack, or
+     * (not counting class loaders of generated reflection implementation classes)
-     * null if only code from the null class loader is on the stack.  This
+     * up the execution stack, or null if only code from the bootstrap and
-     * method is also called via reflection by the following RMI-IIOP class:
+     * platform class loader is on the stack.
+     * This method is also called via reflection by the following RMI-IIOP class:
      *
      *     com.sun.corba.se.internal.util.JDKClassLoader
      *

jdk/src/java.base/share/classes/jdk/internal/misc/VM.java

@@ -390,10 +390,25 @@ public class VM {
     private static final int JVMTI_THREAD_STATE_WAITING_WITH_TIMEOUT = 0x0020;
 
     /*
-     * Returns the first non-null class loader up the execution stack,
+     * Returns the first user-defined class loader up the execution stack,
-     * or null if only code from the null class loader is on the stack.
+     * or the platform class loader if only code from the platform or
+     * bootstrap class loader is on the stack.
      */
-    public static native ClassLoader latestUserDefinedLoader();
+    public static ClassLoader latestUserDefinedLoader() {
+        ClassLoader loader = latestUserDefinedLoader0();
+        return loader != null ? loader : ClassLoader.getPlatformClassLoader();
+    }
+
+    /*
+     * Returns the first user-defined class loader up the execution stack,
+     * or null if only code from the platform or bootstrap class loader is
+     * on the stack.  VM does not keep a reference of platform loader and so
+     * it returns null.
+     *
+     * This method should be replaced with StackWalker::walk and then we can
+     * remove the logic in the VM.
+     */
+    private static native ClassLoader latestUserDefinedLoader0();
 
     /**
      * Returns {@code true} if we are in a set UID program.

jdk/src/java.base/share/native/libjava/VM.c

@@ -36,7 +36,7 @@ static JNINativeMethod methods[] = {
 };
 
 JNIEXPORT jobject JNICALL
-Java_jdk_internal_misc_VM_latestUserDefinedLoader(JNIEnv *env, jclass cls) {
+Java_jdk_internal_misc_VM_latestUserDefinedLoader0(JNIEnv *env, jclass cls) {
     return JVM_LatestUserDefinedLoader(env);
 }
 

jdk/src/java.rmi/share/classes/sun/rmi/server/MarshalInputStream.java

@@ -194,7 +194,7 @@ public class MarshalInputStream extends ObjectInputStream {
         /*
          * Unless we were told to skip this consideration, choose the
          * "default loader" to simulate the default ObjectInputStream
-         * resolveClass mechanism (that is, choose the first non-null
+         * resolveClass mechanism (that is, choose the first non-platform
          * loader on the execution stack) to maximize the likelihood of
          * type compatibility with calling code.  (This consideration
          * is skipped during server parameter unmarshalling using the 1.2
@@ -268,8 +268,9 @@ public class MarshalInputStream extends ObjectInputStream {
     }
 
     /*
-     * Returns the first non-null class loader up the execution stack, or null
+     * Returns the first non-platform class loader up the execution stack,
-     * if only code from the null class loader is on the stack.
+     * or platform class loader if only code from the platform class loader or null
+     * is on the stack.
      */
     private static ClassLoader latestUserDefinedLoader() {
         return jdk.internal.misc.VM.latestUserDefinedLoader();