8318737: Fallback linker passes bad JNI handle

Reviewed-by: alanb

src/hotspot/share/runtime/jniHandles.cpp

@@ -199,13 +199,9 @@ jobjectRefType JNIHandles::handle_type(JavaThread* thread, jobject handle) {
     default:
       ShouldNotReachHere();
     }
-  } else {
+  } else if (is_local_handle(thread, handle) || is_frame_handle(thread, handle)) {
     // Not in global storage.  Might be a local handle.
-    if (is_local_handle(thread, handle) || is_frame_handle(thread, handle)) {
+    result = JNILocalRefType;
-      result = JNILocalRefType;
-    } else {
-      ShouldNotReachHere();
-    }
   }
   return result;
 }

src/java.base/share/classes/jdk/internal/foreign/abi/fallback/LibFallback.java

@@ -43,11 +43,14 @@ final class LibFallback {
                     public Boolean run() {
                         try {
                             System.loadLibrary("fallbackLinker");
-                            init();
-                            return true;
                         } catch (UnsatisfiedLinkError ule) {
                             return false;
                         }
+                        if (!init()) {
+                            // library failed to initialize. Do not silently mark as unsupported
+                            throw new ExceptionInInitializerError("Fallback library failed to initialize");
+                        }
+                        return true;
                     }
                 });
     }
@@ -201,7 +204,7 @@ final class LibFallback {
         }
     }
 
-    private static native void init();
+    private static native boolean init();
 
     private static native long sizeofCif();
 

src/java.base/share/native/libfallbackLinker/fallbackLinker.c

@@ -42,12 +42,29 @@ static jclass LibFallback_class;
 static jmethodID LibFallback_doUpcall_ID;
 static const char* LibFallback_doUpcall_sig = "(JJLjava/lang/invoke/MethodHandle;)V";
 
-JNIEXPORT void JNICALL
+#define CHECK_NULL(expr) \
+  if (expr == NULL) { \
+    return JNI_FALSE; \
+  }
+
+JNIEXPORT jboolean JNICALL
 Java_jdk_internal_foreign_abi_fallback_LibFallback_init(JNIEnv* env, jclass cls) {
-  (*env)->GetJavaVM(env, &VM);
+  jint result = (*env)->GetJavaVM(env, &VM);
-  LibFallback_class = (*env)->FindClass(env, "jdk/internal/foreign/abi/fallback/LibFallback");
+  if (result != 0) {
+    return JNI_FALSE;
+  }
+
+  jclass LibFallback_class_local = (*env)->FindClass(env, "jdk/internal/foreign/abi/fallback/LibFallback");
+  CHECK_NULL(LibFallback_class_local)
+
+  LibFallback_class = (*env)->NewGlobalRef(env, LibFallback_class_local);
+  CHECK_NULL(LibFallback_class)
+
   LibFallback_doUpcall_ID = (*env)->GetStaticMethodID(env,
     LibFallback_class, "doUpcall", LibFallback_doUpcall_sig);
+  CHECK_NULL(LibFallback_doUpcall_ID)
+
+  return JNI_TRUE;
 }
 
 JNIEXPORT jlong JNICALL

test/jdk/java/foreign/TestDowncallScope.java

@@ -26,7 +26,7 @@
  * @modules java.base/jdk.internal.foreign
  * @build NativeTestHelper CallGeneratorHelper TestDowncallBase
  *
- * @run testng/othervm -XX:+IgnoreUnrecognizedVMOptions -XX:-VerifyDependencies
+ * @run testng/othervm -Xcheck:jni -XX:+IgnoreUnrecognizedVMOptions -XX:-VerifyDependencies
  *   --enable-native-access=ALL-UNNAMED -Dgenerator.sample.factor=17
  *   TestDowncallScope
  *

test/jdk/java/foreign/TestDowncallStack.java

@@ -26,7 +26,7 @@
  * @modules java.base/jdk.internal.foreign
  * @build NativeTestHelper CallGeneratorHelper TestDowncallBase
  *
- * @run testng/othervm -XX:+IgnoreUnrecognizedVMOptions -XX:-VerifyDependencies
+ * @run testng/othervm -Xcheck:jni -XX:+IgnoreUnrecognizedVMOptions -XX:-VerifyDependencies
  *   --enable-native-access=ALL-UNNAMED -Dgenerator.sample.factor=17
  *   TestDowncallStack
  */

test/jdk/java/foreign/TestUpcallScope.java

@@ -26,7 +26,7 @@
  * @modules java.base/jdk.internal.foreign
  * @build NativeTestHelper CallGeneratorHelper TestUpcallBase
  *
- * @run testng/othervm -XX:+IgnoreUnrecognizedVMOptions -XX:-VerifyDependencies
+ * @run testng/othervm -Xcheck:jni -XX:+IgnoreUnrecognizedVMOptions -XX:-VerifyDependencies
  *   --enable-native-access=ALL-UNNAMED -Dgenerator.sample.factor=17
  *   TestUpcallScope
  */

test/jdk/java/foreign/TestUpcallStack.java

@@ -27,7 +27,7 @@
  * @modules java.base/jdk.internal.foreign
  * @build NativeTestHelper CallGeneratorHelper TestUpcallBase
  *
- * @run testng/othervm -XX:+IgnoreUnrecognizedVMOptions -XX:-VerifyDependencies
+ * @run testng/othervm -Xcheck:jni -XX:+IgnoreUnrecognizedVMOptions -XX:-VerifyDependencies
  *   --enable-native-access=ALL-UNNAMED -Dgenerator.sample.factor=17
  *   TestUpcallStack
  */