8357826: Avoid running some jtreg tests when asan is configured

Reviewed-by: sspitsyn, amitkumar, lmesnik, syan, lucy, cjplummer

src/hotspot/share/prims/whitebox.cpp

@@ -1075,6 +1075,22 @@ bool WhiteBox::validate_cgroup(bool cgroups_v2_enabled,
 }
 #endif
 
+bool WhiteBox::is_asan_enabled() {
+#ifdef ADDRESS_SANITIZER
+  return true;
+#else
+  return false;
+#endif
+}
+
+bool WhiteBox::is_ubsan_enabled() {
+#ifdef UNDEFINED_BEHAVIOR_SANITIZER
+  return true;
+#else
+  return false;
+#endif
+}
+
 bool WhiteBox::compile_method(Method* method, int comp_level, int bci, JavaThread* THREAD) {
   // Screen for unavailable/bad comp level or null method
   AbstractCompiler* comp = CompileBroker::compiler(comp_level);
@@ -1886,6 +1902,14 @@ WB_ENTRY(jboolean, WB_IsMonitorInflated(JNIEnv* env, jobject wb, jobject obj))
   return (jboolean) obj_oop->mark().has_monitor();
 WB_END
 
+WB_ENTRY(jboolean, WB_IsAsanEnabled(JNIEnv* env))
+  return (jboolean) WhiteBox::is_asan_enabled();
+WB_END
+
+WB_ENTRY(jboolean, WB_IsUbsanEnabled(JNIEnv* env))
+  return (jboolean) WhiteBox::is_ubsan_enabled();
+WB_END
+
 WB_ENTRY(jlong, WB_getInUseMonitorCount(JNIEnv* env, jobject wb))
   return (jlong) WhiteBox::get_in_use_monitor_count();
 WB_END
@@ -2882,6 +2906,8 @@ static JNINativeMethod methods[] = {
                                                       (void*)&WB_AddModuleExportsToAll },
   {CC"deflateIdleMonitors", CC"()Z",                  (void*)&WB_DeflateIdleMonitors },
   {CC"isMonitorInflated0", CC"(Ljava/lang/Object;)Z", (void*)&WB_IsMonitorInflated  },
+  {CC"isAsanEnabled", CC"()Z",                        (void*)&WB_IsAsanEnabled },
+  {CC"isUbsanEnabled", CC"()Z",                       (void*)&WB_IsUbsanEnabled },
   {CC"getInUseMonitorCount", CC"()J", (void*)&WB_getInUseMonitorCount  },
   {CC"getLockStackCapacity", CC"()I",                 (void*)&WB_getLockStackCapacity },
   {CC"supportsRecursiveLightweightLocking", CC"()Z",  (void*)&WB_supportsRecursiveLightweightLocking },

src/hotspot/share/prims/whitebox.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -72,6 +72,9 @@ class WhiteBox : public AllStatic {
 #ifdef LINUX
   static bool validate_cgroup(bool cgroups_v2_enabled, const char* controllers_file, const char* proc_self_cgroup, const char* proc_self_mountinfo, u1* cg_flags);
 #endif
+  // provide info about enabling of Address Sanitizer / Undefined Behavior Sanitizer
+  static bool is_asan_enabled();
+  static bool is_ubsan_enabled();
 };
 
 #endif // SHARE_PRIMS_WHITEBOX_HPP

test/hotspot/jtreg/TEST.ROOT

@@ -91,6 +91,8 @@ requires.properties= \
     vm.compiler1.enabled \
     vm.compiler2.enabled \
     vm.musl \
+    vm.asan \
+    vm.ubsan \
     vm.flagless \
     container.support \
     systemd.support \

test/hotspot/jtreg/gc/arguments/TestUseCompressedOopsFlagsWithUlimit.java

@@ -31,6 +31,8 @@ package gc.arguments;
  * @library /test/lib
  * @library /
  * @requires vm.bits == "64"
+ * @comment ulimit clashes with the memory requirements of ASAN
+ * @requires !vm.asan
  * @requires os.family == "linux"
  * @requires vm.gc != "Z"
  * @requires vm.opt.UseCompressedOops == null

test/hotspot/jtreg/runtime/Thread/TestBreakSignalThreadDump.java

@@ -38,6 +38,8 @@
  * @requires os.family != "windows" & os.family != "aix"
  * @comment TODO: Decide libjsig support on static JDK with 8351367
  * @requires !jdk.static
+ * @comment loading of the jsig lib does currently not work well with ASAN lib
+ * @requires !vm.asan
  * @library /vmTestbase
  *          /test/lib
  * @run driver TestBreakSignalThreadDump load_libjsig

test/hotspot/jtreg/runtime/XCheckJniJsig/XCheckJSig.java

@@ -29,6 +29,8 @@
  * @modules java.base/jdk.internal.misc
  *          java.management
  * @requires os.family == "linux" | os.family == "mac"
+ * @comment loading of the jsig lib does currently not work well with ASAN lib
+ * @requires !vm.asan
  * @comment TODO: Decide libjsig support on static JDK with 8351367
  * @requires !jdk.static
  * @run driver XCheckJSig

test/hotspot/jtreg/serviceability/dcmd/vm/SystemDumpMapTest.java

@@ -36,6 +36,8 @@ import java.util.regex.Pattern;
  * @summary Test of diagnostic command System.map
  * @library /test/lib
  * @requires (os.family == "linux" | os.family == "windows" | os.family == "mac")
+ * @comment ASAN changes the memory map dump slightly, but the test has rather strict requirements
+ * @requires !vm.asan
  * @requires os.arch != "riscv64" | !(vm.cpu.features ~= ".*qemu.*")
  * @modules java.base/jdk.internal.misc
  *          java.compiler

test/hotspot/jtreg/serviceability/dcmd/vm/SystemMapTest.java

@@ -32,6 +32,8 @@ import jdk.test.lib.process.OutputAnalyzer;
  * @summary Test of diagnostic command System.map
  * @library /test/lib
  * @requires (vm.gc != "Z") & (os.family == "linux" | os.family == "windows" | os.family == "mac")
+ * @comment ASAN changes the memory map dump slightly, but the test has rather strict requirements
+ * @requires !vm.asan
  * @modules java.base/jdk.internal.misc
  *          java.compiler
  *          java.management
@@ -47,6 +49,8 @@ import jdk.test.lib.process.OutputAnalyzer;
  * @summary Test of diagnostic command System.map using ZGC
  * @library /test/lib
  * @requires vm.gc.Z & (os.family == "linux" | os.family == "windows" | os.family == "mac")
+ * @comment ASAN changes the memory map dump slightly, but the test has rather strict requirements
+ * @requires !vm.asan
  * @modules java.base/jdk.internal.misc
  *          java.compiler
  *          java.management

test/jdk/TEST.ROOT

@@ -102,6 +102,8 @@ requires.properties= \
     vm.cds.write.archived.java.heap \
     vm.continuations \
     vm.musl \
+    vm.asan \
+    vm.ubsan \
     vm.debug \
     vm.hasSA \
     vm.hasJFR \

test/jdk/build/AbsPathsInImage.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -40,6 +40,8 @@ import java.util.zip.ZipInputStream;
  * @bug 8226346
  * @summary Check all output files for absolute path fragments
  * @requires !vm.debug
+ * @comment ASAN keeps the 'unwanted' paths in the binaries because of its build options
+ * @requires !vm.asan
  * @run main/othervm -Xmx900m AbsPathsInImage
  */
 public class AbsPathsInImage {

test/jtreg-ext/requires/VMProps.java

@@ -138,6 +138,8 @@ public class VMProps implements Callable<Map<String, String>> {
         map.put("container.support", this::containerSupport);
         map.put("systemd.support", this::systemdSupport);
         map.put("vm.musl", this::isMusl);
+        map.put("vm.asan", this::isAsanEnabled);
+        map.put("vm.ubsan", this::isUbsanEnabled);
         map.put("release.implementor", this::implementor);
         map.put("jdk.containerized", this::jdkContainerized);
         map.put("vm.flagless", this::isFlagless);
@@ -728,6 +730,15 @@ public class VMProps implements Callable<Map<String, String>> {
         return Boolean.toString(WB.getLibcName().contains("musl"));
     }
 
+    // Sanitizer support
+    protected String isAsanEnabled() {
+        return "" + WB.isAsanEnabled();
+    }
+
+    protected String isUbsanEnabled() {
+        return "" + WB.isUbsanEnabled();
+    }
+
     private String implementor() {
         try (InputStream in = new BufferedInputStream(new FileInputStream(
                 System.getProperty("java.home") + "/release"))) {

test/lib/jdk/test/whitebox/WhiteBox.java

@@ -322,6 +322,10 @@ public class WhiteBox {
   public native void NMTFreeArena(long arena);
   public native void NMTArenaMalloc(long arena, long size);
 
+  // Sanitizers
+  public native boolean isAsanEnabled();
+  public native boolean isUbsanEnabled();
+
   // Compiler
 
   // Determines if the libgraal shared library file is present.