8280761: UseCompressedOops should be set after limit_heap_by_allocatable_memory

Reviewed-by: ayang, tschatzl

src/hotspot/share/runtime/arguments.cpp

@@ -1760,6 +1760,18 @@ void Arguments::set_heap_size() {
       reasonable_max = MIN2(reasonable_max, (julong)ErgoHeapSizeLimit);
     }
 
+    reasonable_max = limit_heap_by_allocatable_memory(reasonable_max);
+
+    if (!FLAG_IS_DEFAULT(InitialHeapSize)) {
+      // An initial heap size was specified on the command line,
+      // so be sure that the maximum size is consistent.  Done
+      // after call to limit_heap_by_allocatable_memory because that
+      // method might reduce the allocation size.
+      reasonable_max = MAX2(reasonable_max, (julong)InitialHeapSize);
+    } else if (!FLAG_IS_DEFAULT(MinHeapSize)) {
+      reasonable_max = MAX2(reasonable_max, (julong)MinHeapSize);
+    }
+
 #ifdef _LP64
     if (UseCompressedOops || UseCompressedClassPointers) {
       // HeapBaseMinAddress can be greater than default but not less than.
@@ -1806,18 +1818,6 @@ void Arguments::set_heap_size() {
     }
 #endif // _LP64
 
-    reasonable_max = limit_heap_by_allocatable_memory(reasonable_max);
-
-    if (!FLAG_IS_DEFAULT(InitialHeapSize)) {
-      // An initial heap size was specified on the command line,
-      // so be sure that the maximum size is consistent.  Done
-      // after call to limit_heap_by_allocatable_memory because that
-      // method might reduce the allocation size.
-      reasonable_max = MAX2(reasonable_max, (julong)InitialHeapSize);
-    } else if (!FLAG_IS_DEFAULT(MinHeapSize)) {
-      reasonable_max = MAX2(reasonable_max, (julong)MinHeapSize);
-    }
-
     log_trace(gc, heap)("  Maximum heap size " SIZE_FORMAT, (size_t) reasonable_max);
     FLAG_SET_ERGO(MaxHeapSize, (size_t)reasonable_max);
   }

test/hotspot/jtreg/gc/arguments/TestUseCompressedOopsFlagsWithUlimit.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.arguments;
+
+/*
+ * @test TestUseCompressedOopsFlagsWithUlimit
+ * @bug 8280761
+ * @summary Verify correct UseCompressedOops when MaxRAM and MaxRAMPercentage
+ * are specified with ulimit -v.
+ * @library /test/lib
+ * @library /
+ * @requires vm.bits == "64"
+ * @requires os.family != "aix" & os.family != "windows"
+ * @run driver gc.arguments.TestUseCompressedOopsFlagsWithUlimit
+ */
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+
+public class TestUseCompressedOopsFlagsWithUlimit {
+
+  private static void checkFlag(long ulimit, long maxram, int maxrampercent, boolean expectcoop) throws Exception {
+
+    ArrayList<String> args = new ArrayList<String>();
+    args.add("-XX:MaxRAM=" + maxram);
+    args.add("-XX:MaxRAMPercentage=" + maxrampercent);
+    args.add("-XX:+PrintFlagsFinal");
+    args.add("-version");
+
+    // Convert bytes to kbytes for ulimit -v
+    var ulimit_prefix = "ulimit -v " + (ulimit / 1024);
+
+    String cmd = ProcessTools.getCommandLine(ProcessTools.createTestJvm(args.toArray(String[]::new)));
+    ProcessBuilder pb = new ProcessBuilder("sh", "-c", ulimit_prefix + ";" + cmd);
+    OutputAnalyzer output = new OutputAnalyzer(pb.start());
+    output.shouldHaveExitValue(0);
+    String stdout = output.getStdout();
+
+    boolean actualcoop = getFlagBoolValue("UseCompressedOops", stdout);
+    if (actualcoop != expectcoop) {
+      throw new RuntimeException("UseCompressedOops set to " + actualcoop +
+        ", expected " + expectcoop + " when running with the following flags: " + Arrays.asList(args).toString());
+    }
+  }
+
+  private static boolean getFlagBoolValue(String flag, String where) {
+    Matcher m = Pattern.compile(flag + "\\s+:?= (true|false)").matcher(where);
+    if (!m.find()) {
+      throw new RuntimeException("Could not find value for flag " + flag + " in output string");
+    }
+    return m.group(1).equals("true");
+  }
+
+  public static void main(String args[]) throws Exception {
+    // Tests
+    // Verify that UseCompressedOops Ergo follows ulimit -v setting.
+
+    long oneG = 1L * 1024L * 1024L * 1024L;
+
+    // Args: ulimit, max_ram, max_ram_percent, expected_coop
+    // Setting MaxRAMPercentage explicitly to make the test more resilient.
+    checkFlag(10 * oneG, 32 * oneG, 100, true);
+    checkFlag(10 * oneG, 128 * oneG, 100, true);
+  }
+}